kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,1984 @@
1
+ """Render an ExtractionBundle as a single, structured knowledge document.
2
+
3
+ Unlike the other renderers (which produce a multi-file bundle: obsidian note,
4
+ markdown notes, summary, skill package, JSON), this renderer asks MiniMax to
5
+ rewrite the raw transcript into ONE polished markdown file. It targets the
6
+ "give me a single, clean, knowledge document" workflow: no skill package
7
+ scaffolding, no per-atom JSON, no per-section files. Just the distilled
8
+ knowledge in the reader's preferred language.
9
+
10
+ Output: a single .md file with overview, core concepts, workflows, code
11
+ samples, configuration, and timestamp-anchored citations. Filler words
12
+ ("uh", "so", "you know", ...) are removed, fragmented segments are merged,
13
+ and the document is written in the configured language (Chinese or English).
14
+
15
+ Language selection (in priority order):
16
+ 1. Explicit ``YTK_DOC_LANGUAGE`` env var ("zh" or "en")
17
+ 2. The user's shell locale ($LANG / $LC_ALL / $LC_MESSAGES, sampled for
18
+ "zh" or "en")
19
+ 3. Default: "zh" (preserves v1.0.0 behavior for existing users)
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import hashlib
25
+ import json
26
+ import logging
27
+ import math
28
+ import os
29
+ import re
30
+ import tempfile
31
+ import threading
32
+ import time
33
+ from collections.abc import Callable
34
+ from contextlib import contextmanager
35
+ from dataclasses import dataclass
36
+ from pathlib import Path
37
+ from typing import Any, ClassVar, Literal, TypedDict
38
+
39
+ import httpx
40
+
41
+ from ytk.config import AppSettings
42
+ from ytk.renderers.citation_alignment import (
43
+ CitationAlignmentError,
44
+ validate_citation_alignment,
45
+ )
46
+
47
+ logger = logging.getLogger(__name__)
48
+
49
+ DocLanguage = Literal["zh", "en"]
50
+
51
+
52
+ class _ProgressBusy(RuntimeError):
53
+ """Another process currently owns this output's model dispatch."""
54
+
55
+
56
+ def complete_doc_status(output_dir: Path) -> bool:
57
+ """Check the status and exact knowledge.md bytes before reporting completion."""
58
+ try:
59
+ status = json.loads((output_dir / ".knowledge-doc-status.json").read_text(encoding="utf-8"))
60
+ output = output_dir / "knowledge.md"
61
+ return (status.get("schema") == 1 and status.get("state") == "COMPLETE"
62
+ and status.get("output_file") == output.name
63
+ and status.get("output_sha256") == hashlib.sha256(output.read_bytes()).hexdigest())
64
+ except (OSError, UnicodeError, ValueError, AttributeError, TypeError):
65
+ return False
66
+
67
+
68
+ class _SectionLabels(TypedDict):
69
+ """Localization labels for one output language."""
70
+
71
+ title_template: str
72
+ source: str
73
+ overview: str
74
+ no_transcript: str
75
+ topic_segments: str
76
+ knowledge_atoms: str
77
+ type_titles: dict[str, str]
78
+ full_transcript: str
79
+ overview_separator: str
80
+
81
+
82
+ # Mirror ytk.ingestion.providers.minimax._BASE_URLS so this renderer targets
83
+ # the same backend the configured region was issued for.
84
+ _MINIMAX_BASE_URLS: dict[str, str] = {
85
+ "cn": "https://api.minimaxi.com",
86
+ "global": "https://api.minimax.io",
87
+ }
88
+
89
+ # Same regex as ytk.ingestion.providers.minimax._THINK_RE so the leading
90
+ # `<think>...</think>` block the model emits in thinking mode is stripped
91
+ # before we write the document to disk.
92
+ _THINK_RE = re.compile(r"(?is)^\s*<think\b.*?</think\s*>\s*")
93
+
94
+
95
+ _PROMPT_TEMPLATES: dict[str, str] = {
96
+ "zh": """你是知识整理助手。下面是一个 YouTube 教程的转录(带时间戳,格式 `{ts_format} 文本`)。请把它改写成一篇干净、结构化、**中文**知识文档,直接保存为 markdown 文件使用。
97
+
98
+ # 视频元数据
99
+ - 标题: {title}
100
+ - URL: {url}
101
+ - 时长: {duration} 秒
102
+
103
+ # 转录(带时间戳)
104
+ {transcript}
105
+
106
+ # 要求
107
+ 1. **必须用中文输出**(专有名词如产品名、API、命令保留英文原文)。**专有名词必须逐字沿用转录里的原始拼写**(产品名、模型名、游戏名、数据集名、人名、API、命令、URL),转录怎么写就怎么写;**严禁**用同领域更知名的缩写或同类术语去"纠正"、"归一"或"补全"它(反例:转录里的 `Jev` 不得改写成 `JEPA`,转录里的 `howlight` 不得"更正"成别的游戏名)。拼写存疑时保留转录原样,并在该名词后用括号标注 `(拼写依转录)`,不要自行替换。
108
+ 2. **去口语化**:删除 "uh", "so", "you know", "ok", "right", "alright", "well", "I mean", "like", "basically", "let's go ahead and", "you know what", "kinda", "sort of" 等填充词和口头禅
109
+ 3. **整合碎片**:把同一主题的多个片段合并成连贯段落;不要按时间顺序罗列,要按主题组织
110
+ 4. **结构化**:使用以下 sections(按需保留相关 sections,去掉空 sections):
111
+ - `# 概述`(一段话总结视频讲什么、为什么重要)
112
+ - `## 核心概念`(关键技术名词 + 简短解释)
113
+ - `## 工作流程`(步骤化、清晰的指令,去掉所有口语)
114
+ - `## 代码与命令`(具体可执行的内容,标注 `bash` 代码块)
115
+ - `## 配置与设置`(配置文件、参数)
116
+ - `## 关键时间点`(标注 `{citation_format}` 时间戳,方便回看,列 5-10 个最重要的时刻)
117
+ 5. **不引入视频外的内容**——只整理视频里讲的
118
+ 6. **不使用 emoji**
119
+ 7. 引用关键事实时附 `{citation_format}` 时间戳,**写在括号内**而不是塞在句子里
120
+ 8. 全文使用中文标点(,。;:)而非英文标点
121
+ 9. 文档长度 1500-4000 中文字
122
+ 10. **直接输出 markdown**,不要加前言(如"以下是..."),不要加元评论
123
+ 11. {hour_rule}
124
+
125
+ # 输出格式
126
+ 直接以 `# 概述` 开头。
127
+ """,
128
+ "en": """You are a knowledge-curation assistant. Below is the transcript of a YouTube tutorial with timestamps in `{ts_format}` format. Please rewrite it into a clean, structured, **English** knowledge document, ready to be saved as a markdown file.
129
+
130
+ # Video metadata
131
+ - Title: {title}
132
+ - URL: {url}
133
+ - Duration: {duration} seconds
134
+
135
+ # Transcript (with timestamps)
136
+ {transcript}
137
+
138
+ # Requirements
139
+ 1. **Output MUST be in English** (keep product names, APIs, and commands in their original English form). **Preserve every proper noun exactly as spelled in the transcript** (product, model, game, dataset, and people names, APIs, commands, URLs). **Never** "correct", normalize, or expand a name into a more famous acronym or a similar term from the same field (counter-example: `Jev` in the transcript must NOT become `JEPA`). When the spelling is unclear, keep the transcript's form and append `(spelling per transcript)`.
140
+ 2. **Remove filler words**: drop "uh", "so", "you know", "ok", "right", "alright", "well", "I mean", "like", "basically", "let's go ahead and", "you know what", "kinda", "sort of", and similar verbal tics
141
+ 3. **Merge fragments**: combine multiple segments on the same topic into coherent paragraphs; organize by topic, not chronologically
142
+ 4. **Structure**: use the following sections (keep relevant ones, drop empty ones):
143
+ - `# Overview` (one paragraph: what the video covers and why it matters)
144
+ - `## Core Concepts` (key technical terms + short explanations)
145
+ - `## Workflow` (step-by-step instructions, no spoken-language fluff)
146
+ - `## Code & Commands` (concrete runnable content, marked with `bash` code blocks)
147
+ - `## Configuration` (config files, parameters)
148
+ - `## Key Moments` (annotate `{citation_format}` timestamps so the reader can jump back, list 5-10 most important moments)
149
+ 5. **Do not introduce content from outside the video** — only reorganize what was said
150
+ 6. **No emoji**
151
+ 7. When citing key facts, attach `{citation_format}` timestamps in parentheses, not inline in the sentence
152
+ 8. Use English punctuation (.,;:)
153
+ 9. Document length 1500-4000 English words
154
+ 10. **Output the markdown directly** — no preamble (e.g. "Below is..."), no meta-commentary
155
+ 11. {hour_rule}
156
+
157
+ # Output format
158
+ Start directly with `# Overview`.
159
+ """,
160
+ }
161
+
162
+
163
+ _HOUR_RULE_ZH_SHORT = (
164
+ "时间戳引证统一使用 `[mm:ss]` 格式(本视频不足 1 小时,无需小时位)"
165
+ )
166
+ _HOUR_RULE_ZH_LONG = (
167
+ "**时间戳硬约束:本视频时长超过 1 小时,所有时间戳引证必须写成 `[h:mm:ss]` 全局格式"
168
+ "(必须含小时位),严禁丢掉小时位(反例:把 1:02:01 写成 02:01);"
169
+ "输出前逐一自查每条时间戳都带小时位且为视频全局时间**"
170
+ )
171
+ _HOUR_RULE_EN_SHORT = (
172
+ "Cite timestamps as `[mm:ss]` (this video is under one hour; no hour component needed)"
173
+ )
174
+ _HOUR_RULE_EN_LONG = (
175
+ "**Hard timestamp rule: this video is longer than one hour, so every timestamp citation "
176
+ "must be written as a global `[h:mm:ss]` timestamp (hour component mandatory). Never drop "
177
+ "the hour (counter-example: 1:02:01 written as 02:01); self-check every timestamp before "
178
+ "finishing**"
179
+ )
180
+
181
+
182
+ # ---------------------------------------------------------------------------
183
+ # Reviewer-scaffolding block
184
+ #
185
+ # When the CLI is invoked with `--with-reviewer-scaffolding`, the renderer
186
+ # appends a deterministic footer to the knowledge doc so that downstream
187
+ # reviewers (multi-model 4-rubric scoring) see:
188
+ # 1. An A/B/C source list. A-class and C-class entries are populated from
189
+ # real bundle metadata (the source video URL is both the primary source
190
+ # and the project-experience source). B-class is an explicitly-marked
191
+ # empty placeholder, because the extractor cannot know which community
192
+ # sources a human editor will judge relevant.
193
+ # 2. A generic image-asset manifest template (cover / mindmap / flowchart /
194
+ # code image). The content column is a placeholder slot, NOT a
195
+ # topic-specific description — the extractor does not know the video's
196
+ # subject, so it must not invent one (e.g. it must not claim a "four
197
+ # primitives" cover for a cooking video).
198
+ # 3. A closing CTA that points the reader at the doc body (which always
199
+ # exists) rather than at a named section that may or may not be emitted
200
+ # by the LLM/fallback path.
201
+ #
202
+ # Content is bilingual-ready via the ``language`` arg (zh / en). The builder
203
+ # is a pure function of (language, title, source_url) — no LLM call, no
204
+ # datetime, no random — so reviewer scores are reproducible across runs.
205
+ # ---------------------------------------------------------------------------
206
+
207
+ # Marker used in the B-class list and the image-manifest content slots to
208
+ # make it visually obvious to a human editor (and to a downstream reviewer)
209
+ # that the slot is a template waiting to be populated, not real evidence.
210
+ _PLACEHOLDER_MARKER_ZH = "_(发布前请补充)_" # noqa: RUF001 - intentional wide underscore
211
+ _PLACEHOLDER_MARKER_EN = "_(populate before publishing)_"
212
+
213
+
214
+ def _a_class_entries_zh(title: str, source_url: str) -> str:
215
+ """A-class: the source video is itself a primary source. Always non-empty."""
216
+ label = title.strip() or "源视频"
217
+ url = source_url.strip() or "_(源 URL 缺失)_"
218
+ return f"- [{label}]({url}) — 视频本身就是一手素材(讲解者亲自演示)"
219
+
220
+
221
+ def _a_class_entries_en(title: str, source_url: str) -> str:
222
+ label = title.strip() or "Source video"
223
+ url = source_url.strip() or "_(source URL missing)_"
224
+ return f"- [{label}]({url}) — the video itself is the primary material (presenter's own demo)"
225
+
226
+
227
+ def _c_class_entries_zh(source_url: str) -> str:
228
+ """C-class: the video's demos ARE project experience. Always non-empty."""
229
+ url = source_url.strip()
230
+ if not url:
231
+ return "- _(源 URL 缺失)_ — 视频里呈现的实战演示属于 C 类(项目经验),可按时间戳标记"
232
+ # Use a descriptive label (not the bare URL) so the link renders cleanly
233
+ # even if the URL contains markdown-significant characters.
234
+ return f"- [视频作者的实战演示(C 类项目经验)]({url}) — 可按时间戳标记"
235
+
236
+
237
+ def _c_class_entries_en(source_url: str) -> str:
238
+ url = source_url.strip()
239
+ if not url:
240
+ return "- _(source URL missing)_ — demos shown in the video are C-class (project experience); tag with timestamps"
241
+ return f"- [Presenter's live demo (C-class project experience)]({url}) — tag with timestamps"
242
+
243
+
244
+ # The image manifest is intentionally generic. Each row's "content" column is
245
+ # a placeholder slot (the renderer does not know the video's subject), and the
246
+ # "renderer" column names the tool that would produce that asset type. A human
247
+ # editor (or a downstream image pipeline) replaces the placeholder text with
248
+ # topic-specific descriptions before publishing.
249
+ _IMAGE_MANIFEST_ROWS_ZH = (
250
+ "| 1 | `00-cover.png` | {_ph} 封面图 | drawio MCP / PIL |",
251
+ "| 2 | `01-mindmap-overview.png` | {_ph} 核心概念思维导图 | drawio MCP |",
252
+ "| 3 | `02-flowchart.png` | {_ph} 关键工作流图 | mmdc |",
253
+ "| 4 | `03-code-snippet.png` | {_ph} 关键代码或命令截图(若适用) | freeze + PIL |",
254
+ )
255
+ _IMAGE_MANIFEST_ROWS_EN = (
256
+ "| 1 | `00-cover.png` | {_ph} cover art | drawio MCP / PIL |",
257
+ "| 2 | `01-mindmap-overview.png` | {_ph} mindmap of core concepts | drawio MCP |",
258
+ "| 3 | `02-flowchart.png` | {_ph} key workflow diagram | mmdc |",
259
+ "| 4 | `03-code-snippet.png` | {_ph} key code/command screenshot (if applicable) | freeze + PIL |",
260
+ )
261
+
262
+
263
+ def _image_manifest_zh() -> str:
264
+ rows = "\n".join(row.format(_ph=_PLACEHOLDER_MARKER_ZH) for row in _IMAGE_MANIFEST_ROWS_ZH)
265
+ return "| # | 文件 | 内容 | 渲染方式 |\n| --- | --- | --- | --- |\n" + rows
266
+
267
+
268
+ def _image_manifest_en() -> str:
269
+ rows = "\n".join(row.format(_ph=_PLACEHOLDER_MARKER_EN) for row in _IMAGE_MANIFEST_ROWS_EN)
270
+ return "| # | File | Content | Renderer |\n| --- | --- | --- | --- |\n" + rows
271
+
272
+
273
+ _CTA_BODY_ZH = (
274
+ "1. 回到本文开头的概述,用一句话向同事复述这篇文档解决什么问题。\n"
275
+ "2. 从正文中挑一条最具体的命令或概念,在本地照做一次,记录实际输出。\n"
276
+ "3. 把你照做时遇到的偏差(命令不存在、参数变了、结果不一致)写回这份文档的复盘区。\n\n"
277
+ "完成这三步,这份文档就从『读过的』变成『跑通的』。\n"
278
+ )
279
+ _CTA_BODY_EN = (
280
+ "1. Re-read the overview at the top and explain in one sentence what this doc solves.\n"
281
+ "2. Pick the most concrete command or concept in the body, run it locally, and record the actual output.\n"
282
+ "3. Write any deviation you hit (missing command, changed flag, mismatched result) back into this doc.\n\n"
283
+ "After these three steps the doc has gone from 'read' to 'run'.\n"
284
+ )
285
+
286
+
287
+ def build_reviewer_scaffolding(
288
+ language: DocLanguage,
289
+ *,
290
+ title: str = "",
291
+ source_url: str = "",
292
+ ) -> str:
293
+ """Return the reviewer-scaffolding footer for the given language.
294
+
295
+ The footer is appended to the rendered knowledge doc. It is a pure
296
+ function of (language, title, source_url) — no LLM call, no datetime,
297
+ no random — so reviewer scoring is reproducible across runs.
298
+
299
+ Real metadata (``title``, ``source_url``) populates the A-class and
300
+ C-class source entries. B-class and image-manifest content slots are
301
+ explicitly-marked placeholders, because the extractor cannot know which
302
+ community sources a human editor will judge relevant, nor the video's
303
+ subject matter.
304
+ """
305
+ # Treat anything that is not explicitly "en" as Chinese (the documented
306
+ # default — matches detect_doc_language's fallback). This preserves the
307
+ # contract that unknown / "auto" / None → zh.
308
+ is_zh = language != "en"
309
+ parts: list[str] = []
310
+ # Leading blank line separates the footer from the body.
311
+ parts.append("")
312
+
313
+ # --- A / B / C source list ---
314
+ if is_zh:
315
+ parts.append("## 参考来源(A / B / C 三类)")
316
+ parts.append("视频本身就是一手素材;下列清单按来源可信度分三类,B 类由人工补充:")
317
+ else:
318
+ parts.append("## References (A / B / C classes)")
319
+ parts.append("The video itself is primary material; the list below splits sources by credibility. B-class is populated by a human editor:")
320
+ parts.append("")
321
+
322
+ if is_zh:
323
+ parts.append("### A 类:官方 / 一手")
324
+ parts.append(_a_class_entries_zh(title, source_url))
325
+ parts.append("")
326
+ parts.append("### B 类:社区 / 开源")
327
+ parts.append(f"- {_PLACEHOLDER_MARKER_ZH} 视频提到的第三方仓库、博文或工具链接")
328
+ parts.append("")
329
+ parts.append("### C 类:项目经验(视频作者演示)")
330
+ parts.append(_c_class_entries_zh(source_url))
331
+ else:
332
+ parts.append("### A class: Official / primary")
333
+ parts.append(_a_class_entries_en(title, source_url))
334
+ parts.append("")
335
+ parts.append("### B class: Community / OSS")
336
+ parts.append(f"- {_PLACEHOLDER_MARKER_EN} third-party repos, blog posts, or tools mentioned in the video")
337
+ parts.append("")
338
+ parts.append("### C class: Project experience (presenter's demo)")
339
+ parts.append(_c_class_entries_en(source_url))
340
+ parts.append("")
341
+
342
+ # --- Image-asset manifest (generic template) ---
343
+ if is_zh:
344
+ parts.append("## 配套图示(模板)")
345
+ parts.append("下列图示为通用模板,发布前请把「内容」列的占位符替换为本视频的具体描述:")
346
+ parts.append("")
347
+ parts.append(_image_manifest_zh())
348
+ else:
349
+ parts.append("## Companion visuals (template)")
350
+ parts.append("The visual plan below is a generic template; replace the Content-column placeholders with this video's specifics before publishing:")
351
+ parts.append("")
352
+ parts.append(_image_manifest_en())
353
+ parts.append("")
354
+
355
+ # --- Closing CTA (references the body, not a named section) ---
356
+ if is_zh:
357
+ parts.append("## 现在就开始")
358
+ parts.append(_CTA_BODY_ZH)
359
+ else:
360
+ parts.append("## Get started now")
361
+ parts.append(_CTA_BODY_EN)
362
+
363
+ return "\n".join(parts) + "\n"
364
+
365
+
366
+ _SECTION_TITLES: dict[str, _SectionLabels] = {
367
+ "zh": {
368
+ "title_template": "# {title}",
369
+ "source": "来源:{url}",
370
+ "overview": "## 概述",
371
+ "no_transcript": "(无转录内容)",
372
+ "topic_segments": "## 主题分段",
373
+ "knowledge_atoms": "## 提取的知识要点",
374
+ "type_titles": {
375
+ "core_concept": "核心概念",
376
+ "workflow_step": "工作流程",
377
+ "terminal_command": "命令",
378
+ "command_sequence": "命令序列",
379
+ "configuration_snippet": "配置",
380
+ "code_block": "代码块",
381
+ "best_practice": "最佳实践",
382
+ },
383
+ "full_transcript": "## 完整转录(去填充词)",
384
+ "overview_separator": "",
385
+ },
386
+ "en": {
387
+ "title_template": "# {title}",
388
+ "source": "Source: {url}",
389
+ "overview": "## Overview",
390
+ "no_transcript": "(no transcript content)",
391
+ "topic_segments": "## Topic Segments",
392
+ "knowledge_atoms": "## Extracted Knowledge Atoms",
393
+ "type_titles": {
394
+ "core_concept": "Core Concepts",
395
+ "workflow_step": "Workflow Steps",
396
+ "terminal_command": "Commands",
397
+ "command_sequence": "Command Sequences",
398
+ "configuration_snippet": "Configuration",
399
+ "code_block": "Code Blocks",
400
+ "best_practice": "Best Practices",
401
+ },
402
+ "full_transcript": "## Full Transcript (filler words removed)",
403
+ "overview_separator": "",
404
+ },
405
+ }
406
+
407
+
408
+ _TIMESTAMP_HEAD_RE = re.compile(r"^\[(\d{1,2}:\d{1,2}(?::\d{1,2})?)\]")
409
+
410
+
411
+ def _transcript_has_hour_anchor(transcript: str) -> bool:
412
+ """True when any transcript line is anchored at >=1h (the `[h:mm:ss]`
413
+ contract's single decision point)."""
414
+ for line in transcript.splitlines():
415
+ match = _TIMESTAMP_HEAD_RE.match(line.strip())
416
+ if not match:
417
+ continue
418
+ parts = [int(p) for p in match.group(1).split(":")]
419
+ seconds = (
420
+ parts[0] * 3600 + parts[1] * 60 + parts[2]
421
+ if len(parts) == 3
422
+ else parts[0] * 60 + parts[1]
423
+ )
424
+ if seconds >= 3600:
425
+ return True
426
+ return False
427
+
428
+
429
+ def _citation_format(expect_hours: bool) -> str:
430
+ return "[h:mm:ss]" if expect_hours else "[mm:ss]"
431
+
432
+
433
+ def _fmt_ts(seconds: float) -> str:
434
+ """Format seconds as [mm:ss] (or [h:mm:ss] for >1h)."""
435
+ total = int(seconds)
436
+ h, rem = divmod(total, 3600)
437
+ m, s = divmod(rem, 60)
438
+ if h > 0:
439
+ return f"{h}:{m:02d}:{s:02d}"
440
+ return f"{m:02d}:{s:02d}"
441
+
442
+
443
+ # ---------------------------------------------------------------------------
444
+ # Chunked rendering (roadmap 2026-10-06 §3.1.3)
445
+ #
446
+ # A >=1h transcript no longer goes through one MiniMax call (the 10/04
447
+ # kLZgQWjnUz0 single call hit the 16K-token output cap and dropped every
448
+ # hour component). Chunk boundary rule, locked by Q14c: real video chapters
449
+ # when available, fixed time windows otherwise. Embedding-derived topic
450
+ # blocks are never chunk boundaries (their boundary quality is unstable —
451
+ # the 10/04 final embedding block spanned 133s→14916s).
452
+ # ---------------------------------------------------------------------------
453
+
454
+ CHUNK_WINDOW_SECONDS = 1800.0 # 30-minute fixed windows when no chapters
455
+ BOUNDED_CHUNK_CHARS = 12000 # rendered transcript characters, not tokens
456
+ UNBOUND_BOUNDED_PLANNER = "bounded-transcript-v1"
457
+ BOUNDED_PLANNER = "bounded-transcript-v2"
458
+
459
+
460
+ @dataclass(frozen=True)
461
+ class ChunkWindow:
462
+ start: float
463
+ end: float
464
+ title: str
465
+
466
+
467
+ def valid_chapter_list(chapters: list | None) -> list[dict]:
468
+ """Normalize chapter dicts to usable {start,end,title} windows.
469
+
470
+ Idempotent: accepts both the raw yt-dlp spelling (start_time/end_time)
471
+ and its own output (start/end), so a caller that already validated the
472
+ list can pass it straight through (2026-10-06 review: the list was
473
+ computed twice with two independent code paths)."""
474
+ valid: list[dict] = []
475
+ for chapter in chapters or []:
476
+ if not isinstance(chapter, dict):
477
+ continue
478
+ start = chapter.get("start_time", chapter.get("start"))
479
+ end = chapter.get("end_time", chapter.get("end"))
480
+ title = chapter.get("title")
481
+ if (
482
+ not isinstance(start, int | float)
483
+ or not isinstance(end, int | float)
484
+ or not isinstance(title, str)
485
+ ):
486
+ continue
487
+ if start < 0 or end <= start:
488
+ continue
489
+ valid.append({"start": float(start), "end": float(end), "title": title})
490
+ return valid
491
+
492
+
493
+ def _generic_window_title(index: int, start: float, end: float, is_zh: bool) -> str:
494
+ if is_zh:
495
+ return f"第 {index} 段({_fmt_ts(start)}–{_fmt_ts(end)})"
496
+ return f"Part {index} ({_fmt_ts(start)}–{_fmt_ts(end)})"
497
+
498
+
499
+ def plan_chunk_windows(
500
+ chapters: list | None,
501
+ max_time: float,
502
+ *,
503
+ window_seconds: float = CHUNK_WINDOW_SECONDS,
504
+ language: str = "zh",
505
+ ) -> list[ChunkWindow]:
506
+ """Tile [0, max_time] with render windows: chapters when usable, else
507
+ fixed windows of ``window_seconds``. Windows never gap or overlap, so
508
+ every transcript segment lands in exactly one chunk (last segment at the
509
+ final boundary rides the last window)."""
510
+ is_zh = language != "en"
511
+ max_time = float(max_time)
512
+ valid = valid_chapter_list(chapters)
513
+ starts = sorted(
514
+ {c["start"] for c in valid if c["start"] < max_time}
515
+ )
516
+ if starts:
517
+ if starts[0] > 0:
518
+ starts.insert(0, 0.0)
519
+ title_by_start = {c["start"]: c["title"].strip() for c in valid}
520
+ windows: list[ChunkWindow] = []
521
+ for index, start in enumerate(starts):
522
+ end = starts[index + 1] if index + 1 < len(starts) else max(max_time, start + 1.0)
523
+ title = title_by_start.get(start) or _generic_window_title(
524
+ index + 1, start, end, is_zh
525
+ )
526
+ windows.append(ChunkWindow(start=start, end=end, title=title))
527
+ return windows
528
+
529
+ total = max(1, math.ceil(max_time / window_seconds)) if max_time > 0 else 1
530
+ return [
531
+ ChunkWindow(
532
+ start=i * window_seconds,
533
+ end=(i + 1) * window_seconds,
534
+ title=_generic_window_title(i + 1, i * window_seconds, (i + 1) * window_seconds, is_zh),
535
+ )
536
+ for i in range(total)
537
+ ]
538
+
539
+
540
+ def _dedupe_titles(titles: list[str], is_zh: bool) -> list[str]:
541
+ """Give duplicate headings a unique numeric suffix so the assembled doc
542
+ has no duplicated section headings."""
543
+ seen: dict[str, int] = {}
544
+ out: list[str] = []
545
+ for title in titles:
546
+ count = seen.get(title, 0) + 1
547
+ seen[title] = count
548
+ if count == 1:
549
+ out.append(title)
550
+ elif is_zh:
551
+ out.append(f"{title}({count})")
552
+ else:
553
+ out.append(f"{title} ({count})")
554
+ return out
555
+
556
+
557
+ _HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*$")
558
+
559
+
560
+ def _normalize_chunk_content(
561
+ content: str,
562
+ expected_title: str,
563
+ emitted_headings: set[str],
564
+ ) -> str:
565
+ """Normalize one chunk's LLM output to start with its ``## expected_title``
566
+ heading: keep a matching heading, drop one that duplicates an already
567
+ emitted heading (seam duplication), otherwise prepend the heading."""
568
+ lines = content.strip().splitlines()
569
+ first_idx = next((i for i, line in enumerate(lines) if line.strip()), None)
570
+ if first_idx is not None:
571
+ match = _HEADING_RE.match(lines[first_idx].strip())
572
+ if match:
573
+ heading_text = match.group(2).strip()
574
+ if heading_text == expected_title:
575
+ emitted_headings.add(expected_title)
576
+ return "\n".join(lines).strip()
577
+ if heading_text in emitted_headings:
578
+ # Seam duplication: drop the echoed heading, fall through to
579
+ # prepend the chunk's own heading.
580
+ del lines[first_idx]
581
+ else:
582
+ # Unexpected heading: keep it below the expected heading.
583
+ lines.insert(first_idx, f"## {expected_title}")
584
+ emitted_headings.add(expected_title)
585
+ return "\n".join(lines).strip()
586
+ emitted_headings.add(expected_title)
587
+ return "\n".join([f"## {expected_title}", *lines]).strip()
588
+
589
+
590
+ def assemble_chunked_doc(
591
+ *,
592
+ title: str,
593
+ url: str,
594
+ windows: list[ChunkWindow],
595
+ contents: list[str],
596
+ language: str,
597
+ boundary_source: str,
598
+ citation_format: str,
599
+ final_titles: list[str] | None = None,
600
+ ) -> str:
601
+ """Assemble validated chunk outputs into one knowledge document with a
602
+ deterministic overview map and a machine-readable provenance comment."""
603
+ is_zh = language != "en"
604
+ labels = _SECTION_TITLES[language]
605
+ if final_titles is None:
606
+ final_titles = _dedupe_titles([w.title for w in windows], is_zh)
607
+ parts: list[str] = [labels["title_template"].format(title=title), ""]
608
+ if url:
609
+ parts += [labels["source"].format(url=url), ""]
610
+ parts += [
611
+ (
612
+ f"<!-- ytk-knowledge-doc: chunked; boundary={boundary_source}; "
613
+ f"chunks={len(windows)}; citation_format={citation_format} -->"
614
+ ),
615
+ "",
616
+ labels["overview"],
617
+ "",
618
+ ]
619
+ # strict=False on both zips below: final_titles/contents are caller contracts
620
+ # (1:1 with windows at every in-repo call site); strict=True would raise on a
621
+ # length mismatch where the previous code truncated.
622
+ for window, final_title in zip(windows, final_titles, strict=False):
623
+ if is_zh:
624
+ parts.append(f"- {final_title}({_fmt_ts(window.start)}–{_fmt_ts(window.end)})")
625
+ else:
626
+ parts.append(f"- {final_title} ({_fmt_ts(window.start)}–{_fmt_ts(window.end)})")
627
+ parts.append("")
628
+ emitted: set[str] = set()
629
+ for _window, final_title, content in zip(windows, final_titles, contents, strict=False):
630
+ parts.append(
631
+ _normalize_chunk_content(content, final_title, emitted)
632
+ )
633
+ parts.append("")
634
+ return "\n".join(parts).rstrip()
635
+
636
+
637
+ def _strip_leading_heading(content: str) -> str:
638
+ lines = content.strip().splitlines()
639
+ first_idx = next((i for i, line in enumerate(lines) if line.strip()), None)
640
+ if first_idx is not None and _HEADING_RE.match(lines[first_idx].strip()):
641
+ del lines[first_idx]
642
+ return "\n".join(lines)
643
+
644
+
645
+ def verify_chunk_assembly(
646
+ assembled: str,
647
+ windows: list[ChunkWindow],
648
+ contents: list[str],
649
+ *,
650
+ language: str = "zh",
651
+ final_titles: list[str] | None = None,
652
+ ) -> list[str]:
653
+ """Seam-consistency check (acceptance §3.1 ⑤): every chunk body must
654
+ appear exactly once and every expected heading exactly once.
655
+
656
+ ``final_titles`` lets a caller that already de-duplicated the headings
657
+ hand the SAME list to assembler and verifier — recomputing it here
658
+ would let the two drift (the check would then validate headings that
659
+ were never written; 2026-10-06 review finding)."""
660
+ is_zh = language != "en"
661
+ if final_titles is None:
662
+ final_titles = _dedupe_titles([w.title for w in windows], is_zh)
663
+ problems: list[str] = []
664
+ # strict=False: a caller-supplied final_titles may be shorter than contents;
665
+ # the verifier must report the gap as a problem instead of raising.
666
+ for index, (content, final_title) in enumerate(
667
+ zip(contents, final_titles, strict=False), start=1
668
+ ):
669
+ body = _strip_leading_heading(content).strip()
670
+ if not body:
671
+ problems.append(f"chunk {index} ({final_title}): empty content")
672
+ continue
673
+ occurrences = assembled.count(body)
674
+ if occurrences == 0:
675
+ problems.append(f"chunk {index} ({final_title}): content missing from assembled doc")
676
+ elif occurrences > 1:
677
+ problems.append(
678
+ f"chunk {index} ({final_title}): content duplicated {occurrences}x"
679
+ )
680
+ heading = f"## {final_title}"
681
+ heading_count = len(
682
+ re.findall(f"^{re.escape(heading)}$", assembled, flags=re.MULTILINE)
683
+ )
684
+ if heading_count != 1:
685
+ problems.append(
686
+ f"chunk {index}: heading '{heading}' present {heading_count}x (expected 1)"
687
+ )
688
+ return problems
689
+
690
+
691
+ def detect_doc_language(explicit: str | None = None) -> DocLanguage:
692
+ """Resolve the doc output language.
693
+
694
+ Resolution order (highest priority first):
695
+ 1. ``explicit`` argument (typically the ``YTK_DOC_LANGUAGE`` env var
696
+ value, or a constructor argument the CLI passed in).
697
+ 2. Shell locale: $LC_MESSAGES → $LC_ALL → $LANG. If any of these
698
+ contains "zh" we return "zh"; if it contains "en" we return "en".
699
+ (Locale strings look like ``zh_CN.UTF-8`` or ``en_US.UTF-8``.)
700
+ 3. Default: "zh" (preserves v1.0.0 behavior for existing users).
701
+
702
+ Only the canonical codes "zh" and "en" are accepted. Any other explicit
703
+ value is logged and treated as "auto" (fall through to locale probing).
704
+ """
705
+ if explicit:
706
+ candidate = explicit.strip().lower()
707
+ if candidate in ("zh", "en"):
708
+ return candidate # type: ignore[return-value]
709
+ if candidate != "auto":
710
+ logger.warning(
711
+ "YTK_DOC_LANGUAGE=%r is not one of {zh, en, auto}; ignoring.",
712
+ explicit,
713
+ )
714
+
715
+ for env_var in ("LC_MESSAGES", "LC_ALL", "LANG"):
716
+ value = os.environ.get(env_var, "")
717
+ if not value:
718
+ continue
719
+ normalized = value.lower().replace("-", "_")
720
+ if normalized.startswith("zh"):
721
+ return "zh"
722
+ if normalized.startswith("en"):
723
+ return "en"
724
+ return "zh"
725
+
726
+
727
+
728
+ # ---------------------------------------------------------------------------
729
+ # v2 knowledge-doc extension (T33.3)
730
+ #
731
+ # The v1 path above stays byte-identical. v2 bundles are planned from their
732
+ # ContentBundle blocks instead of raw-transcript.json:
733
+ # * video -> time-range blocks become (start, text) segments and reuse
734
+ # the existing bounded time chunker unchanged;
735
+ # * image_note -> blocks are grouped in reading order under the same
736
+ # character budget, with no block dropped or duplicated, and
737
+ # citations use 图 N / 第 N 段 (never a timestamp, A22).
738
+ # Prompt classification (tutorial / general knowledge / image note) is applied
739
+ # through ``_classification_suffix``; with no v2 render in flight it returns ""
740
+ # so every v1 prompt and checkpoint stays byte-identical.
741
+ # ---------------------------------------------------------------------------
742
+
743
+ IMAGE_NOTE_PLANNER = "image-note-chunk-v1"
744
+
745
+ #: Atom types that make a source actionable enough to drive a tutorial doc.
746
+ ACTIONABLE_ATOM_TYPES: frozenset[str] = frozenset(
747
+ {
748
+ "terminal_command",
749
+ "command_sequence",
750
+ "configuration_snippet",
751
+ "code_block",
752
+ "workflow_step",
753
+ }
754
+ )
755
+
756
+ DocKind = Literal["tutorial", "general", "image_note"]
757
+
758
+ _IMAGE_CITATION_FORMAT: dict[str, str] = {
759
+ "zh": "[图 N] / [第 N 段]",
760
+ "en": "[Image N] / [Paragraph N]",
761
+ }
762
+
763
+ _IMAGE_NOTE_TEMPLATES: dict[str, str] = {
764
+ "zh": """你是知识整理助手。下面是一篇**图文笔记**(来自小红书等平台)的内容块,按原帖阅读顺序排列,每块以定位标记开头(格式 `[图 N]` 或 `[第 N 段]`)。请把它改写成一篇干净、结构化的**中文**知识文档,直接保存为 markdown 文件使用。
765
+
766
+ # 笔记元数据
767
+ - 标题: {title}
768
+ - URL: {url}
769
+
770
+ # 内容块(按原帖顺序)
771
+ {blocks}
772
+
773
+ # 要求
774
+ 1. **必须用中文输出**(专有名词保留原文)。
775
+ 2. 按图片与正文的原始顺序组织内容,不要打乱图片顺序。
776
+ 3. 引用内容时使用 `{citation_format}` 形式的定位标记,写在括号内;**禁止出现任何时间戳**(图文没有时间轴)。
777
+ 4. 不引入笔记之外的内容;没有操作步骤时不要编造步骤。
778
+ 5. 不使用 emoji;直接输出 markdown,不要加前言。
779
+
780
+ # 输出格式
781
+ 直接以 `# 概述` 开头。
782
+ """,
783
+ "en": """You are a knowledge-curation assistant. Below is an **image note** (e.g. a Xiaohongshu post) given as content blocks in the original reading order; each block starts with its locator marker (`[Image N]` or `[Paragraph N]`). Rewrite it into a clean, structured **English** markdown knowledge document.
784
+
785
+ # Note metadata
786
+ - Title: {title}
787
+ - URL: {url}
788
+
789
+ # Content blocks (original order)
790
+ {blocks}
791
+
792
+ # Requirements
793
+ 1. **Output MUST be in English** (keep proper nouns as written).
794
+ 2. Keep the original image/paragraph order; never reorder images.
795
+ 3. Cite with `{citation_format}` locator markers in parentheses; **never emit a timestamp** (an image note has no timeline).
796
+ 4. Do not introduce content from outside the note; never invent steps when none exist.
797
+ 5. No emoji; output markdown directly with no preamble.
798
+
799
+ # Output format
800
+ Start directly with `# Overview`.
801
+ """,
802
+ }
803
+
804
+
805
+ @dataclass(frozen=True)
806
+ class V2DocSource:
807
+ """Normalized v2 input for the knowledge-doc renderer."""
808
+
809
+ kind: Literal["video", "image_note"]
810
+ blocks: list[Any]
811
+ segments: list[tuple[float, str]]
812
+ metadata: dict[str, Any]
813
+ primary: Any = None
814
+ content: Any = None
815
+
816
+
817
+ @dataclass(frozen=True)
818
+ class ImageNoteChunk:
819
+ """One bounded image-note plan unit (blocks keep their reading order)."""
820
+
821
+ index: int
822
+ block_ids: tuple[str, ...]
823
+ lines: tuple[str, ...]
824
+
825
+
826
+ @dataclass(frozen=True)
827
+ class _VideoDocShim:
828
+ """Minimal v1-compatible view for the existing LLM doc pipeline."""
829
+
830
+ metadata: dict[str, Any]
831
+ video_id: str
832
+
833
+
834
+ def is_v2_bundle(bundle: Any) -> bool:
835
+ """True for an :class:`ExtractionBundleV2` (v2 render path)."""
836
+ from ytk.models.content import ExtractionBundleV2
837
+
838
+ return isinstance(bundle, ExtractionBundleV2)
839
+
840
+
841
+ def classify_doc_kind(bundle: Any) -> DocKind:
842
+ """Classify the source for prompt selection: tutorial / general / image note."""
843
+ content_types = {
844
+ content.source.content_type for content in getattr(bundle, "contents", [])
845
+ }
846
+ if "image_note" in content_types:
847
+ return "image_note"
848
+ atom_types = {atom.atom_type for atom in getattr(bundle, "knowledge", [])}
849
+ if atom_types & ACTIONABLE_ATOM_TYPES:
850
+ return "tutorial"
851
+ return "general"
852
+
853
+
854
+ def image_note_block_line(block: Any, *, language: str = "zh") -> str:
855
+ """One image-note block as ``[图 N] text`` / ``[第 N 段] text``."""
856
+ from ytk.renderers.source_links import readable_locator
857
+
858
+ text = (block.cleaned_text or block.raw_text).strip()
859
+ marker = readable_locator(block.locator, language=language)
860
+ return f"[{marker}] {text}" if marker else text
861
+
862
+
863
+ def plan_image_note_chunks(
864
+ blocks: list[Any], *, max_chars: int = BOUNDED_CHUNK_CHARS, language: str = "zh"
865
+ ) -> list[ImageNoteChunk]:
866
+ """Group image-note blocks in order under a character budget.
867
+
868
+ Blocks are never split, dropped or duplicated: every input block lands in
869
+ exactly one chunk, in its original order. A single block larger than the
870
+ budget fails *before* any paid dispatch (same contract as the transcript
871
+ planner).
872
+ """
873
+ chunks: list[ImageNoteChunk] = []
874
+ current_ids: list[str] = []
875
+ current_lines: list[str] = []
876
+ size = 0
877
+ for block in blocks:
878
+ line = image_note_block_line(block, language=language)
879
+ if len(line) > max_chars:
880
+ raise RuntimeError("an image-note block exceeds the bounded chunk size")
881
+ if current_lines and size + len(line) + 1 > max_chars:
882
+ chunks.append(
883
+ ImageNoteChunk(len(chunks) + 1, tuple(current_ids), tuple(current_lines))
884
+ )
885
+ current_ids, current_lines, size = [], [], 0
886
+ current_ids.append(block.block_id)
887
+ current_lines.append(line)
888
+ size += len(line) + 1
889
+ if current_lines:
890
+ chunks.append(ImageNoteChunk(len(chunks) + 1, tuple(current_ids), tuple(current_lines)))
891
+ return chunks
892
+
893
+
894
+ def verify_image_note_plan(chunks: list[ImageNoteChunk], blocks: list[Any]) -> list[str]:
895
+ """Check the plan covers every block exactly once, in order."""
896
+ problems: list[str] = []
897
+ planned = [block_id for chunk in chunks for block_id in chunk.block_ids]
898
+ expected = [block.block_id for block in blocks]
899
+ if planned != expected:
900
+ problems.append(
901
+ f"image-note plan covers blocks {planned!r}, expected {expected!r} "
902
+ "(dropped, duplicated or reordered blocks)"
903
+ )
904
+ return problems
905
+
906
+
907
+ def _image_chunk_title(index: int, language: str) -> str:
908
+ if language == "en":
909
+ return f"Part {index}"
910
+ return f"第 {index} 部分"
911
+
912
+
913
+ def assemble_image_note_doc(
914
+ *,
915
+ title: str,
916
+ url: str,
917
+ chunks: list[ImageNoteChunk],
918
+ contents: list[str],
919
+ language: str,
920
+ final_titles: list[str] | None = None,
921
+ ) -> str:
922
+ """Assemble validated image-note chunk outputs (no timestamps anywhere)."""
923
+ is_zh = language != "en"
924
+ labels = _SECTION_TITLES[language]
925
+ if final_titles is None:
926
+ final_titles = _dedupe_titles(
927
+ [_image_chunk_title(chunk.index, language) for chunk in chunks], is_zh
928
+ )
929
+ parts: list[str] = [labels["title_template"].format(title=title), ""]
930
+ if url:
931
+ parts += [labels["source"].format(url=url), ""]
932
+ parts += [
933
+ (
934
+ f"<!-- ytk-knowledge-doc: image-note; chunks={len(chunks)}; "
935
+ f"citation_format={_IMAGE_CITATION_FORMAT[language]} -->"
936
+ ),
937
+ "",
938
+ labels["overview"],
939
+ "",
940
+ ]
941
+ # strict=False: final_titles/contents are caller contracts (1:1 with chunks at
942
+ # every in-repo call site); strict=True would raise where the previous code
943
+ # truncated.
944
+ for chunk, final_title in zip(chunks, final_titles, strict=False):
945
+ if is_zh:
946
+ parts.append(f"- {final_title}({len(chunk.block_ids)} 个内容块)")
947
+ else:
948
+ parts.append(f"- {final_title} ({len(chunk.block_ids)} content blocks)")
949
+ parts.append("")
950
+ emitted: set[str] = set()
951
+ for final_title, content in zip(final_titles, contents, strict=False):
952
+ parts.append(_normalize_chunk_content(content, final_title, emitted))
953
+ parts.append("")
954
+ return "\n".join(parts).rstrip()
955
+
956
+
957
+ class KnowledgeDocRenderer:
958
+ """Rewrite a transcript into a single knowledge document via MiniMax.
959
+
960
+ Output language is selectable (zh/en). See module docstring for the
961
+ resolution order.
962
+ """
963
+
964
+ SUPPORTED_LANGUAGES: ClassVar[tuple[str, ...]] = ("zh", "en")
965
+
966
+ def __init__(self, language: str | None = None, *, settings: AppSettings | None = None) -> None:
967
+ self._api_key: str | None = settings.minimax_api_key if settings is not None else os.environ.get("YTK_MINIMAX_API_KEY") or None
968
+ self._region: str = settings.minimax_region if settings is not None else os.environ.get("YTK_MINIMAX_REGION", "cn")
969
+ self._model: str = settings.minimax_model if settings is not None else os.environ.get("YTK_MINIMAX_MODEL", "MiniMax-M3")
970
+ self._base_url: str = _MINIMAX_BASE_URLS.get(self._region, _MINIMAX_BASE_URLS["cn"])
971
+ self._timeout: float = settings.knowledge_doc_timeout if settings is not None else float(os.environ.get("YTK_KNOWLEDGE_DOC_TIMEOUT", "300.0"))
972
+ self._total_timeout: float = float(os.environ.get("YTK_KNOWLEDGE_DOC_TOTAL_TIMEOUT", "1200.0"))
973
+ if not math.isfinite(self._total_timeout) or self._total_timeout <= 0:
974
+ raise ValueError("YTK_KNOWLEDGE_DOC_TOTAL_TIMEOUT must be positive and finite")
975
+ self._checkpoint_path: Path | None = None
976
+ self._deadline: float | None = None
977
+ self._doc_kind: DocKind | None = None
978
+ self._instance_lock = threading.Lock()
979
+ # Allow longer outputs than the ingestion path — the knowledge doc is
980
+ # often the only artifact the user keeps, so trimming it hurts.
981
+ self._max_tokens: int = settings.knowledge_doc_max_tokens if settings is not None else int(os.environ.get("YTK_KNOWLEDGE_DOC_MAX_TOKENS", "16384"))
982
+ # Resolve language: explicit ctor arg wins; otherwise YTK_DOC_LANGUAGE
983
+ # env var, otherwise locale probe, otherwise "zh".
984
+ explicit_env = settings.doc_language if settings is not None else os.environ.get("YTK_DOC_LANGUAGE")
985
+ self._language: DocLanguage = detect_doc_language(language or explicit_env)
986
+
987
+ @property
988
+ def language(self) -> DocLanguage:
989
+ """Return the resolved output language (zh or en)."""
990
+ return self._language
991
+
992
+ def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
993
+ if not self._instance_lock.acquire(blocking=False):
994
+ raise _ProgressBusy("knowledge-doc renderer is already in use")
995
+ try:
996
+ output_path.parent.mkdir(parents=True, exist_ok=True)
997
+ self._checkpoint_path = output_path.parent / ".knowledge-doc-progress.json"
998
+ with self._progress_lock(self._checkpoint_path):
999
+ self._render_locked(bundle, output_path, **kwargs)
1000
+ finally:
1001
+ self._instance_lock.release()
1002
+
1003
+ def _render_locked(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
1004
+ status_path = output_path.parent / ".knowledge-doc-status.json"
1005
+ self._write_progress(status_path, {"schema": 1, "state": "RUNNING"})
1006
+ if is_v2_bundle(bundle):
1007
+ self._render_locked_v2(bundle, output_path, status_path, **kwargs)
1008
+ return
1009
+ segments = self._load_transcript_segments(output_path.parent / "raw-transcript.json")
1010
+ # Build the transcript text ONCE (it was re-parsed from disk up to
1011
+ # three times per render — 2026-10-06 review finding).
1012
+ transcript = self._transcript_text(segments)
1013
+ reason: str | None = None
1014
+ if not self._api_key:
1015
+ # No LLM key — fall back to a deterministic, no-fluff document
1016
+ # built from the raw transcript and the bundle's existing
1017
+ # knowledge atoms. Useful for offline demos / fixture runs.
1018
+ logger.warning(
1019
+ "YTK_MINIMAX_API_KEY not set; KnowledgeDocRenderer is falling back "
1020
+ "to a deterministic document built from the raw transcript. "
1021
+ "Set YTK_MINIMAX_API_KEY in .env for the LLM-rewritten version."
1022
+ )
1023
+ content = self._fallback_doc(bundle, transcript)
1024
+ reason = "NO_API_KEY"
1025
+ else:
1026
+ try:
1027
+ content = self._render_llm_document(bundle, segments)
1028
+ except RuntimeError as exc:
1029
+ # LLM failure OR citation-alignment failure: the deterministic
1030
+ # fallback keeps the doc usable, and the warning keeps the
1031
+ # degradation non-silent (roadmap §3.1.2: misaligned LLM
1032
+ # output must never land as-is).
1033
+ logger.warning(
1034
+ "MiniMax knowledge-doc rewrite rejected; falling back to the "
1035
+ "deterministic document: %s",
1036
+ exc,
1037
+ )
1038
+ content = self._fallback_doc(bundle, transcript)
1039
+ message = str(exc)
1040
+ if message == "knowledge-doc total time budget exhausted":
1041
+ reason = "TOTAL_TIMEOUT"
1042
+ elif message == "knowledge-doc chunk has an unresolved or invalid prior dispatch":
1043
+ reason = "UNRESOLVED_OR_INVALID_DISPATCH"
1044
+ elif message == "knowledge-doc progress cannot be safely reused":
1045
+ reason = "PROGRESS_MISMATCH"
1046
+ else:
1047
+ reason = "REWRITE_FAILED"
1048
+ try:
1049
+ checkpoint_path = self._checkpoint_path
1050
+ if checkpoint_path is None: # defensive: no checkpoint on this path
1051
+ raise OSError("knowledge-doc checkpoint path is not configured")
1052
+ progress = json.loads(checkpoint_path.read_text(encoding="utf-8"))
1053
+ chunks = progress.get("chunks", [])
1054
+ if isinstance(chunks, list) and chunks and isinstance(chunks[-1], dict) and chunks[-1].get("state") == "DISPATCHING":
1055
+ reason = "TOTAL_TIMEOUT_WITH_UNRESOLVED_DISPATCH" if reason == "TOTAL_TIMEOUT" else "UNRESOLVED_OR_INVALID_DISPATCH"
1056
+ except (OSError, ValueError, TypeError, AttributeError):
1057
+ pass
1058
+ finally:
1059
+ self._deadline = None
1060
+
1061
+ if kwargs.get("with_reviewer_scaffolding"):
1062
+ # Pull real metadata from the bundle so A-class and C-class source
1063
+ # entries are actual citations (the source video URL), not
1064
+ # placeholder text. B-class and the image manifest stay as
1065
+ # explicitly-marked template slots.
1066
+ metadata = getattr(bundle, "metadata", None) or {}
1067
+ title = str(metadata.get("title", "") or "")
1068
+ source_url = str(metadata.get("canonical_url", "") or "")
1069
+ content = (
1070
+ content.rstrip()
1071
+ + "\n"
1072
+ + build_reviewer_scaffolding(
1073
+ self._language,
1074
+ title=title,
1075
+ source_url=source_url,
1076
+ )
1077
+ )
1078
+
1079
+ if reason:
1080
+ content = f"<!-- ytk-knowledge-doc: fallback; reason={reason} -->\n\n" + content
1081
+ rendered = content.strip() + "\n"
1082
+ destination = output_path
1083
+ if reason and output_path.exists():
1084
+ destination = output_path.with_name(output_path.stem + ".fallback" + output_path.suffix)
1085
+ self._write_text_atomic(destination, rendered)
1086
+ self._write_progress(status_path, {"schema": 1, "state": "FALLBACK" if reason else "COMPLETE",
1087
+ "reason": reason, "output_file": destination.name,
1088
+ "output_sha256": hashlib.sha256(rendered.encode()).hexdigest()})
1089
+
1090
+ def _fallback_doc(self, bundle: Any, transcript: str) -> str:
1091
+ """Build a deterministic knowledge document without LLM.
1092
+
1093
+ Strips common fillers from the raw transcript, lists each topic
1094
+ segment with its first sentence, and dumps the surviving knowledge
1095
+ atoms grouped by type. This is a degraded mode — the LLM-rewritten
1096
+ path produces a far better document, but this fallback keeps
1097
+ `ytk extract --format doc` working in offline / fixture scenarios
1098
+ where no API key is available.
1099
+
1100
+ Output language follows ``self._language``.
1101
+ """
1102
+ labels = _SECTION_TITLES[self._language]
1103
+ title = str(bundle.metadata.get("title", bundle.video_id))
1104
+ url = str(bundle.metadata.get("canonical_url", ""))
1105
+
1106
+ filler_re = re.compile(
1107
+ r"\b(?:uh+|um+|ah+|er+|hmm|"
1108
+ r"so|so basically|basically|"
1109
+ r"you know|i mean|like|"
1110
+ r"right|alright|okay|ok|"
1111
+ r"well|honestly|literally|"
1112
+ r"kinda|sort of|"
1113
+ r"and then|so then|"
1114
+ r"let's go ahead and|now let's)\b[,]?",
1115
+ re.IGNORECASE,
1116
+ )
1117
+
1118
+ cleaned_lines: list[str] = []
1119
+ for line in transcript.splitlines():
1120
+ stripped = filler_re.sub("", line).strip()
1121
+ stripped = re.sub(r"\s{2,}", " ", stripped)
1122
+ # Strip leading/trailing punctuation left over from filler removal.
1123
+ stripped = re.sub(r"^[\s,.;:]+", "", stripped)
1124
+ stripped = re.sub(r"[\s,.;:]+$", "", stripped)
1125
+ if stripped:
1126
+ cleaned_lines.append(stripped)
1127
+
1128
+ sections: list[str] = [labels["title_template"].format(title=title), ""]
1129
+ if url:
1130
+ sections.append(labels["source"].format(url=url))
1131
+ sections.append("")
1132
+
1133
+ sections.append(labels["overview"])
1134
+ sections.append("")
1135
+ overview = " ".join(cleaned_lines[:3]).strip() or labels["no_transcript"]
1136
+ sections.append(overview)
1137
+ sections.append("")
1138
+
1139
+ if bundle.topic_segments:
1140
+ sections.append(labels["topic_segments"])
1141
+ sections.append("")
1142
+ for topic in bundle.topic_segments:
1143
+ first_sentence = topic.sentences[0] if topic.sentences else ""
1144
+ sections.append(
1145
+ f"- `{topic.title}` ({topic.start:.0f}s-{topic.end:.0f}s)"
1146
+ + (f" — {first_sentence}" if first_sentence else "")
1147
+ )
1148
+ sections.append("")
1149
+
1150
+ if bundle.knowledge_atoms:
1151
+ sections.append(labels["knowledge_atoms"])
1152
+ sections.append("")
1153
+ by_type: dict[str, list] = {}
1154
+ for atom in bundle.knowledge_atoms:
1155
+ by_type.setdefault(atom.atom_type, []).append(atom)
1156
+ for atom_type, atoms in by_type.items():
1157
+ localized = labels["type_titles"].get(atom_type, atom_type)
1158
+ sections.append(f"### {localized}")
1159
+ sections.append("")
1160
+ for atom in atoms:
1161
+ evidence = atom.evidence[0] if atom.evidence else None
1162
+ cite = (
1163
+ f" `[{_fmt_ts(evidence.start)}]`"
1164
+ if evidence is not None
1165
+ else ""
1166
+ )
1167
+ sections.append(f"- {atom.content}{cite}")
1168
+ sections.append("")
1169
+
1170
+ sections.append(labels["full_transcript"])
1171
+ sections.append("")
1172
+ sections.extend(cleaned_lines)
1173
+
1174
+ return "\n".join(sections)
1175
+
1176
+ @staticmethod
1177
+ def _load_transcript_segments(raw_path: Path) -> list[tuple[float, str]]:
1178
+ """Load raw-transcript.json as (start_seconds, text) pairs."""
1179
+ if not raw_path.exists():
1180
+ logger.warning("raw-transcript.json not found at %s; emitting empty doc", raw_path)
1181
+ return []
1182
+ try:
1183
+ payload = json.loads(raw_path.read_text(encoding="utf-8"))
1184
+ except json.JSONDecodeError as exc:
1185
+ logger.warning("Failed to parse raw-transcript.json: %s", exc)
1186
+ return []
1187
+ segments = payload.get("segments") if isinstance(payload, dict) else payload
1188
+ if not isinstance(segments, list):
1189
+ return []
1190
+ loaded: list[tuple[float, str]] = []
1191
+ for seg in segments:
1192
+ if not isinstance(seg, dict):
1193
+ continue
1194
+ text = str(seg.get("text", "")).strip()
1195
+ if not text:
1196
+ continue
1197
+ loaded.append((float(seg.get("start", 0.0)), text))
1198
+ return loaded
1199
+
1200
+ @staticmethod
1201
+ def _transcript_text(segments: list[tuple[float, str]]) -> str:
1202
+ """Render loaded segments as `[mm:ss] text` lines. Timestamps use
1203
+ `_fmt_ts`, so transcripts of >=1h videos already carry `[h:mm:ss]`
1204
+ anchors regardless of the prompt format contract."""
1205
+ return "\n".join(
1206
+ f"[{_fmt_ts(start)}] {text}" for start, text in segments
1207
+ )
1208
+
1209
+ @classmethod
1210
+ def _load_transcript(cls, raw_path: Path) -> str:
1211
+ """Concatenate raw-transcript.json segments with timestamp prefixes."""
1212
+ return cls._transcript_text(cls._load_transcript_segments(raw_path))
1213
+
1214
+ def _build_prompt(
1215
+ self, bundle: Any, transcript: str, *, expect_hours: bool | None = None
1216
+ ) -> str:
1217
+ title = str(bundle.metadata.get("title", bundle.video_id))
1218
+ url = str(bundle.metadata.get("canonical_url", ""))
1219
+ duration = bundle.metadata.get("duration")
1220
+ if duration is None:
1221
+ # fall back to last segment end if available
1222
+ duration = "未知" if self._language == "zh" else "unknown"
1223
+ # Dynamic citation format (roadmap §3.1.1): >=1h videos cite [h:mm:ss].
1224
+ # Single source of truth for the decision (the transcript's own
1225
+ # `_fmt_ts` anchors are authoritative; metadata duration is the
1226
+ # hint). The render path already computed it from the segments and
1227
+ # passes it in so the two can never disagree.
1228
+ if expect_hours is None:
1229
+ expect_hours = _transcript_has_hour_anchor(transcript)
1230
+ citation_format = _citation_format(expect_hours)
1231
+ if self._language == "zh":
1232
+ hour_rule = _HOUR_RULE_ZH_LONG if expect_hours else _HOUR_RULE_ZH_SHORT
1233
+ else:
1234
+ hour_rule = _HOUR_RULE_EN_LONG if expect_hours else _HOUR_RULE_EN_SHORT
1235
+ template = _PROMPT_TEMPLATES[self._language]
1236
+ prompt = template.format(
1237
+ title=title,
1238
+ url=url,
1239
+ duration=duration,
1240
+ transcript=transcript,
1241
+ ts_format=citation_format,
1242
+ citation_format=citation_format,
1243
+ hour_rule=hour_rule,
1244
+ )
1245
+ # v2 prompt classification; "" for every v1 render, so existing
1246
+ # prompts and checkpoints stay byte-identical.
1247
+ return prompt + self._classification_suffix()
1248
+
1249
+ def _chunk_scope_block(
1250
+ self, index: int, total: int, window: ChunkWindow, citation_format: str
1251
+ ) -> str:
1252
+ """Chunk framing appended to the base prompt: absolute global times
1253
+ only, expected heading, and no document-level structure."""
1254
+ span = f"{_fmt_ts(window.start)}–{_fmt_ts(window.end)}"
1255
+ if self._language == "zh":
1256
+ return (
1257
+ f"\n\n# 本次改写范围(分段渲染 {index}/{total})\n"
1258
+ f"- 本段标题:{window.title}\n"
1259
+ f"- 本段范围:{span}(视频全局时间)\n"
1260
+ f"- 只改写本段转录内容;**所有时间戳引证必须使用视频全局时间"
1261
+ f"(不是段内相对时间),格式 {citation_format}**\n"
1262
+ f"- 输出以 `## {window.title}` 小节标题开头,直接写正文;"
1263
+ f"不要输出 `# 概述` 等文档级结构,不要重复视频标题,"
1264
+ f"不要写其他段的内容"
1265
+ )
1266
+ return (
1267
+ f"\n\n# Rewrite scope (chunked rendering {index}/{total})\n"
1268
+ f"- Chunk title: {window.title}\n"
1269
+ f"- Chunk span: {span} (global video time)\n"
1270
+ f"- Rewrite THIS chunk's transcript only; **every timestamp citation "
1271
+ f"must use global video time (not chunk-relative time), "
1272
+ f"formatted {citation_format}**\n"
1273
+ f"- Start your output with the `## {window.title}` heading and go "
1274
+ f"straight into content; do NOT emit document-level structure like "
1275
+ f"`# Overview`, do NOT repeat the video title, do NOT write other "
1276
+ f"chunks' content"
1277
+ )
1278
+
1279
+ def _render_llm_document(self, bundle: Any, segments: list[tuple[float, str]]) -> str:
1280
+ """Plan bounded work before dispatch, preserving older durable plans."""
1281
+ if not segments:
1282
+ raise RuntimeError("empty transcript cannot be sent for knowledge-doc rewriting")
1283
+ starts = [start for start, _ in segments]
1284
+ max_start = max(starts) if starts else 0.0
1285
+ duration = bundle.metadata.get("duration")
1286
+ expect_hours = (
1287
+ isinstance(duration, int | float) and duration >= 3600
1288
+ ) or max_start >= 3600
1289
+ # (passed to the prompt/chunk prompts so the format decision is made
1290
+ # exactly once per render — see _build_prompt's expect_hours arg)
1291
+
1292
+ # Old progress files have no planner marker. Preserve their exact old
1293
+ # prompts and checkpoint identity, including UNKNOWN dispatches; never
1294
+ # turn a timeout into a fresh set of paid calls.
1295
+ planner: str | None = BOUNDED_PLANNER
1296
+ if self._checkpoint_path is not None and self._checkpoint_path.exists():
1297
+ try:
1298
+ old_progress = json.loads(self._checkpoint_path.read_text(encoding="utf-8"))
1299
+ if not isinstance(old_progress, dict):
1300
+ raise ValueError("invalid progress")
1301
+ planner = old_progress.get("planner")
1302
+ if planner not in (None, UNBOUND_BOUNDED_PLANNER, BOUNDED_PLANNER):
1303
+ raise ValueError("unknown planner")
1304
+ except (OSError, UnicodeError, ValueError, TypeError):
1305
+ raise RuntimeError("knowledge-doc progress cannot be safely reused") from None
1306
+
1307
+ source_sha: str | None = None
1308
+ if planner == BOUNDED_PLANNER:
1309
+ try:
1310
+ source = json.dumps(segments, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
1311
+ except (TypeError, ValueError):
1312
+ raise RuntimeError("transcript segments cannot be safely bound") from None
1313
+ source_sha = hashlib.sha256(source.encode("utf-8")).hexdigest()
1314
+
1315
+ transcript_chars = sum(len(f"[{_fmt_ts(start)}] {text}\n") for start, text in segments)
1316
+ long_enough = (isinstance(duration, int | float) and duration >= CHUNK_WINDOW_SECONDS) or max_start >= CHUNK_WINDOW_SECONDS
1317
+ should_chunk = expect_hours or (planner is not None and (long_enough or transcript_chars > BOUNDED_CHUNK_CHARS))
1318
+ if should_chunk:
1319
+ chapters = bundle.metadata.get("chapters")
1320
+ chapters = chapters if isinstance(chapters, list) else None
1321
+ return self._render_chunked(bundle, segments, chapters, expect_hours,
1322
+ bounded=planner is not None, planner=planner, source_sha=source_sha)
1323
+
1324
+ transcript = "\n".join(f"[{_fmt_ts(start)}] {text}" for start, text in segments)
1325
+ prompt = self._build_prompt(bundle, transcript, expect_hours=expect_hours)
1326
+ return self._execute_rewrite_plan([prompt], [segments], expect_hours,
1327
+ planner=planner, source_sha=source_sha)[0]
1328
+
1329
+ @staticmethod
1330
+ @contextmanager
1331
+ def _progress_lock(path: Path):
1332
+ """Use an OS-held lock; process death releases it, the journal does not."""
1333
+ lock_path = path.with_name(path.name + ".lock")
1334
+ flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
1335
+ try:
1336
+ fd = os.open(lock_path, flags, 0o600)
1337
+ except OSError as exc:
1338
+ raise _ProgressBusy("unable to open knowledge-doc progress lock") from exc
1339
+ locked = False
1340
+ try:
1341
+ try:
1342
+ if os.name == "nt":
1343
+ import msvcrt
1344
+ if os.fstat(fd).st_size == 0:
1345
+ os.write(fd, b"0")
1346
+ os.lseek(fd, 0, os.SEEK_SET)
1347
+ # typeshed ships an empty msvcrt stub off Windows; these two
1348
+ # calls only ever run when os.name == "nt".
1349
+ msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) # type: ignore[attr-defined]
1350
+ else:
1351
+ import fcntl
1352
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1353
+ locked = True
1354
+ except OSError as exc:
1355
+ raise _ProgressBusy("knowledge-doc output is already being rendered") from exc
1356
+ yield
1357
+ finally:
1358
+ if locked:
1359
+ if os.name == "nt":
1360
+ import msvcrt
1361
+ os.lseek(fd, 0, os.SEEK_SET)
1362
+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) # type: ignore[attr-defined]
1363
+ else:
1364
+ import fcntl
1365
+ fcntl.flock(fd, fcntl.LOCK_UN)
1366
+ os.close(fd)
1367
+
1368
+ @staticmethod
1369
+ def _write_text_atomic(path: Path, content: str) -> None:
1370
+ tmp_path: str | None = None
1371
+ try:
1372
+ with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent,
1373
+ prefix=".knowledge-doc-", delete=False) as tmp:
1374
+ tmp_path = tmp.name
1375
+ tmp.write(content)
1376
+ tmp.flush()
1377
+ os.fsync(tmp.fileno())
1378
+ os.replace(tmp_path, path)
1379
+ tmp_path = None
1380
+ if os.name != "nt":
1381
+ fd = os.open(path.parent, os.O_RDONLY)
1382
+ try:
1383
+ os.fsync(fd)
1384
+ finally:
1385
+ os.close(fd)
1386
+ except OSError as exc:
1387
+ raise RuntimeError("unable to persist knowledge-doc progress") from exc
1388
+ finally:
1389
+ if tmp_path is not None:
1390
+ try:
1391
+ os.unlink(tmp_path)
1392
+ except FileNotFoundError:
1393
+ pass
1394
+
1395
+ @classmethod
1396
+ def _write_progress(cls, path: Path, progress: dict[str, Any]) -> None:
1397
+ """Persist the dispatch intent before the paid call, and results afterwards."""
1398
+ cls._write_text_atomic(path, json.dumps(progress, ensure_ascii=False))
1399
+
1400
+ def _execute_rewrite_plan(
1401
+ self, prompts: list[str], line_buckets: list[list[tuple[float, str]]], expect_hours: bool,
1402
+ planner: str | None = None, source_sha: str | None = None,
1403
+ validator: Callable[[int, str, list[tuple[float, str]]], None] | None = None,
1404
+ ) -> list[str]:
1405
+ total = len(prompts)
1406
+ plan = {"prompts": prompts, "model": self._model, "base_url": self._base_url,
1407
+ "max_tokens": self._max_tokens}
1408
+ if source_sha is not None:
1409
+ plan["segment_source_sha256"] = source_sha
1410
+ plan_sha = hashlib.sha256(json.dumps(plan, ensure_ascii=False, sort_keys=True).encode()).hexdigest()
1411
+ path = self._checkpoint_path
1412
+ progress: dict[str, Any] = {"schema": 1, "plan_sha256": plan_sha, "chunks": []}
1413
+ if planner:
1414
+ progress["planner"] = planner
1415
+ if source_sha is not None:
1416
+ progress["segment_source_sha256"] = source_sha
1417
+ if path is not None and path.exists():
1418
+ try:
1419
+ progress = json.loads(path.read_text(encoding="utf-8"))
1420
+ if (not isinstance(progress, dict) or progress.get("schema") != 1
1421
+ or progress.get("plan_sha256") != plan_sha
1422
+ or progress.get("planner") != planner
1423
+ or progress.get("segment_source_sha256") != source_sha
1424
+ or not isinstance(progress.get("chunks"), list)
1425
+ or len(progress["chunks"]) > total):
1426
+ raise ValueError("progress does not match current inputs")
1427
+ except (OSError, UnicodeError, ValueError, TypeError) as exc:
1428
+ raise RuntimeError("knowledge-doc progress cannot be safely reused") from exc
1429
+
1430
+ contents: list[str] = []
1431
+ self._deadline = time.monotonic() + self._total_timeout
1432
+ # strict=False: prompts and line_buckets are built in lockstep by every
1433
+ # in-repo caller; strict=True would raise where the previous code truncated.
1434
+ for index, (lines, prompt) in enumerate(
1435
+ zip(line_buckets, prompts, strict=False), start=1
1436
+ ):
1437
+ entries = progress["chunks"]
1438
+ prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()
1439
+ if index <= len(entries):
1440
+ entry = entries[index - 1]
1441
+ if (not isinstance(entry, dict) or entry.get("index") != index
1442
+ or entry.get("prompt_sha256") != prompt_sha
1443
+ or entry.get("state") != "COMPLETE"
1444
+ or not isinstance(entry.get("content"), str)
1445
+ or hashlib.sha256(entry["content"].encode()).hexdigest() != entry.get("content_sha256")):
1446
+ raise RuntimeError("knowledge-doc chunk has an unresolved or invalid prior dispatch")
1447
+ raw = entry["content"]
1448
+ else:
1449
+ if time.monotonic() >= self._deadline:
1450
+ raise RuntimeError("knowledge-doc total time budget exhausted")
1451
+ entries.append({"index": index, "prompt_sha256": prompt_sha, "state": "DISPATCHING"})
1452
+ if path is not None:
1453
+ self._write_progress(path, progress)
1454
+ raw = self._call_minimax(prompt)
1455
+ if validator is not None:
1456
+ # v2 image-note chunks are locator-based (图/段), not time-based;
1457
+ # the caller supplies the locator-aware validator.
1458
+ validator(index, raw, lines)
1459
+ else:
1460
+ # zyr: validate the chunk against ITS OWN window's segments —
1461
+ # the full-video start list let any chunk's citation satisfy
1462
+ # itself against a segment anywhere in a multi-hour video.
1463
+ chunk_starts = [start for start, _ in lines]
1464
+ report = validate_citation_alignment(
1465
+ raw, chunk_starts, expect_hours=expect_hours,
1466
+ require_citations=True,
1467
+ segment_texts=[text for _, text in lines])
1468
+ if not report.ok:
1469
+ raise CitationAlignmentError(report)
1470
+ if index == len(entries) and entries[index - 1].get("state") == "DISPATCHING":
1471
+ entries[index - 1] = {"index": index, "prompt_sha256": prompt_sha, "state": "COMPLETE",
1472
+ "content": raw, "content_sha256": hashlib.sha256(raw.encode()).hexdigest()}
1473
+ if path is not None:
1474
+ self._write_progress(path, progress)
1475
+ contents.append(raw)
1476
+
1477
+ return contents
1478
+
1479
+ def _render_chunked(
1480
+ self,
1481
+ bundle: Any,
1482
+ segments: list[tuple[float, str]],
1483
+ chapters: list | None,
1484
+ expect_hours: bool,
1485
+ *, bounded: bool = False, planner: str | None = None, source_sha: str | None = None,
1486
+ ) -> str:
1487
+ starts = [start for start, _ in segments]
1488
+ max_time = max(starts) if starts else 0.0
1489
+ usable_chapters = valid_chapter_list(chapters)
1490
+ boundary_source = "chapter" if usable_chapters else "fixed_window"
1491
+ planned = plan_chunk_windows(usable_chapters, max_time, language=self._language)
1492
+
1493
+ # Assign every segment to exactly one window (last-segment-at-final-
1494
+ # boundary rides the last window).
1495
+ assigned: list[tuple[ChunkWindow, list[tuple[float, str]]]] = [
1496
+ (window, []) for window in planned
1497
+ ]
1498
+ for start, text in segments:
1499
+ for window, bucket in assigned:
1500
+ if window.start <= start < window.end:
1501
+ bucket.append((start, text))
1502
+ break
1503
+ else:
1504
+ assigned[-1][1].append((start, text))
1505
+
1506
+ if bounded:
1507
+ # Keep real chapter boundaries and global timestamps. Split long
1508
+ # chapter buckets by input size; additionally split sub-hour and
1509
+ # single-chapter spans by time so a long sparse chapter does not
1510
+ # become a single unbounded request. Segment text is never cut.
1511
+ rebucketed: list[tuple[ChunkWindow, list[tuple[float, str]]]] = []
1512
+ bound_time = not expect_hours or len(usable_chapters) <= 1
1513
+ for window, lines in assigned:
1514
+ groups: list[list[tuple[float, str]]] = []
1515
+ group: list[tuple[float, str]] = []
1516
+ size = 0
1517
+ for start, text in lines:
1518
+ line_size = len(f"[{_fmt_ts(start)}] {text}\n")
1519
+ if line_size > BOUNDED_CHUNK_CHARS:
1520
+ raise RuntimeError("a transcript segment exceeds the bounded chunk size")
1521
+ if group and (size + line_size > BOUNDED_CHUNK_CHARS
1522
+ or (bound_time and start - group[0][0] >= CHUNK_WINDOW_SECONDS)):
1523
+ groups.append(group)
1524
+ group, size = [], 0
1525
+ group.append((start, text))
1526
+ size += line_size
1527
+ if group:
1528
+ groups.append(group)
1529
+ for index, group in enumerate(groups):
1530
+ start = window.start if index == 0 else group[0][0]
1531
+ end = groups[index + 1][0][0] if index + 1 < len(groups) else window.end
1532
+ rebucketed.append((ChunkWindow(start, end, window.title), group))
1533
+ assigned = rebucketed
1534
+
1535
+ rendered = [(window, lines) for window, lines in assigned if lines]
1536
+ citation_format = _citation_format(expect_hours)
1537
+ total = len(rendered)
1538
+ prompts: list[str] = []
1539
+ for index, (window, lines) in enumerate(rendered, start=1):
1540
+ chunk_transcript = "\n".join(
1541
+ f"[{_fmt_ts(start)}] {text}" for start, text in lines
1542
+ )
1543
+ prompts.append(self._build_prompt(
1544
+ bundle, chunk_transcript, expect_hours=expect_hours
1545
+ ) + self._chunk_scope_block(index, total, window, citation_format))
1546
+
1547
+ contents = self._execute_rewrite_plan(prompts, [lines for _, lines in rendered], expect_hours,
1548
+ planner, source_sha)
1549
+
1550
+ windows = [window for window, _ in rendered]
1551
+ final_titles = _dedupe_titles([w.title for w in windows], self._language != "en")
1552
+ assembled = assemble_chunked_doc(
1553
+ title=str(bundle.metadata.get("title", bundle.video_id)),
1554
+ url=str(bundle.metadata.get("canonical_url", "")),
1555
+ windows=windows,
1556
+ contents=contents,
1557
+ language=self._language,
1558
+ boundary_source=boundary_source,
1559
+ citation_format=citation_format,
1560
+ final_titles=final_titles,
1561
+ )
1562
+ problems = verify_chunk_assembly(
1563
+ assembled, windows, contents, language=self._language,
1564
+ final_titles=final_titles,
1565
+ )
1566
+ if problems:
1567
+ raise RuntimeError(
1568
+ "chunk assembly consistency check failed: " + "; ".join(problems)
1569
+ )
1570
+ return assembled
1571
+
1572
+ # ------------------------------------------------------------------
1573
+ # v2 path (T33.3) — additive; the v1 path above is untouched
1574
+ # ------------------------------------------------------------------
1575
+
1576
+ def _classification_suffix(self) -> str:
1577
+ """Prompt-classification note for v2 renders (empty for every v1 render)."""
1578
+ kind = getattr(self, "_doc_kind", None)
1579
+ if not kind or kind == "image_note":
1580
+ # The image-note template already states "no timestamps"; adding a
1581
+ # second note would only drift from it.
1582
+ return ""
1583
+ if self._language == "zh":
1584
+ if kind == "tutorial":
1585
+ return (
1586
+ "\n\n# 内容类型\n"
1587
+ "本内容是可操作教程:优先整理可执行步骤、命令与配置;步骤顺序不得打乱,"
1588
+ "没有依据的步骤不要补写。"
1589
+ )
1590
+ return (
1591
+ "\n\n# 内容类型\n"
1592
+ "本内容是通用知识:侧重概念、原理与结论;来源没有操作步骤时不要编造步骤。"
1593
+ )
1594
+ if kind == "tutorial":
1595
+ return (
1596
+ "\n\n# Content type\n"
1597
+ "This source is an actionable tutorial: prioritise executable steps, commands "
1598
+ "and configuration; keep step order and never invent missing steps."
1599
+ )
1600
+ return (
1601
+ "\n\n# Content type\n"
1602
+ "This source is general knowledge: focus on concepts, principles and conclusions; "
1603
+ "do not invent steps the source does not contain."
1604
+ )
1605
+
1606
+ def _v2_doc_source(self, bundle: Any, raw_transcript_path: Path) -> V2DocSource:
1607
+ """Normalize a v2 bundle into blocks/segments (+ display metadata)."""
1608
+ from ytk.models.content import TimeRangeLocator, VideoFrameLocator
1609
+ from ytk.renderers.source_links import display_title
1610
+
1611
+ contents = list(getattr(bundle, "contents", []))
1612
+ content = contents[0] if contents else None
1613
+ primary = content.source if content is not None else None
1614
+ metadata: dict[str, Any] = dict(getattr(bundle, "metadata", {}) or {})
1615
+ title = metadata.get("title")
1616
+ if not isinstance(title, str) or not title.strip():
1617
+ title = display_title(primary) if primary is not None else "Knowledge"
1618
+ metadata["title"] = title
1619
+ if primary is not None:
1620
+ metadata["canonical_url"] = primary.canonical_url
1621
+ metadata.setdefault("content_type", primary.content_type)
1622
+ blocks = list(content.blocks) if content is not None else []
1623
+ blocks.sort(key=lambda block: (block.order, block.block_id))
1624
+ is_image_note = primary is not None and primary.content_type == "image_note"
1625
+ if is_image_note:
1626
+ return V2DocSource(
1627
+ kind="image_note",
1628
+ blocks=blocks,
1629
+ segments=[],
1630
+ metadata=metadata,
1631
+ primary=primary,
1632
+ content=content,
1633
+ )
1634
+ segments: list[tuple[float, str]] = []
1635
+ for block in blocks:
1636
+ text = (block.cleaned_text or block.raw_text).strip()
1637
+ if not text:
1638
+ continue
1639
+ locator = block.locator
1640
+ if isinstance(locator, TimeRangeLocator):
1641
+ segments.append((locator.time.start, text))
1642
+ elif isinstance(locator, VideoFrameLocator):
1643
+ segments.append((locator.time, text))
1644
+ if content is not None and isinstance(content.metadata, dict):
1645
+ duration = content.metadata.get("duration")
1646
+ if isinstance(duration, int | float) and not isinstance(duration, bool):
1647
+ metadata["duration"] = duration
1648
+ chapters = content.metadata.get("chapters")
1649
+ if isinstance(chapters, list):
1650
+ metadata["chapters"] = chapters
1651
+ if not segments:
1652
+ segments = self._load_transcript_segments(raw_transcript_path)
1653
+ return V2DocSource(
1654
+ kind="video",
1655
+ blocks=blocks,
1656
+ segments=segments,
1657
+ metadata=metadata,
1658
+ primary=primary,
1659
+ content=content,
1660
+ )
1661
+
1662
+ def _render_locked_v2(
1663
+ self, bundle: Any, output_path: Path, status_path: Path, **kwargs: Any
1664
+ ) -> None:
1665
+ """v2 branch of :meth:`_render_locked` (same journal/status contract)."""
1666
+ source = self._v2_doc_source(bundle, output_path.parent / "raw-transcript.json")
1667
+ reason: str | None = None
1668
+ # Cloud text generation is opt-in (plan §4.6 / A25): the caller decides
1669
+ # from the resolved platform settings; a local-only run must never POST
1670
+ # the transcript to a cloud provider just because an API key is present.
1671
+ allow_cloud = bool(kwargs.get("allow_cloud", True))
1672
+ if not allow_cloud:
1673
+ logger.warning(
1674
+ "cloud text generation is disabled for this platform; falling back to the "
1675
+ "deterministic v2 document (no network call)."
1676
+ )
1677
+ content = self._fallback_doc_v2(bundle, source)
1678
+ reason = "CLOUD_TEXT_DISABLED"
1679
+ elif not self._api_key:
1680
+ logger.warning(
1681
+ "YTK_MINIMAX_API_KEY not set; KnowledgeDocRenderer is falling back "
1682
+ "to the deterministic v2 document."
1683
+ )
1684
+ content = self._fallback_doc_v2(bundle, source)
1685
+ reason = "NO_API_KEY"
1686
+ else:
1687
+ try:
1688
+ content = self._render_v2_document(bundle, source)
1689
+ except RuntimeError as exc:
1690
+ logger.warning(
1691
+ "MiniMax v2 knowledge-doc rewrite rejected; falling back to the "
1692
+ "deterministic document: %s",
1693
+ exc,
1694
+ )
1695
+ content = self._fallback_doc_v2(bundle, source)
1696
+ reason = self._failure_reason(exc)
1697
+ finally:
1698
+ self._deadline = None
1699
+ self._doc_kind = None
1700
+
1701
+ if kwargs.get("with_reviewer_scaffolding"):
1702
+ metadata = source.metadata
1703
+ content = content.rstrip() + "\n" + build_reviewer_scaffolding(
1704
+ self._language,
1705
+ title=str(metadata.get("title", "") or ""),
1706
+ source_url=str(metadata.get("canonical_url", "") or ""),
1707
+ )
1708
+
1709
+ if reason:
1710
+ content = f"<!-- ytk-knowledge-doc: fallback; reason={reason} -->\n\n" + content
1711
+ rendered = content.strip() + "\n"
1712
+ destination = output_path
1713
+ if reason and output_path.exists():
1714
+ destination = output_path.with_name(output_path.stem + ".fallback" + output_path.suffix)
1715
+ self._write_text_atomic(destination, rendered)
1716
+ self._write_progress(
1717
+ status_path,
1718
+ {
1719
+ "schema": 1,
1720
+ "state": "FALLBACK" if reason else "COMPLETE",
1721
+ "reason": reason,
1722
+ "output_file": destination.name,
1723
+ "output_sha256": hashlib.sha256(rendered.encode()).hexdigest(),
1724
+ },
1725
+ )
1726
+
1727
+ def _failure_reason(self, exc: RuntimeError) -> str:
1728
+ """Map a v2 rewrite failure to the same reason codes as the v1 path."""
1729
+ message = str(exc)
1730
+ if message == "knowledge-doc total time budget exhausted":
1731
+ reason = "TOTAL_TIMEOUT"
1732
+ elif message == "knowledge-doc chunk has an unresolved or invalid prior dispatch":
1733
+ reason = "UNRESOLVED_OR_INVALID_DISPATCH"
1734
+ elif message == "knowledge-doc progress cannot be safely reused":
1735
+ reason = "PROGRESS_MISMATCH"
1736
+ else:
1737
+ reason = "REWRITE_FAILED"
1738
+ try:
1739
+ checkpoint_path = self._checkpoint_path
1740
+ if checkpoint_path is None: # defensive: no checkpoint on this path
1741
+ raise OSError("knowledge-doc checkpoint path is not configured")
1742
+ progress = json.loads(checkpoint_path.read_text(encoding="utf-8"))
1743
+ chunks = progress.get("chunks", [])
1744
+ if (
1745
+ isinstance(chunks, list)
1746
+ and chunks
1747
+ and isinstance(chunks[-1], dict)
1748
+ and chunks[-1].get("state") == "DISPATCHING"
1749
+ ):
1750
+ reason = (
1751
+ "TOTAL_TIMEOUT_WITH_UNRESOLVED_DISPATCH"
1752
+ if reason == "TOTAL_TIMEOUT"
1753
+ else "UNRESOLVED_OR_INVALID_DISPATCH"
1754
+ )
1755
+ except (OSError, ValueError, TypeError, AttributeError):
1756
+ pass
1757
+ return reason
1758
+
1759
+ def _render_v2_document(self, bundle: Any, source: V2DocSource) -> str:
1760
+ """Plan and rewrite a v2 source (video: time chunks; image note: blocks)."""
1761
+ self._doc_kind = classify_doc_kind(bundle)
1762
+ if source.kind == "image_note":
1763
+ return self._render_v2_image_note(bundle, source)
1764
+ shim = _VideoDocShim(
1765
+ metadata=dict(source.metadata),
1766
+ video_id=str(source.primary.content_id if source.primary is not None else "video"),
1767
+ )
1768
+ return self._render_llm_document(shim, source.segments)
1769
+
1770
+ def _render_v2_image_note(self, bundle: Any, source: V2DocSource) -> str:
1771
+ """Bounded block-based rewrite for image notes (no timestamps)."""
1772
+ from ytk.renderers.citation_alignment import validate_citation_alignment_v2
1773
+
1774
+ if not source.blocks:
1775
+ raise RuntimeError("empty image note cannot be sent for knowledge-doc rewriting")
1776
+ chunks = plan_image_note_chunks(source.blocks, language=self._language)
1777
+ problems = verify_image_note_plan(chunks, source.blocks)
1778
+ if problems:
1779
+ raise RuntimeError("image-note chunk plan is inconsistent: " + "; ".join(problems))
1780
+ source_sha = hashlib.sha256(
1781
+ json.dumps(
1782
+ [(chunk.block_ids, chunk.lines) for chunk in chunks],
1783
+ ensure_ascii=False,
1784
+ separators=(",", ":"),
1785
+ ).encode("utf-8")
1786
+ ).hexdigest()
1787
+ total = len(chunks)
1788
+ prompts = [
1789
+ self._build_image_note_prompt(source.metadata, chunk, total) for chunk in chunks
1790
+ ]
1791
+ buckets: list[list[tuple[float, str]]] = [[] for _ in chunks]
1792
+ block_ids_by_index = {chunk.index: set(chunk.block_ids) for chunk in chunks}
1793
+ source_id = source.primary.source_id if source.primary is not None else None
1794
+
1795
+ def validate_chunk(index: int, raw: str, lines: list[tuple[float, str]]) -> None:
1796
+ report = validate_citation_alignment_v2(
1797
+ raw,
1798
+ bundle,
1799
+ source_id=source_id,
1800
+ block_ids=block_ids_by_index.get(index),
1801
+ require_citations=True,
1802
+ )
1803
+ if not report.ok:
1804
+ raise CitationAlignmentError(report)
1805
+
1806
+ contents = self._execute_rewrite_plan(
1807
+ prompts,
1808
+ buckets,
1809
+ False,
1810
+ planner=IMAGE_NOTE_PLANNER,
1811
+ source_sha=source_sha,
1812
+ validator=validate_chunk,
1813
+ )
1814
+ final_titles = _dedupe_titles(
1815
+ [_image_chunk_title(chunk.index, self._language) for chunk in chunks],
1816
+ self._language != "en",
1817
+ )
1818
+ assembled = assemble_image_note_doc(
1819
+ title=str(source.metadata.get("title") or "Image note"),
1820
+ url=str(source.metadata.get("canonical_url") or ""),
1821
+ chunks=chunks,
1822
+ contents=contents,
1823
+ language=self._language,
1824
+ final_titles=final_titles,
1825
+ )
1826
+ window_shims = [ChunkWindow(0.0, 0.0, title) for title in final_titles]
1827
+ problems = verify_chunk_assembly(
1828
+ assembled,
1829
+ window_shims,
1830
+ contents,
1831
+ language=self._language,
1832
+ final_titles=final_titles,
1833
+ )
1834
+ if problems:
1835
+ raise RuntimeError(
1836
+ "image-note assembly consistency check failed: " + "; ".join(problems)
1837
+ )
1838
+ return assembled
1839
+
1840
+ def _build_image_note_prompt(
1841
+ self, metadata: dict[str, Any], chunk: ImageNoteChunk, total: int
1842
+ ) -> str:
1843
+ template = _IMAGE_NOTE_TEMPLATES[self._language]
1844
+ citation_format = _IMAGE_CITATION_FORMAT[self._language]
1845
+ title = str(metadata.get("title") or "Image note")
1846
+ url = str(metadata.get("canonical_url") or "")
1847
+ prompt = template.format(
1848
+ title=title,
1849
+ url=url or ("(无)" if self._language == "zh" else "(none)"),
1850
+ blocks="\n".join(chunk.lines),
1851
+ citation_format=citation_format,
1852
+ )
1853
+ return prompt + self._image_chunk_scope_block(chunk, total, citation_format)
1854
+
1855
+ def _image_chunk_scope_block(
1856
+ self, chunk: ImageNoteChunk, total: int, citation_format: str
1857
+ ) -> str:
1858
+ title = _image_chunk_title(chunk.index, self._language)
1859
+ if self._language == "zh":
1860
+ return (
1861
+ f"\n\n# 本次改写范围(图文分块 {chunk.index}/{total})\n"
1862
+ f"- 本部分标题:{title}\n"
1863
+ f"- 只改写本部分内容块;引用使用 {citation_format} 定位标记\n"
1864
+ f"- **禁止出现任何时间戳**(图文没有时间轴)\n"
1865
+ f"- 输出以 `## {title}` 开头,直接写正文;不要输出 `# 概述` 等文档级结构,"
1866
+ f"不要写其他部分的内容"
1867
+ )
1868
+ return (
1869
+ f"\n\n# Rewrite scope (image-note chunk {chunk.index}/{total})\n"
1870
+ f"- Chunk title: {title}\n"
1871
+ f"- Rewrite THIS chunk's blocks only; cite with {citation_format} locator markers\n"
1872
+ f"- **Never emit a timestamp** (an image note has no timeline)\n"
1873
+ f"- Start your output with the `## {title}` heading and go straight into content; "
1874
+ f"do NOT emit document-level structure like `# Overview`, do NOT write other "
1875
+ f"chunks' content"
1876
+ )
1877
+
1878
+ def _fallback_doc_v2(self, bundle: Any, source: V2DocSource) -> str:
1879
+ """Deterministic v2 document: formatted evidence, no invented timestamps."""
1880
+ from ytk.renderers.evidence_formatter import format_evidence_refs
1881
+ from ytk.renderers.source_links import format_seconds, image_markdown_ref
1882
+
1883
+ labels = _SECTION_TITLES[self._language]
1884
+ title = str(source.metadata.get("title") or "Knowledge")
1885
+ url = str(source.metadata.get("canonical_url") or "")
1886
+ sections: list[str] = [labels["title_template"].format(title=title), ""]
1887
+ if url:
1888
+ sections += [labels["source"].format(url=url), ""]
1889
+ sections += [labels["overview"], ""]
1890
+ first_texts = [
1891
+ (block.cleaned_text or block.raw_text).strip() for block in source.blocks[:3]
1892
+ ]
1893
+ overview = " ".join(text for text in first_texts if text) or labels["no_transcript"]
1894
+ sections += [overview, ""]
1895
+
1896
+ image_assets = [
1897
+ asset
1898
+ for asset in (getattr(source.content, "assets", []) or [])
1899
+ if asset.media_type == "image"
1900
+ ]
1901
+ if image_assets:
1902
+ sections += ["## 图片" if self._language != "en" else "## Images", ""]
1903
+ for index, asset in enumerate(image_assets, start=1):
1904
+ alt = f"图 {index}" if self._language != "en" else f"Image {index}"
1905
+ sections.append(image_markdown_ref(asset.relative_path, alt=alt))
1906
+ sections.append("")
1907
+
1908
+ knowledge = list(getattr(bundle, "knowledge", []))
1909
+ if knowledge:
1910
+ sections += [labels["knowledge_atoms"], ""]
1911
+ by_type: dict[str, list[Any]] = {}
1912
+ for atom in knowledge:
1913
+ by_type.setdefault(atom.atom_type, []).append(atom)
1914
+ for atom_type, atoms in by_type.items():
1915
+ localized = labels["type_titles"].get(atom_type, atom_type)
1916
+ sections += [f"### {localized}", ""]
1917
+ for atom in atoms:
1918
+ evidence = [
1919
+ bundle.evidence_index[evidence_id]
1920
+ for evidence_id in atom.evidence_ids
1921
+ if evidence_id in bundle.evidence_index
1922
+ ]
1923
+ citations = format_evidence_refs(
1924
+ evidence, bundle, language=self._language, validate=False
1925
+ )
1926
+ suffix = f" {' '.join(citations)}" if citations else ""
1927
+ sections.append(f"- {atom.content}{suffix}")
1928
+ sections.append("")
1929
+
1930
+ sections += [labels["full_transcript"], ""]
1931
+ if source.kind == "image_note":
1932
+ sections.extend(image_note_block_line(block, language=self._language) for block in source.blocks)
1933
+ else:
1934
+ sections.extend(f"[{format_seconds(start)}] {text}" for start, text in source.segments)
1935
+ if not source.blocks and not source.segments:
1936
+ sections.append(labels["no_transcript"])
1937
+ return "\n".join(sections)
1938
+
1939
+ def _call_minimax(self, prompt: str) -> str:
1940
+ url = f"{self._base_url}/v1/chat/completions"
1941
+ headers = {"Authorization": f"Bearer {self._api_key}"}
1942
+ payload = {
1943
+ "model": self._model,
1944
+ "messages": [{"role": "user", "content": prompt}],
1945
+ "max_completion_tokens": self._max_tokens,
1946
+ }
1947
+ try:
1948
+ if self._deadline is None:
1949
+ with httpx.Client(timeout=self._timeout) as client:
1950
+ resp = client.post(url, headers=headers, json=payload)
1951
+ resp.raise_for_status()
1952
+ result = resp.json()
1953
+ else:
1954
+ remaining = self._deadline - time.monotonic()
1955
+ if remaining <= 0:
1956
+ raise RuntimeError("knowledge-doc total time budget exhausted")
1957
+ # Inspect every received fragment against the wall-clock deadline.
1958
+ # The read inactivity bound limits overshoot even for slow streams.
1959
+ timeout = min(self._timeout, remaining, 60.0)
1960
+ chunks: list[bytes] = []
1961
+ received = 0
1962
+ with httpx.Client(timeout=timeout) as client:
1963
+ stream_headers = {**headers, "Accept-Encoding": "identity"}
1964
+ with client.stream("POST", url, headers=stream_headers, json=payload) as resp:
1965
+ resp.raise_for_status()
1966
+ if resp.headers.get("Content-Encoding", "identity").lower() not in ("", "identity"):
1967
+ raise RuntimeError("knowledge-doc response has unexpected compression")
1968
+ for chunk in resp.iter_raw():
1969
+ if time.monotonic() >= self._deadline:
1970
+ raise RuntimeError("knowledge-doc total time budget exhausted")
1971
+ received += len(chunk)
1972
+ if received > 10 * 1024 * 1024:
1973
+ raise RuntimeError("knowledge-doc response exceeds size limit")
1974
+ chunks.append(chunk)
1975
+ if time.monotonic() >= self._deadline:
1976
+ raise RuntimeError("knowledge-doc total time budget exhausted")
1977
+ result = json.loads(b"".join(chunks))
1978
+ choice = result["choices"][0]
1979
+ if choice.get("finish_reason") != "stop":
1980
+ raise RuntimeError("knowledge-doc response was incomplete or has no completion reason")
1981
+ raw = choice["message"]["content"]
1982
+ except (KeyError, json.JSONDecodeError, httpx.HTTPError) as exc:
1983
+ raise RuntimeError(f"MiniMax knowledge-doc call failed: {exc}") from exc
1984
+ return _THINK_RE.sub("", raw).strip()