kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,1984 @@
|
|
|
1
|
+
"""Render an ExtractionBundle as a single, structured knowledge document.
|
|
2
|
+
|
|
3
|
+
Unlike the other renderers (which produce a multi-file bundle: obsidian note,
|
|
4
|
+
markdown notes, summary, skill package, JSON), this renderer asks MiniMax to
|
|
5
|
+
rewrite the raw transcript into ONE polished markdown file. It targets the
|
|
6
|
+
"give me a single, clean, knowledge document" workflow: no skill package
|
|
7
|
+
scaffolding, no per-atom JSON, no per-section files. Just the distilled
|
|
8
|
+
knowledge in the reader's preferred language.
|
|
9
|
+
|
|
10
|
+
Output: a single .md file with overview, core concepts, workflows, code
|
|
11
|
+
samples, configuration, and timestamp-anchored citations. Filler words
|
|
12
|
+
("uh", "so", "you know", ...) are removed, fragmented segments are merged,
|
|
13
|
+
and the document is written in the configured language (Chinese or English).
|
|
14
|
+
|
|
15
|
+
Language selection (in priority order):
|
|
16
|
+
1. Explicit ``YTK_DOC_LANGUAGE`` env var ("zh" or "en")
|
|
17
|
+
2. The user's shell locale ($LANG / $LC_ALL / $LC_MESSAGES, sampled for
|
|
18
|
+
"zh" or "en")
|
|
19
|
+
3. Default: "zh" (preserves v1.0.0 behavior for existing users)
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import hashlib
|
|
25
|
+
import json
|
|
26
|
+
import logging
|
|
27
|
+
import math
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
import tempfile
|
|
31
|
+
import threading
|
|
32
|
+
import time
|
|
33
|
+
from collections.abc import Callable
|
|
34
|
+
from contextlib import contextmanager
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
from typing import Any, ClassVar, Literal, TypedDict
|
|
38
|
+
|
|
39
|
+
import httpx
|
|
40
|
+
|
|
41
|
+
from ytk.config import AppSettings
|
|
42
|
+
from ytk.renderers.citation_alignment import (
|
|
43
|
+
CitationAlignmentError,
|
|
44
|
+
validate_citation_alignment,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
DocLanguage = Literal["zh", "en"]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _ProgressBusy(RuntimeError):
|
|
53
|
+
"""Another process currently owns this output's model dispatch."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def complete_doc_status(output_dir: Path) -> bool:
|
|
57
|
+
"""Check the status and exact knowledge.md bytes before reporting completion."""
|
|
58
|
+
try:
|
|
59
|
+
status = json.loads((output_dir / ".knowledge-doc-status.json").read_text(encoding="utf-8"))
|
|
60
|
+
output = output_dir / "knowledge.md"
|
|
61
|
+
return (status.get("schema") == 1 and status.get("state") == "COMPLETE"
|
|
62
|
+
and status.get("output_file") == output.name
|
|
63
|
+
and status.get("output_sha256") == hashlib.sha256(output.read_bytes()).hexdigest())
|
|
64
|
+
except (OSError, UnicodeError, ValueError, AttributeError, TypeError):
|
|
65
|
+
return False
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class _SectionLabels(TypedDict):
|
|
69
|
+
"""Localization labels for one output language."""
|
|
70
|
+
|
|
71
|
+
title_template: str
|
|
72
|
+
source: str
|
|
73
|
+
overview: str
|
|
74
|
+
no_transcript: str
|
|
75
|
+
topic_segments: str
|
|
76
|
+
knowledge_atoms: str
|
|
77
|
+
type_titles: dict[str, str]
|
|
78
|
+
full_transcript: str
|
|
79
|
+
overview_separator: str
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# Mirror ytk.ingestion.providers.minimax._BASE_URLS so this renderer targets
|
|
83
|
+
# the same backend the configured region was issued for.
|
|
84
|
+
_MINIMAX_BASE_URLS: dict[str, str] = {
|
|
85
|
+
"cn": "https://api.minimaxi.com",
|
|
86
|
+
"global": "https://api.minimax.io",
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
# Same regex as ytk.ingestion.providers.minimax._THINK_RE so the leading
|
|
90
|
+
# `<think>...</think>` block the model emits in thinking mode is stripped
|
|
91
|
+
# before we write the document to disk.
|
|
92
|
+
_THINK_RE = re.compile(r"(?is)^\s*<think\b.*?</think\s*>\s*")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
_PROMPT_TEMPLATES: dict[str, str] = {
|
|
96
|
+
"zh": """你是知识整理助手。下面是一个 YouTube 教程的转录(带时间戳,格式 `{ts_format} 文本`)。请把它改写成一篇干净、结构化、**中文**知识文档,直接保存为 markdown 文件使用。
|
|
97
|
+
|
|
98
|
+
# 视频元数据
|
|
99
|
+
- 标题: {title}
|
|
100
|
+
- URL: {url}
|
|
101
|
+
- 时长: {duration} 秒
|
|
102
|
+
|
|
103
|
+
# 转录(带时间戳)
|
|
104
|
+
{transcript}
|
|
105
|
+
|
|
106
|
+
# 要求
|
|
107
|
+
1. **必须用中文输出**(专有名词如产品名、API、命令保留英文原文)。**专有名词必须逐字沿用转录里的原始拼写**(产品名、模型名、游戏名、数据集名、人名、API、命令、URL),转录怎么写就怎么写;**严禁**用同领域更知名的缩写或同类术语去"纠正"、"归一"或"补全"它(反例:转录里的 `Jev` 不得改写成 `JEPA`,转录里的 `howlight` 不得"更正"成别的游戏名)。拼写存疑时保留转录原样,并在该名词后用括号标注 `(拼写依转录)`,不要自行替换。
|
|
108
|
+
2. **去口语化**:删除 "uh", "so", "you know", "ok", "right", "alright", "well", "I mean", "like", "basically", "let's go ahead and", "you know what", "kinda", "sort of" 等填充词和口头禅
|
|
109
|
+
3. **整合碎片**:把同一主题的多个片段合并成连贯段落;不要按时间顺序罗列,要按主题组织
|
|
110
|
+
4. **结构化**:使用以下 sections(按需保留相关 sections,去掉空 sections):
|
|
111
|
+
- `# 概述`(一段话总结视频讲什么、为什么重要)
|
|
112
|
+
- `## 核心概念`(关键技术名词 + 简短解释)
|
|
113
|
+
- `## 工作流程`(步骤化、清晰的指令,去掉所有口语)
|
|
114
|
+
- `## 代码与命令`(具体可执行的内容,标注 `bash` 代码块)
|
|
115
|
+
- `## 配置与设置`(配置文件、参数)
|
|
116
|
+
- `## 关键时间点`(标注 `{citation_format}` 时间戳,方便回看,列 5-10 个最重要的时刻)
|
|
117
|
+
5. **不引入视频外的内容**——只整理视频里讲的
|
|
118
|
+
6. **不使用 emoji**
|
|
119
|
+
7. 引用关键事实时附 `{citation_format}` 时间戳,**写在括号内**而不是塞在句子里
|
|
120
|
+
8. 全文使用中文标点(,。;:)而非英文标点
|
|
121
|
+
9. 文档长度 1500-4000 中文字
|
|
122
|
+
10. **直接输出 markdown**,不要加前言(如"以下是..."),不要加元评论
|
|
123
|
+
11. {hour_rule}
|
|
124
|
+
|
|
125
|
+
# 输出格式
|
|
126
|
+
直接以 `# 概述` 开头。
|
|
127
|
+
""",
|
|
128
|
+
"en": """You are a knowledge-curation assistant. Below is the transcript of a YouTube tutorial with timestamps in `{ts_format}` format. Please rewrite it into a clean, structured, **English** knowledge document, ready to be saved as a markdown file.
|
|
129
|
+
|
|
130
|
+
# Video metadata
|
|
131
|
+
- Title: {title}
|
|
132
|
+
- URL: {url}
|
|
133
|
+
- Duration: {duration} seconds
|
|
134
|
+
|
|
135
|
+
# Transcript (with timestamps)
|
|
136
|
+
{transcript}
|
|
137
|
+
|
|
138
|
+
# Requirements
|
|
139
|
+
1. **Output MUST be in English** (keep product names, APIs, and commands in their original English form). **Preserve every proper noun exactly as spelled in the transcript** (product, model, game, dataset, and people names, APIs, commands, URLs). **Never** "correct", normalize, or expand a name into a more famous acronym or a similar term from the same field (counter-example: `Jev` in the transcript must NOT become `JEPA`). When the spelling is unclear, keep the transcript's form and append `(spelling per transcript)`.
|
|
140
|
+
2. **Remove filler words**: drop "uh", "so", "you know", "ok", "right", "alright", "well", "I mean", "like", "basically", "let's go ahead and", "you know what", "kinda", "sort of", and similar verbal tics
|
|
141
|
+
3. **Merge fragments**: combine multiple segments on the same topic into coherent paragraphs; organize by topic, not chronologically
|
|
142
|
+
4. **Structure**: use the following sections (keep relevant ones, drop empty ones):
|
|
143
|
+
- `# Overview` (one paragraph: what the video covers and why it matters)
|
|
144
|
+
- `## Core Concepts` (key technical terms + short explanations)
|
|
145
|
+
- `## Workflow` (step-by-step instructions, no spoken-language fluff)
|
|
146
|
+
- `## Code & Commands` (concrete runnable content, marked with `bash` code blocks)
|
|
147
|
+
- `## Configuration` (config files, parameters)
|
|
148
|
+
- `## Key Moments` (annotate `{citation_format}` timestamps so the reader can jump back, list 5-10 most important moments)
|
|
149
|
+
5. **Do not introduce content from outside the video** — only reorganize what was said
|
|
150
|
+
6. **No emoji**
|
|
151
|
+
7. When citing key facts, attach `{citation_format}` timestamps in parentheses, not inline in the sentence
|
|
152
|
+
8. Use English punctuation (.,;:)
|
|
153
|
+
9. Document length 1500-4000 English words
|
|
154
|
+
10. **Output the markdown directly** — no preamble (e.g. "Below is..."), no meta-commentary
|
|
155
|
+
11. {hour_rule}
|
|
156
|
+
|
|
157
|
+
# Output format
|
|
158
|
+
Start directly with `# Overview`.
|
|
159
|
+
""",
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
_HOUR_RULE_ZH_SHORT = (
|
|
164
|
+
"时间戳引证统一使用 `[mm:ss]` 格式(本视频不足 1 小时,无需小时位)"
|
|
165
|
+
)
|
|
166
|
+
_HOUR_RULE_ZH_LONG = (
|
|
167
|
+
"**时间戳硬约束:本视频时长超过 1 小时,所有时间戳引证必须写成 `[h:mm:ss]` 全局格式"
|
|
168
|
+
"(必须含小时位),严禁丢掉小时位(反例:把 1:02:01 写成 02:01);"
|
|
169
|
+
"输出前逐一自查每条时间戳都带小时位且为视频全局时间**"
|
|
170
|
+
)
|
|
171
|
+
_HOUR_RULE_EN_SHORT = (
|
|
172
|
+
"Cite timestamps as `[mm:ss]` (this video is under one hour; no hour component needed)"
|
|
173
|
+
)
|
|
174
|
+
_HOUR_RULE_EN_LONG = (
|
|
175
|
+
"**Hard timestamp rule: this video is longer than one hour, so every timestamp citation "
|
|
176
|
+
"must be written as a global `[h:mm:ss]` timestamp (hour component mandatory). Never drop "
|
|
177
|
+
"the hour (counter-example: 1:02:01 written as 02:01); self-check every timestamp before "
|
|
178
|
+
"finishing**"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# ---------------------------------------------------------------------------
|
|
183
|
+
# Reviewer-scaffolding block
|
|
184
|
+
#
|
|
185
|
+
# When the CLI is invoked with `--with-reviewer-scaffolding`, the renderer
|
|
186
|
+
# appends a deterministic footer to the knowledge doc so that downstream
|
|
187
|
+
# reviewers (multi-model 4-rubric scoring) see:
|
|
188
|
+
# 1. An A/B/C source list. A-class and C-class entries are populated from
|
|
189
|
+
# real bundle metadata (the source video URL is both the primary source
|
|
190
|
+
# and the project-experience source). B-class is an explicitly-marked
|
|
191
|
+
# empty placeholder, because the extractor cannot know which community
|
|
192
|
+
# sources a human editor will judge relevant.
|
|
193
|
+
# 2. A generic image-asset manifest template (cover / mindmap / flowchart /
|
|
194
|
+
# code image). The content column is a placeholder slot, NOT a
|
|
195
|
+
# topic-specific description — the extractor does not know the video's
|
|
196
|
+
# subject, so it must not invent one (e.g. it must not claim a "four
|
|
197
|
+
# primitives" cover for a cooking video).
|
|
198
|
+
# 3. A closing CTA that points the reader at the doc body (which always
|
|
199
|
+
# exists) rather than at a named section that may or may not be emitted
|
|
200
|
+
# by the LLM/fallback path.
|
|
201
|
+
#
|
|
202
|
+
# Content is bilingual-ready via the ``language`` arg (zh / en). The builder
|
|
203
|
+
# is a pure function of (language, title, source_url) — no LLM call, no
|
|
204
|
+
# datetime, no random — so reviewer scores are reproducible across runs.
|
|
205
|
+
# ---------------------------------------------------------------------------
|
|
206
|
+
|
|
207
|
+
# Marker used in the B-class list and the image-manifest content slots to
|
|
208
|
+
# make it visually obvious to a human editor (and to a downstream reviewer)
|
|
209
|
+
# that the slot is a template waiting to be populated, not real evidence.
|
|
210
|
+
_PLACEHOLDER_MARKER_ZH = "_(发布前请补充)_" # noqa: RUF001 - intentional wide underscore
|
|
211
|
+
_PLACEHOLDER_MARKER_EN = "_(populate before publishing)_"
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _a_class_entries_zh(title: str, source_url: str) -> str:
|
|
215
|
+
"""A-class: the source video is itself a primary source. Always non-empty."""
|
|
216
|
+
label = title.strip() or "源视频"
|
|
217
|
+
url = source_url.strip() or "_(源 URL 缺失)_"
|
|
218
|
+
return f"- [{label}]({url}) — 视频本身就是一手素材(讲解者亲自演示)"
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _a_class_entries_en(title: str, source_url: str) -> str:
|
|
222
|
+
label = title.strip() or "Source video"
|
|
223
|
+
url = source_url.strip() or "_(source URL missing)_"
|
|
224
|
+
return f"- [{label}]({url}) — the video itself is the primary material (presenter's own demo)"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _c_class_entries_zh(source_url: str) -> str:
|
|
228
|
+
"""C-class: the video's demos ARE project experience. Always non-empty."""
|
|
229
|
+
url = source_url.strip()
|
|
230
|
+
if not url:
|
|
231
|
+
return "- _(源 URL 缺失)_ — 视频里呈现的实战演示属于 C 类(项目经验),可按时间戳标记"
|
|
232
|
+
# Use a descriptive label (not the bare URL) so the link renders cleanly
|
|
233
|
+
# even if the URL contains markdown-significant characters.
|
|
234
|
+
return f"- [视频作者的实战演示(C 类项目经验)]({url}) — 可按时间戳标记"
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _c_class_entries_en(source_url: str) -> str:
|
|
238
|
+
url = source_url.strip()
|
|
239
|
+
if not url:
|
|
240
|
+
return "- _(source URL missing)_ — demos shown in the video are C-class (project experience); tag with timestamps"
|
|
241
|
+
return f"- [Presenter's live demo (C-class project experience)]({url}) — tag with timestamps"
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
# The image manifest is intentionally generic. Each row's "content" column is
|
|
245
|
+
# a placeholder slot (the renderer does not know the video's subject), and the
|
|
246
|
+
# "renderer" column names the tool that would produce that asset type. A human
|
|
247
|
+
# editor (or a downstream image pipeline) replaces the placeholder text with
|
|
248
|
+
# topic-specific descriptions before publishing.
|
|
249
|
+
_IMAGE_MANIFEST_ROWS_ZH = (
|
|
250
|
+
"| 1 | `00-cover.png` | {_ph} 封面图 | drawio MCP / PIL |",
|
|
251
|
+
"| 2 | `01-mindmap-overview.png` | {_ph} 核心概念思维导图 | drawio MCP |",
|
|
252
|
+
"| 3 | `02-flowchart.png` | {_ph} 关键工作流图 | mmdc |",
|
|
253
|
+
"| 4 | `03-code-snippet.png` | {_ph} 关键代码或命令截图(若适用) | freeze + PIL |",
|
|
254
|
+
)
|
|
255
|
+
_IMAGE_MANIFEST_ROWS_EN = (
|
|
256
|
+
"| 1 | `00-cover.png` | {_ph} cover art | drawio MCP / PIL |",
|
|
257
|
+
"| 2 | `01-mindmap-overview.png` | {_ph} mindmap of core concepts | drawio MCP |",
|
|
258
|
+
"| 3 | `02-flowchart.png` | {_ph} key workflow diagram | mmdc |",
|
|
259
|
+
"| 4 | `03-code-snippet.png` | {_ph} key code/command screenshot (if applicable) | freeze + PIL |",
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _image_manifest_zh() -> str:
|
|
264
|
+
rows = "\n".join(row.format(_ph=_PLACEHOLDER_MARKER_ZH) for row in _IMAGE_MANIFEST_ROWS_ZH)
|
|
265
|
+
return "| # | 文件 | 内容 | 渲染方式 |\n| --- | --- | --- | --- |\n" + rows
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _image_manifest_en() -> str:
|
|
269
|
+
rows = "\n".join(row.format(_ph=_PLACEHOLDER_MARKER_EN) for row in _IMAGE_MANIFEST_ROWS_EN)
|
|
270
|
+
return "| # | File | Content | Renderer |\n| --- | --- | --- | --- |\n" + rows
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
_CTA_BODY_ZH = (
|
|
274
|
+
"1. 回到本文开头的概述,用一句话向同事复述这篇文档解决什么问题。\n"
|
|
275
|
+
"2. 从正文中挑一条最具体的命令或概念,在本地照做一次,记录实际输出。\n"
|
|
276
|
+
"3. 把你照做时遇到的偏差(命令不存在、参数变了、结果不一致)写回这份文档的复盘区。\n\n"
|
|
277
|
+
"完成这三步,这份文档就从『读过的』变成『跑通的』。\n"
|
|
278
|
+
)
|
|
279
|
+
_CTA_BODY_EN = (
|
|
280
|
+
"1. Re-read the overview at the top and explain in one sentence what this doc solves.\n"
|
|
281
|
+
"2. Pick the most concrete command or concept in the body, run it locally, and record the actual output.\n"
|
|
282
|
+
"3. Write any deviation you hit (missing command, changed flag, mismatched result) back into this doc.\n\n"
|
|
283
|
+
"After these three steps the doc has gone from 'read' to 'run'.\n"
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def build_reviewer_scaffolding(
|
|
288
|
+
language: DocLanguage,
|
|
289
|
+
*,
|
|
290
|
+
title: str = "",
|
|
291
|
+
source_url: str = "",
|
|
292
|
+
) -> str:
|
|
293
|
+
"""Return the reviewer-scaffolding footer for the given language.
|
|
294
|
+
|
|
295
|
+
The footer is appended to the rendered knowledge doc. It is a pure
|
|
296
|
+
function of (language, title, source_url) — no LLM call, no datetime,
|
|
297
|
+
no random — so reviewer scoring is reproducible across runs.
|
|
298
|
+
|
|
299
|
+
Real metadata (``title``, ``source_url``) populates the A-class and
|
|
300
|
+
C-class source entries. B-class and image-manifest content slots are
|
|
301
|
+
explicitly-marked placeholders, because the extractor cannot know which
|
|
302
|
+
community sources a human editor will judge relevant, nor the video's
|
|
303
|
+
subject matter.
|
|
304
|
+
"""
|
|
305
|
+
# Treat anything that is not explicitly "en" as Chinese (the documented
|
|
306
|
+
# default — matches detect_doc_language's fallback). This preserves the
|
|
307
|
+
# contract that unknown / "auto" / None → zh.
|
|
308
|
+
is_zh = language != "en"
|
|
309
|
+
parts: list[str] = []
|
|
310
|
+
# Leading blank line separates the footer from the body.
|
|
311
|
+
parts.append("")
|
|
312
|
+
|
|
313
|
+
# --- A / B / C source list ---
|
|
314
|
+
if is_zh:
|
|
315
|
+
parts.append("## 参考来源(A / B / C 三类)")
|
|
316
|
+
parts.append("视频本身就是一手素材;下列清单按来源可信度分三类,B 类由人工补充:")
|
|
317
|
+
else:
|
|
318
|
+
parts.append("## References (A / B / C classes)")
|
|
319
|
+
parts.append("The video itself is primary material; the list below splits sources by credibility. B-class is populated by a human editor:")
|
|
320
|
+
parts.append("")
|
|
321
|
+
|
|
322
|
+
if is_zh:
|
|
323
|
+
parts.append("### A 类:官方 / 一手")
|
|
324
|
+
parts.append(_a_class_entries_zh(title, source_url))
|
|
325
|
+
parts.append("")
|
|
326
|
+
parts.append("### B 类:社区 / 开源")
|
|
327
|
+
parts.append(f"- {_PLACEHOLDER_MARKER_ZH} 视频提到的第三方仓库、博文或工具链接")
|
|
328
|
+
parts.append("")
|
|
329
|
+
parts.append("### C 类:项目经验(视频作者演示)")
|
|
330
|
+
parts.append(_c_class_entries_zh(source_url))
|
|
331
|
+
else:
|
|
332
|
+
parts.append("### A class: Official / primary")
|
|
333
|
+
parts.append(_a_class_entries_en(title, source_url))
|
|
334
|
+
parts.append("")
|
|
335
|
+
parts.append("### B class: Community / OSS")
|
|
336
|
+
parts.append(f"- {_PLACEHOLDER_MARKER_EN} third-party repos, blog posts, or tools mentioned in the video")
|
|
337
|
+
parts.append("")
|
|
338
|
+
parts.append("### C class: Project experience (presenter's demo)")
|
|
339
|
+
parts.append(_c_class_entries_en(source_url))
|
|
340
|
+
parts.append("")
|
|
341
|
+
|
|
342
|
+
# --- Image-asset manifest (generic template) ---
|
|
343
|
+
if is_zh:
|
|
344
|
+
parts.append("## 配套图示(模板)")
|
|
345
|
+
parts.append("下列图示为通用模板,发布前请把「内容」列的占位符替换为本视频的具体描述:")
|
|
346
|
+
parts.append("")
|
|
347
|
+
parts.append(_image_manifest_zh())
|
|
348
|
+
else:
|
|
349
|
+
parts.append("## Companion visuals (template)")
|
|
350
|
+
parts.append("The visual plan below is a generic template; replace the Content-column placeholders with this video's specifics before publishing:")
|
|
351
|
+
parts.append("")
|
|
352
|
+
parts.append(_image_manifest_en())
|
|
353
|
+
parts.append("")
|
|
354
|
+
|
|
355
|
+
# --- Closing CTA (references the body, not a named section) ---
|
|
356
|
+
if is_zh:
|
|
357
|
+
parts.append("## 现在就开始")
|
|
358
|
+
parts.append(_CTA_BODY_ZH)
|
|
359
|
+
else:
|
|
360
|
+
parts.append("## Get started now")
|
|
361
|
+
parts.append(_CTA_BODY_EN)
|
|
362
|
+
|
|
363
|
+
return "\n".join(parts) + "\n"
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
_SECTION_TITLES: dict[str, _SectionLabels] = {
|
|
367
|
+
"zh": {
|
|
368
|
+
"title_template": "# {title}",
|
|
369
|
+
"source": "来源:{url}",
|
|
370
|
+
"overview": "## 概述",
|
|
371
|
+
"no_transcript": "(无转录内容)",
|
|
372
|
+
"topic_segments": "## 主题分段",
|
|
373
|
+
"knowledge_atoms": "## 提取的知识要点",
|
|
374
|
+
"type_titles": {
|
|
375
|
+
"core_concept": "核心概念",
|
|
376
|
+
"workflow_step": "工作流程",
|
|
377
|
+
"terminal_command": "命令",
|
|
378
|
+
"command_sequence": "命令序列",
|
|
379
|
+
"configuration_snippet": "配置",
|
|
380
|
+
"code_block": "代码块",
|
|
381
|
+
"best_practice": "最佳实践",
|
|
382
|
+
},
|
|
383
|
+
"full_transcript": "## 完整转录(去填充词)",
|
|
384
|
+
"overview_separator": "",
|
|
385
|
+
},
|
|
386
|
+
"en": {
|
|
387
|
+
"title_template": "# {title}",
|
|
388
|
+
"source": "Source: {url}",
|
|
389
|
+
"overview": "## Overview",
|
|
390
|
+
"no_transcript": "(no transcript content)",
|
|
391
|
+
"topic_segments": "## Topic Segments",
|
|
392
|
+
"knowledge_atoms": "## Extracted Knowledge Atoms",
|
|
393
|
+
"type_titles": {
|
|
394
|
+
"core_concept": "Core Concepts",
|
|
395
|
+
"workflow_step": "Workflow Steps",
|
|
396
|
+
"terminal_command": "Commands",
|
|
397
|
+
"command_sequence": "Command Sequences",
|
|
398
|
+
"configuration_snippet": "Configuration",
|
|
399
|
+
"code_block": "Code Blocks",
|
|
400
|
+
"best_practice": "Best Practices",
|
|
401
|
+
},
|
|
402
|
+
"full_transcript": "## Full Transcript (filler words removed)",
|
|
403
|
+
"overview_separator": "",
|
|
404
|
+
},
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
_TIMESTAMP_HEAD_RE = re.compile(r"^\[(\d{1,2}:\d{1,2}(?::\d{1,2})?)\]")
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _transcript_has_hour_anchor(transcript: str) -> bool:
|
|
412
|
+
"""True when any transcript line is anchored at >=1h (the `[h:mm:ss]`
|
|
413
|
+
contract's single decision point)."""
|
|
414
|
+
for line in transcript.splitlines():
|
|
415
|
+
match = _TIMESTAMP_HEAD_RE.match(line.strip())
|
|
416
|
+
if not match:
|
|
417
|
+
continue
|
|
418
|
+
parts = [int(p) for p in match.group(1).split(":")]
|
|
419
|
+
seconds = (
|
|
420
|
+
parts[0] * 3600 + parts[1] * 60 + parts[2]
|
|
421
|
+
if len(parts) == 3
|
|
422
|
+
else parts[0] * 60 + parts[1]
|
|
423
|
+
)
|
|
424
|
+
if seconds >= 3600:
|
|
425
|
+
return True
|
|
426
|
+
return False
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _citation_format(expect_hours: bool) -> str:
|
|
430
|
+
return "[h:mm:ss]" if expect_hours else "[mm:ss]"
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _fmt_ts(seconds: float) -> str:
|
|
434
|
+
"""Format seconds as [mm:ss] (or [h:mm:ss] for >1h)."""
|
|
435
|
+
total = int(seconds)
|
|
436
|
+
h, rem = divmod(total, 3600)
|
|
437
|
+
m, s = divmod(rem, 60)
|
|
438
|
+
if h > 0:
|
|
439
|
+
return f"{h}:{m:02d}:{s:02d}"
|
|
440
|
+
return f"{m:02d}:{s:02d}"
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
# ---------------------------------------------------------------------------
|
|
444
|
+
# Chunked rendering (roadmap 2026-10-06 §3.1.3)
|
|
445
|
+
#
|
|
446
|
+
# A >=1h transcript no longer goes through one MiniMax call (the 10/04
|
|
447
|
+
# kLZgQWjnUz0 single call hit the 16K-token output cap and dropped every
|
|
448
|
+
# hour component). Chunk boundary rule, locked by Q14c: real video chapters
|
|
449
|
+
# when available, fixed time windows otherwise. Embedding-derived topic
|
|
450
|
+
# blocks are never chunk boundaries (their boundary quality is unstable —
|
|
451
|
+
# the 10/04 final embedding block spanned 133s→14916s).
|
|
452
|
+
# ---------------------------------------------------------------------------
|
|
453
|
+
|
|
454
|
+
CHUNK_WINDOW_SECONDS = 1800.0 # 30-minute fixed windows when no chapters
|
|
455
|
+
BOUNDED_CHUNK_CHARS = 12000 # rendered transcript characters, not tokens
|
|
456
|
+
UNBOUND_BOUNDED_PLANNER = "bounded-transcript-v1"
|
|
457
|
+
BOUNDED_PLANNER = "bounded-transcript-v2"
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
@dataclass(frozen=True)
|
|
461
|
+
class ChunkWindow:
|
|
462
|
+
start: float
|
|
463
|
+
end: float
|
|
464
|
+
title: str
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def valid_chapter_list(chapters: list | None) -> list[dict]:
|
|
468
|
+
"""Normalize chapter dicts to usable {start,end,title} windows.
|
|
469
|
+
|
|
470
|
+
Idempotent: accepts both the raw yt-dlp spelling (start_time/end_time)
|
|
471
|
+
and its own output (start/end), so a caller that already validated the
|
|
472
|
+
list can pass it straight through (2026-10-06 review: the list was
|
|
473
|
+
computed twice with two independent code paths)."""
|
|
474
|
+
valid: list[dict] = []
|
|
475
|
+
for chapter in chapters or []:
|
|
476
|
+
if not isinstance(chapter, dict):
|
|
477
|
+
continue
|
|
478
|
+
start = chapter.get("start_time", chapter.get("start"))
|
|
479
|
+
end = chapter.get("end_time", chapter.get("end"))
|
|
480
|
+
title = chapter.get("title")
|
|
481
|
+
if (
|
|
482
|
+
not isinstance(start, int | float)
|
|
483
|
+
or not isinstance(end, int | float)
|
|
484
|
+
or not isinstance(title, str)
|
|
485
|
+
):
|
|
486
|
+
continue
|
|
487
|
+
if start < 0 or end <= start:
|
|
488
|
+
continue
|
|
489
|
+
valid.append({"start": float(start), "end": float(end), "title": title})
|
|
490
|
+
return valid
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _generic_window_title(index: int, start: float, end: float, is_zh: bool) -> str:
|
|
494
|
+
if is_zh:
|
|
495
|
+
return f"第 {index} 段({_fmt_ts(start)}–{_fmt_ts(end)})"
|
|
496
|
+
return f"Part {index} ({_fmt_ts(start)}–{_fmt_ts(end)})"
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def plan_chunk_windows(
|
|
500
|
+
chapters: list | None,
|
|
501
|
+
max_time: float,
|
|
502
|
+
*,
|
|
503
|
+
window_seconds: float = CHUNK_WINDOW_SECONDS,
|
|
504
|
+
language: str = "zh",
|
|
505
|
+
) -> list[ChunkWindow]:
|
|
506
|
+
"""Tile [0, max_time] with render windows: chapters when usable, else
|
|
507
|
+
fixed windows of ``window_seconds``. Windows never gap or overlap, so
|
|
508
|
+
every transcript segment lands in exactly one chunk (last segment at the
|
|
509
|
+
final boundary rides the last window)."""
|
|
510
|
+
is_zh = language != "en"
|
|
511
|
+
max_time = float(max_time)
|
|
512
|
+
valid = valid_chapter_list(chapters)
|
|
513
|
+
starts = sorted(
|
|
514
|
+
{c["start"] for c in valid if c["start"] < max_time}
|
|
515
|
+
)
|
|
516
|
+
if starts:
|
|
517
|
+
if starts[0] > 0:
|
|
518
|
+
starts.insert(0, 0.0)
|
|
519
|
+
title_by_start = {c["start"]: c["title"].strip() for c in valid}
|
|
520
|
+
windows: list[ChunkWindow] = []
|
|
521
|
+
for index, start in enumerate(starts):
|
|
522
|
+
end = starts[index + 1] if index + 1 < len(starts) else max(max_time, start + 1.0)
|
|
523
|
+
title = title_by_start.get(start) or _generic_window_title(
|
|
524
|
+
index + 1, start, end, is_zh
|
|
525
|
+
)
|
|
526
|
+
windows.append(ChunkWindow(start=start, end=end, title=title))
|
|
527
|
+
return windows
|
|
528
|
+
|
|
529
|
+
total = max(1, math.ceil(max_time / window_seconds)) if max_time > 0 else 1
|
|
530
|
+
return [
|
|
531
|
+
ChunkWindow(
|
|
532
|
+
start=i * window_seconds,
|
|
533
|
+
end=(i + 1) * window_seconds,
|
|
534
|
+
title=_generic_window_title(i + 1, i * window_seconds, (i + 1) * window_seconds, is_zh),
|
|
535
|
+
)
|
|
536
|
+
for i in range(total)
|
|
537
|
+
]
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _dedupe_titles(titles: list[str], is_zh: bool) -> list[str]:
|
|
541
|
+
"""Give duplicate headings a unique numeric suffix so the assembled doc
|
|
542
|
+
has no duplicated section headings."""
|
|
543
|
+
seen: dict[str, int] = {}
|
|
544
|
+
out: list[str] = []
|
|
545
|
+
for title in titles:
|
|
546
|
+
count = seen.get(title, 0) + 1
|
|
547
|
+
seen[title] = count
|
|
548
|
+
if count == 1:
|
|
549
|
+
out.append(title)
|
|
550
|
+
elif is_zh:
|
|
551
|
+
out.append(f"{title}({count})")
|
|
552
|
+
else:
|
|
553
|
+
out.append(f"{title} ({count})")
|
|
554
|
+
return out
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*$")
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _normalize_chunk_content(
|
|
561
|
+
content: str,
|
|
562
|
+
expected_title: str,
|
|
563
|
+
emitted_headings: set[str],
|
|
564
|
+
) -> str:
|
|
565
|
+
"""Normalize one chunk's LLM output to start with its ``## expected_title``
|
|
566
|
+
heading: keep a matching heading, drop one that duplicates an already
|
|
567
|
+
emitted heading (seam duplication), otherwise prepend the heading."""
|
|
568
|
+
lines = content.strip().splitlines()
|
|
569
|
+
first_idx = next((i for i, line in enumerate(lines) if line.strip()), None)
|
|
570
|
+
if first_idx is not None:
|
|
571
|
+
match = _HEADING_RE.match(lines[first_idx].strip())
|
|
572
|
+
if match:
|
|
573
|
+
heading_text = match.group(2).strip()
|
|
574
|
+
if heading_text == expected_title:
|
|
575
|
+
emitted_headings.add(expected_title)
|
|
576
|
+
return "\n".join(lines).strip()
|
|
577
|
+
if heading_text in emitted_headings:
|
|
578
|
+
# Seam duplication: drop the echoed heading, fall through to
|
|
579
|
+
# prepend the chunk's own heading.
|
|
580
|
+
del lines[first_idx]
|
|
581
|
+
else:
|
|
582
|
+
# Unexpected heading: keep it below the expected heading.
|
|
583
|
+
lines.insert(first_idx, f"## {expected_title}")
|
|
584
|
+
emitted_headings.add(expected_title)
|
|
585
|
+
return "\n".join(lines).strip()
|
|
586
|
+
emitted_headings.add(expected_title)
|
|
587
|
+
return "\n".join([f"## {expected_title}", *lines]).strip()
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def assemble_chunked_doc(
|
|
591
|
+
*,
|
|
592
|
+
title: str,
|
|
593
|
+
url: str,
|
|
594
|
+
windows: list[ChunkWindow],
|
|
595
|
+
contents: list[str],
|
|
596
|
+
language: str,
|
|
597
|
+
boundary_source: str,
|
|
598
|
+
citation_format: str,
|
|
599
|
+
final_titles: list[str] | None = None,
|
|
600
|
+
) -> str:
|
|
601
|
+
"""Assemble validated chunk outputs into one knowledge document with a
|
|
602
|
+
deterministic overview map and a machine-readable provenance comment."""
|
|
603
|
+
is_zh = language != "en"
|
|
604
|
+
labels = _SECTION_TITLES[language]
|
|
605
|
+
if final_titles is None:
|
|
606
|
+
final_titles = _dedupe_titles([w.title for w in windows], is_zh)
|
|
607
|
+
parts: list[str] = [labels["title_template"].format(title=title), ""]
|
|
608
|
+
if url:
|
|
609
|
+
parts += [labels["source"].format(url=url), ""]
|
|
610
|
+
parts += [
|
|
611
|
+
(
|
|
612
|
+
f"<!-- ytk-knowledge-doc: chunked; boundary={boundary_source}; "
|
|
613
|
+
f"chunks={len(windows)}; citation_format={citation_format} -->"
|
|
614
|
+
),
|
|
615
|
+
"",
|
|
616
|
+
labels["overview"],
|
|
617
|
+
"",
|
|
618
|
+
]
|
|
619
|
+
# strict=False on both zips below: final_titles/contents are caller contracts
|
|
620
|
+
# (1:1 with windows at every in-repo call site); strict=True would raise on a
|
|
621
|
+
# length mismatch where the previous code truncated.
|
|
622
|
+
for window, final_title in zip(windows, final_titles, strict=False):
|
|
623
|
+
if is_zh:
|
|
624
|
+
parts.append(f"- {final_title}({_fmt_ts(window.start)}–{_fmt_ts(window.end)})")
|
|
625
|
+
else:
|
|
626
|
+
parts.append(f"- {final_title} ({_fmt_ts(window.start)}–{_fmt_ts(window.end)})")
|
|
627
|
+
parts.append("")
|
|
628
|
+
emitted: set[str] = set()
|
|
629
|
+
for _window, final_title, content in zip(windows, final_titles, contents, strict=False):
|
|
630
|
+
parts.append(
|
|
631
|
+
_normalize_chunk_content(content, final_title, emitted)
|
|
632
|
+
)
|
|
633
|
+
parts.append("")
|
|
634
|
+
return "\n".join(parts).rstrip()
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _strip_leading_heading(content: str) -> str:
|
|
638
|
+
lines = content.strip().splitlines()
|
|
639
|
+
first_idx = next((i for i, line in enumerate(lines) if line.strip()), None)
|
|
640
|
+
if first_idx is not None and _HEADING_RE.match(lines[first_idx].strip()):
|
|
641
|
+
del lines[first_idx]
|
|
642
|
+
return "\n".join(lines)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def verify_chunk_assembly(
|
|
646
|
+
assembled: str,
|
|
647
|
+
windows: list[ChunkWindow],
|
|
648
|
+
contents: list[str],
|
|
649
|
+
*,
|
|
650
|
+
language: str = "zh",
|
|
651
|
+
final_titles: list[str] | None = None,
|
|
652
|
+
) -> list[str]:
|
|
653
|
+
"""Seam-consistency check (acceptance §3.1 ⑤): every chunk body must
|
|
654
|
+
appear exactly once and every expected heading exactly once.
|
|
655
|
+
|
|
656
|
+
``final_titles`` lets a caller that already de-duplicated the headings
|
|
657
|
+
hand the SAME list to assembler and verifier — recomputing it here
|
|
658
|
+
would let the two drift (the check would then validate headings that
|
|
659
|
+
were never written; 2026-10-06 review finding)."""
|
|
660
|
+
is_zh = language != "en"
|
|
661
|
+
if final_titles is None:
|
|
662
|
+
final_titles = _dedupe_titles([w.title for w in windows], is_zh)
|
|
663
|
+
problems: list[str] = []
|
|
664
|
+
# strict=False: a caller-supplied final_titles may be shorter than contents;
|
|
665
|
+
# the verifier must report the gap as a problem instead of raising.
|
|
666
|
+
for index, (content, final_title) in enumerate(
|
|
667
|
+
zip(contents, final_titles, strict=False), start=1
|
|
668
|
+
):
|
|
669
|
+
body = _strip_leading_heading(content).strip()
|
|
670
|
+
if not body:
|
|
671
|
+
problems.append(f"chunk {index} ({final_title}): empty content")
|
|
672
|
+
continue
|
|
673
|
+
occurrences = assembled.count(body)
|
|
674
|
+
if occurrences == 0:
|
|
675
|
+
problems.append(f"chunk {index} ({final_title}): content missing from assembled doc")
|
|
676
|
+
elif occurrences > 1:
|
|
677
|
+
problems.append(
|
|
678
|
+
f"chunk {index} ({final_title}): content duplicated {occurrences}x"
|
|
679
|
+
)
|
|
680
|
+
heading = f"## {final_title}"
|
|
681
|
+
heading_count = len(
|
|
682
|
+
re.findall(f"^{re.escape(heading)}$", assembled, flags=re.MULTILINE)
|
|
683
|
+
)
|
|
684
|
+
if heading_count != 1:
|
|
685
|
+
problems.append(
|
|
686
|
+
f"chunk {index}: heading '{heading}' present {heading_count}x (expected 1)"
|
|
687
|
+
)
|
|
688
|
+
return problems
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def detect_doc_language(explicit: str | None = None) -> DocLanguage:
|
|
692
|
+
"""Resolve the doc output language.
|
|
693
|
+
|
|
694
|
+
Resolution order (highest priority first):
|
|
695
|
+
1. ``explicit`` argument (typically the ``YTK_DOC_LANGUAGE`` env var
|
|
696
|
+
value, or a constructor argument the CLI passed in).
|
|
697
|
+
2. Shell locale: $LC_MESSAGES → $LC_ALL → $LANG. If any of these
|
|
698
|
+
contains "zh" we return "zh"; if it contains "en" we return "en".
|
|
699
|
+
(Locale strings look like ``zh_CN.UTF-8`` or ``en_US.UTF-8``.)
|
|
700
|
+
3. Default: "zh" (preserves v1.0.0 behavior for existing users).
|
|
701
|
+
|
|
702
|
+
Only the canonical codes "zh" and "en" are accepted. Any other explicit
|
|
703
|
+
value is logged and treated as "auto" (fall through to locale probing).
|
|
704
|
+
"""
|
|
705
|
+
if explicit:
|
|
706
|
+
candidate = explicit.strip().lower()
|
|
707
|
+
if candidate in ("zh", "en"):
|
|
708
|
+
return candidate # type: ignore[return-value]
|
|
709
|
+
if candidate != "auto":
|
|
710
|
+
logger.warning(
|
|
711
|
+
"YTK_DOC_LANGUAGE=%r is not one of {zh, en, auto}; ignoring.",
|
|
712
|
+
explicit,
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
for env_var in ("LC_MESSAGES", "LC_ALL", "LANG"):
|
|
716
|
+
value = os.environ.get(env_var, "")
|
|
717
|
+
if not value:
|
|
718
|
+
continue
|
|
719
|
+
normalized = value.lower().replace("-", "_")
|
|
720
|
+
if normalized.startswith("zh"):
|
|
721
|
+
return "zh"
|
|
722
|
+
if normalized.startswith("en"):
|
|
723
|
+
return "en"
|
|
724
|
+
return "zh"
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
# ---------------------------------------------------------------------------
|
|
729
|
+
# v2 knowledge-doc extension (T33.3)
|
|
730
|
+
#
|
|
731
|
+
# The v1 path above stays byte-identical. v2 bundles are planned from their
|
|
732
|
+
# ContentBundle blocks instead of raw-transcript.json:
|
|
733
|
+
# * video -> time-range blocks become (start, text) segments and reuse
|
|
734
|
+
# the existing bounded time chunker unchanged;
|
|
735
|
+
# * image_note -> blocks are grouped in reading order under the same
|
|
736
|
+
# character budget, with no block dropped or duplicated, and
|
|
737
|
+
# citations use 图 N / 第 N 段 (never a timestamp, A22).
|
|
738
|
+
# Prompt classification (tutorial / general knowledge / image note) is applied
|
|
739
|
+
# through ``_classification_suffix``; with no v2 render in flight it returns ""
|
|
740
|
+
# so every v1 prompt and checkpoint stays byte-identical.
|
|
741
|
+
# ---------------------------------------------------------------------------
|
|
742
|
+
|
|
743
|
+
IMAGE_NOTE_PLANNER = "image-note-chunk-v1"
|
|
744
|
+
|
|
745
|
+
#: Atom types that make a source actionable enough to drive a tutorial doc.
|
|
746
|
+
ACTIONABLE_ATOM_TYPES: frozenset[str] = frozenset(
|
|
747
|
+
{
|
|
748
|
+
"terminal_command",
|
|
749
|
+
"command_sequence",
|
|
750
|
+
"configuration_snippet",
|
|
751
|
+
"code_block",
|
|
752
|
+
"workflow_step",
|
|
753
|
+
}
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
DocKind = Literal["tutorial", "general", "image_note"]
|
|
757
|
+
|
|
758
|
+
_IMAGE_CITATION_FORMAT: dict[str, str] = {
|
|
759
|
+
"zh": "[图 N] / [第 N 段]",
|
|
760
|
+
"en": "[Image N] / [Paragraph N]",
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
_IMAGE_NOTE_TEMPLATES: dict[str, str] = {
|
|
764
|
+
"zh": """你是知识整理助手。下面是一篇**图文笔记**(来自小红书等平台)的内容块,按原帖阅读顺序排列,每块以定位标记开头(格式 `[图 N]` 或 `[第 N 段]`)。请把它改写成一篇干净、结构化的**中文**知识文档,直接保存为 markdown 文件使用。
|
|
765
|
+
|
|
766
|
+
# 笔记元数据
|
|
767
|
+
- 标题: {title}
|
|
768
|
+
- URL: {url}
|
|
769
|
+
|
|
770
|
+
# 内容块(按原帖顺序)
|
|
771
|
+
{blocks}
|
|
772
|
+
|
|
773
|
+
# 要求
|
|
774
|
+
1. **必须用中文输出**(专有名词保留原文)。
|
|
775
|
+
2. 按图片与正文的原始顺序组织内容,不要打乱图片顺序。
|
|
776
|
+
3. 引用内容时使用 `{citation_format}` 形式的定位标记,写在括号内;**禁止出现任何时间戳**(图文没有时间轴)。
|
|
777
|
+
4. 不引入笔记之外的内容;没有操作步骤时不要编造步骤。
|
|
778
|
+
5. 不使用 emoji;直接输出 markdown,不要加前言。
|
|
779
|
+
|
|
780
|
+
# 输出格式
|
|
781
|
+
直接以 `# 概述` 开头。
|
|
782
|
+
""",
|
|
783
|
+
"en": """You are a knowledge-curation assistant. Below is an **image note** (e.g. a Xiaohongshu post) given as content blocks in the original reading order; each block starts with its locator marker (`[Image N]` or `[Paragraph N]`). Rewrite it into a clean, structured **English** markdown knowledge document.
|
|
784
|
+
|
|
785
|
+
# Note metadata
|
|
786
|
+
- Title: {title}
|
|
787
|
+
- URL: {url}
|
|
788
|
+
|
|
789
|
+
# Content blocks (original order)
|
|
790
|
+
{blocks}
|
|
791
|
+
|
|
792
|
+
# Requirements
|
|
793
|
+
1. **Output MUST be in English** (keep proper nouns as written).
|
|
794
|
+
2. Keep the original image/paragraph order; never reorder images.
|
|
795
|
+
3. Cite with `{citation_format}` locator markers in parentheses; **never emit a timestamp** (an image note has no timeline).
|
|
796
|
+
4. Do not introduce content from outside the note; never invent steps when none exist.
|
|
797
|
+
5. No emoji; output markdown directly with no preamble.
|
|
798
|
+
|
|
799
|
+
# Output format
|
|
800
|
+
Start directly with `# Overview`.
|
|
801
|
+
""",
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
@dataclass(frozen=True)
|
|
806
|
+
class V2DocSource:
|
|
807
|
+
"""Normalized v2 input for the knowledge-doc renderer."""
|
|
808
|
+
|
|
809
|
+
kind: Literal["video", "image_note"]
|
|
810
|
+
blocks: list[Any]
|
|
811
|
+
segments: list[tuple[float, str]]
|
|
812
|
+
metadata: dict[str, Any]
|
|
813
|
+
primary: Any = None
|
|
814
|
+
content: Any = None
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
@dataclass(frozen=True)
|
|
818
|
+
class ImageNoteChunk:
|
|
819
|
+
"""One bounded image-note plan unit (blocks keep their reading order)."""
|
|
820
|
+
|
|
821
|
+
index: int
|
|
822
|
+
block_ids: tuple[str, ...]
|
|
823
|
+
lines: tuple[str, ...]
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
@dataclass(frozen=True)
|
|
827
|
+
class _VideoDocShim:
|
|
828
|
+
"""Minimal v1-compatible view for the existing LLM doc pipeline."""
|
|
829
|
+
|
|
830
|
+
metadata: dict[str, Any]
|
|
831
|
+
video_id: str
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def is_v2_bundle(bundle: Any) -> bool:
|
|
835
|
+
"""True for an :class:`ExtractionBundleV2` (v2 render path)."""
|
|
836
|
+
from ytk.models.content import ExtractionBundleV2
|
|
837
|
+
|
|
838
|
+
return isinstance(bundle, ExtractionBundleV2)
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def classify_doc_kind(bundle: Any) -> DocKind:
|
|
842
|
+
"""Classify the source for prompt selection: tutorial / general / image note."""
|
|
843
|
+
content_types = {
|
|
844
|
+
content.source.content_type for content in getattr(bundle, "contents", [])
|
|
845
|
+
}
|
|
846
|
+
if "image_note" in content_types:
|
|
847
|
+
return "image_note"
|
|
848
|
+
atom_types = {atom.atom_type for atom in getattr(bundle, "knowledge", [])}
|
|
849
|
+
if atom_types & ACTIONABLE_ATOM_TYPES:
|
|
850
|
+
return "tutorial"
|
|
851
|
+
return "general"
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def image_note_block_line(block: Any, *, language: str = "zh") -> str:
|
|
855
|
+
"""One image-note block as ``[图 N] text`` / ``[第 N 段] text``."""
|
|
856
|
+
from ytk.renderers.source_links import readable_locator
|
|
857
|
+
|
|
858
|
+
text = (block.cleaned_text or block.raw_text).strip()
|
|
859
|
+
marker = readable_locator(block.locator, language=language)
|
|
860
|
+
return f"[{marker}] {text}" if marker else text
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def plan_image_note_chunks(
|
|
864
|
+
blocks: list[Any], *, max_chars: int = BOUNDED_CHUNK_CHARS, language: str = "zh"
|
|
865
|
+
) -> list[ImageNoteChunk]:
|
|
866
|
+
"""Group image-note blocks in order under a character budget.
|
|
867
|
+
|
|
868
|
+
Blocks are never split, dropped or duplicated: every input block lands in
|
|
869
|
+
exactly one chunk, in its original order. A single block larger than the
|
|
870
|
+
budget fails *before* any paid dispatch (same contract as the transcript
|
|
871
|
+
planner).
|
|
872
|
+
"""
|
|
873
|
+
chunks: list[ImageNoteChunk] = []
|
|
874
|
+
current_ids: list[str] = []
|
|
875
|
+
current_lines: list[str] = []
|
|
876
|
+
size = 0
|
|
877
|
+
for block in blocks:
|
|
878
|
+
line = image_note_block_line(block, language=language)
|
|
879
|
+
if len(line) > max_chars:
|
|
880
|
+
raise RuntimeError("an image-note block exceeds the bounded chunk size")
|
|
881
|
+
if current_lines and size + len(line) + 1 > max_chars:
|
|
882
|
+
chunks.append(
|
|
883
|
+
ImageNoteChunk(len(chunks) + 1, tuple(current_ids), tuple(current_lines))
|
|
884
|
+
)
|
|
885
|
+
current_ids, current_lines, size = [], [], 0
|
|
886
|
+
current_ids.append(block.block_id)
|
|
887
|
+
current_lines.append(line)
|
|
888
|
+
size += len(line) + 1
|
|
889
|
+
if current_lines:
|
|
890
|
+
chunks.append(ImageNoteChunk(len(chunks) + 1, tuple(current_ids), tuple(current_lines)))
|
|
891
|
+
return chunks
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def verify_image_note_plan(chunks: list[ImageNoteChunk], blocks: list[Any]) -> list[str]:
|
|
895
|
+
"""Check the plan covers every block exactly once, in order."""
|
|
896
|
+
problems: list[str] = []
|
|
897
|
+
planned = [block_id for chunk in chunks for block_id in chunk.block_ids]
|
|
898
|
+
expected = [block.block_id for block in blocks]
|
|
899
|
+
if planned != expected:
|
|
900
|
+
problems.append(
|
|
901
|
+
f"image-note plan covers blocks {planned!r}, expected {expected!r} "
|
|
902
|
+
"(dropped, duplicated or reordered blocks)"
|
|
903
|
+
)
|
|
904
|
+
return problems
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
def _image_chunk_title(index: int, language: str) -> str:
|
|
908
|
+
if language == "en":
|
|
909
|
+
return f"Part {index}"
|
|
910
|
+
return f"第 {index} 部分"
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def assemble_image_note_doc(
|
|
914
|
+
*,
|
|
915
|
+
title: str,
|
|
916
|
+
url: str,
|
|
917
|
+
chunks: list[ImageNoteChunk],
|
|
918
|
+
contents: list[str],
|
|
919
|
+
language: str,
|
|
920
|
+
final_titles: list[str] | None = None,
|
|
921
|
+
) -> str:
|
|
922
|
+
"""Assemble validated image-note chunk outputs (no timestamps anywhere)."""
|
|
923
|
+
is_zh = language != "en"
|
|
924
|
+
labels = _SECTION_TITLES[language]
|
|
925
|
+
if final_titles is None:
|
|
926
|
+
final_titles = _dedupe_titles(
|
|
927
|
+
[_image_chunk_title(chunk.index, language) for chunk in chunks], is_zh
|
|
928
|
+
)
|
|
929
|
+
parts: list[str] = [labels["title_template"].format(title=title), ""]
|
|
930
|
+
if url:
|
|
931
|
+
parts += [labels["source"].format(url=url), ""]
|
|
932
|
+
parts += [
|
|
933
|
+
(
|
|
934
|
+
f"<!-- ytk-knowledge-doc: image-note; chunks={len(chunks)}; "
|
|
935
|
+
f"citation_format={_IMAGE_CITATION_FORMAT[language]} -->"
|
|
936
|
+
),
|
|
937
|
+
"",
|
|
938
|
+
labels["overview"],
|
|
939
|
+
"",
|
|
940
|
+
]
|
|
941
|
+
# strict=False: final_titles/contents are caller contracts (1:1 with chunks at
|
|
942
|
+
# every in-repo call site); strict=True would raise where the previous code
|
|
943
|
+
# truncated.
|
|
944
|
+
for chunk, final_title in zip(chunks, final_titles, strict=False):
|
|
945
|
+
if is_zh:
|
|
946
|
+
parts.append(f"- {final_title}({len(chunk.block_ids)} 个内容块)")
|
|
947
|
+
else:
|
|
948
|
+
parts.append(f"- {final_title} ({len(chunk.block_ids)} content blocks)")
|
|
949
|
+
parts.append("")
|
|
950
|
+
emitted: set[str] = set()
|
|
951
|
+
for final_title, content in zip(final_titles, contents, strict=False):
|
|
952
|
+
parts.append(_normalize_chunk_content(content, final_title, emitted))
|
|
953
|
+
parts.append("")
|
|
954
|
+
return "\n".join(parts).rstrip()
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
class KnowledgeDocRenderer:
|
|
958
|
+
"""Rewrite a transcript into a single knowledge document via MiniMax.
|
|
959
|
+
|
|
960
|
+
Output language is selectable (zh/en). See module docstring for the
|
|
961
|
+
resolution order.
|
|
962
|
+
"""
|
|
963
|
+
|
|
964
|
+
SUPPORTED_LANGUAGES: ClassVar[tuple[str, ...]] = ("zh", "en")
|
|
965
|
+
|
|
966
|
+
def __init__(self, language: str | None = None, *, settings: AppSettings | None = None) -> None:
|
|
967
|
+
self._api_key: str | None = settings.minimax_api_key if settings is not None else os.environ.get("YTK_MINIMAX_API_KEY") or None
|
|
968
|
+
self._region: str = settings.minimax_region if settings is not None else os.environ.get("YTK_MINIMAX_REGION", "cn")
|
|
969
|
+
self._model: str = settings.minimax_model if settings is not None else os.environ.get("YTK_MINIMAX_MODEL", "MiniMax-M3")
|
|
970
|
+
self._base_url: str = _MINIMAX_BASE_URLS.get(self._region, _MINIMAX_BASE_URLS["cn"])
|
|
971
|
+
self._timeout: float = settings.knowledge_doc_timeout if settings is not None else float(os.environ.get("YTK_KNOWLEDGE_DOC_TIMEOUT", "300.0"))
|
|
972
|
+
self._total_timeout: float = float(os.environ.get("YTK_KNOWLEDGE_DOC_TOTAL_TIMEOUT", "1200.0"))
|
|
973
|
+
if not math.isfinite(self._total_timeout) or self._total_timeout <= 0:
|
|
974
|
+
raise ValueError("YTK_KNOWLEDGE_DOC_TOTAL_TIMEOUT must be positive and finite")
|
|
975
|
+
self._checkpoint_path: Path | None = None
|
|
976
|
+
self._deadline: float | None = None
|
|
977
|
+
self._doc_kind: DocKind | None = None
|
|
978
|
+
self._instance_lock = threading.Lock()
|
|
979
|
+
# Allow longer outputs than the ingestion path — the knowledge doc is
|
|
980
|
+
# often the only artifact the user keeps, so trimming it hurts.
|
|
981
|
+
self._max_tokens: int = settings.knowledge_doc_max_tokens if settings is not None else int(os.environ.get("YTK_KNOWLEDGE_DOC_MAX_TOKENS", "16384"))
|
|
982
|
+
# Resolve language: explicit ctor arg wins; otherwise YTK_DOC_LANGUAGE
|
|
983
|
+
# env var, otherwise locale probe, otherwise "zh".
|
|
984
|
+
explicit_env = settings.doc_language if settings is not None else os.environ.get("YTK_DOC_LANGUAGE")
|
|
985
|
+
self._language: DocLanguage = detect_doc_language(language or explicit_env)
|
|
986
|
+
|
|
987
|
+
@property
|
|
988
|
+
def language(self) -> DocLanguage:
|
|
989
|
+
"""Return the resolved output language (zh or en)."""
|
|
990
|
+
return self._language
|
|
991
|
+
|
|
992
|
+
def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
|
|
993
|
+
if not self._instance_lock.acquire(blocking=False):
|
|
994
|
+
raise _ProgressBusy("knowledge-doc renderer is already in use")
|
|
995
|
+
try:
|
|
996
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
997
|
+
self._checkpoint_path = output_path.parent / ".knowledge-doc-progress.json"
|
|
998
|
+
with self._progress_lock(self._checkpoint_path):
|
|
999
|
+
self._render_locked(bundle, output_path, **kwargs)
|
|
1000
|
+
finally:
|
|
1001
|
+
self._instance_lock.release()
|
|
1002
|
+
|
|
1003
|
+
def _render_locked(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
|
|
1004
|
+
status_path = output_path.parent / ".knowledge-doc-status.json"
|
|
1005
|
+
self._write_progress(status_path, {"schema": 1, "state": "RUNNING"})
|
|
1006
|
+
if is_v2_bundle(bundle):
|
|
1007
|
+
self._render_locked_v2(bundle, output_path, status_path, **kwargs)
|
|
1008
|
+
return
|
|
1009
|
+
segments = self._load_transcript_segments(output_path.parent / "raw-transcript.json")
|
|
1010
|
+
# Build the transcript text ONCE (it was re-parsed from disk up to
|
|
1011
|
+
# three times per render — 2026-10-06 review finding).
|
|
1012
|
+
transcript = self._transcript_text(segments)
|
|
1013
|
+
reason: str | None = None
|
|
1014
|
+
if not self._api_key:
|
|
1015
|
+
# No LLM key — fall back to a deterministic, no-fluff document
|
|
1016
|
+
# built from the raw transcript and the bundle's existing
|
|
1017
|
+
# knowledge atoms. Useful for offline demos / fixture runs.
|
|
1018
|
+
logger.warning(
|
|
1019
|
+
"YTK_MINIMAX_API_KEY not set; KnowledgeDocRenderer is falling back "
|
|
1020
|
+
"to a deterministic document built from the raw transcript. "
|
|
1021
|
+
"Set YTK_MINIMAX_API_KEY in .env for the LLM-rewritten version."
|
|
1022
|
+
)
|
|
1023
|
+
content = self._fallback_doc(bundle, transcript)
|
|
1024
|
+
reason = "NO_API_KEY"
|
|
1025
|
+
else:
|
|
1026
|
+
try:
|
|
1027
|
+
content = self._render_llm_document(bundle, segments)
|
|
1028
|
+
except RuntimeError as exc:
|
|
1029
|
+
# LLM failure OR citation-alignment failure: the deterministic
|
|
1030
|
+
# fallback keeps the doc usable, and the warning keeps the
|
|
1031
|
+
# degradation non-silent (roadmap §3.1.2: misaligned LLM
|
|
1032
|
+
# output must never land as-is).
|
|
1033
|
+
logger.warning(
|
|
1034
|
+
"MiniMax knowledge-doc rewrite rejected; falling back to the "
|
|
1035
|
+
"deterministic document: %s",
|
|
1036
|
+
exc,
|
|
1037
|
+
)
|
|
1038
|
+
content = self._fallback_doc(bundle, transcript)
|
|
1039
|
+
message = str(exc)
|
|
1040
|
+
if message == "knowledge-doc total time budget exhausted":
|
|
1041
|
+
reason = "TOTAL_TIMEOUT"
|
|
1042
|
+
elif message == "knowledge-doc chunk has an unresolved or invalid prior dispatch":
|
|
1043
|
+
reason = "UNRESOLVED_OR_INVALID_DISPATCH"
|
|
1044
|
+
elif message == "knowledge-doc progress cannot be safely reused":
|
|
1045
|
+
reason = "PROGRESS_MISMATCH"
|
|
1046
|
+
else:
|
|
1047
|
+
reason = "REWRITE_FAILED"
|
|
1048
|
+
try:
|
|
1049
|
+
checkpoint_path = self._checkpoint_path
|
|
1050
|
+
if checkpoint_path is None: # defensive: no checkpoint on this path
|
|
1051
|
+
raise OSError("knowledge-doc checkpoint path is not configured")
|
|
1052
|
+
progress = json.loads(checkpoint_path.read_text(encoding="utf-8"))
|
|
1053
|
+
chunks = progress.get("chunks", [])
|
|
1054
|
+
if isinstance(chunks, list) and chunks and isinstance(chunks[-1], dict) and chunks[-1].get("state") == "DISPATCHING":
|
|
1055
|
+
reason = "TOTAL_TIMEOUT_WITH_UNRESOLVED_DISPATCH" if reason == "TOTAL_TIMEOUT" else "UNRESOLVED_OR_INVALID_DISPATCH"
|
|
1056
|
+
except (OSError, ValueError, TypeError, AttributeError):
|
|
1057
|
+
pass
|
|
1058
|
+
finally:
|
|
1059
|
+
self._deadline = None
|
|
1060
|
+
|
|
1061
|
+
if kwargs.get("with_reviewer_scaffolding"):
|
|
1062
|
+
# Pull real metadata from the bundle so A-class and C-class source
|
|
1063
|
+
# entries are actual citations (the source video URL), not
|
|
1064
|
+
# placeholder text. B-class and the image manifest stay as
|
|
1065
|
+
# explicitly-marked template slots.
|
|
1066
|
+
metadata = getattr(bundle, "metadata", None) or {}
|
|
1067
|
+
title = str(metadata.get("title", "") or "")
|
|
1068
|
+
source_url = str(metadata.get("canonical_url", "") or "")
|
|
1069
|
+
content = (
|
|
1070
|
+
content.rstrip()
|
|
1071
|
+
+ "\n"
|
|
1072
|
+
+ build_reviewer_scaffolding(
|
|
1073
|
+
self._language,
|
|
1074
|
+
title=title,
|
|
1075
|
+
source_url=source_url,
|
|
1076
|
+
)
|
|
1077
|
+
)
|
|
1078
|
+
|
|
1079
|
+
if reason:
|
|
1080
|
+
content = f"<!-- ytk-knowledge-doc: fallback; reason={reason} -->\n\n" + content
|
|
1081
|
+
rendered = content.strip() + "\n"
|
|
1082
|
+
destination = output_path
|
|
1083
|
+
if reason and output_path.exists():
|
|
1084
|
+
destination = output_path.with_name(output_path.stem + ".fallback" + output_path.suffix)
|
|
1085
|
+
self._write_text_atomic(destination, rendered)
|
|
1086
|
+
self._write_progress(status_path, {"schema": 1, "state": "FALLBACK" if reason else "COMPLETE",
|
|
1087
|
+
"reason": reason, "output_file": destination.name,
|
|
1088
|
+
"output_sha256": hashlib.sha256(rendered.encode()).hexdigest()})
|
|
1089
|
+
|
|
1090
|
+
def _fallback_doc(self, bundle: Any, transcript: str) -> str:
|
|
1091
|
+
"""Build a deterministic knowledge document without LLM.
|
|
1092
|
+
|
|
1093
|
+
Strips common fillers from the raw transcript, lists each topic
|
|
1094
|
+
segment with its first sentence, and dumps the surviving knowledge
|
|
1095
|
+
atoms grouped by type. This is a degraded mode — the LLM-rewritten
|
|
1096
|
+
path produces a far better document, but this fallback keeps
|
|
1097
|
+
`ytk extract --format doc` working in offline / fixture scenarios
|
|
1098
|
+
where no API key is available.
|
|
1099
|
+
|
|
1100
|
+
Output language follows ``self._language``.
|
|
1101
|
+
"""
|
|
1102
|
+
labels = _SECTION_TITLES[self._language]
|
|
1103
|
+
title = str(bundle.metadata.get("title", bundle.video_id))
|
|
1104
|
+
url = str(bundle.metadata.get("canonical_url", ""))
|
|
1105
|
+
|
|
1106
|
+
filler_re = re.compile(
|
|
1107
|
+
r"\b(?:uh+|um+|ah+|er+|hmm|"
|
|
1108
|
+
r"so|so basically|basically|"
|
|
1109
|
+
r"you know|i mean|like|"
|
|
1110
|
+
r"right|alright|okay|ok|"
|
|
1111
|
+
r"well|honestly|literally|"
|
|
1112
|
+
r"kinda|sort of|"
|
|
1113
|
+
r"and then|so then|"
|
|
1114
|
+
r"let's go ahead and|now let's)\b[,]?",
|
|
1115
|
+
re.IGNORECASE,
|
|
1116
|
+
)
|
|
1117
|
+
|
|
1118
|
+
cleaned_lines: list[str] = []
|
|
1119
|
+
for line in transcript.splitlines():
|
|
1120
|
+
stripped = filler_re.sub("", line).strip()
|
|
1121
|
+
stripped = re.sub(r"\s{2,}", " ", stripped)
|
|
1122
|
+
# Strip leading/trailing punctuation left over from filler removal.
|
|
1123
|
+
stripped = re.sub(r"^[\s,.;:]+", "", stripped)
|
|
1124
|
+
stripped = re.sub(r"[\s,.;:]+$", "", stripped)
|
|
1125
|
+
if stripped:
|
|
1126
|
+
cleaned_lines.append(stripped)
|
|
1127
|
+
|
|
1128
|
+
sections: list[str] = [labels["title_template"].format(title=title), ""]
|
|
1129
|
+
if url:
|
|
1130
|
+
sections.append(labels["source"].format(url=url))
|
|
1131
|
+
sections.append("")
|
|
1132
|
+
|
|
1133
|
+
sections.append(labels["overview"])
|
|
1134
|
+
sections.append("")
|
|
1135
|
+
overview = " ".join(cleaned_lines[:3]).strip() or labels["no_transcript"]
|
|
1136
|
+
sections.append(overview)
|
|
1137
|
+
sections.append("")
|
|
1138
|
+
|
|
1139
|
+
if bundle.topic_segments:
|
|
1140
|
+
sections.append(labels["topic_segments"])
|
|
1141
|
+
sections.append("")
|
|
1142
|
+
for topic in bundle.topic_segments:
|
|
1143
|
+
first_sentence = topic.sentences[0] if topic.sentences else ""
|
|
1144
|
+
sections.append(
|
|
1145
|
+
f"- `{topic.title}` ({topic.start:.0f}s-{topic.end:.0f}s)"
|
|
1146
|
+
+ (f" — {first_sentence}" if first_sentence else "")
|
|
1147
|
+
)
|
|
1148
|
+
sections.append("")
|
|
1149
|
+
|
|
1150
|
+
if bundle.knowledge_atoms:
|
|
1151
|
+
sections.append(labels["knowledge_atoms"])
|
|
1152
|
+
sections.append("")
|
|
1153
|
+
by_type: dict[str, list] = {}
|
|
1154
|
+
for atom in bundle.knowledge_atoms:
|
|
1155
|
+
by_type.setdefault(atom.atom_type, []).append(atom)
|
|
1156
|
+
for atom_type, atoms in by_type.items():
|
|
1157
|
+
localized = labels["type_titles"].get(atom_type, atom_type)
|
|
1158
|
+
sections.append(f"### {localized}")
|
|
1159
|
+
sections.append("")
|
|
1160
|
+
for atom in atoms:
|
|
1161
|
+
evidence = atom.evidence[0] if atom.evidence else None
|
|
1162
|
+
cite = (
|
|
1163
|
+
f" `[{_fmt_ts(evidence.start)}]`"
|
|
1164
|
+
if evidence is not None
|
|
1165
|
+
else ""
|
|
1166
|
+
)
|
|
1167
|
+
sections.append(f"- {atom.content}{cite}")
|
|
1168
|
+
sections.append("")
|
|
1169
|
+
|
|
1170
|
+
sections.append(labels["full_transcript"])
|
|
1171
|
+
sections.append("")
|
|
1172
|
+
sections.extend(cleaned_lines)
|
|
1173
|
+
|
|
1174
|
+
return "\n".join(sections)
|
|
1175
|
+
|
|
1176
|
+
@staticmethod
|
|
1177
|
+
def _load_transcript_segments(raw_path: Path) -> list[tuple[float, str]]:
|
|
1178
|
+
"""Load raw-transcript.json as (start_seconds, text) pairs."""
|
|
1179
|
+
if not raw_path.exists():
|
|
1180
|
+
logger.warning("raw-transcript.json not found at %s; emitting empty doc", raw_path)
|
|
1181
|
+
return []
|
|
1182
|
+
try:
|
|
1183
|
+
payload = json.loads(raw_path.read_text(encoding="utf-8"))
|
|
1184
|
+
except json.JSONDecodeError as exc:
|
|
1185
|
+
logger.warning("Failed to parse raw-transcript.json: %s", exc)
|
|
1186
|
+
return []
|
|
1187
|
+
segments = payload.get("segments") if isinstance(payload, dict) else payload
|
|
1188
|
+
if not isinstance(segments, list):
|
|
1189
|
+
return []
|
|
1190
|
+
loaded: list[tuple[float, str]] = []
|
|
1191
|
+
for seg in segments:
|
|
1192
|
+
if not isinstance(seg, dict):
|
|
1193
|
+
continue
|
|
1194
|
+
text = str(seg.get("text", "")).strip()
|
|
1195
|
+
if not text:
|
|
1196
|
+
continue
|
|
1197
|
+
loaded.append((float(seg.get("start", 0.0)), text))
|
|
1198
|
+
return loaded
|
|
1199
|
+
|
|
1200
|
+
@staticmethod
|
|
1201
|
+
def _transcript_text(segments: list[tuple[float, str]]) -> str:
|
|
1202
|
+
"""Render loaded segments as `[mm:ss] text` lines. Timestamps use
|
|
1203
|
+
`_fmt_ts`, so transcripts of >=1h videos already carry `[h:mm:ss]`
|
|
1204
|
+
anchors regardless of the prompt format contract."""
|
|
1205
|
+
return "\n".join(
|
|
1206
|
+
f"[{_fmt_ts(start)}] {text}" for start, text in segments
|
|
1207
|
+
)
|
|
1208
|
+
|
|
1209
|
+
@classmethod
|
|
1210
|
+
def _load_transcript(cls, raw_path: Path) -> str:
|
|
1211
|
+
"""Concatenate raw-transcript.json segments with timestamp prefixes."""
|
|
1212
|
+
return cls._transcript_text(cls._load_transcript_segments(raw_path))
|
|
1213
|
+
|
|
1214
|
+
def _build_prompt(
|
|
1215
|
+
self, bundle: Any, transcript: str, *, expect_hours: bool | None = None
|
|
1216
|
+
) -> str:
|
|
1217
|
+
title = str(bundle.metadata.get("title", bundle.video_id))
|
|
1218
|
+
url = str(bundle.metadata.get("canonical_url", ""))
|
|
1219
|
+
duration = bundle.metadata.get("duration")
|
|
1220
|
+
if duration is None:
|
|
1221
|
+
# fall back to last segment end if available
|
|
1222
|
+
duration = "未知" if self._language == "zh" else "unknown"
|
|
1223
|
+
# Dynamic citation format (roadmap §3.1.1): >=1h videos cite [h:mm:ss].
|
|
1224
|
+
# Single source of truth for the decision (the transcript's own
|
|
1225
|
+
# `_fmt_ts` anchors are authoritative; metadata duration is the
|
|
1226
|
+
# hint). The render path already computed it from the segments and
|
|
1227
|
+
# passes it in so the two can never disagree.
|
|
1228
|
+
if expect_hours is None:
|
|
1229
|
+
expect_hours = _transcript_has_hour_anchor(transcript)
|
|
1230
|
+
citation_format = _citation_format(expect_hours)
|
|
1231
|
+
if self._language == "zh":
|
|
1232
|
+
hour_rule = _HOUR_RULE_ZH_LONG if expect_hours else _HOUR_RULE_ZH_SHORT
|
|
1233
|
+
else:
|
|
1234
|
+
hour_rule = _HOUR_RULE_EN_LONG if expect_hours else _HOUR_RULE_EN_SHORT
|
|
1235
|
+
template = _PROMPT_TEMPLATES[self._language]
|
|
1236
|
+
prompt = template.format(
|
|
1237
|
+
title=title,
|
|
1238
|
+
url=url,
|
|
1239
|
+
duration=duration,
|
|
1240
|
+
transcript=transcript,
|
|
1241
|
+
ts_format=citation_format,
|
|
1242
|
+
citation_format=citation_format,
|
|
1243
|
+
hour_rule=hour_rule,
|
|
1244
|
+
)
|
|
1245
|
+
# v2 prompt classification; "" for every v1 render, so existing
|
|
1246
|
+
# prompts and checkpoints stay byte-identical.
|
|
1247
|
+
return prompt + self._classification_suffix()
|
|
1248
|
+
|
|
1249
|
+
def _chunk_scope_block(
|
|
1250
|
+
self, index: int, total: int, window: ChunkWindow, citation_format: str
|
|
1251
|
+
) -> str:
|
|
1252
|
+
"""Chunk framing appended to the base prompt: absolute global times
|
|
1253
|
+
only, expected heading, and no document-level structure."""
|
|
1254
|
+
span = f"{_fmt_ts(window.start)}–{_fmt_ts(window.end)}"
|
|
1255
|
+
if self._language == "zh":
|
|
1256
|
+
return (
|
|
1257
|
+
f"\n\n# 本次改写范围(分段渲染 {index}/{total})\n"
|
|
1258
|
+
f"- 本段标题:{window.title}\n"
|
|
1259
|
+
f"- 本段范围:{span}(视频全局时间)\n"
|
|
1260
|
+
f"- 只改写本段转录内容;**所有时间戳引证必须使用视频全局时间"
|
|
1261
|
+
f"(不是段内相对时间),格式 {citation_format}**\n"
|
|
1262
|
+
f"- 输出以 `## {window.title}` 小节标题开头,直接写正文;"
|
|
1263
|
+
f"不要输出 `# 概述` 等文档级结构,不要重复视频标题,"
|
|
1264
|
+
f"不要写其他段的内容"
|
|
1265
|
+
)
|
|
1266
|
+
return (
|
|
1267
|
+
f"\n\n# Rewrite scope (chunked rendering {index}/{total})\n"
|
|
1268
|
+
f"- Chunk title: {window.title}\n"
|
|
1269
|
+
f"- Chunk span: {span} (global video time)\n"
|
|
1270
|
+
f"- Rewrite THIS chunk's transcript only; **every timestamp citation "
|
|
1271
|
+
f"must use global video time (not chunk-relative time), "
|
|
1272
|
+
f"formatted {citation_format}**\n"
|
|
1273
|
+
f"- Start your output with the `## {window.title}` heading and go "
|
|
1274
|
+
f"straight into content; do NOT emit document-level structure like "
|
|
1275
|
+
f"`# Overview`, do NOT repeat the video title, do NOT write other "
|
|
1276
|
+
f"chunks' content"
|
|
1277
|
+
)
|
|
1278
|
+
|
|
1279
|
+
def _render_llm_document(self, bundle: Any, segments: list[tuple[float, str]]) -> str:
|
|
1280
|
+
"""Plan bounded work before dispatch, preserving older durable plans."""
|
|
1281
|
+
if not segments:
|
|
1282
|
+
raise RuntimeError("empty transcript cannot be sent for knowledge-doc rewriting")
|
|
1283
|
+
starts = [start for start, _ in segments]
|
|
1284
|
+
max_start = max(starts) if starts else 0.0
|
|
1285
|
+
duration = bundle.metadata.get("duration")
|
|
1286
|
+
expect_hours = (
|
|
1287
|
+
isinstance(duration, int | float) and duration >= 3600
|
|
1288
|
+
) or max_start >= 3600
|
|
1289
|
+
# (passed to the prompt/chunk prompts so the format decision is made
|
|
1290
|
+
# exactly once per render — see _build_prompt's expect_hours arg)
|
|
1291
|
+
|
|
1292
|
+
# Old progress files have no planner marker. Preserve their exact old
|
|
1293
|
+
# prompts and checkpoint identity, including UNKNOWN dispatches; never
|
|
1294
|
+
# turn a timeout into a fresh set of paid calls.
|
|
1295
|
+
planner: str | None = BOUNDED_PLANNER
|
|
1296
|
+
if self._checkpoint_path is not None and self._checkpoint_path.exists():
|
|
1297
|
+
try:
|
|
1298
|
+
old_progress = json.loads(self._checkpoint_path.read_text(encoding="utf-8"))
|
|
1299
|
+
if not isinstance(old_progress, dict):
|
|
1300
|
+
raise ValueError("invalid progress")
|
|
1301
|
+
planner = old_progress.get("planner")
|
|
1302
|
+
if planner not in (None, UNBOUND_BOUNDED_PLANNER, BOUNDED_PLANNER):
|
|
1303
|
+
raise ValueError("unknown planner")
|
|
1304
|
+
except (OSError, UnicodeError, ValueError, TypeError):
|
|
1305
|
+
raise RuntimeError("knowledge-doc progress cannot be safely reused") from None
|
|
1306
|
+
|
|
1307
|
+
source_sha: str | None = None
|
|
1308
|
+
if planner == BOUNDED_PLANNER:
|
|
1309
|
+
try:
|
|
1310
|
+
source = json.dumps(segments, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
|
|
1311
|
+
except (TypeError, ValueError):
|
|
1312
|
+
raise RuntimeError("transcript segments cannot be safely bound") from None
|
|
1313
|
+
source_sha = hashlib.sha256(source.encode("utf-8")).hexdigest()
|
|
1314
|
+
|
|
1315
|
+
transcript_chars = sum(len(f"[{_fmt_ts(start)}] {text}\n") for start, text in segments)
|
|
1316
|
+
long_enough = (isinstance(duration, int | float) and duration >= CHUNK_WINDOW_SECONDS) or max_start >= CHUNK_WINDOW_SECONDS
|
|
1317
|
+
should_chunk = expect_hours or (planner is not None and (long_enough or transcript_chars > BOUNDED_CHUNK_CHARS))
|
|
1318
|
+
if should_chunk:
|
|
1319
|
+
chapters = bundle.metadata.get("chapters")
|
|
1320
|
+
chapters = chapters if isinstance(chapters, list) else None
|
|
1321
|
+
return self._render_chunked(bundle, segments, chapters, expect_hours,
|
|
1322
|
+
bounded=planner is not None, planner=planner, source_sha=source_sha)
|
|
1323
|
+
|
|
1324
|
+
transcript = "\n".join(f"[{_fmt_ts(start)}] {text}" for start, text in segments)
|
|
1325
|
+
prompt = self._build_prompt(bundle, transcript, expect_hours=expect_hours)
|
|
1326
|
+
return self._execute_rewrite_plan([prompt], [segments], expect_hours,
|
|
1327
|
+
planner=planner, source_sha=source_sha)[0]
|
|
1328
|
+
|
|
1329
|
+
@staticmethod
|
|
1330
|
+
@contextmanager
|
|
1331
|
+
def _progress_lock(path: Path):
|
|
1332
|
+
"""Use an OS-held lock; process death releases it, the journal does not."""
|
|
1333
|
+
lock_path = path.with_name(path.name + ".lock")
|
|
1334
|
+
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
|
|
1335
|
+
try:
|
|
1336
|
+
fd = os.open(lock_path, flags, 0o600)
|
|
1337
|
+
except OSError as exc:
|
|
1338
|
+
raise _ProgressBusy("unable to open knowledge-doc progress lock") from exc
|
|
1339
|
+
locked = False
|
|
1340
|
+
try:
|
|
1341
|
+
try:
|
|
1342
|
+
if os.name == "nt":
|
|
1343
|
+
import msvcrt
|
|
1344
|
+
if os.fstat(fd).st_size == 0:
|
|
1345
|
+
os.write(fd, b"0")
|
|
1346
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
1347
|
+
# typeshed ships an empty msvcrt stub off Windows; these two
|
|
1348
|
+
# calls only ever run when os.name == "nt".
|
|
1349
|
+
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) # type: ignore[attr-defined]
|
|
1350
|
+
else:
|
|
1351
|
+
import fcntl
|
|
1352
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1353
|
+
locked = True
|
|
1354
|
+
except OSError as exc:
|
|
1355
|
+
raise _ProgressBusy("knowledge-doc output is already being rendered") from exc
|
|
1356
|
+
yield
|
|
1357
|
+
finally:
|
|
1358
|
+
if locked:
|
|
1359
|
+
if os.name == "nt":
|
|
1360
|
+
import msvcrt
|
|
1361
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
1362
|
+
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) # type: ignore[attr-defined]
|
|
1363
|
+
else:
|
|
1364
|
+
import fcntl
|
|
1365
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
1366
|
+
os.close(fd)
|
|
1367
|
+
|
|
1368
|
+
@staticmethod
|
|
1369
|
+
def _write_text_atomic(path: Path, content: str) -> None:
|
|
1370
|
+
tmp_path: str | None = None
|
|
1371
|
+
try:
|
|
1372
|
+
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent,
|
|
1373
|
+
prefix=".knowledge-doc-", delete=False) as tmp:
|
|
1374
|
+
tmp_path = tmp.name
|
|
1375
|
+
tmp.write(content)
|
|
1376
|
+
tmp.flush()
|
|
1377
|
+
os.fsync(tmp.fileno())
|
|
1378
|
+
os.replace(tmp_path, path)
|
|
1379
|
+
tmp_path = None
|
|
1380
|
+
if os.name != "nt":
|
|
1381
|
+
fd = os.open(path.parent, os.O_RDONLY)
|
|
1382
|
+
try:
|
|
1383
|
+
os.fsync(fd)
|
|
1384
|
+
finally:
|
|
1385
|
+
os.close(fd)
|
|
1386
|
+
except OSError as exc:
|
|
1387
|
+
raise RuntimeError("unable to persist knowledge-doc progress") from exc
|
|
1388
|
+
finally:
|
|
1389
|
+
if tmp_path is not None:
|
|
1390
|
+
try:
|
|
1391
|
+
os.unlink(tmp_path)
|
|
1392
|
+
except FileNotFoundError:
|
|
1393
|
+
pass
|
|
1394
|
+
|
|
1395
|
+
@classmethod
|
|
1396
|
+
def _write_progress(cls, path: Path, progress: dict[str, Any]) -> None:
|
|
1397
|
+
"""Persist the dispatch intent before the paid call, and results afterwards."""
|
|
1398
|
+
cls._write_text_atomic(path, json.dumps(progress, ensure_ascii=False))
|
|
1399
|
+
|
|
1400
|
+
def _execute_rewrite_plan(
|
|
1401
|
+
self, prompts: list[str], line_buckets: list[list[tuple[float, str]]], expect_hours: bool,
|
|
1402
|
+
planner: str | None = None, source_sha: str | None = None,
|
|
1403
|
+
validator: Callable[[int, str, list[tuple[float, str]]], None] | None = None,
|
|
1404
|
+
) -> list[str]:
|
|
1405
|
+
total = len(prompts)
|
|
1406
|
+
plan = {"prompts": prompts, "model": self._model, "base_url": self._base_url,
|
|
1407
|
+
"max_tokens": self._max_tokens}
|
|
1408
|
+
if source_sha is not None:
|
|
1409
|
+
plan["segment_source_sha256"] = source_sha
|
|
1410
|
+
plan_sha = hashlib.sha256(json.dumps(plan, ensure_ascii=False, sort_keys=True).encode()).hexdigest()
|
|
1411
|
+
path = self._checkpoint_path
|
|
1412
|
+
progress: dict[str, Any] = {"schema": 1, "plan_sha256": plan_sha, "chunks": []}
|
|
1413
|
+
if planner:
|
|
1414
|
+
progress["planner"] = planner
|
|
1415
|
+
if source_sha is not None:
|
|
1416
|
+
progress["segment_source_sha256"] = source_sha
|
|
1417
|
+
if path is not None and path.exists():
|
|
1418
|
+
try:
|
|
1419
|
+
progress = json.loads(path.read_text(encoding="utf-8"))
|
|
1420
|
+
if (not isinstance(progress, dict) or progress.get("schema") != 1
|
|
1421
|
+
or progress.get("plan_sha256") != plan_sha
|
|
1422
|
+
or progress.get("planner") != planner
|
|
1423
|
+
or progress.get("segment_source_sha256") != source_sha
|
|
1424
|
+
or not isinstance(progress.get("chunks"), list)
|
|
1425
|
+
or len(progress["chunks"]) > total):
|
|
1426
|
+
raise ValueError("progress does not match current inputs")
|
|
1427
|
+
except (OSError, UnicodeError, ValueError, TypeError) as exc:
|
|
1428
|
+
raise RuntimeError("knowledge-doc progress cannot be safely reused") from exc
|
|
1429
|
+
|
|
1430
|
+
contents: list[str] = []
|
|
1431
|
+
self._deadline = time.monotonic() + self._total_timeout
|
|
1432
|
+
# strict=False: prompts and line_buckets are built in lockstep by every
|
|
1433
|
+
# in-repo caller; strict=True would raise where the previous code truncated.
|
|
1434
|
+
for index, (lines, prompt) in enumerate(
|
|
1435
|
+
zip(line_buckets, prompts, strict=False), start=1
|
|
1436
|
+
):
|
|
1437
|
+
entries = progress["chunks"]
|
|
1438
|
+
prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()
|
|
1439
|
+
if index <= len(entries):
|
|
1440
|
+
entry = entries[index - 1]
|
|
1441
|
+
if (not isinstance(entry, dict) or entry.get("index") != index
|
|
1442
|
+
or entry.get("prompt_sha256") != prompt_sha
|
|
1443
|
+
or entry.get("state") != "COMPLETE"
|
|
1444
|
+
or not isinstance(entry.get("content"), str)
|
|
1445
|
+
or hashlib.sha256(entry["content"].encode()).hexdigest() != entry.get("content_sha256")):
|
|
1446
|
+
raise RuntimeError("knowledge-doc chunk has an unresolved or invalid prior dispatch")
|
|
1447
|
+
raw = entry["content"]
|
|
1448
|
+
else:
|
|
1449
|
+
if time.monotonic() >= self._deadline:
|
|
1450
|
+
raise RuntimeError("knowledge-doc total time budget exhausted")
|
|
1451
|
+
entries.append({"index": index, "prompt_sha256": prompt_sha, "state": "DISPATCHING"})
|
|
1452
|
+
if path is not None:
|
|
1453
|
+
self._write_progress(path, progress)
|
|
1454
|
+
raw = self._call_minimax(prompt)
|
|
1455
|
+
if validator is not None:
|
|
1456
|
+
# v2 image-note chunks are locator-based (图/段), not time-based;
|
|
1457
|
+
# the caller supplies the locator-aware validator.
|
|
1458
|
+
validator(index, raw, lines)
|
|
1459
|
+
else:
|
|
1460
|
+
# zyr: validate the chunk against ITS OWN window's segments —
|
|
1461
|
+
# the full-video start list let any chunk's citation satisfy
|
|
1462
|
+
# itself against a segment anywhere in a multi-hour video.
|
|
1463
|
+
chunk_starts = [start for start, _ in lines]
|
|
1464
|
+
report = validate_citation_alignment(
|
|
1465
|
+
raw, chunk_starts, expect_hours=expect_hours,
|
|
1466
|
+
require_citations=True,
|
|
1467
|
+
segment_texts=[text for _, text in lines])
|
|
1468
|
+
if not report.ok:
|
|
1469
|
+
raise CitationAlignmentError(report)
|
|
1470
|
+
if index == len(entries) and entries[index - 1].get("state") == "DISPATCHING":
|
|
1471
|
+
entries[index - 1] = {"index": index, "prompt_sha256": prompt_sha, "state": "COMPLETE",
|
|
1472
|
+
"content": raw, "content_sha256": hashlib.sha256(raw.encode()).hexdigest()}
|
|
1473
|
+
if path is not None:
|
|
1474
|
+
self._write_progress(path, progress)
|
|
1475
|
+
contents.append(raw)
|
|
1476
|
+
|
|
1477
|
+
return contents
|
|
1478
|
+
|
|
1479
|
+
def _render_chunked(
|
|
1480
|
+
self,
|
|
1481
|
+
bundle: Any,
|
|
1482
|
+
segments: list[tuple[float, str]],
|
|
1483
|
+
chapters: list | None,
|
|
1484
|
+
expect_hours: bool,
|
|
1485
|
+
*, bounded: bool = False, planner: str | None = None, source_sha: str | None = None,
|
|
1486
|
+
) -> str:
|
|
1487
|
+
starts = [start for start, _ in segments]
|
|
1488
|
+
max_time = max(starts) if starts else 0.0
|
|
1489
|
+
usable_chapters = valid_chapter_list(chapters)
|
|
1490
|
+
boundary_source = "chapter" if usable_chapters else "fixed_window"
|
|
1491
|
+
planned = plan_chunk_windows(usable_chapters, max_time, language=self._language)
|
|
1492
|
+
|
|
1493
|
+
# Assign every segment to exactly one window (last-segment-at-final-
|
|
1494
|
+
# boundary rides the last window).
|
|
1495
|
+
assigned: list[tuple[ChunkWindow, list[tuple[float, str]]]] = [
|
|
1496
|
+
(window, []) for window in planned
|
|
1497
|
+
]
|
|
1498
|
+
for start, text in segments:
|
|
1499
|
+
for window, bucket in assigned:
|
|
1500
|
+
if window.start <= start < window.end:
|
|
1501
|
+
bucket.append((start, text))
|
|
1502
|
+
break
|
|
1503
|
+
else:
|
|
1504
|
+
assigned[-1][1].append((start, text))
|
|
1505
|
+
|
|
1506
|
+
if bounded:
|
|
1507
|
+
# Keep real chapter boundaries and global timestamps. Split long
|
|
1508
|
+
# chapter buckets by input size; additionally split sub-hour and
|
|
1509
|
+
# single-chapter spans by time so a long sparse chapter does not
|
|
1510
|
+
# become a single unbounded request. Segment text is never cut.
|
|
1511
|
+
rebucketed: list[tuple[ChunkWindow, list[tuple[float, str]]]] = []
|
|
1512
|
+
bound_time = not expect_hours or len(usable_chapters) <= 1
|
|
1513
|
+
for window, lines in assigned:
|
|
1514
|
+
groups: list[list[tuple[float, str]]] = []
|
|
1515
|
+
group: list[tuple[float, str]] = []
|
|
1516
|
+
size = 0
|
|
1517
|
+
for start, text in lines:
|
|
1518
|
+
line_size = len(f"[{_fmt_ts(start)}] {text}\n")
|
|
1519
|
+
if line_size > BOUNDED_CHUNK_CHARS:
|
|
1520
|
+
raise RuntimeError("a transcript segment exceeds the bounded chunk size")
|
|
1521
|
+
if group and (size + line_size > BOUNDED_CHUNK_CHARS
|
|
1522
|
+
or (bound_time and start - group[0][0] >= CHUNK_WINDOW_SECONDS)):
|
|
1523
|
+
groups.append(group)
|
|
1524
|
+
group, size = [], 0
|
|
1525
|
+
group.append((start, text))
|
|
1526
|
+
size += line_size
|
|
1527
|
+
if group:
|
|
1528
|
+
groups.append(group)
|
|
1529
|
+
for index, group in enumerate(groups):
|
|
1530
|
+
start = window.start if index == 0 else group[0][0]
|
|
1531
|
+
end = groups[index + 1][0][0] if index + 1 < len(groups) else window.end
|
|
1532
|
+
rebucketed.append((ChunkWindow(start, end, window.title), group))
|
|
1533
|
+
assigned = rebucketed
|
|
1534
|
+
|
|
1535
|
+
rendered = [(window, lines) for window, lines in assigned if lines]
|
|
1536
|
+
citation_format = _citation_format(expect_hours)
|
|
1537
|
+
total = len(rendered)
|
|
1538
|
+
prompts: list[str] = []
|
|
1539
|
+
for index, (window, lines) in enumerate(rendered, start=1):
|
|
1540
|
+
chunk_transcript = "\n".join(
|
|
1541
|
+
f"[{_fmt_ts(start)}] {text}" for start, text in lines
|
|
1542
|
+
)
|
|
1543
|
+
prompts.append(self._build_prompt(
|
|
1544
|
+
bundle, chunk_transcript, expect_hours=expect_hours
|
|
1545
|
+
) + self._chunk_scope_block(index, total, window, citation_format))
|
|
1546
|
+
|
|
1547
|
+
contents = self._execute_rewrite_plan(prompts, [lines for _, lines in rendered], expect_hours,
|
|
1548
|
+
planner, source_sha)
|
|
1549
|
+
|
|
1550
|
+
windows = [window for window, _ in rendered]
|
|
1551
|
+
final_titles = _dedupe_titles([w.title for w in windows], self._language != "en")
|
|
1552
|
+
assembled = assemble_chunked_doc(
|
|
1553
|
+
title=str(bundle.metadata.get("title", bundle.video_id)),
|
|
1554
|
+
url=str(bundle.metadata.get("canonical_url", "")),
|
|
1555
|
+
windows=windows,
|
|
1556
|
+
contents=contents,
|
|
1557
|
+
language=self._language,
|
|
1558
|
+
boundary_source=boundary_source,
|
|
1559
|
+
citation_format=citation_format,
|
|
1560
|
+
final_titles=final_titles,
|
|
1561
|
+
)
|
|
1562
|
+
problems = verify_chunk_assembly(
|
|
1563
|
+
assembled, windows, contents, language=self._language,
|
|
1564
|
+
final_titles=final_titles,
|
|
1565
|
+
)
|
|
1566
|
+
if problems:
|
|
1567
|
+
raise RuntimeError(
|
|
1568
|
+
"chunk assembly consistency check failed: " + "; ".join(problems)
|
|
1569
|
+
)
|
|
1570
|
+
return assembled
|
|
1571
|
+
|
|
1572
|
+
# ------------------------------------------------------------------
|
|
1573
|
+
# v2 path (T33.3) — additive; the v1 path above is untouched
|
|
1574
|
+
# ------------------------------------------------------------------
|
|
1575
|
+
|
|
1576
|
+
def _classification_suffix(self) -> str:
|
|
1577
|
+
"""Prompt-classification note for v2 renders (empty for every v1 render)."""
|
|
1578
|
+
kind = getattr(self, "_doc_kind", None)
|
|
1579
|
+
if not kind or kind == "image_note":
|
|
1580
|
+
# The image-note template already states "no timestamps"; adding a
|
|
1581
|
+
# second note would only drift from it.
|
|
1582
|
+
return ""
|
|
1583
|
+
if self._language == "zh":
|
|
1584
|
+
if kind == "tutorial":
|
|
1585
|
+
return (
|
|
1586
|
+
"\n\n# 内容类型\n"
|
|
1587
|
+
"本内容是可操作教程:优先整理可执行步骤、命令与配置;步骤顺序不得打乱,"
|
|
1588
|
+
"没有依据的步骤不要补写。"
|
|
1589
|
+
)
|
|
1590
|
+
return (
|
|
1591
|
+
"\n\n# 内容类型\n"
|
|
1592
|
+
"本内容是通用知识:侧重概念、原理与结论;来源没有操作步骤时不要编造步骤。"
|
|
1593
|
+
)
|
|
1594
|
+
if kind == "tutorial":
|
|
1595
|
+
return (
|
|
1596
|
+
"\n\n# Content type\n"
|
|
1597
|
+
"This source is an actionable tutorial: prioritise executable steps, commands "
|
|
1598
|
+
"and configuration; keep step order and never invent missing steps."
|
|
1599
|
+
)
|
|
1600
|
+
return (
|
|
1601
|
+
"\n\n# Content type\n"
|
|
1602
|
+
"This source is general knowledge: focus on concepts, principles and conclusions; "
|
|
1603
|
+
"do not invent steps the source does not contain."
|
|
1604
|
+
)
|
|
1605
|
+
|
|
1606
|
+
def _v2_doc_source(self, bundle: Any, raw_transcript_path: Path) -> V2DocSource:
|
|
1607
|
+
"""Normalize a v2 bundle into blocks/segments (+ display metadata)."""
|
|
1608
|
+
from ytk.models.content import TimeRangeLocator, VideoFrameLocator
|
|
1609
|
+
from ytk.renderers.source_links import display_title
|
|
1610
|
+
|
|
1611
|
+
contents = list(getattr(bundle, "contents", []))
|
|
1612
|
+
content = contents[0] if contents else None
|
|
1613
|
+
primary = content.source if content is not None else None
|
|
1614
|
+
metadata: dict[str, Any] = dict(getattr(bundle, "metadata", {}) or {})
|
|
1615
|
+
title = metadata.get("title")
|
|
1616
|
+
if not isinstance(title, str) or not title.strip():
|
|
1617
|
+
title = display_title(primary) if primary is not None else "Knowledge"
|
|
1618
|
+
metadata["title"] = title
|
|
1619
|
+
if primary is not None:
|
|
1620
|
+
metadata["canonical_url"] = primary.canonical_url
|
|
1621
|
+
metadata.setdefault("content_type", primary.content_type)
|
|
1622
|
+
blocks = list(content.blocks) if content is not None else []
|
|
1623
|
+
blocks.sort(key=lambda block: (block.order, block.block_id))
|
|
1624
|
+
is_image_note = primary is not None and primary.content_type == "image_note"
|
|
1625
|
+
if is_image_note:
|
|
1626
|
+
return V2DocSource(
|
|
1627
|
+
kind="image_note",
|
|
1628
|
+
blocks=blocks,
|
|
1629
|
+
segments=[],
|
|
1630
|
+
metadata=metadata,
|
|
1631
|
+
primary=primary,
|
|
1632
|
+
content=content,
|
|
1633
|
+
)
|
|
1634
|
+
segments: list[tuple[float, str]] = []
|
|
1635
|
+
for block in blocks:
|
|
1636
|
+
text = (block.cleaned_text or block.raw_text).strip()
|
|
1637
|
+
if not text:
|
|
1638
|
+
continue
|
|
1639
|
+
locator = block.locator
|
|
1640
|
+
if isinstance(locator, TimeRangeLocator):
|
|
1641
|
+
segments.append((locator.time.start, text))
|
|
1642
|
+
elif isinstance(locator, VideoFrameLocator):
|
|
1643
|
+
segments.append((locator.time, text))
|
|
1644
|
+
if content is not None and isinstance(content.metadata, dict):
|
|
1645
|
+
duration = content.metadata.get("duration")
|
|
1646
|
+
if isinstance(duration, int | float) and not isinstance(duration, bool):
|
|
1647
|
+
metadata["duration"] = duration
|
|
1648
|
+
chapters = content.metadata.get("chapters")
|
|
1649
|
+
if isinstance(chapters, list):
|
|
1650
|
+
metadata["chapters"] = chapters
|
|
1651
|
+
if not segments:
|
|
1652
|
+
segments = self._load_transcript_segments(raw_transcript_path)
|
|
1653
|
+
return V2DocSource(
|
|
1654
|
+
kind="video",
|
|
1655
|
+
blocks=blocks,
|
|
1656
|
+
segments=segments,
|
|
1657
|
+
metadata=metadata,
|
|
1658
|
+
primary=primary,
|
|
1659
|
+
content=content,
|
|
1660
|
+
)
|
|
1661
|
+
|
|
1662
|
+
def _render_locked_v2(
|
|
1663
|
+
self, bundle: Any, output_path: Path, status_path: Path, **kwargs: Any
|
|
1664
|
+
) -> None:
|
|
1665
|
+
"""v2 branch of :meth:`_render_locked` (same journal/status contract)."""
|
|
1666
|
+
source = self._v2_doc_source(bundle, output_path.parent / "raw-transcript.json")
|
|
1667
|
+
reason: str | None = None
|
|
1668
|
+
# Cloud text generation is opt-in (plan §4.6 / A25): the caller decides
|
|
1669
|
+
# from the resolved platform settings; a local-only run must never POST
|
|
1670
|
+
# the transcript to a cloud provider just because an API key is present.
|
|
1671
|
+
allow_cloud = bool(kwargs.get("allow_cloud", True))
|
|
1672
|
+
if not allow_cloud:
|
|
1673
|
+
logger.warning(
|
|
1674
|
+
"cloud text generation is disabled for this platform; falling back to the "
|
|
1675
|
+
"deterministic v2 document (no network call)."
|
|
1676
|
+
)
|
|
1677
|
+
content = self._fallback_doc_v2(bundle, source)
|
|
1678
|
+
reason = "CLOUD_TEXT_DISABLED"
|
|
1679
|
+
elif not self._api_key:
|
|
1680
|
+
logger.warning(
|
|
1681
|
+
"YTK_MINIMAX_API_KEY not set; KnowledgeDocRenderer is falling back "
|
|
1682
|
+
"to the deterministic v2 document."
|
|
1683
|
+
)
|
|
1684
|
+
content = self._fallback_doc_v2(bundle, source)
|
|
1685
|
+
reason = "NO_API_KEY"
|
|
1686
|
+
else:
|
|
1687
|
+
try:
|
|
1688
|
+
content = self._render_v2_document(bundle, source)
|
|
1689
|
+
except RuntimeError as exc:
|
|
1690
|
+
logger.warning(
|
|
1691
|
+
"MiniMax v2 knowledge-doc rewrite rejected; falling back to the "
|
|
1692
|
+
"deterministic document: %s",
|
|
1693
|
+
exc,
|
|
1694
|
+
)
|
|
1695
|
+
content = self._fallback_doc_v2(bundle, source)
|
|
1696
|
+
reason = self._failure_reason(exc)
|
|
1697
|
+
finally:
|
|
1698
|
+
self._deadline = None
|
|
1699
|
+
self._doc_kind = None
|
|
1700
|
+
|
|
1701
|
+
if kwargs.get("with_reviewer_scaffolding"):
|
|
1702
|
+
metadata = source.metadata
|
|
1703
|
+
content = content.rstrip() + "\n" + build_reviewer_scaffolding(
|
|
1704
|
+
self._language,
|
|
1705
|
+
title=str(metadata.get("title", "") or ""),
|
|
1706
|
+
source_url=str(metadata.get("canonical_url", "") or ""),
|
|
1707
|
+
)
|
|
1708
|
+
|
|
1709
|
+
if reason:
|
|
1710
|
+
content = f"<!-- ytk-knowledge-doc: fallback; reason={reason} -->\n\n" + content
|
|
1711
|
+
rendered = content.strip() + "\n"
|
|
1712
|
+
destination = output_path
|
|
1713
|
+
if reason and output_path.exists():
|
|
1714
|
+
destination = output_path.with_name(output_path.stem + ".fallback" + output_path.suffix)
|
|
1715
|
+
self._write_text_atomic(destination, rendered)
|
|
1716
|
+
self._write_progress(
|
|
1717
|
+
status_path,
|
|
1718
|
+
{
|
|
1719
|
+
"schema": 1,
|
|
1720
|
+
"state": "FALLBACK" if reason else "COMPLETE",
|
|
1721
|
+
"reason": reason,
|
|
1722
|
+
"output_file": destination.name,
|
|
1723
|
+
"output_sha256": hashlib.sha256(rendered.encode()).hexdigest(),
|
|
1724
|
+
},
|
|
1725
|
+
)
|
|
1726
|
+
|
|
1727
|
+
def _failure_reason(self, exc: RuntimeError) -> str:
|
|
1728
|
+
"""Map a v2 rewrite failure to the same reason codes as the v1 path."""
|
|
1729
|
+
message = str(exc)
|
|
1730
|
+
if message == "knowledge-doc total time budget exhausted":
|
|
1731
|
+
reason = "TOTAL_TIMEOUT"
|
|
1732
|
+
elif message == "knowledge-doc chunk has an unresolved or invalid prior dispatch":
|
|
1733
|
+
reason = "UNRESOLVED_OR_INVALID_DISPATCH"
|
|
1734
|
+
elif message == "knowledge-doc progress cannot be safely reused":
|
|
1735
|
+
reason = "PROGRESS_MISMATCH"
|
|
1736
|
+
else:
|
|
1737
|
+
reason = "REWRITE_FAILED"
|
|
1738
|
+
try:
|
|
1739
|
+
checkpoint_path = self._checkpoint_path
|
|
1740
|
+
if checkpoint_path is None: # defensive: no checkpoint on this path
|
|
1741
|
+
raise OSError("knowledge-doc checkpoint path is not configured")
|
|
1742
|
+
progress = json.loads(checkpoint_path.read_text(encoding="utf-8"))
|
|
1743
|
+
chunks = progress.get("chunks", [])
|
|
1744
|
+
if (
|
|
1745
|
+
isinstance(chunks, list)
|
|
1746
|
+
and chunks
|
|
1747
|
+
and isinstance(chunks[-1], dict)
|
|
1748
|
+
and chunks[-1].get("state") == "DISPATCHING"
|
|
1749
|
+
):
|
|
1750
|
+
reason = (
|
|
1751
|
+
"TOTAL_TIMEOUT_WITH_UNRESOLVED_DISPATCH"
|
|
1752
|
+
if reason == "TOTAL_TIMEOUT"
|
|
1753
|
+
else "UNRESOLVED_OR_INVALID_DISPATCH"
|
|
1754
|
+
)
|
|
1755
|
+
except (OSError, ValueError, TypeError, AttributeError):
|
|
1756
|
+
pass
|
|
1757
|
+
return reason
|
|
1758
|
+
|
|
1759
|
+
def _render_v2_document(self, bundle: Any, source: V2DocSource) -> str:
|
|
1760
|
+
"""Plan and rewrite a v2 source (video: time chunks; image note: blocks)."""
|
|
1761
|
+
self._doc_kind = classify_doc_kind(bundle)
|
|
1762
|
+
if source.kind == "image_note":
|
|
1763
|
+
return self._render_v2_image_note(bundle, source)
|
|
1764
|
+
shim = _VideoDocShim(
|
|
1765
|
+
metadata=dict(source.metadata),
|
|
1766
|
+
video_id=str(source.primary.content_id if source.primary is not None else "video"),
|
|
1767
|
+
)
|
|
1768
|
+
return self._render_llm_document(shim, source.segments)
|
|
1769
|
+
|
|
1770
|
+
def _render_v2_image_note(self, bundle: Any, source: V2DocSource) -> str:
|
|
1771
|
+
"""Bounded block-based rewrite for image notes (no timestamps)."""
|
|
1772
|
+
from ytk.renderers.citation_alignment import validate_citation_alignment_v2
|
|
1773
|
+
|
|
1774
|
+
if not source.blocks:
|
|
1775
|
+
raise RuntimeError("empty image note cannot be sent for knowledge-doc rewriting")
|
|
1776
|
+
chunks = plan_image_note_chunks(source.blocks, language=self._language)
|
|
1777
|
+
problems = verify_image_note_plan(chunks, source.blocks)
|
|
1778
|
+
if problems:
|
|
1779
|
+
raise RuntimeError("image-note chunk plan is inconsistent: " + "; ".join(problems))
|
|
1780
|
+
source_sha = hashlib.sha256(
|
|
1781
|
+
json.dumps(
|
|
1782
|
+
[(chunk.block_ids, chunk.lines) for chunk in chunks],
|
|
1783
|
+
ensure_ascii=False,
|
|
1784
|
+
separators=(",", ":"),
|
|
1785
|
+
).encode("utf-8")
|
|
1786
|
+
).hexdigest()
|
|
1787
|
+
total = len(chunks)
|
|
1788
|
+
prompts = [
|
|
1789
|
+
self._build_image_note_prompt(source.metadata, chunk, total) for chunk in chunks
|
|
1790
|
+
]
|
|
1791
|
+
buckets: list[list[tuple[float, str]]] = [[] for _ in chunks]
|
|
1792
|
+
block_ids_by_index = {chunk.index: set(chunk.block_ids) for chunk in chunks}
|
|
1793
|
+
source_id = source.primary.source_id if source.primary is not None else None
|
|
1794
|
+
|
|
1795
|
+
def validate_chunk(index: int, raw: str, lines: list[tuple[float, str]]) -> None:
|
|
1796
|
+
report = validate_citation_alignment_v2(
|
|
1797
|
+
raw,
|
|
1798
|
+
bundle,
|
|
1799
|
+
source_id=source_id,
|
|
1800
|
+
block_ids=block_ids_by_index.get(index),
|
|
1801
|
+
require_citations=True,
|
|
1802
|
+
)
|
|
1803
|
+
if not report.ok:
|
|
1804
|
+
raise CitationAlignmentError(report)
|
|
1805
|
+
|
|
1806
|
+
contents = self._execute_rewrite_plan(
|
|
1807
|
+
prompts,
|
|
1808
|
+
buckets,
|
|
1809
|
+
False,
|
|
1810
|
+
planner=IMAGE_NOTE_PLANNER,
|
|
1811
|
+
source_sha=source_sha,
|
|
1812
|
+
validator=validate_chunk,
|
|
1813
|
+
)
|
|
1814
|
+
final_titles = _dedupe_titles(
|
|
1815
|
+
[_image_chunk_title(chunk.index, self._language) for chunk in chunks],
|
|
1816
|
+
self._language != "en",
|
|
1817
|
+
)
|
|
1818
|
+
assembled = assemble_image_note_doc(
|
|
1819
|
+
title=str(source.metadata.get("title") or "Image note"),
|
|
1820
|
+
url=str(source.metadata.get("canonical_url") or ""),
|
|
1821
|
+
chunks=chunks,
|
|
1822
|
+
contents=contents,
|
|
1823
|
+
language=self._language,
|
|
1824
|
+
final_titles=final_titles,
|
|
1825
|
+
)
|
|
1826
|
+
window_shims = [ChunkWindow(0.0, 0.0, title) for title in final_titles]
|
|
1827
|
+
problems = verify_chunk_assembly(
|
|
1828
|
+
assembled,
|
|
1829
|
+
window_shims,
|
|
1830
|
+
contents,
|
|
1831
|
+
language=self._language,
|
|
1832
|
+
final_titles=final_titles,
|
|
1833
|
+
)
|
|
1834
|
+
if problems:
|
|
1835
|
+
raise RuntimeError(
|
|
1836
|
+
"image-note assembly consistency check failed: " + "; ".join(problems)
|
|
1837
|
+
)
|
|
1838
|
+
return assembled
|
|
1839
|
+
|
|
1840
|
+
def _build_image_note_prompt(
|
|
1841
|
+
self, metadata: dict[str, Any], chunk: ImageNoteChunk, total: int
|
|
1842
|
+
) -> str:
|
|
1843
|
+
template = _IMAGE_NOTE_TEMPLATES[self._language]
|
|
1844
|
+
citation_format = _IMAGE_CITATION_FORMAT[self._language]
|
|
1845
|
+
title = str(metadata.get("title") or "Image note")
|
|
1846
|
+
url = str(metadata.get("canonical_url") or "")
|
|
1847
|
+
prompt = template.format(
|
|
1848
|
+
title=title,
|
|
1849
|
+
url=url or ("(无)" if self._language == "zh" else "(none)"),
|
|
1850
|
+
blocks="\n".join(chunk.lines),
|
|
1851
|
+
citation_format=citation_format,
|
|
1852
|
+
)
|
|
1853
|
+
return prompt + self._image_chunk_scope_block(chunk, total, citation_format)
|
|
1854
|
+
|
|
1855
|
+
def _image_chunk_scope_block(
|
|
1856
|
+
self, chunk: ImageNoteChunk, total: int, citation_format: str
|
|
1857
|
+
) -> str:
|
|
1858
|
+
title = _image_chunk_title(chunk.index, self._language)
|
|
1859
|
+
if self._language == "zh":
|
|
1860
|
+
return (
|
|
1861
|
+
f"\n\n# 本次改写范围(图文分块 {chunk.index}/{total})\n"
|
|
1862
|
+
f"- 本部分标题:{title}\n"
|
|
1863
|
+
f"- 只改写本部分内容块;引用使用 {citation_format} 定位标记\n"
|
|
1864
|
+
f"- **禁止出现任何时间戳**(图文没有时间轴)\n"
|
|
1865
|
+
f"- 输出以 `## {title}` 开头,直接写正文;不要输出 `# 概述` 等文档级结构,"
|
|
1866
|
+
f"不要写其他部分的内容"
|
|
1867
|
+
)
|
|
1868
|
+
return (
|
|
1869
|
+
f"\n\n# Rewrite scope (image-note chunk {chunk.index}/{total})\n"
|
|
1870
|
+
f"- Chunk title: {title}\n"
|
|
1871
|
+
f"- Rewrite THIS chunk's blocks only; cite with {citation_format} locator markers\n"
|
|
1872
|
+
f"- **Never emit a timestamp** (an image note has no timeline)\n"
|
|
1873
|
+
f"- Start your output with the `## {title}` heading and go straight into content; "
|
|
1874
|
+
f"do NOT emit document-level structure like `# Overview`, do NOT write other "
|
|
1875
|
+
f"chunks' content"
|
|
1876
|
+
)
|
|
1877
|
+
|
|
1878
|
+
def _fallback_doc_v2(self, bundle: Any, source: V2DocSource) -> str:
|
|
1879
|
+
"""Deterministic v2 document: formatted evidence, no invented timestamps."""
|
|
1880
|
+
from ytk.renderers.evidence_formatter import format_evidence_refs
|
|
1881
|
+
from ytk.renderers.source_links import format_seconds, image_markdown_ref
|
|
1882
|
+
|
|
1883
|
+
labels = _SECTION_TITLES[self._language]
|
|
1884
|
+
title = str(source.metadata.get("title") or "Knowledge")
|
|
1885
|
+
url = str(source.metadata.get("canonical_url") or "")
|
|
1886
|
+
sections: list[str] = [labels["title_template"].format(title=title), ""]
|
|
1887
|
+
if url:
|
|
1888
|
+
sections += [labels["source"].format(url=url), ""]
|
|
1889
|
+
sections += [labels["overview"], ""]
|
|
1890
|
+
first_texts = [
|
|
1891
|
+
(block.cleaned_text or block.raw_text).strip() for block in source.blocks[:3]
|
|
1892
|
+
]
|
|
1893
|
+
overview = " ".join(text for text in first_texts if text) or labels["no_transcript"]
|
|
1894
|
+
sections += [overview, ""]
|
|
1895
|
+
|
|
1896
|
+
image_assets = [
|
|
1897
|
+
asset
|
|
1898
|
+
for asset in (getattr(source.content, "assets", []) or [])
|
|
1899
|
+
if asset.media_type == "image"
|
|
1900
|
+
]
|
|
1901
|
+
if image_assets:
|
|
1902
|
+
sections += ["## 图片" if self._language != "en" else "## Images", ""]
|
|
1903
|
+
for index, asset in enumerate(image_assets, start=1):
|
|
1904
|
+
alt = f"图 {index}" if self._language != "en" else f"Image {index}"
|
|
1905
|
+
sections.append(image_markdown_ref(asset.relative_path, alt=alt))
|
|
1906
|
+
sections.append("")
|
|
1907
|
+
|
|
1908
|
+
knowledge = list(getattr(bundle, "knowledge", []))
|
|
1909
|
+
if knowledge:
|
|
1910
|
+
sections += [labels["knowledge_atoms"], ""]
|
|
1911
|
+
by_type: dict[str, list[Any]] = {}
|
|
1912
|
+
for atom in knowledge:
|
|
1913
|
+
by_type.setdefault(atom.atom_type, []).append(atom)
|
|
1914
|
+
for atom_type, atoms in by_type.items():
|
|
1915
|
+
localized = labels["type_titles"].get(atom_type, atom_type)
|
|
1916
|
+
sections += [f"### {localized}", ""]
|
|
1917
|
+
for atom in atoms:
|
|
1918
|
+
evidence = [
|
|
1919
|
+
bundle.evidence_index[evidence_id]
|
|
1920
|
+
for evidence_id in atom.evidence_ids
|
|
1921
|
+
if evidence_id in bundle.evidence_index
|
|
1922
|
+
]
|
|
1923
|
+
citations = format_evidence_refs(
|
|
1924
|
+
evidence, bundle, language=self._language, validate=False
|
|
1925
|
+
)
|
|
1926
|
+
suffix = f" {' '.join(citations)}" if citations else ""
|
|
1927
|
+
sections.append(f"- {atom.content}{suffix}")
|
|
1928
|
+
sections.append("")
|
|
1929
|
+
|
|
1930
|
+
sections += [labels["full_transcript"], ""]
|
|
1931
|
+
if source.kind == "image_note":
|
|
1932
|
+
sections.extend(image_note_block_line(block, language=self._language) for block in source.blocks)
|
|
1933
|
+
else:
|
|
1934
|
+
sections.extend(f"[{format_seconds(start)}] {text}" for start, text in source.segments)
|
|
1935
|
+
if not source.blocks and not source.segments:
|
|
1936
|
+
sections.append(labels["no_transcript"])
|
|
1937
|
+
return "\n".join(sections)
|
|
1938
|
+
|
|
1939
|
+
def _call_minimax(self, prompt: str) -> str:
|
|
1940
|
+
url = f"{self._base_url}/v1/chat/completions"
|
|
1941
|
+
headers = {"Authorization": f"Bearer {self._api_key}"}
|
|
1942
|
+
payload = {
|
|
1943
|
+
"model": self._model,
|
|
1944
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
1945
|
+
"max_completion_tokens": self._max_tokens,
|
|
1946
|
+
}
|
|
1947
|
+
try:
|
|
1948
|
+
if self._deadline is None:
|
|
1949
|
+
with httpx.Client(timeout=self._timeout) as client:
|
|
1950
|
+
resp = client.post(url, headers=headers, json=payload)
|
|
1951
|
+
resp.raise_for_status()
|
|
1952
|
+
result = resp.json()
|
|
1953
|
+
else:
|
|
1954
|
+
remaining = self._deadline - time.monotonic()
|
|
1955
|
+
if remaining <= 0:
|
|
1956
|
+
raise RuntimeError("knowledge-doc total time budget exhausted")
|
|
1957
|
+
# Inspect every received fragment against the wall-clock deadline.
|
|
1958
|
+
# The read inactivity bound limits overshoot even for slow streams.
|
|
1959
|
+
timeout = min(self._timeout, remaining, 60.0)
|
|
1960
|
+
chunks: list[bytes] = []
|
|
1961
|
+
received = 0
|
|
1962
|
+
with httpx.Client(timeout=timeout) as client:
|
|
1963
|
+
stream_headers = {**headers, "Accept-Encoding": "identity"}
|
|
1964
|
+
with client.stream("POST", url, headers=stream_headers, json=payload) as resp:
|
|
1965
|
+
resp.raise_for_status()
|
|
1966
|
+
if resp.headers.get("Content-Encoding", "identity").lower() not in ("", "identity"):
|
|
1967
|
+
raise RuntimeError("knowledge-doc response has unexpected compression")
|
|
1968
|
+
for chunk in resp.iter_raw():
|
|
1969
|
+
if time.monotonic() >= self._deadline:
|
|
1970
|
+
raise RuntimeError("knowledge-doc total time budget exhausted")
|
|
1971
|
+
received += len(chunk)
|
|
1972
|
+
if received > 10 * 1024 * 1024:
|
|
1973
|
+
raise RuntimeError("knowledge-doc response exceeds size limit")
|
|
1974
|
+
chunks.append(chunk)
|
|
1975
|
+
if time.monotonic() >= self._deadline:
|
|
1976
|
+
raise RuntimeError("knowledge-doc total time budget exhausted")
|
|
1977
|
+
result = json.loads(b"".join(chunks))
|
|
1978
|
+
choice = result["choices"][0]
|
|
1979
|
+
if choice.get("finish_reason") != "stop":
|
|
1980
|
+
raise RuntimeError("knowledge-doc response was incomplete or has no completion reason")
|
|
1981
|
+
raw = choice["message"]["content"]
|
|
1982
|
+
except (KeyError, json.JSONDecodeError, httpx.HTTPError) as exc:
|
|
1983
|
+
raise RuntimeError(f"MiniMax knowledge-doc call failed: {exc}") from exc
|
|
1984
|
+
return _THINK_RE.sub("", raw).strip()
|