kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
ytk/cli.py
ADDED
|
@@ -0,0 +1,1281 @@
|
|
|
1
|
+
"""Command-line interface (T43.1).
|
|
2
|
+
|
|
3
|
+
Surface (v2.0)::
|
|
4
|
+
|
|
5
|
+
ytk extract URL [--format ...] [--config TOML] [--fixture-content JSON]
|
|
6
|
+
[--fixture-transcript JSON] [--refresh] [--platform NAME]
|
|
7
|
+
[--content-type video|image_note] [--out DIR] ...
|
|
8
|
+
ytk batch run --urls-file FILE [--format ...] [--resume] [--refresh] ...
|
|
9
|
+
ytk platforms [--json]
|
|
10
|
+
ytk probe URL [--json] [--platform NAME] [--content-type TYPE]
|
|
11
|
+
ytk doctor [--offline] [--platform all|NAME] [--strict]
|
|
12
|
+
ytk cache status|clear
|
|
13
|
+
ytk skill build ...
|
|
14
|
+
ytk merge ...
|
|
15
|
+
|
|
16
|
+
Routing: a plain YouTube link (no ``--fixture-content``) keeps the historical
|
|
17
|
+
v1 pipeline byte-for-byte (A01 compatibility); TikTok / Douyin / Bilibili /
|
|
18
|
+
Xiaohongshu and every ``--fixture-content`` run use the v2 multi-platform
|
|
19
|
+
pipeline. Exit codes: 0 for COMPLETE / PARTIAL / NO_KNOWLEDGE, 1 for
|
|
20
|
+
BLOCKED / FAILED / every-requested-format-not-applicable
|
|
21
|
+
(``OUTPUTS_NOT_APPLICABLE``).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import inspect
|
|
27
|
+
import json
|
|
28
|
+
import logging
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
import typer
|
|
33
|
+
from pydantic import ValidationError
|
|
34
|
+
|
|
35
|
+
from ytk.config import AppSettings, ConfigFileError
|
|
36
|
+
from ytk.doctor import DoctorService, format_doctor_report
|
|
37
|
+
from ytk.dotenv import load_dotenv
|
|
38
|
+
from ytk.errors import UserFacingError
|
|
39
|
+
from ytk.logging import configure_logging
|
|
40
|
+
from ytk.models.source import SUPPORTED_PLATFORMS, AccessContext, ContentRef
|
|
41
|
+
from ytk.pipeline.orchestrator import build_default_pipeline, load_bundle
|
|
42
|
+
from ytk.pipeline.state import ArtifactStore, derive_overall_status
|
|
43
|
+
|
|
44
|
+
LOGGER = logging.getLogger(__name__)
|
|
45
|
+
|
|
46
|
+
app = typer.Typer(
|
|
47
|
+
help="Knowledge Distiller (萃知): distill videos and image notes into "
|
|
48
|
+
"evidence-linked knowledge and reusable AI-agent skills."
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _show_version(value: bool) -> None:
|
|
53
|
+
if value:
|
|
54
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
55
|
+
|
|
56
|
+
try:
|
|
57
|
+
installed_version = version("kd-engine")
|
|
58
|
+
except PackageNotFoundError:
|
|
59
|
+
installed_version = "1.0.0" # source checkout; synchronized with pyproject.toml
|
|
60
|
+
typer.echo(f"kd-engine {installed_version}")
|
|
61
|
+
raise typer.Exit()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.callback()
|
|
65
|
+
def _bootstrap_env(
|
|
66
|
+
version: bool = typer.Option(
|
|
67
|
+
False, "--version", callback=_show_version, is_eager=True,
|
|
68
|
+
help="Show the installed kd-engine version and exit.",
|
|
69
|
+
),
|
|
70
|
+
) -> None:
|
|
71
|
+
"""Load the project .env for every entry point (console script included).
|
|
72
|
+
|
|
73
|
+
``python -m ytk`` loads it in ``__main__``; the installed ``ytk`` console
|
|
74
|
+
script calls ``app`` directly, so without this callback the two entry points
|
|
75
|
+
silently diverge (A32). ``YTK_NO_DOTENV=1`` is honoured before any read.
|
|
76
|
+
"""
|
|
77
|
+
try:
|
|
78
|
+
from ytk.dotenv import dotenv_disabled, load_dotenv
|
|
79
|
+
|
|
80
|
+
if dotenv_disabled():
|
|
81
|
+
return
|
|
82
|
+
load_dotenv(Path(__file__).resolve().parent.parent.parent / ".env")
|
|
83
|
+
except Exception: # pragma: no cover - env loading must never break the CLI
|
|
84
|
+
return
|
|
85
|
+
skill_app = typer.Typer(help="Build skills from extraction artifacts.")
|
|
86
|
+
batch_app = typer.Typer(help="Batch-process content links from multiple platforms.")
|
|
87
|
+
cache_app = typer.Typer(help="Manage caches (transcript v1 + layered v2).")
|
|
88
|
+
|
|
89
|
+
app.add_typer(skill_app, name="skill")
|
|
90
|
+
app.add_typer(batch_app, name="batch")
|
|
91
|
+
app.add_typer(cache_app, name="cache")
|
|
92
|
+
|
|
93
|
+
V2_VALIDATOR_VERSION = "v2-1" # kept local: importing pipeline.v2 here would load v2 deps eagerly
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# --------------------------------------------------------------------------- #
|
|
97
|
+
# Environment loading (historical behaviour preserved; NO_DOTENV honoured)
|
|
98
|
+
# --------------------------------------------------------------------------- #
|
|
99
|
+
def _load_env_file(path: Path) -> None:
|
|
100
|
+
"""Load the project ``.env`` (best effort) before the CLI runs.
|
|
101
|
+
|
|
102
|
+
Root cause fix 2026-08-18: the ybs cron ran ``python -m ytk extract`` from an
|
|
103
|
+
arbitrary cwd where ``YTK_MINIMAX_API_KEY`` was NOT exported — the renderer
|
|
104
|
+
silently fell back to the deterministic document. Loading the project
|
|
105
|
+
``.env`` at CLI entry makes every caller get the same keys as the ``.env``
|
|
106
|
+
author intended. The loader lives in :mod:`ytk.dotenv` so ``YTK_NO_DOTENV=1``
|
|
107
|
+
is honoured *before* the file is opened (offline tests / diagnostics).
|
|
108
|
+
"""
|
|
109
|
+
load_dotenv(path)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# --------------------------------------------------------------------------- #
|
|
113
|
+
# Cache commands
|
|
114
|
+
# --------------------------------------------------------------------------- #
|
|
115
|
+
@cache_app.command("status")
|
|
116
|
+
def cache_status() -> None:
|
|
117
|
+
"""Show cache location, entry count, size, and per-video detail."""
|
|
118
|
+
from ytk.ingestion.transcript_cache import TranscriptCache
|
|
119
|
+
|
|
120
|
+
settings = AppSettings.load()
|
|
121
|
+
cache = TranscriptCache(
|
|
122
|
+
cache_dir=settings.transcript_cache_dir,
|
|
123
|
+
enabled=settings.transcript_cache_enabled,
|
|
124
|
+
)
|
|
125
|
+
stats = cache.stats()
|
|
126
|
+
size_kb = stats["total_bytes"] / 1024
|
|
127
|
+
print(f"cache dir: {stats['dir']}")
|
|
128
|
+
print(f"enabled: {settings.transcript_cache_enabled}")
|
|
129
|
+
print(f"entries: {stats['entries']} ({size_kb:.1f} KB)")
|
|
130
|
+
for entry in stats["detail"]:
|
|
131
|
+
langs = ",".join(entry["languages"]) or "?"
|
|
132
|
+
print(
|
|
133
|
+
f" {entry['video_id']} {entry['bytes'] / 1024:8.1f} KB "
|
|
134
|
+
f"src={entry['source']} lang={langs} segments={entry['segments']}"
|
|
135
|
+
)
|
|
136
|
+
_print_v2_cache_stats(settings)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _print_v2_cache_stats(settings: AppSettings) -> None:
|
|
140
|
+
try:
|
|
141
|
+
from ytk.cache import LayeredCache
|
|
142
|
+
|
|
143
|
+
v2 = LayeredCache(root=settings.cache_dir)
|
|
144
|
+
stats = v2.stats()
|
|
145
|
+
except Exception: # noqa: BLE001 - cache introspection must never break the CLI
|
|
146
|
+
return
|
|
147
|
+
size_mb = stats["total_bytes"] / (1024 * 1024)
|
|
148
|
+
print(f"v2 cache: {stats['root']}")
|
|
149
|
+
print(f"v2 entries: {stats['entries']} ({size_mb:.2f} MB, max {stats['max_bytes']} bytes)")
|
|
150
|
+
for namespace, bucket in stats["namespaces"].items():
|
|
151
|
+
if bucket["entries"]:
|
|
152
|
+
print(f" {namespace}: {bucket['entries']} entr(ies), {bucket['bytes'] / 1024:.1f} KB")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@cache_app.command("clear")
|
|
156
|
+
def cache_clear(
|
|
157
|
+
v2: bool = typer.Option(
|
|
158
|
+
False, "--v2", help="Also clear the v2 layered cache (media/subtitle/asr/ocr/generation)."
|
|
159
|
+
),
|
|
160
|
+
) -> None:
|
|
161
|
+
"""Delete every cached transcript bundle (safe: cache is regenerable)."""
|
|
162
|
+
from ytk.ingestion.transcript_cache import TranscriptCache
|
|
163
|
+
|
|
164
|
+
settings = AppSettings.load()
|
|
165
|
+
cache = TranscriptCache(
|
|
166
|
+
cache_dir=settings.transcript_cache_dir,
|
|
167
|
+
enabled=settings.transcript_cache_enabled,
|
|
168
|
+
)
|
|
169
|
+
removed = cache.clear()
|
|
170
|
+
print(f"removed {removed} cached transcript(s)")
|
|
171
|
+
if v2:
|
|
172
|
+
from ytk.cache import LayeredCache
|
|
173
|
+
|
|
174
|
+
removed_v2 = LayeredCache(root=settings.cache_dir, enabled=True).clear()
|
|
175
|
+
print(f"removed {removed_v2} v2 cache entr(ies)")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
# --------------------------------------------------------------------------- #
|
|
179
|
+
# extract
|
|
180
|
+
# --------------------------------------------------------------------------- #
|
|
181
|
+
@app.command()
|
|
182
|
+
def extract(
|
|
183
|
+
url: str,
|
|
184
|
+
out: Path = typer.Option(Path("./outputs"), "--out"),
|
|
185
|
+
format: str | None = typer.Option(
|
|
186
|
+
None,
|
|
187
|
+
"--format",
|
|
188
|
+
help="Comma-separated outputs. Omit for per-platform defaults "
|
|
189
|
+
"(YouTube keeps its legacy default: json,md,skill; new platforms: json,md).",
|
|
190
|
+
),
|
|
191
|
+
fixture_transcript: Path | None = typer.Option(None, "--fixture-transcript"),
|
|
192
|
+
fixture_content: Path | None = typer.Option(
|
|
193
|
+
None,
|
|
194
|
+
"--fixture-content",
|
|
195
|
+
"--acquired-content",
|
|
196
|
+
help="JSON descriptor of recorded content or a measured browser video acquisition; browser media identity, bytes and streams are revalidated.",
|
|
197
|
+
),
|
|
198
|
+
proxy: str | None = typer.Option(
|
|
199
|
+
None,
|
|
200
|
+
"--proxy",
|
|
201
|
+
help="HTTP/HTTPS proxy URL (e.g. http://127.0.0.1:7890)",
|
|
202
|
+
),
|
|
203
|
+
cookies: Path | None = typer.Option(
|
|
204
|
+
None,
|
|
205
|
+
"--cookies",
|
|
206
|
+
help="Path to Netscape formatted YouTube cookies file for yt-dlp.",
|
|
207
|
+
),
|
|
208
|
+
cookies_from_browser: str | None = typer.Option(
|
|
209
|
+
None,
|
|
210
|
+
"--cookies-from-browser",
|
|
211
|
+
help="Browser cookies source for yt-dlp, e.g. chrome:default",
|
|
212
|
+
),
|
|
213
|
+
coarse_atoms: bool = typer.Option(
|
|
214
|
+
False,
|
|
215
|
+
"--coarse-atoms",
|
|
216
|
+
help="When a source yields no structured knowledge, emit explicitly "
|
|
217
|
+
"labelled low-confidence segment summaries instead of NO_KNOWLEDGE. "
|
|
218
|
+
"Off by default: never silently turns an empty ledger into a fake one.",
|
|
219
|
+
),
|
|
220
|
+
doc_language: str | None = typer.Option(
|
|
221
|
+
None,
|
|
222
|
+
"--doc-language",
|
|
223
|
+
help="Output language for the 'doc' format. One of: zh, en, auto. "
|
|
224
|
+
"When 'auto' (or unset), the skill samples $LANG/$LC_ALL — defaults to 'zh'.",
|
|
225
|
+
),
|
|
226
|
+
with_reviewer_scaffolding: bool = typer.Option(
|
|
227
|
+
False,
|
|
228
|
+
"--with-reviewer-scaffolding",
|
|
229
|
+
help="Append an A/B/C reference list, image-asset manifest, and "
|
|
230
|
+
"closing CTA to the `doc` output. Designed to lift the `doc` "
|
|
231
|
+
"across the 4-model review rubric when downstream penflow will "
|
|
232
|
+
"reuse the doc as a knowledge source.",
|
|
233
|
+
),
|
|
234
|
+
skip_existing: bool = typer.Option(
|
|
235
|
+
False, "--skip-existing",
|
|
236
|
+
help="Exit 0 without reprocessing when the output dir already has "
|
|
237
|
+
"knowledge.json (cron idempotence for single-video reruns).",
|
|
238
|
+
),
|
|
239
|
+
config: Path | None = typer.Option(
|
|
240
|
+
None, "--config", help="Optional TOML config file (explicit field priority)."
|
|
241
|
+
),
|
|
242
|
+
refresh: bool = typer.Option(
|
|
243
|
+
False, "--refresh",
|
|
244
|
+
help="Re-resolve the source and re-acquire only material whose digest changed; "
|
|
245
|
+
"never forgets an unresolved remote submission.",
|
|
246
|
+
),
|
|
247
|
+
platform: str | None = typer.Option(
|
|
248
|
+
None, "--platform", help="Force the platform (youtube|tiktok|douyin|bilibili|xiaohongshu)."
|
|
249
|
+
),
|
|
250
|
+
content_type: str | None = typer.Option(
|
|
251
|
+
None, "--content-type", help="Force the content type (video|image_note)."
|
|
252
|
+
),
|
|
253
|
+
asr_model: str | None = typer.Option(None, "--asr-model", help="Explicitly prepared local faster-whisper model directory; never downloads during extraction."),
|
|
254
|
+
asr_language: str | None = typer.Option(None, "--asr-language", help="Audio language code, e.g. zh or en; auto enables language detection."),
|
|
255
|
+
asr_beam_size: int | None = typer.Option(None, "--asr-beam-size", min=1, max=20),
|
|
256
|
+
asr_hotwords: str | None = typer.Option(None, "--asr-hotwords", help="Optional domain vocabulary, not corrected answers or a transcript."),
|
|
257
|
+
asr_initial_prompt: str | None = typer.Option(None, "--asr-initial-prompt", help="Optional audio context; never a supplied reference transcript."),
|
|
258
|
+
debug: bool = typer.Option(False, "--debug"),
|
|
259
|
+
) -> None:
|
|
260
|
+
try:
|
|
261
|
+
formats = _parse_formats(format) if format is not None else None
|
|
262
|
+
_validate_doc_language(doc_language)
|
|
263
|
+
platform_name = _normalise_platform(platform)
|
|
264
|
+
content_type_name = _normalise_content_type(content_type)
|
|
265
|
+
if fixture_transcript is not None and fixture_content is not None:
|
|
266
|
+
raise typer.BadParameter(
|
|
267
|
+
"--fixture-transcript and --fixture-content are mutually exclusive"
|
|
268
|
+
)
|
|
269
|
+
configure_logging(debug=debug)
|
|
270
|
+
settings = _build_settings(
|
|
271
|
+
config_path=config,
|
|
272
|
+
proxy=proxy,
|
|
273
|
+
cookies=cookies,
|
|
274
|
+
cookies_from_browser=cookies_from_browser,
|
|
275
|
+
debug=debug,
|
|
276
|
+
platform=platform_name,
|
|
277
|
+
asr_model=asr_model,
|
|
278
|
+
asr_language=asr_language,
|
|
279
|
+
asr_beam_size=asr_beam_size,
|
|
280
|
+
asr_hotwords=asr_hotwords,
|
|
281
|
+
asr_initial_prompt=asr_initial_prompt,
|
|
282
|
+
)
|
|
283
|
+
route = _extract_route(
|
|
284
|
+
url,
|
|
285
|
+
fixture_content=fixture_content,
|
|
286
|
+
platform=platform_name,
|
|
287
|
+
content_type=content_type_name,
|
|
288
|
+
)
|
|
289
|
+
if route == "v1":
|
|
290
|
+
_extract_legacy(
|
|
291
|
+
url=url,
|
|
292
|
+
out=out,
|
|
293
|
+
formats=formats,
|
|
294
|
+
fixture_transcript=fixture_transcript,
|
|
295
|
+
doc_language=doc_language,
|
|
296
|
+
with_reviewer_scaffolding=with_reviewer_scaffolding,
|
|
297
|
+
skip_existing=skip_existing,
|
|
298
|
+
settings=settings,
|
|
299
|
+
)
|
|
300
|
+
return
|
|
301
|
+
if fixture_transcript is not None:
|
|
302
|
+
raise typer.BadParameter(
|
|
303
|
+
"--fixture-transcript is only supported for YouTube; use --fixture-content "
|
|
304
|
+
"for the multi-platform pipeline"
|
|
305
|
+
)
|
|
306
|
+
_extract_v2(
|
|
307
|
+
url=url,
|
|
308
|
+
out=out,
|
|
309
|
+
settings=settings,
|
|
310
|
+
formats=formats,
|
|
311
|
+
fixture_content=fixture_content,
|
|
312
|
+
refresh=refresh,
|
|
313
|
+
doc_language=doc_language,
|
|
314
|
+
platform=platform_name,
|
|
315
|
+
content_type=content_type_name,
|
|
316
|
+
skip_existing=skip_existing,
|
|
317
|
+
coarse_atoms=coarse_atoms,
|
|
318
|
+
)
|
|
319
|
+
except typer.BadParameter:
|
|
320
|
+
raise
|
|
321
|
+
except typer.Exit:
|
|
322
|
+
raise
|
|
323
|
+
except (UserFacingError, ConfigFileError, ValidationError) as exc:
|
|
324
|
+
_exit_with_runtime_error(str(exc))
|
|
325
|
+
except Exception as exc:
|
|
326
|
+
_exit_with_runtime_error(f"Extraction failed: {exc}")
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _extract_legacy(
|
|
330
|
+
*,
|
|
331
|
+
url: str,
|
|
332
|
+
out: Path,
|
|
333
|
+
formats: set[str] | None,
|
|
334
|
+
fixture_transcript: Path | None,
|
|
335
|
+
doc_language: str | None,
|
|
336
|
+
with_reviewer_scaffolding: bool,
|
|
337
|
+
skip_existing: bool,
|
|
338
|
+
settings: AppSettings,
|
|
339
|
+
) -> None:
|
|
340
|
+
"""The historical YouTube path — kept byte-for-byte (A01)."""
|
|
341
|
+
legacy_formats = formats if formats is not None else _parse_formats("json,md,skill")
|
|
342
|
+
if skip_existing:
|
|
343
|
+
if "json" not in legacy_formats:
|
|
344
|
+
# knowledge.json is only written when json ∈ formats; without it the
|
|
345
|
+
# flag would be a silent no-op forever (adversarial review 2026-08-15).
|
|
346
|
+
typer.echo(
|
|
347
|
+
"WARNING: --skip-existing requires the 'json' format "
|
|
348
|
+
"(knowledge.json is the completion marker); without it "
|
|
349
|
+
"nothing is ever skipped — add json to --format",
|
|
350
|
+
err=True,
|
|
351
|
+
)
|
|
352
|
+
elif _existing_output_matches(out, url) and (
|
|
353
|
+
"doc" not in legacy_formats or _verified_doc(out)
|
|
354
|
+
):
|
|
355
|
+
typer.echo(f"skip-existing: {out / 'knowledge.json'} matches url — nothing to do")
|
|
356
|
+
return
|
|
357
|
+
pipeline = build_default_pipeline(settings=settings)
|
|
358
|
+
pipeline.run(
|
|
359
|
+
url=url,
|
|
360
|
+
output_dir=out,
|
|
361
|
+
formats=legacy_formats,
|
|
362
|
+
fixture_transcript=fixture_transcript,
|
|
363
|
+
doc_language=doc_language,
|
|
364
|
+
with_reviewer_scaffolding=with_reviewer_scaffolding,
|
|
365
|
+
)
|
|
366
|
+
if "doc" in legacy_formats and (out / ".knowledge-doc-status.json").exists() and not _verified_doc(out):
|
|
367
|
+
typer.echo(
|
|
368
|
+
"WARNING: knowledge document is FALLBACK or unverified; see .knowledge-doc-status.json",
|
|
369
|
+
err=True,
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _extract_v2(
|
|
374
|
+
*,
|
|
375
|
+
url: str,
|
|
376
|
+
out: Path,
|
|
377
|
+
settings: AppSettings,
|
|
378
|
+
formats: set[str] | None,
|
|
379
|
+
fixture_content: Path | None,
|
|
380
|
+
refresh: bool,
|
|
381
|
+
doc_language: str | None,
|
|
382
|
+
platform: str | None,
|
|
383
|
+
content_type: str | None,
|
|
384
|
+
skip_existing: bool,
|
|
385
|
+
coarse_atoms: bool = False,
|
|
386
|
+
) -> None:
|
|
387
|
+
# An external descriptor can change independently of its URL. Revalidate
|
|
388
|
+
# identity, copied bytes and streams before trusting any earlier outputs.
|
|
389
|
+
if skip_existing and fixture_content is None and not refresh and _v2_outputs_complete(
|
|
390
|
+
out, formats, expected_source_id=_requested_v2_source_id(url, fixture_content),
|
|
391
|
+
settings=settings, doc_language=doc_language,
|
|
392
|
+
):
|
|
393
|
+
typer.echo(f"skip-existing: {out} already has verified outputs — nothing to do")
|
|
394
|
+
return
|
|
395
|
+
result = _run_v2(
|
|
396
|
+
url=url,
|
|
397
|
+
out=out,
|
|
398
|
+
settings=settings,
|
|
399
|
+
formats=formats,
|
|
400
|
+
fixture_content=fixture_content,
|
|
401
|
+
refresh=refresh,
|
|
402
|
+
doc_language=doc_language,
|
|
403
|
+
platform=platform,
|
|
404
|
+
content_type=content_type,
|
|
405
|
+
coarse_atoms=coarse_atoms,
|
|
406
|
+
)
|
|
407
|
+
outcome, reasons = _v2_outcome(result, out, formats)
|
|
408
|
+
if outcome == "COMPLETE":
|
|
409
|
+
return
|
|
410
|
+
if outcome == "PARTIAL":
|
|
411
|
+
message = "; ".join(reasons) if reasons else "some formats incomplete"
|
|
412
|
+
typer.echo(f"PARTIAL: {message}")
|
|
413
|
+
return
|
|
414
|
+
if outcome == "NO_KNOWLEDGE":
|
|
415
|
+
typer.echo("NO_KNOWLEDGE: content complete but no knowledge atoms were extracted")
|
|
416
|
+
return
|
|
417
|
+
for reason in reasons:
|
|
418
|
+
typer.echo(reason, err=True)
|
|
419
|
+
if outcome == "OUTPUTS_NOT_APPLICABLE":
|
|
420
|
+
typer.echo(f"{outcome}: no requested output format is applicable", err=True)
|
|
421
|
+
else:
|
|
422
|
+
typer.echo(f"{outcome or 'BLOCKED'}: extraction did not complete", err=True)
|
|
423
|
+
raise typer.Exit(code=1)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _build_v2_pipeline(*, settings: AppSettings | None = None,
|
|
427
|
+
coarse_atoms: bool = False) -> Any:
|
|
428
|
+
"""Factory used by the CLI; lazy so the v1-only path never imports v2."""
|
|
429
|
+
from ytk.pipeline.v2 import build_v2_pipeline
|
|
430
|
+
|
|
431
|
+
return build_v2_pipeline(settings=settings, coarse_atoms=coarse_atoms)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _run_v2(
|
|
435
|
+
*,
|
|
436
|
+
url: str,
|
|
437
|
+
out: Path,
|
|
438
|
+
settings: AppSettings,
|
|
439
|
+
formats: set[str] | None,
|
|
440
|
+
fixture_content: Path | None,
|
|
441
|
+
refresh: bool,
|
|
442
|
+
doc_language: str | None,
|
|
443
|
+
platform: str | None,
|
|
444
|
+
content_type: str | None,
|
|
445
|
+
coarse_atoms: bool = False,
|
|
446
|
+
) -> Any:
|
|
447
|
+
pipeline = _build_v2_pipeline(settings=settings, coarse_atoms=coarse_atoms)
|
|
448
|
+
kwargs: dict[str, Any] = {
|
|
449
|
+
"output_dir": out,
|
|
450
|
+
"formats": formats,
|
|
451
|
+
"fixture_content": fixture_content,
|
|
452
|
+
"refresh": refresh,
|
|
453
|
+
"platform": platform,
|
|
454
|
+
"content_type": content_type,
|
|
455
|
+
}
|
|
456
|
+
if doc_language is not None:
|
|
457
|
+
parameters: Any
|
|
458
|
+
try:
|
|
459
|
+
parameters = inspect.signature(pipeline.run).parameters
|
|
460
|
+
except (TypeError, ValueError): # pragma: no cover - exotic callables
|
|
461
|
+
parameters = {}
|
|
462
|
+
if "doc_language" in parameters:
|
|
463
|
+
kwargs["doc_language"] = doc_language
|
|
464
|
+
return pipeline.run(url, **kwargs)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _v2_outcome(
|
|
468
|
+
result: Any, out: Path, formats: set[str] | None
|
|
469
|
+
) -> tuple[str, list[str]]:
|
|
470
|
+
"""Map a V2RunResult (or manifest fallback) to the CLI outcome vocabulary."""
|
|
471
|
+
outcome_value = _result_field(result, "outcome")
|
|
472
|
+
reasons = [str(reason) for reason in (_result_field(result, "reasons") or [])]
|
|
473
|
+
format_status = dict(_result_field(result, "format_status") or {})
|
|
474
|
+
outcome = str(outcome_value).upper() if outcome_value is not None else ""
|
|
475
|
+
if not outcome:
|
|
476
|
+
store = ArtifactStore(out)
|
|
477
|
+
manifest = store.read_manifest()
|
|
478
|
+
if manifest is not None:
|
|
479
|
+
overall, derived_reasons = derive_overall_status(
|
|
480
|
+
manifest, requested=sorted(formats) if formats else None
|
|
481
|
+
)
|
|
482
|
+
outcome = overall
|
|
483
|
+
reasons = reasons or derived_reasons
|
|
484
|
+
format_status = format_status or {
|
|
485
|
+
fmt: manifest.outputs[fmt].status for fmt in manifest.outputs
|
|
486
|
+
}
|
|
487
|
+
if not outcome:
|
|
488
|
+
outcome = "COMPLETE"
|
|
489
|
+
if outcome == "NO_KNOWLEDGE":
|
|
490
|
+
return "NO_KNOWLEDGE", reasons
|
|
491
|
+
if format_status and all(status == "not_applicable" for status in format_status.values()):
|
|
492
|
+
return "OUTPUTS_NOT_APPLICABLE", reasons
|
|
493
|
+
return outcome, reasons
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _result_field(result: Any, name: str) -> Any:
|
|
497
|
+
if isinstance(result, dict):
|
|
498
|
+
return result.get(name)
|
|
499
|
+
return getattr(result, name, None)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _requested_v2_source_id(url: str, fixture_content: Path | None) -> str | None:
|
|
503
|
+
try:
|
|
504
|
+
if fixture_content is not None:
|
|
505
|
+
from ytk.pipeline.v2 import MultiPlatformPipeline
|
|
506
|
+
|
|
507
|
+
return MultiPlatformPipeline._load_fixture_content(fixture_content).source.source_id
|
|
508
|
+
return resolve_source_safe(url).source_id
|
|
509
|
+
except (UserFacingError, OSError, ValueError):
|
|
510
|
+
return None
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _v2_outputs_complete(
|
|
514
|
+
out: Path, formats: set[str] | None, *, expected_source_id: str | None = None,
|
|
515
|
+
settings: AppSettings | None = None, doc_language: str | None = None,
|
|
516
|
+
) -> bool:
|
|
517
|
+
try:
|
|
518
|
+
store = ArtifactStore(out)
|
|
519
|
+
manifest = store.read_manifest()
|
|
520
|
+
if manifest is None:
|
|
521
|
+
return False
|
|
522
|
+
if expected_source_id is None or manifest.source_ids != [expected_source_id]:
|
|
523
|
+
return False
|
|
524
|
+
content_cache = getattr(manifest, "content_cache", None)
|
|
525
|
+
if settings is not None and isinstance(content_cache, dict):
|
|
526
|
+
from ytk.pipeline.v2 import MultiPlatformPipeline
|
|
527
|
+
|
|
528
|
+
if content_cache.get("processing_fingerprint") != MultiPlatformPipeline(settings=settings)._processing_fingerprint():
|
|
529
|
+
return False
|
|
530
|
+
requested = formats if formats is not None else set(manifest.outputs)
|
|
531
|
+
if not requested:
|
|
532
|
+
return False
|
|
533
|
+
for fmt in requested:
|
|
534
|
+
if settings is not None and fmt != "json" and fmt in manifest.outputs and manifest.outputs[fmt].status == "complete":
|
|
535
|
+
from ytk.pipeline.v2 import render_request_fingerprint
|
|
536
|
+
|
|
537
|
+
platform = expected_source_id.split(":", 1)[0]
|
|
538
|
+
if getattr(manifest, "render_fingerprints", {}).get(fmt) != render_request_fingerprint(fmt, settings, doc_language, platform):
|
|
539
|
+
return False
|
|
540
|
+
skip, _reason = store.should_skip_format(
|
|
541
|
+
fmt,
|
|
542
|
+
request_fingerprint=manifest.request_fingerprint,
|
|
543
|
+
evidence_fingerprint=manifest.source_revision,
|
|
544
|
+
checker_version=(
|
|
545
|
+
manifest.outputs[fmt].checker_version
|
|
546
|
+
if fmt in manifest.outputs
|
|
547
|
+
else None
|
|
548
|
+
),
|
|
549
|
+
validator_version=V2_VALIDATOR_VERSION,
|
|
550
|
+
source_revision=manifest.source_revision,
|
|
551
|
+
)
|
|
552
|
+
if not skip:
|
|
553
|
+
return False
|
|
554
|
+
return True
|
|
555
|
+
except Exception: # noqa: BLE001 - resume probe must degrade to "run it"
|
|
556
|
+
return False
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _extract_route(
|
|
560
|
+
url: str,
|
|
561
|
+
*,
|
|
562
|
+
fixture_content: Path | None,
|
|
563
|
+
platform: str | None,
|
|
564
|
+
content_type: str | None,
|
|
565
|
+
) -> str:
|
|
566
|
+
"""``"v1"`` for legacy YouTube, ``"v2"`` for everything else."""
|
|
567
|
+
if fixture_content is not None:
|
|
568
|
+
return "v2"
|
|
569
|
+
if platform is not None:
|
|
570
|
+
return "v1" if platform == "youtube" else "v2"
|
|
571
|
+
try:
|
|
572
|
+
ref = resolve_source_safe(url, content_type=content_type)
|
|
573
|
+
except (UserFacingError, ValueError):
|
|
574
|
+
return "v2" # let the v2 pipeline raise the actionable error
|
|
575
|
+
return "v1" if ref.platform == "youtube" else "v2"
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def resolve_source_safe(value: str, *, content_type: str | None = None) -> ContentRef:
|
|
579
|
+
from ytk.ingestion.resolver import resolve_source
|
|
580
|
+
|
|
581
|
+
return resolve_source(value, content_type=content_type) # type: ignore[arg-type]
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
@app.command("import-file")
|
|
585
|
+
def import_file_cmd(
|
|
586
|
+
input_file: Path,
|
|
587
|
+
out: Path = typer.Option(..., "--out"),
|
|
588
|
+
source_url: str | None = typer.Option(None, "--source-url", help="Optional original video URL; pairing is user asserted, never live verification."),
|
|
589
|
+
subtitle_file: Path | None = typer.Option(None, "--subtitle-file", help="SRT/VTT/JSON3 subtitles accompanying a local video."),
|
|
590
|
+
visual_notes: Path | None = typer.Option(None, "--visual-notes", help="Observations bound to exported frame hashes and timestamps."),
|
|
591
|
+
format: str = typer.Option("json,md", "--format"),
|
|
592
|
+
resume: bool = typer.Option(False, "--resume", help="Explicitly resume an output owned by this same source."),
|
|
593
|
+
coarse_atoms: bool = typer.Option(False, "--coarse-atoms"),
|
|
594
|
+
) -> None:
|
|
595
|
+
"""Extract from an owned local video, audio or timed subtitle file without platform access."""
|
|
596
|
+
import tempfile
|
|
597
|
+
from ytk.ingestion.local_import import ImportInputError, check_output, prepare_import
|
|
598
|
+
|
|
599
|
+
try:
|
|
600
|
+
if not resume:
|
|
601
|
+
check_output(out)
|
|
602
|
+
payload = prepare_import(input_file, source_url=source_url, subtitle_file=subtitle_file, visual_notes=visual_notes)
|
|
603
|
+
source = ContentRef.model_validate(payload["source"])
|
|
604
|
+
check_output(out, resume=resume, source_id=source.source_id,
|
|
605
|
+
import_fingerprint=payload["metadata"]["import_fingerprint"])
|
|
606
|
+
settings = _build_settings()
|
|
607
|
+
# A stored provider key is not consent to upload an imported private file.
|
|
608
|
+
settings = settings.model_copy(update={"cloud_text_enabled": False, "cloud_media_enabled": False})
|
|
609
|
+
formats = _parse_formats(format)
|
|
610
|
+
with tempfile.TemporaryDirectory(prefix="kd-user-import-") as directory:
|
|
611
|
+
descriptor = Path(directory) / "input.json"
|
|
612
|
+
descriptor.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
|
613
|
+
result = _run_v2(url=source.canonical_url, out=out, settings=settings, formats=formats,
|
|
614
|
+
fixture_content=descriptor, refresh=False, doc_language=None,
|
|
615
|
+
platform=None, content_type="video", coarse_atoms=coarse_atoms)
|
|
616
|
+
outcome, reasons = _v2_outcome(result, out, formats)
|
|
617
|
+
typer.echo(json.dumps({"outcome": outcome, "source_id": source.source_id,
|
|
618
|
+
"acquisition_mode": "user-import", "output": str(out),
|
|
619
|
+
"reasons": reasons}, ensure_ascii=False))
|
|
620
|
+
if outcome not in {"COMPLETE", "PARTIAL", "NO_KNOWLEDGE"}:
|
|
621
|
+
raise typer.Exit(code=1)
|
|
622
|
+
except typer.Exit:
|
|
623
|
+
raise
|
|
624
|
+
except ImportInputError as exc:
|
|
625
|
+
typer.echo(json.dumps({"outcome": "BLOCKED", "reason_code": exc.code,
|
|
626
|
+
"message": str(exc), "recovery": exc.action}, ensure_ascii=False))
|
|
627
|
+
raise typer.Exit(code=1) from None
|
|
628
|
+
except (OSError, ValueError, UserFacingError) as exc:
|
|
629
|
+
from ytk.logging import redact_payload
|
|
630
|
+
typer.echo(json.dumps({"outcome": "BLOCKED", "reason_code": getattr(exc, "reason_code", "IMPORT_FAILED"),
|
|
631
|
+
"message": redact_payload(str(exc)),
|
|
632
|
+
"recovery": "检查本地依赖与模型,保留原文件后用新目录重试;模型缺失时按连接器README显式准备。"}, ensure_ascii=False))
|
|
633
|
+
raise typer.Exit(code=1) from None
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
@app.command("frames")
|
|
637
|
+
def frames_cmd(
|
|
638
|
+
input_file: Path,
|
|
639
|
+
out: Path = typer.Option(..., "--out"),
|
|
640
|
+
max_frames: int = typer.Option(8, "--max-frames", min=1, max=12),
|
|
641
|
+
at: list[float] | None = typer.Option(None, "--at", help="Specific seconds to inspect; repeat up to 12 times in increasing order."),
|
|
642
|
+
) -> None:
|
|
643
|
+
"""Export real timestamped frames for a model/user to inspect; never invent frame observations."""
|
|
644
|
+
from ytk.ingestion.video_frames import export_frames
|
|
645
|
+
try:
|
|
646
|
+
manifest = export_frames(input_file, out, max_frames=max_frames, timestamps=at)
|
|
647
|
+
typer.echo(json.dumps({"outcome": "VISUAL_REVIEW_REQUIRED", "manifest": str(out / "frames-manifest.json"),
|
|
648
|
+
"frames": len(manifest["frames"]),
|
|
649
|
+
"next": "在WorkBuddy读取实际帧,写入含video_sha256、frames_manifest、reviewer与observations[{frame_index,text}]的JSON,然后 import-file --visual-notes;只代表抽样帧,不能声称看完全片。"}, ensure_ascii=False))
|
|
650
|
+
except (OSError, ValueError, UserFacingError) as exc:
|
|
651
|
+
from ytk.logging import redact_payload
|
|
652
|
+
typer.echo(json.dumps({"outcome": "BLOCKED", "reason_code": getattr(exc, "reason_code", "FRAME_EXPORT_FAILED"),
|
|
653
|
+
"message": redact_payload(str(exc))}, ensure_ascii=False))
|
|
654
|
+
raise typer.Exit(code=1) from None
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
# --------------------------------------------------------------------------- #
|
|
658
|
+
# skill / merge (unchanged)
|
|
659
|
+
# --------------------------------------------------------------------------- #
|
|
660
|
+
|
|
661
|
+
def _load_bundle_for_skill(path: Path):
|
|
662
|
+
"""Load a knowledge bundle of any supported schema version for `skill build`."""
|
|
663
|
+
from ytk.models.migrations import load_bundle_any
|
|
664
|
+
|
|
665
|
+
try:
|
|
666
|
+
return load_bundle_any(path)
|
|
667
|
+
except UserFacingError:
|
|
668
|
+
raise
|
|
669
|
+
except FileNotFoundError as exc:
|
|
670
|
+
raise UserFacingError(f"Knowledge bundle not found: {path}") from exc
|
|
671
|
+
except json.JSONDecodeError as exc:
|
|
672
|
+
raise UserFacingError(f"Knowledge bundle is not valid JSON: {path}") from exc
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
@skill_app.command("build")
|
|
676
|
+
def skill_build(
|
|
677
|
+
knowledge_json: Path,
|
|
678
|
+
out: Path = typer.Option(..., "--out"),
|
|
679
|
+
package_kind: str = typer.Option("generic", "--package-kind"),
|
|
680
|
+
target: str = typer.Option("workbuddy", "--target",
|
|
681
|
+
help="Output spec: 'workbuddy' (default) or 'generic'."),
|
|
682
|
+
) -> None:
|
|
683
|
+
if target not in ("workbuddy", "generic"):
|
|
684
|
+
raise typer.BadParameter(f"--target 只接受 workbuddy|generic,收到 {target!r}")
|
|
685
|
+
try:
|
|
686
|
+
# v2-aware: the default pipeline writes schema_version 2 with atom_id /
|
|
687
|
+
# evidence_ids / topics, which the v1-only loader rejects outright.
|
|
688
|
+
# models.migrations.load_bundle_any already exists and the SkillBuilder
|
|
689
|
+
# coerces both shapes, so only the entry point was wired to the old one.
|
|
690
|
+
bundle = _load_bundle_for_skill(knowledge_json)
|
|
691
|
+
pipeline = build_default_pipeline()
|
|
692
|
+
pipeline.skill_builder.build(bundle, out, package_kind=package_kind, target=target)
|
|
693
|
+
except typer.BadParameter:
|
|
694
|
+
raise
|
|
695
|
+
except (UserFacingError, ValidationError) as exc:
|
|
696
|
+
_exit_with_runtime_error(str(exc))
|
|
697
|
+
except Exception as exc:
|
|
698
|
+
_exit_with_runtime_error(f"Skill build failed: {exc}")
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
@app.command("merge")
|
|
702
|
+
def merge_cmd(
|
|
703
|
+
input_dir: Path = typer.Argument(..., help="Directory containing video output subdirectories"),
|
|
704
|
+
output: Path = typer.Option("./merged/knowledge.json", "--out", help="Output path for merged knowledge.json"),
|
|
705
|
+
) -> None:
|
|
706
|
+
try:
|
|
707
|
+
from ytk.commands.merge import merge_knowledge_directories
|
|
708
|
+
|
|
709
|
+
result = merge_knowledge_directories(input_dir, output)
|
|
710
|
+
typer.echo(f"Merged {result['source_count']} videos, {len(result['atoms'])} unique atoms → {output}")
|
|
711
|
+
except Exception as exc:
|
|
712
|
+
_exit_with_runtime_error(f"Merge failed: {exc}")
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
# --------------------------------------------------------------------------- #
|
|
716
|
+
# batch run
|
|
717
|
+
# --------------------------------------------------------------------------- #
|
|
718
|
+
@batch_app.command("run")
|
|
719
|
+
def batch_run(
|
|
720
|
+
urls_file: Path | None = typer.Option(
|
|
721
|
+
None,
|
|
722
|
+
"--urls-file",
|
|
723
|
+
help="Path to file with one URL per line.",
|
|
724
|
+
),
|
|
725
|
+
out: Path = typer.Option(Path("./outputs"), "--out"),
|
|
726
|
+
format: str | None = typer.Option(
|
|
727
|
+
None,
|
|
728
|
+
"--format",
|
|
729
|
+
help="Comma-separated outputs applied to every source. Omit for per-source "
|
|
730
|
+
"defaults (YouTube: json,md,skill; other platforms: json,md).",
|
|
731
|
+
),
|
|
732
|
+
fixture_transcript: Path | None = typer.Option(None, "--fixture-transcript"),
|
|
733
|
+
fixture_content: Path | None = typer.Option(
|
|
734
|
+
None, "--fixture-content", help="JSON descriptor of already-acquired content (offline v2 path)."
|
|
735
|
+
),
|
|
736
|
+
proxy: str | None = typer.Option(
|
|
737
|
+
None,
|
|
738
|
+
"--proxy",
|
|
739
|
+
help="HTTP/HTTPS proxy URL (e.g. http://127.0.0.1:7890)",
|
|
740
|
+
),
|
|
741
|
+
cookies: Path | None = typer.Option(
|
|
742
|
+
None,
|
|
743
|
+
"--cookies",
|
|
744
|
+
help="Path to Netscape formatted YouTube cookies file for yt-dlp.",
|
|
745
|
+
),
|
|
746
|
+
cookies_from_browser: str | None = typer.Option(
|
|
747
|
+
None,
|
|
748
|
+
"--cookies-from-browser",
|
|
749
|
+
help="Browser cookies source for yt-dlp, e.g. chrome:default",
|
|
750
|
+
),
|
|
751
|
+
search: str | None = typer.Option(
|
|
752
|
+
None,
|
|
753
|
+
"--search",
|
|
754
|
+
help="YouTube search query to generate URL list instead of reading from file.",
|
|
755
|
+
),
|
|
756
|
+
doc_language: str | None = typer.Option(
|
|
757
|
+
None,
|
|
758
|
+
"--doc-language",
|
|
759
|
+
help="Output language for the 'doc' format. One of: zh, en, auto. "
|
|
760
|
+
"When 'auto' (or unset), the skill samples $LANG/$LC_ALL — defaults to 'zh'.",
|
|
761
|
+
),
|
|
762
|
+
with_reviewer_scaffolding: bool = typer.Option(
|
|
763
|
+
False,
|
|
764
|
+
"--with-reviewer-scaffolding",
|
|
765
|
+
help="Append an A/B/C reference list, image-asset manifest, and "
|
|
766
|
+
"closing CTA to the `doc` output of every video in the batch. "
|
|
767
|
+
"Applied per-video (same flag as `extract`).",
|
|
768
|
+
),
|
|
769
|
+
resume: bool = typer.Option(
|
|
770
|
+
False, "--resume",
|
|
771
|
+
help="Reuse verified outputs per format (fingerprint/revision/digest/validator) "
|
|
772
|
+
"and skip only what is genuinely complete. Interrupted batches resume.",
|
|
773
|
+
),
|
|
774
|
+
refresh: bool = typer.Option(
|
|
775
|
+
False, "--refresh",
|
|
776
|
+
help="Re-resolve and re-acquire changed material; never forgets an unresolved "
|
|
777
|
+
"remote submission, never reuses stale outputs.",
|
|
778
|
+
),
|
|
779
|
+
config: Path | None = typer.Option(
|
|
780
|
+
None, "--config", help="Optional TOML config file (explicit field priority)."
|
|
781
|
+
),
|
|
782
|
+
debug: bool = typer.Option(False, "--debug"),
|
|
783
|
+
) -> None:
|
|
784
|
+
try:
|
|
785
|
+
if not urls_file and not search:
|
|
786
|
+
_exit_with_runtime_error("Provide --urls-file or --search.")
|
|
787
|
+
return
|
|
788
|
+
formats_explicit = format is not None
|
|
789
|
+
formats = _parse_formats(format) if format is not None else None
|
|
790
|
+
_validate_doc_language(doc_language)
|
|
791
|
+
if fixture_transcript is not None and fixture_content is not None:
|
|
792
|
+
raise typer.BadParameter(
|
|
793
|
+
"--fixture-transcript and --fixture-content are mutually exclusive"
|
|
794
|
+
)
|
|
795
|
+
configure_logging(debug=debug)
|
|
796
|
+
settings = _build_settings(
|
|
797
|
+
config_path=config,
|
|
798
|
+
proxy=proxy,
|
|
799
|
+
cookies=cookies,
|
|
800
|
+
cookies_from_browser=cookies_from_browser,
|
|
801
|
+
debug=debug,
|
|
802
|
+
platform="youtube",
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
if search:
|
|
806
|
+
from ytk.ingestion.search import YouTubeSearchService
|
|
807
|
+
|
|
808
|
+
svc = YouTubeSearchService()
|
|
809
|
+
results = svc.search(search, max_results=10)
|
|
810
|
+
raw_urls = svc.get_urls(results)
|
|
811
|
+
if not raw_urls:
|
|
812
|
+
typer.echo(f"No results found for: {search}", err=True)
|
|
813
|
+
raise typer.Exit(code=1)
|
|
814
|
+
typer.echo(f"Found {len(raw_urls)} videos for: {search}")
|
|
815
|
+
else:
|
|
816
|
+
try:
|
|
817
|
+
raw_urls = urls_file.read_text(encoding="utf-8").splitlines() # type: ignore[union-attr]
|
|
818
|
+
except OSError as exc:
|
|
819
|
+
_exit_with_runtime_error(f"Failed to read URLs file: {exc}")
|
|
820
|
+
return
|
|
821
|
+
|
|
822
|
+
urls = [line.strip() for line in raw_urls if line.strip()]
|
|
823
|
+
report = _run_batch(
|
|
824
|
+
urls,
|
|
825
|
+
out=out,
|
|
826
|
+
settings=settings,
|
|
827
|
+
formats=formats,
|
|
828
|
+
formats_explicit=formats_explicit,
|
|
829
|
+
fixture_transcript=fixture_transcript,
|
|
830
|
+
fixture_content=fixture_content,
|
|
831
|
+
doc_language=doc_language,
|
|
832
|
+
with_reviewer_scaffolding=with_reviewer_scaffolding,
|
|
833
|
+
resume=resume,
|
|
834
|
+
refresh=refresh,
|
|
835
|
+
)
|
|
836
|
+
typer.echo("")
|
|
837
|
+
typer.echo("=" * 40)
|
|
838
|
+
for line in report.summary_lines():
|
|
839
|
+
typer.echo(line)
|
|
840
|
+
if report.exit_code:
|
|
841
|
+
raise typer.Exit(code=report.exit_code)
|
|
842
|
+
except typer.BadParameter:
|
|
843
|
+
raise
|
|
844
|
+
except typer.Exit:
|
|
845
|
+
raise
|
|
846
|
+
except (UserFacingError, ConfigFileError, ValidationError) as exc:
|
|
847
|
+
_exit_with_runtime_error(str(exc))
|
|
848
|
+
except Exception as exc:
|
|
849
|
+
_exit_with_runtime_error(f"Batch run failed: {exc}")
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _run_batch(
|
|
853
|
+
urls: list[str],
|
|
854
|
+
*,
|
|
855
|
+
out: Path,
|
|
856
|
+
settings: AppSettings,
|
|
857
|
+
formats: set[str] | None,
|
|
858
|
+
formats_explicit: bool,
|
|
859
|
+
fixture_transcript: Path | None,
|
|
860
|
+
fixture_content: Path | None,
|
|
861
|
+
doc_language: str | None,
|
|
862
|
+
with_reviewer_scaffolding: bool,
|
|
863
|
+
resume: bool,
|
|
864
|
+
refresh: bool,
|
|
865
|
+
):
|
|
866
|
+
from ytk.batch.runner import run_batch
|
|
867
|
+
|
|
868
|
+
return run_batch(
|
|
869
|
+
urls,
|
|
870
|
+
out_root=out,
|
|
871
|
+
settings=settings,
|
|
872
|
+
formats=formats,
|
|
873
|
+
formats_explicit=formats_explicit,
|
|
874
|
+
fixture_transcript=fixture_transcript,
|
|
875
|
+
fixture_content=fixture_content,
|
|
876
|
+
doc_language=doc_language,
|
|
877
|
+
with_reviewer_scaffolding=with_reviewer_scaffolding,
|
|
878
|
+
resume=resume,
|
|
879
|
+
refresh=refresh,
|
|
880
|
+
legacy_pipeline_factory=build_default_pipeline,
|
|
881
|
+
v2_pipeline_factory=_build_v2_pipeline,
|
|
882
|
+
doc_verifier=_verified_doc,
|
|
883
|
+
echo=typer.echo,
|
|
884
|
+
)
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
# --------------------------------------------------------------------------- #
|
|
888
|
+
# platforms / probe
|
|
889
|
+
# --------------------------------------------------------------------------- #
|
|
890
|
+
@app.command("platforms")
|
|
891
|
+
def platforms_cmd(
|
|
892
|
+
json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON only."),
|
|
893
|
+
config: Path | None = typer.Option(None, "--config"),
|
|
894
|
+
) -> None:
|
|
895
|
+
"""Show the five platform capabilities and local configuration state."""
|
|
896
|
+
try:
|
|
897
|
+
settings = _build_settings(config_path=config)
|
|
898
|
+
payload = _platforms_payload(settings)
|
|
899
|
+
if json_output:
|
|
900
|
+
typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
|
901
|
+
return
|
|
902
|
+
typer.echo(f"{'platform':14} {'enabled':8} {'provider':10} {'formats':16} notes")
|
|
903
|
+
for name, info in payload.items():
|
|
904
|
+
notes = []
|
|
905
|
+
if info["requires_service"]:
|
|
906
|
+
notes.append(
|
|
907
|
+
"service configured" if info["service_configured"] else "service NOT configured"
|
|
908
|
+
)
|
|
909
|
+
if info["cookies_configured"]:
|
|
910
|
+
notes.append("cookies")
|
|
911
|
+
if info["proxy_configured"]:
|
|
912
|
+
notes.append("proxy")
|
|
913
|
+
typer.echo(
|
|
914
|
+
f"{name:14} {str(info['enabled']):8} {info['provider']:10} "
|
|
915
|
+
f"{','.join(info['default_formats']):16} {'; '.join(notes)}"
|
|
916
|
+
)
|
|
917
|
+
except typer.Exit:
|
|
918
|
+
raise
|
|
919
|
+
except (UserFacingError, ConfigFileError, ValidationError) as exc:
|
|
920
|
+
_exit_with_runtime_error(str(exc))
|
|
921
|
+
except Exception as exc:
|
|
922
|
+
_exit_with_runtime_error(f"platforms failed: {exc}")
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def _platforms_payload(settings: AppSettings) -> dict[str, dict[str, Any]]:
|
|
926
|
+
from ytk.ingestion.registry import PlatformRegistry
|
|
927
|
+
from ytk.pipeline.v2 import _builtin_adapters
|
|
928
|
+
|
|
929
|
+
# Reflect what this build can actually acquire with: YouTube/TikTok/Bilibili
|
|
930
|
+
# need no sidecar; Douyin needs DTK and XHS has an optional sidecar.
|
|
931
|
+
registry = PlatformRegistry.default(adapters=_builtin_adapters(settings))
|
|
932
|
+
payload: dict[str, dict[str, Any]] = {}
|
|
933
|
+
for name, capabilities in registry.capabilities_matrix().items():
|
|
934
|
+
ready = bool(capabilities.get("adapter_configured")) and (
|
|
935
|
+
not capabilities.get("requires_service") or bool(settings.base_url_for(name)) # type: ignore[arg-type]
|
|
936
|
+
)
|
|
937
|
+
payload[name] = {
|
|
938
|
+
**capabilities,
|
|
939
|
+
"acquisition_ready": ready,
|
|
940
|
+
"live_access_verified": False,
|
|
941
|
+
"readiness_scope": "local_configuration_only",
|
|
942
|
+
"enabled": settings.is_platform_enabled(name), # type: ignore[arg-type]
|
|
943
|
+
"service_configured": bool(settings.base_url_for(name)), # type: ignore[arg-type]
|
|
944
|
+
"cookies_configured": bool(settings.cookies_file_for(name)), # type: ignore[arg-type]
|
|
945
|
+
"proxy_configured": bool(settings.proxy_for(name)), # type: ignore[arg-type]
|
|
946
|
+
"cloud_text_enabled": settings.cloud_allowed(name, "text"), # type: ignore[arg-type]
|
|
947
|
+
"cloud_media_enabled": settings.cloud_allowed(name, "media"), # type: ignore[arg-type]
|
|
948
|
+
}
|
|
949
|
+
return payload
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
@app.command("probe")
|
|
953
|
+
def probe_cmd(
|
|
954
|
+
url: str,
|
|
955
|
+
json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON only."),
|
|
956
|
+
platform: str | None = typer.Option(None, "--platform"),
|
|
957
|
+
content_type: str | None = typer.Option(
|
|
958
|
+
None, "--content-type", help="video|image_note"
|
|
959
|
+
),
|
|
960
|
+
config: Path | None = typer.Option(None, "--config"),
|
|
961
|
+
) -> None:
|
|
962
|
+
"""Resolve a URL/share text to platform identity + capabilities (no download)."""
|
|
963
|
+
try:
|
|
964
|
+
platform_name = _normalise_platform(platform)
|
|
965
|
+
content_type_name = _normalise_content_type(content_type)
|
|
966
|
+
settings = _build_settings(config_path=config, platform=platform_name)
|
|
967
|
+
ref, identity_source = _probe_identity(
|
|
968
|
+
url,
|
|
969
|
+
settings=settings,
|
|
970
|
+
platform=platform_name,
|
|
971
|
+
content_type=content_type_name,
|
|
972
|
+
)
|
|
973
|
+
capabilities = _platforms_payload(settings).get(ref.platform, {})
|
|
974
|
+
payload = {
|
|
975
|
+
"platform": ref.platform,
|
|
976
|
+
"source_id": ref.source_id,
|
|
977
|
+
"content_type": ref.content_type,
|
|
978
|
+
"canonical_url": ref.canonical_url,
|
|
979
|
+
"variant_id": ref.variant_id,
|
|
980
|
+
"identity_source": identity_source,
|
|
981
|
+
"capabilities": capabilities,
|
|
982
|
+
}
|
|
983
|
+
if json_output:
|
|
984
|
+
typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
|
985
|
+
return
|
|
986
|
+
typer.echo(f"platform: {payload['platform']}")
|
|
987
|
+
typer.echo(f"source_id: {payload['source_id']}")
|
|
988
|
+
typer.echo(f"content_type: {payload['content_type']}")
|
|
989
|
+
typer.echo(f"canonical_url: {payload['canonical_url']}")
|
|
990
|
+
typer.echo(f"identity: {payload['identity_source']}")
|
|
991
|
+
typer.echo(
|
|
992
|
+
f"capabilities: provider={capabilities.get('provider')} "
|
|
993
|
+
f"formats={','.join(capabilities.get('default_formats') or [])} "
|
|
994
|
+
f"subtitle={capabilities.get('subtitle')} asr={capabilities.get('asr')} "
|
|
995
|
+
f"ocr={capabilities.get('ocr')} service={capabilities.get('service_name')}"
|
|
996
|
+
)
|
|
997
|
+
except typer.BadParameter:
|
|
998
|
+
raise
|
|
999
|
+
except typer.Exit:
|
|
1000
|
+
raise
|
|
1001
|
+
except (UserFacingError, ConfigFileError, ValidationError) as exc:
|
|
1002
|
+
_exit_with_runtime_error(str(exc))
|
|
1003
|
+
except Exception as exc:
|
|
1004
|
+
_exit_with_runtime_error(f"Probe failed: {exc}")
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def _probe_identity(
|
|
1008
|
+
url: str,
|
|
1009
|
+
*,
|
|
1010
|
+
settings: AppSettings,
|
|
1011
|
+
platform: str | None,
|
|
1012
|
+
content_type: str | None,
|
|
1013
|
+
) -> tuple[ContentRef, str]:
|
|
1014
|
+
"""Resolve-only identity: adapter when configured, resolver otherwise."""
|
|
1015
|
+
from ytk.ingestion.resolver import detect_platform, extract_urls
|
|
1016
|
+
|
|
1017
|
+
detected = platform
|
|
1018
|
+
if detected is None:
|
|
1019
|
+
try:
|
|
1020
|
+
candidates = extract_urls(url)
|
|
1021
|
+
detected = detect_platform(candidates[0]) if candidates else None
|
|
1022
|
+
except Exception: # noqa: BLE001 - detection is best-effort
|
|
1023
|
+
detected = None
|
|
1024
|
+
adapter = None
|
|
1025
|
+
if detected is not None:
|
|
1026
|
+
try:
|
|
1027
|
+
registry = _registry_with_adapters(settings)
|
|
1028
|
+
adapter = registry.get(detected).adapter
|
|
1029
|
+
except Exception: # noqa: BLE001 - missing adapter is not fatal
|
|
1030
|
+
adapter = None
|
|
1031
|
+
if adapter is not None:
|
|
1032
|
+
context = AccessContext(
|
|
1033
|
+
cookies_file=settings.cookies_file_for(detected), # type: ignore[arg-type]
|
|
1034
|
+
cookies_from_browser=settings.cookies_from_browser_for(detected), # type: ignore[arg-type]
|
|
1035
|
+
proxy=settings.proxy_for(detected), # type: ignore[arg-type]
|
|
1036
|
+
)
|
|
1037
|
+
try:
|
|
1038
|
+
return adapter.resolve(url, context), "adapter"
|
|
1039
|
+
except Exception as exc: # noqa: BLE001 - fall back to the pure resolver
|
|
1040
|
+
LOGGER.debug("adapter resolve failed, falling back to resolver: %s", exc)
|
|
1041
|
+
return resolve_source_safe(url, content_type=content_type), "resolver"
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def _registry_with_adapters(settings: AppSettings):
|
|
1045
|
+
try:
|
|
1046
|
+
from ytk.pipeline.v2 import build_v2_pipeline
|
|
1047
|
+
|
|
1048
|
+
return build_v2_pipeline(settings=settings).registry
|
|
1049
|
+
except Exception: # noqa: BLE001 - probe still works without adapters
|
|
1050
|
+
from ytk.ingestion.registry import PlatformRegistry
|
|
1051
|
+
|
|
1052
|
+
return PlatformRegistry.default()
|
|
1053
|
+
|
|
1054
|
+
|
|
1055
|
+
# --------------------------------------------------------------------------- #
|
|
1056
|
+
# doctor
|
|
1057
|
+
# --------------------------------------------------------------------------- #
|
|
1058
|
+
@app.command()
|
|
1059
|
+
def doctor(
|
|
1060
|
+
offline: bool = typer.Option(
|
|
1061
|
+
False, "--offline", help="Local-only diagnostics: no network probe, no sidecar contact."
|
|
1062
|
+
),
|
|
1063
|
+
platform: str = typer.Option(
|
|
1064
|
+
"all", "--platform", help="all|youtube|tiktok|douyin|bilibili|xiaohongshu"
|
|
1065
|
+
),
|
|
1066
|
+
strict: bool = typer.Option(
|
|
1067
|
+
False, "--strict", help="Exit non-zero when offline local checks are incomplete."
|
|
1068
|
+
),
|
|
1069
|
+
) -> None:
|
|
1070
|
+
try:
|
|
1071
|
+
platform_filter = _normalise_platform(platform, allow_all=True) or "all"
|
|
1072
|
+
report = _doctor_service(offline).run()
|
|
1073
|
+
if offline or platform_filter != "all":
|
|
1074
|
+
report["platforms"] = _platform_diagnostic(platform_filter)
|
|
1075
|
+
typer.echo(format_doctor_report(report))
|
|
1076
|
+
if offline and strict:
|
|
1077
|
+
from ytk.doctor import doctor_report_failures
|
|
1078
|
+
|
|
1079
|
+
failures = doctor_report_failures(report)
|
|
1080
|
+
if failures:
|
|
1081
|
+
typer.echo(
|
|
1082
|
+
"offline checks incomplete: " + ", ".join(sorted(failures)), err=True
|
|
1083
|
+
)
|
|
1084
|
+
raise typer.Exit(code=1)
|
|
1085
|
+
except typer.BadParameter:
|
|
1086
|
+
raise
|
|
1087
|
+
except typer.Exit:
|
|
1088
|
+
raise
|
|
1089
|
+
except Exception as exc:
|
|
1090
|
+
_exit_with_runtime_error(f"Doctor failed: {exc}")
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def _doctor_service(offline: bool):
|
|
1094
|
+
"""Build DoctorService; tolerate older or monkeypatched constructors."""
|
|
1095
|
+
try:
|
|
1096
|
+
return DoctorService(offline=offline)
|
|
1097
|
+
except TypeError:
|
|
1098
|
+
return DoctorService()
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
def _platform_diagnostic(platform_filter: str) -> dict[str, Any]:
|
|
1102
|
+
settings = AppSettings.load()
|
|
1103
|
+
payload = _platforms_payload(settings)
|
|
1104
|
+
if platform_filter != "all":
|
|
1105
|
+
payload = {platform_filter: payload[platform_filter]}
|
|
1106
|
+
problems: list[str] = []
|
|
1107
|
+
for name, info in payload.items():
|
|
1108
|
+
if not info["enabled"]:
|
|
1109
|
+
continue
|
|
1110
|
+
# A service-backed platform whose sidecar is absent is not "ready": the
|
|
1111
|
+
# adapter exists but every acquire would raise ServiceNotConfiguredError.
|
|
1112
|
+
if not info.get("adapter_configured"):
|
|
1113
|
+
problems.append(name)
|
|
1114
|
+
elif info.get("requires_service") and not info.get("service_configured"):
|
|
1115
|
+
problems.append(name)
|
|
1116
|
+
section: dict[str, Any] = {
|
|
1117
|
+
"status": "ok" if not problems else "partial",
|
|
1118
|
+
"verification": "configured_only",
|
|
1119
|
+
"mode": "offline_local",
|
|
1120
|
+
"note": (
|
|
1121
|
+
"local platform configuration only; no network call, no download. "
|
|
1122
|
+
"A configured platform is NOT verified to be reachable."
|
|
1123
|
+
),
|
|
1124
|
+
"platforms": payload,
|
|
1125
|
+
}
|
|
1126
|
+
if problems:
|
|
1127
|
+
section["fix_hint"] = (
|
|
1128
|
+
"not configured locally: "
|
|
1129
|
+
+ ", ".join(problems)
|
|
1130
|
+
+ " | install the optional dependency, set the sidecar base URL, or disable the platform"
|
|
1131
|
+
)
|
|
1132
|
+
return section
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
# --------------------------------------------------------------------------- #
|
|
1136
|
+
# shared helpers
|
|
1137
|
+
# --------------------------------------------------------------------------- #
|
|
1138
|
+
def _existing_output_matches(out: Path, url: str) -> bool:
|
|
1139
|
+
"""True when out/knowledge.json exists AND metadata.json records the SAME
|
|
1140
|
+
video as `url` — a different URL pointed at a populated out dir must NOT
|
|
1141
|
+
exit 0 having processed nothing (adversarial review 2026-08-15 finding)."""
|
|
1142
|
+
if not (out / "knowledge.json").exists():
|
|
1143
|
+
return False
|
|
1144
|
+
try:
|
|
1145
|
+
meta = json.loads((out / "metadata.json").read_text(encoding="utf-8"))
|
|
1146
|
+
except (OSError, json.JSONDecodeError):
|
|
1147
|
+
return False
|
|
1148
|
+
from ytk.ingestion.url_parser import parse_youtube_url
|
|
1149
|
+
|
|
1150
|
+
try:
|
|
1151
|
+
want = parse_youtube_url(url).video_id
|
|
1152
|
+
except Exception:
|
|
1153
|
+
return False
|
|
1154
|
+
return meta.get("video_id") == want
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def _batch_marker_covers(marker: Path, url: str, formats: set[str]) -> bool:
|
|
1158
|
+
"""True when the completion marker matches url AND covers all requested
|
|
1159
|
+
formats (a subset marker means an artifact is still missing)."""
|
|
1160
|
+
try:
|
|
1161
|
+
payload = json.loads(marker.read_text(encoding="utf-8"))
|
|
1162
|
+
except (OSError, json.JSONDecodeError):
|
|
1163
|
+
return False
|
|
1164
|
+
if not isinstance(payload, dict) or payload.get("url") != url:
|
|
1165
|
+
return False
|
|
1166
|
+
return (set(payload.get("formats") or []) >= set(formats)
|
|
1167
|
+
and ("doc" not in formats or _verified_doc(marker.parent)))
|
|
1168
|
+
|
|
1169
|
+
|
|
1170
|
+
def _verified_doc(output_dir: Path) -> bool:
|
|
1171
|
+
from ytk.renderers.knowledge_doc_renderer import complete_doc_status
|
|
1172
|
+
|
|
1173
|
+
return complete_doc_status(output_dir)
|
|
1174
|
+
|
|
1175
|
+
|
|
1176
|
+
def _parse_formats(value: str) -> set[str]:
|
|
1177
|
+
allowed = {"json", "jsonl", "md", "obsidian", "skill", "summary", "doc"}
|
|
1178
|
+
formats = {item.strip() for item in value.split(",") if item.strip()}
|
|
1179
|
+
if not formats:
|
|
1180
|
+
raise typer.BadParameter("At least one output format is required.")
|
|
1181
|
+
unsupported = formats - allowed
|
|
1182
|
+
if unsupported:
|
|
1183
|
+
supported = ", ".join(sorted(allowed))
|
|
1184
|
+
unknown = ", ".join(sorted(unsupported))
|
|
1185
|
+
raise typer.BadParameter(f"Unsupported format(s): {unknown}. Expected one of: {supported}")
|
|
1186
|
+
return formats
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
def _validate_doc_language(value: str | None) -> None:
|
|
1190
|
+
"""Reject obviously invalid --doc-language values early.
|
|
1191
|
+
|
|
1192
|
+
The renderer itself only knows "zh" and "en"; "auto" is a sentinel
|
|
1193
|
+
meaning "use locale / env". Anything else is a user typo.
|
|
1194
|
+
"""
|
|
1195
|
+
if value is None:
|
|
1196
|
+
return
|
|
1197
|
+
candidate = value.strip().lower()
|
|
1198
|
+
if candidate in {"zh", "en", "auto"}:
|
|
1199
|
+
return
|
|
1200
|
+
raise typer.BadParameter(
|
|
1201
|
+
f"--doc-language must be one of: zh, en, auto. Got: {value!r}"
|
|
1202
|
+
)
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def _normalise_platform(value: str | None, *, allow_all: bool = False) -> str | None:
|
|
1206
|
+
if value is None:
|
|
1207
|
+
return None
|
|
1208
|
+
candidate = value.strip().lower()
|
|
1209
|
+
if allow_all and candidate == "all":
|
|
1210
|
+
return "all"
|
|
1211
|
+
if candidate not in SUPPORTED_PLATFORMS:
|
|
1212
|
+
supported = ", ".join(SUPPORTED_PLATFORMS)
|
|
1213
|
+
raise typer.BadParameter(f"--platform must be one of: all, {supported}. Got: {value!r}")
|
|
1214
|
+
return candidate
|
|
1215
|
+
|
|
1216
|
+
|
|
1217
|
+
def _normalise_content_type(value: str | None) -> str | None:
|
|
1218
|
+
if value is None:
|
|
1219
|
+
return None
|
|
1220
|
+
candidate = value.strip().lower()
|
|
1221
|
+
if candidate not in {"video", "image_note"}:
|
|
1222
|
+
raise typer.BadParameter(
|
|
1223
|
+
f"--content-type must be one of: video, image_note. Got: {value!r}"
|
|
1224
|
+
)
|
|
1225
|
+
return candidate
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def _build_settings(
|
|
1229
|
+
*,
|
|
1230
|
+
config_path: Path | None = None,
|
|
1231
|
+
proxy: str | None = None,
|
|
1232
|
+
cookies: Path | None = None,
|
|
1233
|
+
cookies_from_browser: str | None = None,
|
|
1234
|
+
debug: bool = False,
|
|
1235
|
+
platform: str | None = None,
|
|
1236
|
+
asr_model: str | None = None,
|
|
1237
|
+
asr_language: str | None = None,
|
|
1238
|
+
asr_beam_size: int | None = None,
|
|
1239
|
+
asr_hotwords: str | None = None,
|
|
1240
|
+
asr_initial_prompt: str | None = None,
|
|
1241
|
+
) -> AppSettings:
|
|
1242
|
+
"""Build settings with CLI values as the highest-priority tier."""
|
|
1243
|
+
overrides: dict[str, Any] = {}
|
|
1244
|
+
if proxy:
|
|
1245
|
+
overrides.update(
|
|
1246
|
+
{
|
|
1247
|
+
"transcript_proxy_mode": "generic",
|
|
1248
|
+
"transcript_http_proxy_url": proxy,
|
|
1249
|
+
"transcript_https_proxy_url": proxy,
|
|
1250
|
+
}
|
|
1251
|
+
)
|
|
1252
|
+
if platform and platform != "all":
|
|
1253
|
+
overrides[f"{platform}_proxy"] = proxy
|
|
1254
|
+
if cookies is not None:
|
|
1255
|
+
overrides["yt_dlp_cookies_file"] = str(cookies)
|
|
1256
|
+
if platform and platform != "all":
|
|
1257
|
+
overrides[f"{platform}_cookies_file"] = str(cookies)
|
|
1258
|
+
if cookies_from_browser is not None:
|
|
1259
|
+
overrides["yt_dlp_cookies_from_browser"] = cookies_from_browser
|
|
1260
|
+
if debug:
|
|
1261
|
+
overrides["debug"] = True
|
|
1262
|
+
for key, value in {
|
|
1263
|
+
"asr_model": asr_model,
|
|
1264
|
+
"asr_language": asr_language,
|
|
1265
|
+
"asr_beam_size": asr_beam_size,
|
|
1266
|
+
"asr_hotwords": asr_hotwords,
|
|
1267
|
+
"asr_initial_prompt": asr_initial_prompt,
|
|
1268
|
+
}.items():
|
|
1269
|
+
if value is not None:
|
|
1270
|
+
overrides[key] = None if key == "asr_language" and value == "auto" else value
|
|
1271
|
+
return AppSettings.load(config_path=config_path, overrides=overrides)
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
def _exit_with_runtime_error(message: str) -> None:
|
|
1275
|
+
typer.echo(message, err=True)
|
|
1276
|
+
raise typer.Exit(code=1)
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
if __name__ == "__main__":
|
|
1280
|
+
_load_env_file(Path(__file__).resolve().parent.parent.parent / ".env")
|
|
1281
|
+
app()
|