kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
ytk/cli.py ADDED
@@ -0,0 +1,1281 @@
1
+ """Command-line interface (T43.1).
2
+
3
+ Surface (v2.0)::
4
+
5
+ ytk extract URL [--format ...] [--config TOML] [--fixture-content JSON]
6
+ [--fixture-transcript JSON] [--refresh] [--platform NAME]
7
+ [--content-type video|image_note] [--out DIR] ...
8
+ ytk batch run --urls-file FILE [--format ...] [--resume] [--refresh] ...
9
+ ytk platforms [--json]
10
+ ytk probe URL [--json] [--platform NAME] [--content-type TYPE]
11
+ ytk doctor [--offline] [--platform all|NAME] [--strict]
12
+ ytk cache status|clear
13
+ ytk skill build ...
14
+ ytk merge ...
15
+
16
+ Routing: a plain YouTube link (no ``--fixture-content``) keeps the historical
17
+ v1 pipeline byte-for-byte (A01 compatibility); TikTok / Douyin / Bilibili /
18
+ Xiaohongshu and every ``--fixture-content`` run use the v2 multi-platform
19
+ pipeline. Exit codes: 0 for COMPLETE / PARTIAL / NO_KNOWLEDGE, 1 for
20
+ BLOCKED / FAILED / every-requested-format-not-applicable
21
+ (``OUTPUTS_NOT_APPLICABLE``).
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import inspect
27
+ import json
28
+ import logging
29
+ from pathlib import Path
30
+ from typing import Any
31
+
32
+ import typer
33
+ from pydantic import ValidationError
34
+
35
+ from ytk.config import AppSettings, ConfigFileError
36
+ from ytk.doctor import DoctorService, format_doctor_report
37
+ from ytk.dotenv import load_dotenv
38
+ from ytk.errors import UserFacingError
39
+ from ytk.logging import configure_logging
40
+ from ytk.models.source import SUPPORTED_PLATFORMS, AccessContext, ContentRef
41
+ from ytk.pipeline.orchestrator import build_default_pipeline, load_bundle
42
+ from ytk.pipeline.state import ArtifactStore, derive_overall_status
43
+
44
+ LOGGER = logging.getLogger(__name__)
45
+
46
+ app = typer.Typer(
47
+ help="Knowledge Distiller (萃知): distill videos and image notes into "
48
+ "evidence-linked knowledge and reusable AI-agent skills."
49
+ )
50
+
51
+
52
+ def _show_version(value: bool) -> None:
53
+ if value:
54
+ from importlib.metadata import PackageNotFoundError, version
55
+
56
+ try:
57
+ installed_version = version("kd-engine")
58
+ except PackageNotFoundError:
59
+ installed_version = "1.0.0" # source checkout; synchronized with pyproject.toml
60
+ typer.echo(f"kd-engine {installed_version}")
61
+ raise typer.Exit()
62
+
63
+
64
+ @app.callback()
65
+ def _bootstrap_env(
66
+ version: bool = typer.Option(
67
+ False, "--version", callback=_show_version, is_eager=True,
68
+ help="Show the installed kd-engine version and exit.",
69
+ ),
70
+ ) -> None:
71
+ """Load the project .env for every entry point (console script included).
72
+
73
+ ``python -m ytk`` loads it in ``__main__``; the installed ``ytk`` console
74
+ script calls ``app`` directly, so without this callback the two entry points
75
+ silently diverge (A32). ``YTK_NO_DOTENV=1`` is honoured before any read.
76
+ """
77
+ try:
78
+ from ytk.dotenv import dotenv_disabled, load_dotenv
79
+
80
+ if dotenv_disabled():
81
+ return
82
+ load_dotenv(Path(__file__).resolve().parent.parent.parent / ".env")
83
+ except Exception: # pragma: no cover - env loading must never break the CLI
84
+ return
85
+ skill_app = typer.Typer(help="Build skills from extraction artifacts.")
86
+ batch_app = typer.Typer(help="Batch-process content links from multiple platforms.")
87
+ cache_app = typer.Typer(help="Manage caches (transcript v1 + layered v2).")
88
+
89
+ app.add_typer(skill_app, name="skill")
90
+ app.add_typer(batch_app, name="batch")
91
+ app.add_typer(cache_app, name="cache")
92
+
93
+ V2_VALIDATOR_VERSION = "v2-1" # kept local: importing pipeline.v2 here would load v2 deps eagerly
94
+
95
+
96
+ # --------------------------------------------------------------------------- #
97
+ # Environment loading (historical behaviour preserved; NO_DOTENV honoured)
98
+ # --------------------------------------------------------------------------- #
99
+ def _load_env_file(path: Path) -> None:
100
+ """Load the project ``.env`` (best effort) before the CLI runs.
101
+
102
+ Root cause fix 2026-08-18: the ybs cron ran ``python -m ytk extract`` from an
103
+ arbitrary cwd where ``YTK_MINIMAX_API_KEY`` was NOT exported — the renderer
104
+ silently fell back to the deterministic document. Loading the project
105
+ ``.env`` at CLI entry makes every caller get the same keys as the ``.env``
106
+ author intended. The loader lives in :mod:`ytk.dotenv` so ``YTK_NO_DOTENV=1``
107
+ is honoured *before* the file is opened (offline tests / diagnostics).
108
+ """
109
+ load_dotenv(path)
110
+
111
+
112
+ # --------------------------------------------------------------------------- #
113
+ # Cache commands
114
+ # --------------------------------------------------------------------------- #
115
+ @cache_app.command("status")
116
+ def cache_status() -> None:
117
+ """Show cache location, entry count, size, and per-video detail."""
118
+ from ytk.ingestion.transcript_cache import TranscriptCache
119
+
120
+ settings = AppSettings.load()
121
+ cache = TranscriptCache(
122
+ cache_dir=settings.transcript_cache_dir,
123
+ enabled=settings.transcript_cache_enabled,
124
+ )
125
+ stats = cache.stats()
126
+ size_kb = stats["total_bytes"] / 1024
127
+ print(f"cache dir: {stats['dir']}")
128
+ print(f"enabled: {settings.transcript_cache_enabled}")
129
+ print(f"entries: {stats['entries']} ({size_kb:.1f} KB)")
130
+ for entry in stats["detail"]:
131
+ langs = ",".join(entry["languages"]) or "?"
132
+ print(
133
+ f" {entry['video_id']} {entry['bytes'] / 1024:8.1f} KB "
134
+ f"src={entry['source']} lang={langs} segments={entry['segments']}"
135
+ )
136
+ _print_v2_cache_stats(settings)
137
+
138
+
139
+ def _print_v2_cache_stats(settings: AppSettings) -> None:
140
+ try:
141
+ from ytk.cache import LayeredCache
142
+
143
+ v2 = LayeredCache(root=settings.cache_dir)
144
+ stats = v2.stats()
145
+ except Exception: # noqa: BLE001 - cache introspection must never break the CLI
146
+ return
147
+ size_mb = stats["total_bytes"] / (1024 * 1024)
148
+ print(f"v2 cache: {stats['root']}")
149
+ print(f"v2 entries: {stats['entries']} ({size_mb:.2f} MB, max {stats['max_bytes']} bytes)")
150
+ for namespace, bucket in stats["namespaces"].items():
151
+ if bucket["entries"]:
152
+ print(f" {namespace}: {bucket['entries']} entr(ies), {bucket['bytes'] / 1024:.1f} KB")
153
+
154
+
155
+ @cache_app.command("clear")
156
+ def cache_clear(
157
+ v2: bool = typer.Option(
158
+ False, "--v2", help="Also clear the v2 layered cache (media/subtitle/asr/ocr/generation)."
159
+ ),
160
+ ) -> None:
161
+ """Delete every cached transcript bundle (safe: cache is regenerable)."""
162
+ from ytk.ingestion.transcript_cache import TranscriptCache
163
+
164
+ settings = AppSettings.load()
165
+ cache = TranscriptCache(
166
+ cache_dir=settings.transcript_cache_dir,
167
+ enabled=settings.transcript_cache_enabled,
168
+ )
169
+ removed = cache.clear()
170
+ print(f"removed {removed} cached transcript(s)")
171
+ if v2:
172
+ from ytk.cache import LayeredCache
173
+
174
+ removed_v2 = LayeredCache(root=settings.cache_dir, enabled=True).clear()
175
+ print(f"removed {removed_v2} v2 cache entr(ies)")
176
+
177
+
178
+ # --------------------------------------------------------------------------- #
179
+ # extract
180
+ # --------------------------------------------------------------------------- #
181
+ @app.command()
182
+ def extract(
183
+ url: str,
184
+ out: Path = typer.Option(Path("./outputs"), "--out"),
185
+ format: str | None = typer.Option(
186
+ None,
187
+ "--format",
188
+ help="Comma-separated outputs. Omit for per-platform defaults "
189
+ "(YouTube keeps its legacy default: json,md,skill; new platforms: json,md).",
190
+ ),
191
+ fixture_transcript: Path | None = typer.Option(None, "--fixture-transcript"),
192
+ fixture_content: Path | None = typer.Option(
193
+ None,
194
+ "--fixture-content",
195
+ "--acquired-content",
196
+ help="JSON descriptor of recorded content or a measured browser video acquisition; browser media identity, bytes and streams are revalidated.",
197
+ ),
198
+ proxy: str | None = typer.Option(
199
+ None,
200
+ "--proxy",
201
+ help="HTTP/HTTPS proxy URL (e.g. http://127.0.0.1:7890)",
202
+ ),
203
+ cookies: Path | None = typer.Option(
204
+ None,
205
+ "--cookies",
206
+ help="Path to Netscape formatted YouTube cookies file for yt-dlp.",
207
+ ),
208
+ cookies_from_browser: str | None = typer.Option(
209
+ None,
210
+ "--cookies-from-browser",
211
+ help="Browser cookies source for yt-dlp, e.g. chrome:default",
212
+ ),
213
+ coarse_atoms: bool = typer.Option(
214
+ False,
215
+ "--coarse-atoms",
216
+ help="When a source yields no structured knowledge, emit explicitly "
217
+ "labelled low-confidence segment summaries instead of NO_KNOWLEDGE. "
218
+ "Off by default: never silently turns an empty ledger into a fake one.",
219
+ ),
220
+ doc_language: str | None = typer.Option(
221
+ None,
222
+ "--doc-language",
223
+ help="Output language for the 'doc' format. One of: zh, en, auto. "
224
+ "When 'auto' (or unset), the skill samples $LANG/$LC_ALL — defaults to 'zh'.",
225
+ ),
226
+ with_reviewer_scaffolding: bool = typer.Option(
227
+ False,
228
+ "--with-reviewer-scaffolding",
229
+ help="Append an A/B/C reference list, image-asset manifest, and "
230
+ "closing CTA to the `doc` output. Designed to lift the `doc` "
231
+ "across the 4-model review rubric when downstream penflow will "
232
+ "reuse the doc as a knowledge source.",
233
+ ),
234
+ skip_existing: bool = typer.Option(
235
+ False, "--skip-existing",
236
+ help="Exit 0 without reprocessing when the output dir already has "
237
+ "knowledge.json (cron idempotence for single-video reruns).",
238
+ ),
239
+ config: Path | None = typer.Option(
240
+ None, "--config", help="Optional TOML config file (explicit field priority)."
241
+ ),
242
+ refresh: bool = typer.Option(
243
+ False, "--refresh",
244
+ help="Re-resolve the source and re-acquire only material whose digest changed; "
245
+ "never forgets an unresolved remote submission.",
246
+ ),
247
+ platform: str | None = typer.Option(
248
+ None, "--platform", help="Force the platform (youtube|tiktok|douyin|bilibili|xiaohongshu)."
249
+ ),
250
+ content_type: str | None = typer.Option(
251
+ None, "--content-type", help="Force the content type (video|image_note)."
252
+ ),
253
+ asr_model: str | None = typer.Option(None, "--asr-model", help="Explicitly prepared local faster-whisper model directory; never downloads during extraction."),
254
+ asr_language: str | None = typer.Option(None, "--asr-language", help="Audio language code, e.g. zh or en; auto enables language detection."),
255
+ asr_beam_size: int | None = typer.Option(None, "--asr-beam-size", min=1, max=20),
256
+ asr_hotwords: str | None = typer.Option(None, "--asr-hotwords", help="Optional domain vocabulary, not corrected answers or a transcript."),
257
+ asr_initial_prompt: str | None = typer.Option(None, "--asr-initial-prompt", help="Optional audio context; never a supplied reference transcript."),
258
+ debug: bool = typer.Option(False, "--debug"),
259
+ ) -> None:
260
+ try:
261
+ formats = _parse_formats(format) if format is not None else None
262
+ _validate_doc_language(doc_language)
263
+ platform_name = _normalise_platform(platform)
264
+ content_type_name = _normalise_content_type(content_type)
265
+ if fixture_transcript is not None and fixture_content is not None:
266
+ raise typer.BadParameter(
267
+ "--fixture-transcript and --fixture-content are mutually exclusive"
268
+ )
269
+ configure_logging(debug=debug)
270
+ settings = _build_settings(
271
+ config_path=config,
272
+ proxy=proxy,
273
+ cookies=cookies,
274
+ cookies_from_browser=cookies_from_browser,
275
+ debug=debug,
276
+ platform=platform_name,
277
+ asr_model=asr_model,
278
+ asr_language=asr_language,
279
+ asr_beam_size=asr_beam_size,
280
+ asr_hotwords=asr_hotwords,
281
+ asr_initial_prompt=asr_initial_prompt,
282
+ )
283
+ route = _extract_route(
284
+ url,
285
+ fixture_content=fixture_content,
286
+ platform=platform_name,
287
+ content_type=content_type_name,
288
+ )
289
+ if route == "v1":
290
+ _extract_legacy(
291
+ url=url,
292
+ out=out,
293
+ formats=formats,
294
+ fixture_transcript=fixture_transcript,
295
+ doc_language=doc_language,
296
+ with_reviewer_scaffolding=with_reviewer_scaffolding,
297
+ skip_existing=skip_existing,
298
+ settings=settings,
299
+ )
300
+ return
301
+ if fixture_transcript is not None:
302
+ raise typer.BadParameter(
303
+ "--fixture-transcript is only supported for YouTube; use --fixture-content "
304
+ "for the multi-platform pipeline"
305
+ )
306
+ _extract_v2(
307
+ url=url,
308
+ out=out,
309
+ settings=settings,
310
+ formats=formats,
311
+ fixture_content=fixture_content,
312
+ refresh=refresh,
313
+ doc_language=doc_language,
314
+ platform=platform_name,
315
+ content_type=content_type_name,
316
+ skip_existing=skip_existing,
317
+ coarse_atoms=coarse_atoms,
318
+ )
319
+ except typer.BadParameter:
320
+ raise
321
+ except typer.Exit:
322
+ raise
323
+ except (UserFacingError, ConfigFileError, ValidationError) as exc:
324
+ _exit_with_runtime_error(str(exc))
325
+ except Exception as exc:
326
+ _exit_with_runtime_error(f"Extraction failed: {exc}")
327
+
328
+
329
+ def _extract_legacy(
330
+ *,
331
+ url: str,
332
+ out: Path,
333
+ formats: set[str] | None,
334
+ fixture_transcript: Path | None,
335
+ doc_language: str | None,
336
+ with_reviewer_scaffolding: bool,
337
+ skip_existing: bool,
338
+ settings: AppSettings,
339
+ ) -> None:
340
+ """The historical YouTube path — kept byte-for-byte (A01)."""
341
+ legacy_formats = formats if formats is not None else _parse_formats("json,md,skill")
342
+ if skip_existing:
343
+ if "json" not in legacy_formats:
344
+ # knowledge.json is only written when json ∈ formats; without it the
345
+ # flag would be a silent no-op forever (adversarial review 2026-08-15).
346
+ typer.echo(
347
+ "WARNING: --skip-existing requires the 'json' format "
348
+ "(knowledge.json is the completion marker); without it "
349
+ "nothing is ever skipped — add json to --format",
350
+ err=True,
351
+ )
352
+ elif _existing_output_matches(out, url) and (
353
+ "doc" not in legacy_formats or _verified_doc(out)
354
+ ):
355
+ typer.echo(f"skip-existing: {out / 'knowledge.json'} matches url — nothing to do")
356
+ return
357
+ pipeline = build_default_pipeline(settings=settings)
358
+ pipeline.run(
359
+ url=url,
360
+ output_dir=out,
361
+ formats=legacy_formats,
362
+ fixture_transcript=fixture_transcript,
363
+ doc_language=doc_language,
364
+ with_reviewer_scaffolding=with_reviewer_scaffolding,
365
+ )
366
+ if "doc" in legacy_formats and (out / ".knowledge-doc-status.json").exists() and not _verified_doc(out):
367
+ typer.echo(
368
+ "WARNING: knowledge document is FALLBACK or unverified; see .knowledge-doc-status.json",
369
+ err=True,
370
+ )
371
+
372
+
373
+ def _extract_v2(
374
+ *,
375
+ url: str,
376
+ out: Path,
377
+ settings: AppSettings,
378
+ formats: set[str] | None,
379
+ fixture_content: Path | None,
380
+ refresh: bool,
381
+ doc_language: str | None,
382
+ platform: str | None,
383
+ content_type: str | None,
384
+ skip_existing: bool,
385
+ coarse_atoms: bool = False,
386
+ ) -> None:
387
+ # An external descriptor can change independently of its URL. Revalidate
388
+ # identity, copied bytes and streams before trusting any earlier outputs.
389
+ if skip_existing and fixture_content is None and not refresh and _v2_outputs_complete(
390
+ out, formats, expected_source_id=_requested_v2_source_id(url, fixture_content),
391
+ settings=settings, doc_language=doc_language,
392
+ ):
393
+ typer.echo(f"skip-existing: {out} already has verified outputs — nothing to do")
394
+ return
395
+ result = _run_v2(
396
+ url=url,
397
+ out=out,
398
+ settings=settings,
399
+ formats=formats,
400
+ fixture_content=fixture_content,
401
+ refresh=refresh,
402
+ doc_language=doc_language,
403
+ platform=platform,
404
+ content_type=content_type,
405
+ coarse_atoms=coarse_atoms,
406
+ )
407
+ outcome, reasons = _v2_outcome(result, out, formats)
408
+ if outcome == "COMPLETE":
409
+ return
410
+ if outcome == "PARTIAL":
411
+ message = "; ".join(reasons) if reasons else "some formats incomplete"
412
+ typer.echo(f"PARTIAL: {message}")
413
+ return
414
+ if outcome == "NO_KNOWLEDGE":
415
+ typer.echo("NO_KNOWLEDGE: content complete but no knowledge atoms were extracted")
416
+ return
417
+ for reason in reasons:
418
+ typer.echo(reason, err=True)
419
+ if outcome == "OUTPUTS_NOT_APPLICABLE":
420
+ typer.echo(f"{outcome}: no requested output format is applicable", err=True)
421
+ else:
422
+ typer.echo(f"{outcome or 'BLOCKED'}: extraction did not complete", err=True)
423
+ raise typer.Exit(code=1)
424
+
425
+
426
+ def _build_v2_pipeline(*, settings: AppSettings | None = None,
427
+ coarse_atoms: bool = False) -> Any:
428
+ """Factory used by the CLI; lazy so the v1-only path never imports v2."""
429
+ from ytk.pipeline.v2 import build_v2_pipeline
430
+
431
+ return build_v2_pipeline(settings=settings, coarse_atoms=coarse_atoms)
432
+
433
+
434
+ def _run_v2(
435
+ *,
436
+ url: str,
437
+ out: Path,
438
+ settings: AppSettings,
439
+ formats: set[str] | None,
440
+ fixture_content: Path | None,
441
+ refresh: bool,
442
+ doc_language: str | None,
443
+ platform: str | None,
444
+ content_type: str | None,
445
+ coarse_atoms: bool = False,
446
+ ) -> Any:
447
+ pipeline = _build_v2_pipeline(settings=settings, coarse_atoms=coarse_atoms)
448
+ kwargs: dict[str, Any] = {
449
+ "output_dir": out,
450
+ "formats": formats,
451
+ "fixture_content": fixture_content,
452
+ "refresh": refresh,
453
+ "platform": platform,
454
+ "content_type": content_type,
455
+ }
456
+ if doc_language is not None:
457
+ parameters: Any
458
+ try:
459
+ parameters = inspect.signature(pipeline.run).parameters
460
+ except (TypeError, ValueError): # pragma: no cover - exotic callables
461
+ parameters = {}
462
+ if "doc_language" in parameters:
463
+ kwargs["doc_language"] = doc_language
464
+ return pipeline.run(url, **kwargs)
465
+
466
+
467
+ def _v2_outcome(
468
+ result: Any, out: Path, formats: set[str] | None
469
+ ) -> tuple[str, list[str]]:
470
+ """Map a V2RunResult (or manifest fallback) to the CLI outcome vocabulary."""
471
+ outcome_value = _result_field(result, "outcome")
472
+ reasons = [str(reason) for reason in (_result_field(result, "reasons") or [])]
473
+ format_status = dict(_result_field(result, "format_status") or {})
474
+ outcome = str(outcome_value).upper() if outcome_value is not None else ""
475
+ if not outcome:
476
+ store = ArtifactStore(out)
477
+ manifest = store.read_manifest()
478
+ if manifest is not None:
479
+ overall, derived_reasons = derive_overall_status(
480
+ manifest, requested=sorted(formats) if formats else None
481
+ )
482
+ outcome = overall
483
+ reasons = reasons or derived_reasons
484
+ format_status = format_status or {
485
+ fmt: manifest.outputs[fmt].status for fmt in manifest.outputs
486
+ }
487
+ if not outcome:
488
+ outcome = "COMPLETE"
489
+ if outcome == "NO_KNOWLEDGE":
490
+ return "NO_KNOWLEDGE", reasons
491
+ if format_status and all(status == "not_applicable" for status in format_status.values()):
492
+ return "OUTPUTS_NOT_APPLICABLE", reasons
493
+ return outcome, reasons
494
+
495
+
496
+ def _result_field(result: Any, name: str) -> Any:
497
+ if isinstance(result, dict):
498
+ return result.get(name)
499
+ return getattr(result, name, None)
500
+
501
+
502
+ def _requested_v2_source_id(url: str, fixture_content: Path | None) -> str | None:
503
+ try:
504
+ if fixture_content is not None:
505
+ from ytk.pipeline.v2 import MultiPlatformPipeline
506
+
507
+ return MultiPlatformPipeline._load_fixture_content(fixture_content).source.source_id
508
+ return resolve_source_safe(url).source_id
509
+ except (UserFacingError, OSError, ValueError):
510
+ return None
511
+
512
+
513
+ def _v2_outputs_complete(
514
+ out: Path, formats: set[str] | None, *, expected_source_id: str | None = None,
515
+ settings: AppSettings | None = None, doc_language: str | None = None,
516
+ ) -> bool:
517
+ try:
518
+ store = ArtifactStore(out)
519
+ manifest = store.read_manifest()
520
+ if manifest is None:
521
+ return False
522
+ if expected_source_id is None or manifest.source_ids != [expected_source_id]:
523
+ return False
524
+ content_cache = getattr(manifest, "content_cache", None)
525
+ if settings is not None and isinstance(content_cache, dict):
526
+ from ytk.pipeline.v2 import MultiPlatformPipeline
527
+
528
+ if content_cache.get("processing_fingerprint") != MultiPlatformPipeline(settings=settings)._processing_fingerprint():
529
+ return False
530
+ requested = formats if formats is not None else set(manifest.outputs)
531
+ if not requested:
532
+ return False
533
+ for fmt in requested:
534
+ if settings is not None and fmt != "json" and fmt in manifest.outputs and manifest.outputs[fmt].status == "complete":
535
+ from ytk.pipeline.v2 import render_request_fingerprint
536
+
537
+ platform = expected_source_id.split(":", 1)[0]
538
+ if getattr(manifest, "render_fingerprints", {}).get(fmt) != render_request_fingerprint(fmt, settings, doc_language, platform):
539
+ return False
540
+ skip, _reason = store.should_skip_format(
541
+ fmt,
542
+ request_fingerprint=manifest.request_fingerprint,
543
+ evidence_fingerprint=manifest.source_revision,
544
+ checker_version=(
545
+ manifest.outputs[fmt].checker_version
546
+ if fmt in manifest.outputs
547
+ else None
548
+ ),
549
+ validator_version=V2_VALIDATOR_VERSION,
550
+ source_revision=manifest.source_revision,
551
+ )
552
+ if not skip:
553
+ return False
554
+ return True
555
+ except Exception: # noqa: BLE001 - resume probe must degrade to "run it"
556
+ return False
557
+
558
+
559
+ def _extract_route(
560
+ url: str,
561
+ *,
562
+ fixture_content: Path | None,
563
+ platform: str | None,
564
+ content_type: str | None,
565
+ ) -> str:
566
+ """``"v1"`` for legacy YouTube, ``"v2"`` for everything else."""
567
+ if fixture_content is not None:
568
+ return "v2"
569
+ if platform is not None:
570
+ return "v1" if platform == "youtube" else "v2"
571
+ try:
572
+ ref = resolve_source_safe(url, content_type=content_type)
573
+ except (UserFacingError, ValueError):
574
+ return "v2" # let the v2 pipeline raise the actionable error
575
+ return "v1" if ref.platform == "youtube" else "v2"
576
+
577
+
578
+ def resolve_source_safe(value: str, *, content_type: str | None = None) -> ContentRef:
579
+ from ytk.ingestion.resolver import resolve_source
580
+
581
+ return resolve_source(value, content_type=content_type) # type: ignore[arg-type]
582
+
583
+
584
+ @app.command("import-file")
585
+ def import_file_cmd(
586
+ input_file: Path,
587
+ out: Path = typer.Option(..., "--out"),
588
+ source_url: str | None = typer.Option(None, "--source-url", help="Optional original video URL; pairing is user asserted, never live verification."),
589
+ subtitle_file: Path | None = typer.Option(None, "--subtitle-file", help="SRT/VTT/JSON3 subtitles accompanying a local video."),
590
+ visual_notes: Path | None = typer.Option(None, "--visual-notes", help="Observations bound to exported frame hashes and timestamps."),
591
+ format: str = typer.Option("json,md", "--format"),
592
+ resume: bool = typer.Option(False, "--resume", help="Explicitly resume an output owned by this same source."),
593
+ coarse_atoms: bool = typer.Option(False, "--coarse-atoms"),
594
+ ) -> None:
595
+ """Extract from an owned local video, audio or timed subtitle file without platform access."""
596
+ import tempfile
597
+ from ytk.ingestion.local_import import ImportInputError, check_output, prepare_import
598
+
599
+ try:
600
+ if not resume:
601
+ check_output(out)
602
+ payload = prepare_import(input_file, source_url=source_url, subtitle_file=subtitle_file, visual_notes=visual_notes)
603
+ source = ContentRef.model_validate(payload["source"])
604
+ check_output(out, resume=resume, source_id=source.source_id,
605
+ import_fingerprint=payload["metadata"]["import_fingerprint"])
606
+ settings = _build_settings()
607
+ # A stored provider key is not consent to upload an imported private file.
608
+ settings = settings.model_copy(update={"cloud_text_enabled": False, "cloud_media_enabled": False})
609
+ formats = _parse_formats(format)
610
+ with tempfile.TemporaryDirectory(prefix="kd-user-import-") as directory:
611
+ descriptor = Path(directory) / "input.json"
612
+ descriptor.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
613
+ result = _run_v2(url=source.canonical_url, out=out, settings=settings, formats=formats,
614
+ fixture_content=descriptor, refresh=False, doc_language=None,
615
+ platform=None, content_type="video", coarse_atoms=coarse_atoms)
616
+ outcome, reasons = _v2_outcome(result, out, formats)
617
+ typer.echo(json.dumps({"outcome": outcome, "source_id": source.source_id,
618
+ "acquisition_mode": "user-import", "output": str(out),
619
+ "reasons": reasons}, ensure_ascii=False))
620
+ if outcome not in {"COMPLETE", "PARTIAL", "NO_KNOWLEDGE"}:
621
+ raise typer.Exit(code=1)
622
+ except typer.Exit:
623
+ raise
624
+ except ImportInputError as exc:
625
+ typer.echo(json.dumps({"outcome": "BLOCKED", "reason_code": exc.code,
626
+ "message": str(exc), "recovery": exc.action}, ensure_ascii=False))
627
+ raise typer.Exit(code=1) from None
628
+ except (OSError, ValueError, UserFacingError) as exc:
629
+ from ytk.logging import redact_payload
630
+ typer.echo(json.dumps({"outcome": "BLOCKED", "reason_code": getattr(exc, "reason_code", "IMPORT_FAILED"),
631
+ "message": redact_payload(str(exc)),
632
+ "recovery": "检查本地依赖与模型,保留原文件后用新目录重试;模型缺失时按连接器README显式准备。"}, ensure_ascii=False))
633
+ raise typer.Exit(code=1) from None
634
+
635
+
636
+ @app.command("frames")
637
+ def frames_cmd(
638
+ input_file: Path,
639
+ out: Path = typer.Option(..., "--out"),
640
+ max_frames: int = typer.Option(8, "--max-frames", min=1, max=12),
641
+ at: list[float] | None = typer.Option(None, "--at", help="Specific seconds to inspect; repeat up to 12 times in increasing order."),
642
+ ) -> None:
643
+ """Export real timestamped frames for a model/user to inspect; never invent frame observations."""
644
+ from ytk.ingestion.video_frames import export_frames
645
+ try:
646
+ manifest = export_frames(input_file, out, max_frames=max_frames, timestamps=at)
647
+ typer.echo(json.dumps({"outcome": "VISUAL_REVIEW_REQUIRED", "manifest": str(out / "frames-manifest.json"),
648
+ "frames": len(manifest["frames"]),
649
+ "next": "在WorkBuddy读取实际帧,写入含video_sha256、frames_manifest、reviewer与observations[{frame_index,text}]的JSON,然后 import-file --visual-notes;只代表抽样帧,不能声称看完全片。"}, ensure_ascii=False))
650
+ except (OSError, ValueError, UserFacingError) as exc:
651
+ from ytk.logging import redact_payload
652
+ typer.echo(json.dumps({"outcome": "BLOCKED", "reason_code": getattr(exc, "reason_code", "FRAME_EXPORT_FAILED"),
653
+ "message": redact_payload(str(exc))}, ensure_ascii=False))
654
+ raise typer.Exit(code=1) from None
655
+
656
+
657
+ # --------------------------------------------------------------------------- #
658
+ # skill / merge (unchanged)
659
+ # --------------------------------------------------------------------------- #
660
+
661
+ def _load_bundle_for_skill(path: Path):
662
+ """Load a knowledge bundle of any supported schema version for `skill build`."""
663
+ from ytk.models.migrations import load_bundle_any
664
+
665
+ try:
666
+ return load_bundle_any(path)
667
+ except UserFacingError:
668
+ raise
669
+ except FileNotFoundError as exc:
670
+ raise UserFacingError(f"Knowledge bundle not found: {path}") from exc
671
+ except json.JSONDecodeError as exc:
672
+ raise UserFacingError(f"Knowledge bundle is not valid JSON: {path}") from exc
673
+
674
+
675
+ @skill_app.command("build")
676
+ def skill_build(
677
+ knowledge_json: Path,
678
+ out: Path = typer.Option(..., "--out"),
679
+ package_kind: str = typer.Option("generic", "--package-kind"),
680
+ target: str = typer.Option("workbuddy", "--target",
681
+ help="Output spec: 'workbuddy' (default) or 'generic'."),
682
+ ) -> None:
683
+ if target not in ("workbuddy", "generic"):
684
+ raise typer.BadParameter(f"--target 只接受 workbuddy|generic,收到 {target!r}")
685
+ try:
686
+ # v2-aware: the default pipeline writes schema_version 2 with atom_id /
687
+ # evidence_ids / topics, which the v1-only loader rejects outright.
688
+ # models.migrations.load_bundle_any already exists and the SkillBuilder
689
+ # coerces both shapes, so only the entry point was wired to the old one.
690
+ bundle = _load_bundle_for_skill(knowledge_json)
691
+ pipeline = build_default_pipeline()
692
+ pipeline.skill_builder.build(bundle, out, package_kind=package_kind, target=target)
693
+ except typer.BadParameter:
694
+ raise
695
+ except (UserFacingError, ValidationError) as exc:
696
+ _exit_with_runtime_error(str(exc))
697
+ except Exception as exc:
698
+ _exit_with_runtime_error(f"Skill build failed: {exc}")
699
+
700
+
701
+ @app.command("merge")
702
+ def merge_cmd(
703
+ input_dir: Path = typer.Argument(..., help="Directory containing video output subdirectories"),
704
+ output: Path = typer.Option("./merged/knowledge.json", "--out", help="Output path for merged knowledge.json"),
705
+ ) -> None:
706
+ try:
707
+ from ytk.commands.merge import merge_knowledge_directories
708
+
709
+ result = merge_knowledge_directories(input_dir, output)
710
+ typer.echo(f"Merged {result['source_count']} videos, {len(result['atoms'])} unique atoms → {output}")
711
+ except Exception as exc:
712
+ _exit_with_runtime_error(f"Merge failed: {exc}")
713
+
714
+
715
+ # --------------------------------------------------------------------------- #
716
+ # batch run
717
+ # --------------------------------------------------------------------------- #
718
+ @batch_app.command("run")
719
+ def batch_run(
720
+ urls_file: Path | None = typer.Option(
721
+ None,
722
+ "--urls-file",
723
+ help="Path to file with one URL per line.",
724
+ ),
725
+ out: Path = typer.Option(Path("./outputs"), "--out"),
726
+ format: str | None = typer.Option(
727
+ None,
728
+ "--format",
729
+ help="Comma-separated outputs applied to every source. Omit for per-source "
730
+ "defaults (YouTube: json,md,skill; other platforms: json,md).",
731
+ ),
732
+ fixture_transcript: Path | None = typer.Option(None, "--fixture-transcript"),
733
+ fixture_content: Path | None = typer.Option(
734
+ None, "--fixture-content", help="JSON descriptor of already-acquired content (offline v2 path)."
735
+ ),
736
+ proxy: str | None = typer.Option(
737
+ None,
738
+ "--proxy",
739
+ help="HTTP/HTTPS proxy URL (e.g. http://127.0.0.1:7890)",
740
+ ),
741
+ cookies: Path | None = typer.Option(
742
+ None,
743
+ "--cookies",
744
+ help="Path to Netscape formatted YouTube cookies file for yt-dlp.",
745
+ ),
746
+ cookies_from_browser: str | None = typer.Option(
747
+ None,
748
+ "--cookies-from-browser",
749
+ help="Browser cookies source for yt-dlp, e.g. chrome:default",
750
+ ),
751
+ search: str | None = typer.Option(
752
+ None,
753
+ "--search",
754
+ help="YouTube search query to generate URL list instead of reading from file.",
755
+ ),
756
+ doc_language: str | None = typer.Option(
757
+ None,
758
+ "--doc-language",
759
+ help="Output language for the 'doc' format. One of: zh, en, auto. "
760
+ "When 'auto' (or unset), the skill samples $LANG/$LC_ALL — defaults to 'zh'.",
761
+ ),
762
+ with_reviewer_scaffolding: bool = typer.Option(
763
+ False,
764
+ "--with-reviewer-scaffolding",
765
+ help="Append an A/B/C reference list, image-asset manifest, and "
766
+ "closing CTA to the `doc` output of every video in the batch. "
767
+ "Applied per-video (same flag as `extract`).",
768
+ ),
769
+ resume: bool = typer.Option(
770
+ False, "--resume",
771
+ help="Reuse verified outputs per format (fingerprint/revision/digest/validator) "
772
+ "and skip only what is genuinely complete. Interrupted batches resume.",
773
+ ),
774
+ refresh: bool = typer.Option(
775
+ False, "--refresh",
776
+ help="Re-resolve and re-acquire changed material; never forgets an unresolved "
777
+ "remote submission, never reuses stale outputs.",
778
+ ),
779
+ config: Path | None = typer.Option(
780
+ None, "--config", help="Optional TOML config file (explicit field priority)."
781
+ ),
782
+ debug: bool = typer.Option(False, "--debug"),
783
+ ) -> None:
784
+ try:
785
+ if not urls_file and not search:
786
+ _exit_with_runtime_error("Provide --urls-file or --search.")
787
+ return
788
+ formats_explicit = format is not None
789
+ formats = _parse_formats(format) if format is not None else None
790
+ _validate_doc_language(doc_language)
791
+ if fixture_transcript is not None and fixture_content is not None:
792
+ raise typer.BadParameter(
793
+ "--fixture-transcript and --fixture-content are mutually exclusive"
794
+ )
795
+ configure_logging(debug=debug)
796
+ settings = _build_settings(
797
+ config_path=config,
798
+ proxy=proxy,
799
+ cookies=cookies,
800
+ cookies_from_browser=cookies_from_browser,
801
+ debug=debug,
802
+ platform="youtube",
803
+ )
804
+
805
+ if search:
806
+ from ytk.ingestion.search import YouTubeSearchService
807
+
808
+ svc = YouTubeSearchService()
809
+ results = svc.search(search, max_results=10)
810
+ raw_urls = svc.get_urls(results)
811
+ if not raw_urls:
812
+ typer.echo(f"No results found for: {search}", err=True)
813
+ raise typer.Exit(code=1)
814
+ typer.echo(f"Found {len(raw_urls)} videos for: {search}")
815
+ else:
816
+ try:
817
+ raw_urls = urls_file.read_text(encoding="utf-8").splitlines() # type: ignore[union-attr]
818
+ except OSError as exc:
819
+ _exit_with_runtime_error(f"Failed to read URLs file: {exc}")
820
+ return
821
+
822
+ urls = [line.strip() for line in raw_urls if line.strip()]
823
+ report = _run_batch(
824
+ urls,
825
+ out=out,
826
+ settings=settings,
827
+ formats=formats,
828
+ formats_explicit=formats_explicit,
829
+ fixture_transcript=fixture_transcript,
830
+ fixture_content=fixture_content,
831
+ doc_language=doc_language,
832
+ with_reviewer_scaffolding=with_reviewer_scaffolding,
833
+ resume=resume,
834
+ refresh=refresh,
835
+ )
836
+ typer.echo("")
837
+ typer.echo("=" * 40)
838
+ for line in report.summary_lines():
839
+ typer.echo(line)
840
+ if report.exit_code:
841
+ raise typer.Exit(code=report.exit_code)
842
+ except typer.BadParameter:
843
+ raise
844
+ except typer.Exit:
845
+ raise
846
+ except (UserFacingError, ConfigFileError, ValidationError) as exc:
847
+ _exit_with_runtime_error(str(exc))
848
+ except Exception as exc:
849
+ _exit_with_runtime_error(f"Batch run failed: {exc}")
850
+
851
+
852
+ def _run_batch(
853
+ urls: list[str],
854
+ *,
855
+ out: Path,
856
+ settings: AppSettings,
857
+ formats: set[str] | None,
858
+ formats_explicit: bool,
859
+ fixture_transcript: Path | None,
860
+ fixture_content: Path | None,
861
+ doc_language: str | None,
862
+ with_reviewer_scaffolding: bool,
863
+ resume: bool,
864
+ refresh: bool,
865
+ ):
866
+ from ytk.batch.runner import run_batch
867
+
868
+ return run_batch(
869
+ urls,
870
+ out_root=out,
871
+ settings=settings,
872
+ formats=formats,
873
+ formats_explicit=formats_explicit,
874
+ fixture_transcript=fixture_transcript,
875
+ fixture_content=fixture_content,
876
+ doc_language=doc_language,
877
+ with_reviewer_scaffolding=with_reviewer_scaffolding,
878
+ resume=resume,
879
+ refresh=refresh,
880
+ legacy_pipeline_factory=build_default_pipeline,
881
+ v2_pipeline_factory=_build_v2_pipeline,
882
+ doc_verifier=_verified_doc,
883
+ echo=typer.echo,
884
+ )
885
+
886
+
887
+ # --------------------------------------------------------------------------- #
888
+ # platforms / probe
889
+ # --------------------------------------------------------------------------- #
890
+ @app.command("platforms")
891
+ def platforms_cmd(
892
+ json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON only."),
893
+ config: Path | None = typer.Option(None, "--config"),
894
+ ) -> None:
895
+ """Show the five platform capabilities and local configuration state."""
896
+ try:
897
+ settings = _build_settings(config_path=config)
898
+ payload = _platforms_payload(settings)
899
+ if json_output:
900
+ typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
901
+ return
902
+ typer.echo(f"{'platform':14} {'enabled':8} {'provider':10} {'formats':16} notes")
903
+ for name, info in payload.items():
904
+ notes = []
905
+ if info["requires_service"]:
906
+ notes.append(
907
+ "service configured" if info["service_configured"] else "service NOT configured"
908
+ )
909
+ if info["cookies_configured"]:
910
+ notes.append("cookies")
911
+ if info["proxy_configured"]:
912
+ notes.append("proxy")
913
+ typer.echo(
914
+ f"{name:14} {str(info['enabled']):8} {info['provider']:10} "
915
+ f"{','.join(info['default_formats']):16} {'; '.join(notes)}"
916
+ )
917
+ except typer.Exit:
918
+ raise
919
+ except (UserFacingError, ConfigFileError, ValidationError) as exc:
920
+ _exit_with_runtime_error(str(exc))
921
+ except Exception as exc:
922
+ _exit_with_runtime_error(f"platforms failed: {exc}")
923
+
924
+
925
+ def _platforms_payload(settings: AppSettings) -> dict[str, dict[str, Any]]:
926
+ from ytk.ingestion.registry import PlatformRegistry
927
+ from ytk.pipeline.v2 import _builtin_adapters
928
+
929
+ # Reflect what this build can actually acquire with: YouTube/TikTok/Bilibili
930
+ # need no sidecar; Douyin needs DTK and XHS has an optional sidecar.
931
+ registry = PlatformRegistry.default(adapters=_builtin_adapters(settings))
932
+ payload: dict[str, dict[str, Any]] = {}
933
+ for name, capabilities in registry.capabilities_matrix().items():
934
+ ready = bool(capabilities.get("adapter_configured")) and (
935
+ not capabilities.get("requires_service") or bool(settings.base_url_for(name)) # type: ignore[arg-type]
936
+ )
937
+ payload[name] = {
938
+ **capabilities,
939
+ "acquisition_ready": ready,
940
+ "live_access_verified": False,
941
+ "readiness_scope": "local_configuration_only",
942
+ "enabled": settings.is_platform_enabled(name), # type: ignore[arg-type]
943
+ "service_configured": bool(settings.base_url_for(name)), # type: ignore[arg-type]
944
+ "cookies_configured": bool(settings.cookies_file_for(name)), # type: ignore[arg-type]
945
+ "proxy_configured": bool(settings.proxy_for(name)), # type: ignore[arg-type]
946
+ "cloud_text_enabled": settings.cloud_allowed(name, "text"), # type: ignore[arg-type]
947
+ "cloud_media_enabled": settings.cloud_allowed(name, "media"), # type: ignore[arg-type]
948
+ }
949
+ return payload
950
+
951
+
952
+ @app.command("probe")
953
+ def probe_cmd(
954
+ url: str,
955
+ json_output: bool = typer.Option(False, "--json", help="Print machine-readable JSON only."),
956
+ platform: str | None = typer.Option(None, "--platform"),
957
+ content_type: str | None = typer.Option(
958
+ None, "--content-type", help="video|image_note"
959
+ ),
960
+ config: Path | None = typer.Option(None, "--config"),
961
+ ) -> None:
962
+ """Resolve a URL/share text to platform identity + capabilities (no download)."""
963
+ try:
964
+ platform_name = _normalise_platform(platform)
965
+ content_type_name = _normalise_content_type(content_type)
966
+ settings = _build_settings(config_path=config, platform=platform_name)
967
+ ref, identity_source = _probe_identity(
968
+ url,
969
+ settings=settings,
970
+ platform=platform_name,
971
+ content_type=content_type_name,
972
+ )
973
+ capabilities = _platforms_payload(settings).get(ref.platform, {})
974
+ payload = {
975
+ "platform": ref.platform,
976
+ "source_id": ref.source_id,
977
+ "content_type": ref.content_type,
978
+ "canonical_url": ref.canonical_url,
979
+ "variant_id": ref.variant_id,
980
+ "identity_source": identity_source,
981
+ "capabilities": capabilities,
982
+ }
983
+ if json_output:
984
+ typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
985
+ return
986
+ typer.echo(f"platform: {payload['platform']}")
987
+ typer.echo(f"source_id: {payload['source_id']}")
988
+ typer.echo(f"content_type: {payload['content_type']}")
989
+ typer.echo(f"canonical_url: {payload['canonical_url']}")
990
+ typer.echo(f"identity: {payload['identity_source']}")
991
+ typer.echo(
992
+ f"capabilities: provider={capabilities.get('provider')} "
993
+ f"formats={','.join(capabilities.get('default_formats') or [])} "
994
+ f"subtitle={capabilities.get('subtitle')} asr={capabilities.get('asr')} "
995
+ f"ocr={capabilities.get('ocr')} service={capabilities.get('service_name')}"
996
+ )
997
+ except typer.BadParameter:
998
+ raise
999
+ except typer.Exit:
1000
+ raise
1001
+ except (UserFacingError, ConfigFileError, ValidationError) as exc:
1002
+ _exit_with_runtime_error(str(exc))
1003
+ except Exception as exc:
1004
+ _exit_with_runtime_error(f"Probe failed: {exc}")
1005
+
1006
+
1007
+ def _probe_identity(
1008
+ url: str,
1009
+ *,
1010
+ settings: AppSettings,
1011
+ platform: str | None,
1012
+ content_type: str | None,
1013
+ ) -> tuple[ContentRef, str]:
1014
+ """Resolve-only identity: adapter when configured, resolver otherwise."""
1015
+ from ytk.ingestion.resolver import detect_platform, extract_urls
1016
+
1017
+ detected = platform
1018
+ if detected is None:
1019
+ try:
1020
+ candidates = extract_urls(url)
1021
+ detected = detect_platform(candidates[0]) if candidates else None
1022
+ except Exception: # noqa: BLE001 - detection is best-effort
1023
+ detected = None
1024
+ adapter = None
1025
+ if detected is not None:
1026
+ try:
1027
+ registry = _registry_with_adapters(settings)
1028
+ adapter = registry.get(detected).adapter
1029
+ except Exception: # noqa: BLE001 - missing adapter is not fatal
1030
+ adapter = None
1031
+ if adapter is not None:
1032
+ context = AccessContext(
1033
+ cookies_file=settings.cookies_file_for(detected), # type: ignore[arg-type]
1034
+ cookies_from_browser=settings.cookies_from_browser_for(detected), # type: ignore[arg-type]
1035
+ proxy=settings.proxy_for(detected), # type: ignore[arg-type]
1036
+ )
1037
+ try:
1038
+ return adapter.resolve(url, context), "adapter"
1039
+ except Exception as exc: # noqa: BLE001 - fall back to the pure resolver
1040
+ LOGGER.debug("adapter resolve failed, falling back to resolver: %s", exc)
1041
+ return resolve_source_safe(url, content_type=content_type), "resolver"
1042
+
1043
+
1044
+ def _registry_with_adapters(settings: AppSettings):
1045
+ try:
1046
+ from ytk.pipeline.v2 import build_v2_pipeline
1047
+
1048
+ return build_v2_pipeline(settings=settings).registry
1049
+ except Exception: # noqa: BLE001 - probe still works without adapters
1050
+ from ytk.ingestion.registry import PlatformRegistry
1051
+
1052
+ return PlatformRegistry.default()
1053
+
1054
+
1055
+ # --------------------------------------------------------------------------- #
1056
+ # doctor
1057
+ # --------------------------------------------------------------------------- #
1058
+ @app.command()
1059
+ def doctor(
1060
+ offline: bool = typer.Option(
1061
+ False, "--offline", help="Local-only diagnostics: no network probe, no sidecar contact."
1062
+ ),
1063
+ platform: str = typer.Option(
1064
+ "all", "--platform", help="all|youtube|tiktok|douyin|bilibili|xiaohongshu"
1065
+ ),
1066
+ strict: bool = typer.Option(
1067
+ False, "--strict", help="Exit non-zero when offline local checks are incomplete."
1068
+ ),
1069
+ ) -> None:
1070
+ try:
1071
+ platform_filter = _normalise_platform(platform, allow_all=True) or "all"
1072
+ report = _doctor_service(offline).run()
1073
+ if offline or platform_filter != "all":
1074
+ report["platforms"] = _platform_diagnostic(platform_filter)
1075
+ typer.echo(format_doctor_report(report))
1076
+ if offline and strict:
1077
+ from ytk.doctor import doctor_report_failures
1078
+
1079
+ failures = doctor_report_failures(report)
1080
+ if failures:
1081
+ typer.echo(
1082
+ "offline checks incomplete: " + ", ".join(sorted(failures)), err=True
1083
+ )
1084
+ raise typer.Exit(code=1)
1085
+ except typer.BadParameter:
1086
+ raise
1087
+ except typer.Exit:
1088
+ raise
1089
+ except Exception as exc:
1090
+ _exit_with_runtime_error(f"Doctor failed: {exc}")
1091
+
1092
+
1093
+ def _doctor_service(offline: bool):
1094
+ """Build DoctorService; tolerate older or monkeypatched constructors."""
1095
+ try:
1096
+ return DoctorService(offline=offline)
1097
+ except TypeError:
1098
+ return DoctorService()
1099
+
1100
+
1101
+ def _platform_diagnostic(platform_filter: str) -> dict[str, Any]:
1102
+ settings = AppSettings.load()
1103
+ payload = _platforms_payload(settings)
1104
+ if platform_filter != "all":
1105
+ payload = {platform_filter: payload[platform_filter]}
1106
+ problems: list[str] = []
1107
+ for name, info in payload.items():
1108
+ if not info["enabled"]:
1109
+ continue
1110
+ # A service-backed platform whose sidecar is absent is not "ready": the
1111
+ # adapter exists but every acquire would raise ServiceNotConfiguredError.
1112
+ if not info.get("adapter_configured"):
1113
+ problems.append(name)
1114
+ elif info.get("requires_service") and not info.get("service_configured"):
1115
+ problems.append(name)
1116
+ section: dict[str, Any] = {
1117
+ "status": "ok" if not problems else "partial",
1118
+ "verification": "configured_only",
1119
+ "mode": "offline_local",
1120
+ "note": (
1121
+ "local platform configuration only; no network call, no download. "
1122
+ "A configured platform is NOT verified to be reachable."
1123
+ ),
1124
+ "platforms": payload,
1125
+ }
1126
+ if problems:
1127
+ section["fix_hint"] = (
1128
+ "not configured locally: "
1129
+ + ", ".join(problems)
1130
+ + " | install the optional dependency, set the sidecar base URL, or disable the platform"
1131
+ )
1132
+ return section
1133
+
1134
+
1135
+ # --------------------------------------------------------------------------- #
1136
+ # shared helpers
1137
+ # --------------------------------------------------------------------------- #
1138
+ def _existing_output_matches(out: Path, url: str) -> bool:
1139
+ """True when out/knowledge.json exists AND metadata.json records the SAME
1140
+ video as `url` — a different URL pointed at a populated out dir must NOT
1141
+ exit 0 having processed nothing (adversarial review 2026-08-15 finding)."""
1142
+ if not (out / "knowledge.json").exists():
1143
+ return False
1144
+ try:
1145
+ meta = json.loads((out / "metadata.json").read_text(encoding="utf-8"))
1146
+ except (OSError, json.JSONDecodeError):
1147
+ return False
1148
+ from ytk.ingestion.url_parser import parse_youtube_url
1149
+
1150
+ try:
1151
+ want = parse_youtube_url(url).video_id
1152
+ except Exception:
1153
+ return False
1154
+ return meta.get("video_id") == want
1155
+
1156
+
1157
+ def _batch_marker_covers(marker: Path, url: str, formats: set[str]) -> bool:
1158
+ """True when the completion marker matches url AND covers all requested
1159
+ formats (a subset marker means an artifact is still missing)."""
1160
+ try:
1161
+ payload = json.loads(marker.read_text(encoding="utf-8"))
1162
+ except (OSError, json.JSONDecodeError):
1163
+ return False
1164
+ if not isinstance(payload, dict) or payload.get("url") != url:
1165
+ return False
1166
+ return (set(payload.get("formats") or []) >= set(formats)
1167
+ and ("doc" not in formats or _verified_doc(marker.parent)))
1168
+
1169
+
1170
+ def _verified_doc(output_dir: Path) -> bool:
1171
+ from ytk.renderers.knowledge_doc_renderer import complete_doc_status
1172
+
1173
+ return complete_doc_status(output_dir)
1174
+
1175
+
1176
+ def _parse_formats(value: str) -> set[str]:
1177
+ allowed = {"json", "jsonl", "md", "obsidian", "skill", "summary", "doc"}
1178
+ formats = {item.strip() for item in value.split(",") if item.strip()}
1179
+ if not formats:
1180
+ raise typer.BadParameter("At least one output format is required.")
1181
+ unsupported = formats - allowed
1182
+ if unsupported:
1183
+ supported = ", ".join(sorted(allowed))
1184
+ unknown = ", ".join(sorted(unsupported))
1185
+ raise typer.BadParameter(f"Unsupported format(s): {unknown}. Expected one of: {supported}")
1186
+ return formats
1187
+
1188
+
1189
+ def _validate_doc_language(value: str | None) -> None:
1190
+ """Reject obviously invalid --doc-language values early.
1191
+
1192
+ The renderer itself only knows "zh" and "en"; "auto" is a sentinel
1193
+ meaning "use locale / env". Anything else is a user typo.
1194
+ """
1195
+ if value is None:
1196
+ return
1197
+ candidate = value.strip().lower()
1198
+ if candidate in {"zh", "en", "auto"}:
1199
+ return
1200
+ raise typer.BadParameter(
1201
+ f"--doc-language must be one of: zh, en, auto. Got: {value!r}"
1202
+ )
1203
+
1204
+
1205
+ def _normalise_platform(value: str | None, *, allow_all: bool = False) -> str | None:
1206
+ if value is None:
1207
+ return None
1208
+ candidate = value.strip().lower()
1209
+ if allow_all and candidate == "all":
1210
+ return "all"
1211
+ if candidate not in SUPPORTED_PLATFORMS:
1212
+ supported = ", ".join(SUPPORTED_PLATFORMS)
1213
+ raise typer.BadParameter(f"--platform must be one of: all, {supported}. Got: {value!r}")
1214
+ return candidate
1215
+
1216
+
1217
+ def _normalise_content_type(value: str | None) -> str | None:
1218
+ if value is None:
1219
+ return None
1220
+ candidate = value.strip().lower()
1221
+ if candidate not in {"video", "image_note"}:
1222
+ raise typer.BadParameter(
1223
+ f"--content-type must be one of: video, image_note. Got: {value!r}"
1224
+ )
1225
+ return candidate
1226
+
1227
+
1228
+ def _build_settings(
1229
+ *,
1230
+ config_path: Path | None = None,
1231
+ proxy: str | None = None,
1232
+ cookies: Path | None = None,
1233
+ cookies_from_browser: str | None = None,
1234
+ debug: bool = False,
1235
+ platform: str | None = None,
1236
+ asr_model: str | None = None,
1237
+ asr_language: str | None = None,
1238
+ asr_beam_size: int | None = None,
1239
+ asr_hotwords: str | None = None,
1240
+ asr_initial_prompt: str | None = None,
1241
+ ) -> AppSettings:
1242
+ """Build settings with CLI values as the highest-priority tier."""
1243
+ overrides: dict[str, Any] = {}
1244
+ if proxy:
1245
+ overrides.update(
1246
+ {
1247
+ "transcript_proxy_mode": "generic",
1248
+ "transcript_http_proxy_url": proxy,
1249
+ "transcript_https_proxy_url": proxy,
1250
+ }
1251
+ )
1252
+ if platform and platform != "all":
1253
+ overrides[f"{platform}_proxy"] = proxy
1254
+ if cookies is not None:
1255
+ overrides["yt_dlp_cookies_file"] = str(cookies)
1256
+ if platform and platform != "all":
1257
+ overrides[f"{platform}_cookies_file"] = str(cookies)
1258
+ if cookies_from_browser is not None:
1259
+ overrides["yt_dlp_cookies_from_browser"] = cookies_from_browser
1260
+ if debug:
1261
+ overrides["debug"] = True
1262
+ for key, value in {
1263
+ "asr_model": asr_model,
1264
+ "asr_language": asr_language,
1265
+ "asr_beam_size": asr_beam_size,
1266
+ "asr_hotwords": asr_hotwords,
1267
+ "asr_initial_prompt": asr_initial_prompt,
1268
+ }.items():
1269
+ if value is not None:
1270
+ overrides[key] = None if key == "asr_language" and value == "auto" else value
1271
+ return AppSettings.load(config_path=config_path, overrides=overrides)
1272
+
1273
+
1274
+ def _exit_with_runtime_error(message: str) -> None:
1275
+ typer.echo(message, err=True)
1276
+ raise typer.Exit(code=1)
1277
+
1278
+
1279
+ if __name__ == "__main__":
1280
+ _load_env_file(Path(__file__).resolve().parent.parent.parent / ".env")
1281
+ app()