kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
ytk/doctor.py ADDED
@@ -0,0 +1,891 @@
1
+ from __future__ import annotations
2
+
3
+ import importlib.util
4
+ import json
5
+ import os
6
+ import re
7
+ import subprocess
8
+ import threading
9
+ from collections.abc import Callable, Sequence
10
+ from importlib.metadata import PackageNotFoundError, requires, version
11
+ from pathlib import Path
12
+ from shutil import which
13
+ from typing import Any
14
+ from urllib.parse import urlsplit, urlunsplit
15
+
16
+ from ytk.config import AppSettings
17
+ from ytk.ingestion.providers.transcript_api import TranscriptApiProvider, YouTubeTranscriptApi
18
+ from ytk.logging import redact_url
19
+
20
+ BinaryResolver = Callable[[str], str | None]
21
+ VersionRunner = Callable[[Sequence[str]], str]
22
+ PackageVersionGetter = Callable[[str], str]
23
+ TranscriptConnectivityChecker = Callable[..., dict[str, Any]]
24
+ ModuleSpecFinder = Callable[[str], Any]
25
+
26
+ # Verification vocabulary used by the offline diagnostics (T43.2/T00.2).
27
+ # ``configured``/``configured_only`` means "the setting exists, nothing was
28
+ # contacted"; a platform is never reported as verified from configuration.
29
+ VERIFICATION_VERIFIED_LOCALLY = "verified_locally"
30
+ VERIFICATION_CONFIGURED_ONLY = "configured_only"
31
+ VERIFICATION_NOT_VERIFIED = "not_verified"
32
+ VERIFICATION_SKIPPED_OFFLINE = "skipped_offline"
33
+
34
+ FIX_HINT_OFFLINE_TRANSCRIPT = (
35
+ "Offline mode never probes YouTube; run `./scripts/doctor.sh` (online) "
36
+ "with a user-provided sample when connectivity must be verified."
37
+ )
38
+ FIX_HINT_ASR = (
39
+ "pip install 'kd-engine[asr]'"
40
+ )
41
+ FIX_HINT_OCR = (
42
+ "OCR 已随本商品移除:商品边界为视频流(字幕/ASR 文本),不处理图片内容"
43
+ )
44
+ FIX_HINT_FFPROBE = (
45
+ "Install ffprobe (ships with FFmpeg) | macOS: brew install ffmpeg | Ubuntu: sudo apt install ffmpeg"
46
+ )
47
+ FIX_HINT_JS_RUNTIME = (
48
+ "Explicitly prepare `python -m pip install -U 'yt-dlp[default]'` in the connector environment "
49
+ "and install Deno >=2.3.0 (recommended) or Node >=22.0.0. "
50
+ "Node must be enabled with --js-runtimes node; inference never downloads EJS components."
51
+ )
52
+ FIX_HINT_SIDECAR = (
53
+ "Configure the sidecar URL in .env or the optional TOML config; "
54
+ "offline diagnostics only report presence, never reachability"
55
+ )
56
+
57
+ # Sidecar configuration candidates. AppSettings fields win (when the v2
58
+ # config lands) and environment variables are the documented fallback; the
59
+ # offline doctor only reports presence, never reachability.
60
+ _SIDECAR_CONFIG: dict[str, dict[str, tuple[str, ...]]] = {
61
+ "dtk": {
62
+ "attributes": ("dtk_base_url", "douyin_base_url", "dtk_url"),
63
+ "env": ("YTK_DTK_BASE_URL", "DTK_BASE_URL"),
64
+ },
65
+ "xiaohongshu_mcp": {
66
+ "attributes": ("xhs_base_url",),
67
+ "env": ("YTK_XHS_BASE_URL", "YTK_XHS_MCP_BASE_URL"),
68
+ },
69
+ }
70
+
71
+ # fix_hint strings for transcript-connectivity problems. Single source of
72
+ # truth — referenced both by _fix_hint_for_connectivity (the chokepoint that
73
+ # decorates stub-injected results) and by the inline returns in
74
+ # _default_transcript_connectivity_checker (which is also called directly as a
75
+ # @staticmethod, bypassing the chokepoint). Keeping these as module constants
76
+ # prevents the two paths from drifting.
77
+ FIX_HINT_TRANSCRIPT_MISSING = (
78
+ "pip install youtube-transcript-api"
79
+ )
80
+ FIX_HINT_TRANSCRIPT_PROXY = (
81
+ "Set YTK_HTTP_PROXY_URL + YTK_HTTPS_PROXY_URL in .env "
82
+ "| or pass --proxy on the CLI"
83
+ )
84
+ FIX_HINT_TRANSCRIPT_TIMEOUT = (
85
+ "Check connectivity | if behind a firewall, set YTK_HTTP_PROXY_URL in .env"
86
+ )
87
+ FIX_HINT_TRANSCRIPT_PARTIAL = (
88
+ "Some transcript checks only partially succeeded | verify proxy/cookies and rerun ./scripts/doctor.sh"
89
+ )
90
+
91
+
92
+ class DoctorService:
93
+ def __init__(
94
+ self,
95
+ settings: AppSettings | None = None,
96
+ *,
97
+ which: BinaryResolver = which,
98
+ version_runner: VersionRunner | None = None,
99
+ package_version_getter: PackageVersionGetter = version,
100
+ transcript_connectivity_checker: TranscriptConnectivityChecker | None = None,
101
+ module_spec_finder: ModuleSpecFinder | None = None,
102
+ offline: bool = False,
103
+ ) -> None:
104
+ self._settings = settings or AppSettings()
105
+ self._which = which
106
+ self._version_runner = version_runner or self._run_version_command
107
+ self._package_version_getter = package_version_getter
108
+ self._module_spec_finder = module_spec_finder or importlib.util.find_spec
109
+ self._transcript_connectivity_checker = (
110
+ transcript_connectivity_checker or self._default_transcript_connectivity_checker
111
+ )
112
+ # Offline mode is a strict local-only diagnostic: no network probe, no
113
+ # sidecar contact, and every report states how strong its evidence is
114
+ # (verified locally vs merely configured).
115
+ self._offline = offline
116
+
117
+ @property
118
+ def offline(self) -> bool:
119
+ return self._offline
120
+
121
+ def run(self) -> dict[str, dict[str, Any]]:
122
+ transcript_api_report = self._check_distribution("youtube-transcript-api")
123
+ ytdlp_report = self._check_binary(
124
+ self._settings.yt_dlp_binary,
125
+ version_args=("--version",),
126
+ )
127
+ report = {
128
+ "ffmpeg": self._check_binary(self._settings.ffmpeg_binary, version_args=("-version",)),
129
+ "yt-dlp": ytdlp_report,
130
+ "youtube-transcript-api": transcript_api_report,
131
+ "transcript_cache": self._transcript_cache_report(),
132
+ }
133
+ if self._offline:
134
+ # Local-only probes added by T43.2. They are omitted in the
135
+ # default online report so its shape stays stable for callers.
136
+ ytdlp_report["python_module"] = self._check_ytdlp_module()
137
+ ytdlp_report["js_runtime"] = self._check_js_runtime()
138
+ report["ffprobe"] = self._check_ffprobe()
139
+ report["asr"] = self._check_asr_backends()
140
+ report["ocr"] = self._check_ocr_support()
141
+ report["sidecars"] = self._check_sidecar_configuration()
142
+ report["transcript_connectivity"] = {
143
+ "status": "skipped",
144
+ "mode": "offline",
145
+ "verification": VERIFICATION_SKIPPED_OFFLINE,
146
+ "note": (
147
+ "offline mode: no network probe was performed; YouTube "
148
+ "connectivity is NOT verified."
149
+ ),
150
+ "fix_hint": FIX_HINT_OFFLINE_TRANSCRIPT,
151
+ }
152
+ elif transcript_api_report["status"] == "ok":
153
+ report["transcript_connectivity"] = self._check_transcript_connectivity(timeout_seconds=10)
154
+ else:
155
+ report["transcript_connectivity"] = {
156
+ "status": "missing",
157
+ "note": "youtube-transcript-api is not installed.",
158
+ "fix_hint": FIX_HINT_TRANSCRIPT_MISSING,
159
+ }
160
+ whisper_cpp_binary = self._settings.whisper_cpp_binary
161
+ if whisper_cpp_binary:
162
+ report["whisper-cpp"] = self._check_binary(whisper_cpp_binary, version_args=("--help",))
163
+ else:
164
+ report["whisper-cpp"] = {
165
+ "binary": None,
166
+ "status": "not_configured",
167
+ "note": "Set YTK_WHISPER_CPP_BINARY to enable ASR fallback.",
168
+ "fix_hint": "Build whisper.cpp, then set YTK_WHISPER_CPP_BINARY=/path/to/whisper-cli in .env",
169
+ }
170
+ minimax_api_key = self._settings.minimax_api_key
171
+ if minimax_api_key:
172
+ report["minimax"] = {
173
+ "status": "configured",
174
+ "verification": VERIFICATION_CONFIGURED_ONLY,
175
+ "region": self._settings.minimax_region,
176
+ "model": self._settings.minimax_model,
177
+ "max_tokens": self._settings.minimax_max_tokens,
178
+ "video_detail": self._settings.minimax_video_detail,
179
+ "video_fps": self._settings.minimax_video_fps,
180
+ "max_long_side_pixel": self._settings.minimax_max_long_side_pixel,
181
+ "max_video_mb": self._settings.minimax_max_video_mb,
182
+ "api_key_set": True,
183
+ }
184
+ else:
185
+ report["minimax"] = {
186
+ "status": "not_configured",
187
+ "note": "Set YTK_MINIMAX_API_KEY to enable MiniMax video understanding fallback.",
188
+ "fix_hint": "Add YTK_MINIMAX_API_KEY=your-key to .env | get one at https://platform.minimaxi.com",
189
+ }
190
+ # Summary renderer backend — distinct from the ingestion minimax
191
+ # provider. The renderer prefers the same MiniMax key (so a single
192
+ # YTK_MINIMAX_API_KEY covers both transcript extraction and summary
193
+ # generation) and falls back to the OpenAI-compatible endpoint.
194
+ openai_api_key = os.environ.get("OPENAI_API_KEY", "")
195
+ if minimax_api_key:
196
+ report["summary_llm"] = {
197
+ "status": "configured",
198
+ "verification": VERIFICATION_CONFIGURED_ONLY,
199
+ "backend": "minimax",
200
+ "region": self._settings.minimax_region,
201
+ "model": self._settings.minimax_model,
202
+ "openai_api_key_set": bool(openai_api_key),
203
+ "note": "Summary renderer will call MiniMax chat completions (same key as ingestion).",
204
+ }
205
+ elif openai_api_key:
206
+ report["summary_llm"] = {
207
+ "status": "configured",
208
+ "verification": VERIFICATION_CONFIGURED_ONLY,
209
+ "backend": "openai",
210
+ "base_url": redact_url(
211
+ os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1")
212
+ ),
213
+ "model": os.environ.get("OPENAI_MODEL", "gpt-4o-mini"),
214
+ "openai_api_key_set": True,
215
+ "note": "Summary renderer will call OpenAI-compatible endpoint (no MiniMax key).",
216
+ }
217
+ else:
218
+ report["summary_llm"] = {
219
+ "status": "not_configured",
220
+ "backend": "none",
221
+ "note": "Set YTK_MINIMAX_API_KEY (preferred) or OPENAI_API_KEY to enable AI summaries.",
222
+ "fix_hint": "Add YTK_MINIMAX_API_KEY or OPENAI_API_KEY to .env",
223
+ }
224
+ return report
225
+
226
+ def _transcript_cache_report(self) -> dict[str, Any]:
227
+ """Cache observability (tri-evolve-20 round-08): where it lives, how
228
+ big it is, whether it is on. Purely informational — never a failure."""
229
+ from ytk.ingestion.transcript_cache import TranscriptCache
230
+
231
+ cache = TranscriptCache(
232
+ cache_dir=self._settings.transcript_cache_dir,
233
+ enabled=self._settings.transcript_cache_enabled,
234
+ )
235
+ stats = cache.stats()
236
+ return {
237
+ "status": "ok" if self._settings.transcript_cache_enabled else "disabled",
238
+ "enabled": self._settings.transcript_cache_enabled,
239
+ "dir": stats["dir"],
240
+ "entries": stats["entries"],
241
+ "total_bytes": stats["total_bytes"],
242
+ }
243
+
244
+ def _check_binary(self, binary: str, *, version_args: tuple[str, ...]) -> dict[str, Any]:
245
+ # v2 answered "no yt-dlp console script" honestly with yt_dlp_binary="":
246
+ # that means "run sys.executable -m yt_dlp", not "binary missing".
247
+ if binary == "" and self._settings.yt_dlp_binary == "":
248
+ return self._check_python_module("yt_dlp", module="yt-dlp")
249
+
250
+ path = self._which(binary)
251
+ report: dict[str, Any] = {"binary": binary}
252
+
253
+ if path is None:
254
+ report["status"] = "missing"
255
+ report["fix_hint"] = self._fix_hint_for_binary(binary)
256
+ if binary == self._settings.yt_dlp_binary:
257
+ report["auth"] = self._yt_dlp_auth_report()
258
+ return report
259
+
260
+ report["status"] = "ok"
261
+ report["path"] = path
262
+ if binary == self._settings.yt_dlp_binary:
263
+ report["auth"] = self._yt_dlp_auth_report()
264
+
265
+ try:
266
+ report["version"] = self._version_runner([path, *version_args]).strip()
267
+ except Exception as exc:
268
+ report["version_error"] = str(exc)
269
+
270
+ return report
271
+
272
+ def _check_python_module(self, module_name: str, *, module: str) -> dict[str, Any]:
273
+ """Report a dependency that is provided as a Python module (no CLI on PATH)."""
274
+ import importlib.util
275
+
276
+ report: dict[str, Any] = {"binary": "<python -m " + module_name + ">", "module": module_name}
277
+ spec = importlib.util.find_spec(module_name)
278
+ if spec is None:
279
+ report["status"] = "missing"
280
+ report["fix_hint"] = f"pip install {module}"
281
+ if module_name == "yt_dlp":
282
+ report["auth"] = self._yt_dlp_auth_report()
283
+ return report
284
+ report["status"] = "ok"
285
+ report["verification"] = "verified_locally"
286
+ report["path"] = "python -m " + module_name
287
+ try:
288
+ report["version"] = self._package_version_getter(module)
289
+ except PackageNotFoundError:
290
+ # Same contract as ``_check_distribution`` (doctor.py:319): absent
291
+ # metadata is a "missing" report, never an exception-shaped
292
+ # version_error — a bare PackageNotFoundError carries no message, so
293
+ # ``str(exc)`` raises ValueError on CPython 3.13.
294
+ report["status"] = "missing"
295
+ report["fix_hint"] = f"pip install {module}"
296
+ if module_name == "yt_dlp":
297
+ report["auth"] = self._yt_dlp_auth_report()
298
+ return report
299
+ except Exception as exc: # pragma: no cover - defensive
300
+ report["version_error"] = str(exc)
301
+ if module_name == "yt_dlp":
302
+ report["auth"] = self._yt_dlp_auth_report()
303
+ return report
304
+
305
+ @staticmethod
306
+ def _fix_hint_for_binary(binary: str) -> str:
307
+ hints = {
308
+ "ffmpeg": "macOS: brew install ffmpeg | Ubuntu: sudo apt install ffmpeg | Windows: winget install ffmpeg",
309
+ "yt-dlp": "pip install -U yt-dlp | macOS: brew install yt-dlp",
310
+ "whisper-cli": "Build whisper.cpp and set YTK_WHISPER_CPP_BINARY | see https://github.com/ggerganov/whisper.cpp",
311
+ }
312
+ return hints.get(binary, f"Install '{binary}' or ensure it is on your PATH.")
313
+
314
+ def _yt_dlp_auth_report(self) -> dict[str, Any]:
315
+ cookiefile = self._settings.yt_dlp_cookies_file
316
+ cookiesfrombrowser = self._settings.yt_dlp_cookies_from_browser
317
+ return {
318
+ "configured": bool(cookiefile or cookiesfrombrowser),
319
+ "cookiefile": cookiefile,
320
+ "cookiefile_exists": bool(cookiefile and Path(cookiefile).exists()),
321
+ "cookiesfrombrowser": cookiesfrombrowser,
322
+ }
323
+
324
+ def _check_distribution(self, distribution_name: str) -> dict[str, Any]:
325
+ report: dict[str, Any] = {
326
+ "distribution": distribution_name,
327
+ "proxy": self._proxy_report(),
328
+ }
329
+
330
+ try:
331
+ report["version"] = self._package_version_getter(distribution_name)
332
+ except PackageNotFoundError:
333
+ report["status"] = "missing"
334
+ report["fix_hint"] = f"pip install {distribution_name}"
335
+ return report
336
+
337
+ report["status"] = "ok"
338
+ return report
339
+
340
+ # ------------------------------------------------------------------
341
+ # Offline diagnostics (T43.2/T00.2)
342
+ #
343
+ # Everything below is local-only: `which`, package metadata, file system
344
+ # checks and local version commands. Nothing here contacts a platform,
345
+ # downloads a model or opens the project .env. Each section carries a
346
+ # `verification` field so callers can distinguish "present on this
347
+ # machine" from "configuration exists" and never mistake config for a
348
+ # verified platform.
349
+ # ------------------------------------------------------------------
350
+ @staticmethod
351
+ def _aggregate_status(components: dict[str, dict[str, Any]]) -> str:
352
+ statuses = [component.get("status") for component in components.values()]
353
+ if any(status == "ok" for status in statuses):
354
+ return "ok"
355
+ if any(status == "missing" for status in statuses):
356
+ return "missing"
357
+ if statuses and all(status == "not_configured" for status in statuses):
358
+ return "not_configured"
359
+ return "partial"
360
+
361
+ @staticmethod
362
+ def _aggregate_verification(components: dict[str, dict[str, Any]]) -> str:
363
+ verified = [
364
+ component.get("verification")
365
+ for component in components.values()
366
+ if component.get("status") == "ok"
367
+ ]
368
+ if not verified:
369
+ return VERIFICATION_NOT_VERIFIED
370
+ if all(item == VERIFICATION_VERIFIED_LOCALLY for item in verified):
371
+ return VERIFICATION_VERIFIED_LOCALLY
372
+ return VERIFICATION_CONFIGURED_ONLY
373
+
374
+ def _check_ffprobe(self) -> dict[str, Any]:
375
+ ffmpeg = self._settings.ffmpeg_binary
376
+ if os.sep in ffmpeg or (os.altsep is not None and os.altsep in ffmpeg):
377
+ probed_binary = str(Path(ffmpeg).with_name("ffprobe"))
378
+ else:
379
+ probed_binary = "ffprobe"
380
+ report = self._check_binary(probed_binary, version_args=("-version",))
381
+ if report.get("status") != "ok":
382
+ report.setdefault("fix_hint", FIX_HINT_FFPROBE)
383
+ report["verification"] = (
384
+ VERIFICATION_VERIFIED_LOCALLY
385
+ if report.get("status") == "ok"
386
+ else VERIFICATION_NOT_VERIFIED
387
+ )
388
+ return report
389
+
390
+ def _check_ytdlp_module(self) -> dict[str, Any]:
391
+ report: dict[str, Any] = {"distribution": "yt-dlp"}
392
+ try:
393
+ report["version"] = self._package_version_getter("yt-dlp")
394
+ except PackageNotFoundError:
395
+ report["status"] = "missing"
396
+ report["verification"] = VERIFICATION_NOT_VERIFIED
397
+ report["fix_hint"] = "pip install yt-dlp"
398
+ return report
399
+ report["status"] = "ok"
400
+ report["verification"] = VERIFICATION_VERIFIED_LOCALLY
401
+ return report
402
+
403
+ def _check_js_runtime(self) -> dict[str, Any]:
404
+ from ytk.ingestion.ytdlp_acquire import probe_javascript_runtimes
405
+
406
+ runtimes, selected = probe_javascript_runtimes(self._which, self._version_runner)
407
+ ejs = self._check_local_ejs()
408
+ report: dict[str, Any] = {
409
+ "component": "yt-dlp EJS JavaScript runtime",
410
+ "runtimes": runtimes,
411
+ "selected_runtime": selected,
412
+ "ejs": ejs,
413
+ "network_verified": False,
414
+ "note": (
415
+ "Supported runtime and matching local EJS scripts are checked locally only. "
416
+ "This does NOT verify YouTube access, authentication, or successful challenge solving."
417
+ ),
418
+ }
419
+ if selected and ejs["status"] == "ok":
420
+ report["status"] = "ok"
421
+ report["verification"] = VERIFICATION_VERIFIED_LOCALLY
422
+ else:
423
+ report["status"] = "missing"
424
+ report["verification"] = VERIFICATION_NOT_VERIFIED
425
+ report["fix_hint"] = FIX_HINT_JS_RUNTIME
426
+ return report
427
+
428
+ def _check_local_ejs(self) -> dict[str, Any]:
429
+ report: dict[str, Any] = {"status": "missing", "verification": VERIFICATION_NOT_VERIFIED}
430
+ try:
431
+ installed = self._package_version_getter("yt-dlp-ejs")
432
+ report["version"] = installed
433
+ requirements = requires("yt-dlp") or []
434
+ expected = next((re.search(r"yt-dlp-ejs\s*==\s*([\w.+-]+)", item, re.I)
435
+ for item in requirements if re.match(r"yt-dlp-ejs\b", item, re.I)), None)
436
+ if expected is None or installed != expected[1]:
437
+ report["note"] = "EJS version is not verified against this yt-dlp distribution."
438
+ return report
439
+ report["required_version"] = expected[1]
440
+ spec = self._module_spec_finder("yt_dlp_ejs.yt.solver")
441
+ locations = getattr(spec, "submodule_search_locations", ()) or ()
442
+ if not any(all((Path(location) / name).is_file() and (Path(location) / name).stat().st_size > 0
443
+ for name in ("core.min.js", "lib.min.js")) for location in locations):
444
+ report["note"] = "Local EJS core/lib scripts are missing or empty."
445
+ return report
446
+ except (PackageNotFoundError, ImportError, OSError, ValueError):
447
+ return report
448
+ report.update(status="ok", verification=VERIFICATION_VERIFIED_LOCALLY)
449
+ return report
450
+
451
+ def _check_asr_backends(self) -> dict[str, Any]:
452
+ from ytk.errors import ModelUnavailableError
453
+ from ytk.pipeline.runners import resolve_asr_backend, resolve_asr_model
454
+
455
+ selected_backend = resolve_asr_backend(self._settings)
456
+ selected_model = self._settings.asr_model or (
457
+ self._settings.whisper_cpp_model
458
+ if selected_backend == "whisper_cpp"
459
+ else self._settings.faster_whisper_model
460
+ )
461
+ model_error = None
462
+ try:
463
+ selected_model = resolve_asr_model(self._settings)
464
+ except ModelUnavailableError as exc:
465
+ model_error = str(exc)
466
+ backends: dict[str, dict[str, Any]] = {}
467
+
468
+ faster_whisper: dict[str, Any] = {"distribution": "faster-whisper"}
469
+ try:
470
+ faster_whisper["version"] = self._package_version_getter("faster-whisper")
471
+ except PackageNotFoundError:
472
+ faster_whisper["status"] = "missing"
473
+ faster_whisper["verification"] = VERIFICATION_NOT_VERIFIED
474
+ faster_whisper["fix_hint"] = FIX_HINT_ASR
475
+ else:
476
+ faster_whisper["status"] = "ok"
477
+ faster_whisper["verification"] = VERIFICATION_CONFIGURED_ONLY
478
+ model_name = selected_model if selected_backend == "faster_whisper" else self._settings.faster_whisper_model
479
+ model_cached = self._faster_whisper_model_cached(model_name)
480
+ if selected_backend == "faster_whisper" and model_error is not None:
481
+ model_cached = False
482
+ faster_whisper["model"] = model_name
483
+ faster_whisper["model_available_locally"] = model_cached
484
+ if faster_whisper["status"] == "ok" and model_cached:
485
+ faster_whisper["verification"] = VERIFICATION_VERIFIED_LOCALLY
486
+ elif faster_whisper["status"] == "ok":
487
+ faster_whisper["status"] = "partial"
488
+ faster_whisper["fix_hint"] = "Provision nonempty model.bin, config.json and tokenizer.json in the selected local ASR directory."
489
+ backends["faster-whisper"] = faster_whisper
490
+
491
+ whisper_binary = self._settings.whisper_cpp_binary
492
+ whisper_cpp: dict[str, Any] = {"backend": "whisper.cpp", "binary": whisper_binary}
493
+ if not whisper_binary:
494
+ whisper_cpp["status"] = "not_configured"
495
+ whisper_cpp["verification"] = VERIFICATION_NOT_VERIFIED
496
+ whisper_cpp["fix_hint"] = (
497
+ "Build whisper.cpp, then set YTK_WHISPER_CPP_BINARY=/path/to/whisper-cli in .env"
498
+ )
499
+ else:
500
+ resolved = self._which(whisper_binary)
501
+ if resolved is None:
502
+ whisper_cpp["status"] = "missing"
503
+ whisper_cpp["verification"] = VERIFICATION_NOT_VERIFIED
504
+ whisper_cpp["fix_hint"] = FIX_HINT_ASR
505
+ else:
506
+ whisper_cpp["path"] = resolved
507
+ whisper_cpp["status"] = "ok"
508
+ whisper_cpp["verification"] = VERIFICATION_CONFIGURED_ONLY
509
+ whisper_model = selected_model if selected_backend == "whisper_cpp" else self._settings.whisper_cpp_model
510
+ whisper_cpp["model"] = whisper_model
511
+ whisper_cpp["model_exists"] = bool(
512
+ whisper_model and self._nonempty_file(Path(whisper_model).expanduser())
513
+ )
514
+ if selected_backend == "whisper_cpp" and model_error is not None:
515
+ whisper_cpp["model_exists"] = False
516
+ if whisper_cpp["status"] == "ok" and whisper_cpp["model_exists"]:
517
+ whisper_cpp["verification"] = VERIFICATION_VERIFIED_LOCALLY
518
+ elif whisper_cpp["status"] == "ok":
519
+ whisper_cpp["status"] = "partial"
520
+ whisper_cpp["fix_hint"] = "Provision a nonempty local whisper.cpp model file and set YTK_ASR_MODEL or YTK_WHISPER_CPP_MODEL."
521
+ backends["whisper-cpp"] = whisper_cpp
522
+
523
+ selected_report = backends["whisper-cpp" if selected_backend == "whisper_cpp" else "faster-whisper"]
524
+ status = selected_report["status"]
525
+ report: dict[str, Any] = {
526
+ "status": status,
527
+ "verification": selected_report["verification"],
528
+ "selected_backend": selected_backend,
529
+ "selected_model": str(Path(selected_model).expanduser()) if selected_model else None,
530
+ "backends": backends,
531
+ "note": (
532
+ "offline mode: ASR backend distribution/binary presence and "
533
+ "local model files are checked; model downloads are never "
534
+ "attempted and transcription is NOT verified."
535
+ ),
536
+ }
537
+ if model_error is not None:
538
+ report["model_error"] = model_error
539
+ if status != "ok":
540
+ report["fix_hint"] = model_error or selected_report.get("fix_hint", FIX_HINT_ASR)
541
+ return report
542
+
543
+ @staticmethod
544
+ def _nonempty_file(path: Path) -> bool:
545
+ try:
546
+ return path.is_file() and path.stat().st_size > 0
547
+ except OSError:
548
+ return False
549
+
550
+ @staticmethod
551
+ def _hf_cache_roots() -> list[Path]:
552
+ roots: list[Path] = []
553
+ for env_name in ("HUGGINGFACE_HUB_CACHE", "HF_HOME"):
554
+ value = os.environ.get(env_name)
555
+ if not value:
556
+ continue
557
+ path = Path(value).expanduser()
558
+ roots.append(path / "hub" if env_name == "HF_HOME" else path)
559
+ if not roots:
560
+ roots.append(Path.home() / ".cache" / "huggingface" / "hub")
561
+ return roots
562
+
563
+ @classmethod
564
+ def _faster_whisper_model_cached(cls, model: str | None) -> bool:
565
+ if not model:
566
+ return False
567
+ root = Path(model).expanduser()
568
+ return all(cls._nonempty_file(root / name) for name in ("model.bin", "config.json", "tokenizer.json"))
569
+
570
+ def _check_ocr_support(self) -> dict[str, Any]:
571
+ backends: dict[str, dict[str, Any]] = {}
572
+ for distribution in ("rapidocr", "onnxruntime"):
573
+ entry: dict[str, Any] = {"distribution": distribution}
574
+ try:
575
+ entry["version"] = self._package_version_getter(distribution)
576
+ except PackageNotFoundError:
577
+ entry["status"] = "missing"
578
+ entry["verification"] = VERIFICATION_NOT_VERIFIED
579
+ entry["fix_hint"] = FIX_HINT_OCR
580
+ else:
581
+ entry["status"] = "ok"
582
+ entry["verification"] = VERIFICATION_VERIFIED_LOCALLY
583
+ backends[distribution] = entry
584
+
585
+ models = self._rapidocr_model_report()
586
+ status = self._aggregate_status(backends)
587
+ if status == "ok" and models["status"] != "ok":
588
+ status = "partial"
589
+ report: dict[str, Any] = {
590
+ "status": status,
591
+ "verification": (
592
+ VERIFICATION_VERIFIED_LOCALLY if status == "ok" else VERIFICATION_NOT_VERIFIED
593
+ ),
594
+ "backends": backends,
595
+ "models": models,
596
+ "note": (
597
+ "offline mode: OCR runtime distributions and selected local ONNX "
598
+ "model files are checked; inference and model downloads are "
599
+ "NOT exercised."
600
+ ),
601
+ }
602
+ if status != "ok":
603
+ report["fix_hint"] = FIX_HINT_OCR
604
+ return report
605
+
606
+ def _rapidocr_model_report(self) -> dict[str, Any]:
607
+ from ytk.ocr.engine import DEFAULT_MODEL_FILENAMES
608
+
609
+ root = self._settings.resolved_ocr_model_dir()
610
+ selected = [root / name for name in DEFAULT_MODEL_FILENAMES.values()]
611
+ sizes: dict[Path, int] = {}
612
+ for path in selected:
613
+ try:
614
+ size = path.stat().st_size if path.is_file() else 0
615
+ except OSError:
616
+ size = 0
617
+ if size > 0:
618
+ sizes[path] = size
619
+ missing = [path.name for path in selected if path not in sizes]
620
+ report: dict[str, Any] = {
621
+ "component": "RapidOCR ONNX models",
622
+ "model_dir": str(root),
623
+ "status": "missing" if missing else "ok",
624
+ "model_files": len(sizes),
625
+ "total_bytes": sum(sizes.values()),
626
+ "missing": missing,
627
+ "verification": VERIFICATION_NOT_VERIFIED if missing else VERIFICATION_VERIFIED_LOCALLY,
628
+ }
629
+ if missing:
630
+ report["fix_hint"] = f"Provision the three nonempty RapidOCR model files in {root}: {', '.join(missing)}."
631
+ return report
632
+
633
+ def _check_sidecar_configuration(self) -> dict[str, Any]:
634
+ services: dict[str, dict[str, Any]] = {}
635
+ configured_count = 0
636
+ for service_name, candidates in _SIDECAR_CONFIG.items():
637
+ value: str | None = None
638
+ source: str | None = None
639
+ for attribute in candidates["attributes"]:
640
+ candidate = getattr(self._settings, attribute, None)
641
+ if candidate:
642
+ value, source = str(candidate), f"settings.{attribute}"
643
+ break
644
+ if value is None:
645
+ for env_name in candidates["env"]:
646
+ candidate = os.environ.get(env_name)
647
+ if candidate:
648
+ value, source = candidate, f"env.{env_name}"
649
+ break
650
+ entry: dict[str, Any] = {
651
+ "configured": value is not None,
652
+ "required": service_name != "xiaohongshu_mcp",
653
+ "source": source,
654
+ "verification": (
655
+ VERIFICATION_CONFIGURED_ONLY if value else VERIFICATION_NOT_VERIFIED
656
+ ),
657
+ }
658
+ if value is not None:
659
+ configured_count += int(entry["required"])
660
+ entry["url"] = self._redact_service_url(value)
661
+ if service_name == "xiaohongshu_mcp":
662
+ entry.update(native_path="yt-dlp", live_verified=False,
663
+ note="Optional sidecar; native video acquisition may be attempted. Actual platform access remains unverified.")
664
+ services[service_name] = entry
665
+
666
+ required_count = sum(bool(entry["required"]) for entry in services.values())
667
+ if configured_count == required_count:
668
+ status = "configured"
669
+ elif configured_count:
670
+ status = "partial"
671
+ else:
672
+ status = "not_configured"
673
+ report: dict[str, Any] = {
674
+ "status": status,
675
+ "verification": (
676
+ VERIFICATION_CONFIGURED_ONLY if configured_count else VERIFICATION_NOT_VERIFIED
677
+ ),
678
+ "services": services,
679
+ "note": (
680
+ "offline mode: sidecar configuration presence only; DTK/"
681
+ "xiaohongshu-mcp reachability and platform access are NOT "
682
+ "verified."
683
+ ),
684
+ }
685
+ if status != "configured":
686
+ report["fix_hint"] = FIX_HINT_SIDECAR
687
+ return report
688
+
689
+ @staticmethod
690
+ def _redact_service_url(value: str) -> str:
691
+ """Keep scheme+host+port only; never echo credentials or query tokens."""
692
+ try:
693
+ parts = urlsplit(value)
694
+ host = parts.hostname or ""
695
+ if ":" in host:
696
+ host = f"[{host}]"
697
+ netloc = f"{host}:{parts.port}" if parts.port is not None else host
698
+ except ValueError:
699
+ return "***"
700
+ if not parts.scheme or not netloc:
701
+ return "***"
702
+ return f"{parts.scheme}://{netloc}"
703
+
704
+ def _check_transcript_connectivity(self, *, timeout_seconds: int) -> dict[str, Any]:
705
+ result = self._transcript_connectivity_checker(
706
+ self._settings,
707
+ timeout_seconds=timeout_seconds,
708
+ )
709
+ report = dict(result)
710
+ report.setdefault("status", "ok")
711
+ connectivity_hint = self._fix_hint_for_connectivity(report["status"])
712
+ if connectivity_hint is not None:
713
+ report.setdefault("fix_hint", connectivity_hint)
714
+ return report
715
+
716
+ @staticmethod
717
+ def _fix_hint_for_connectivity(status: str) -> str | None:
718
+ if status == "missing":
719
+ return FIX_HINT_TRANSCRIPT_MISSING
720
+ if status == "blocked":
721
+ return FIX_HINT_TRANSCRIPT_PROXY
722
+ if status == "timeout":
723
+ return FIX_HINT_TRANSCRIPT_TIMEOUT
724
+ if status == "partial":
725
+ return FIX_HINT_TRANSCRIPT_PARTIAL
726
+ return None
727
+
728
+ def _proxy_report(self) -> dict[str, Any]:
729
+ mode = self._settings.transcript_proxy_mode
730
+ report: dict[str, Any] = {"mode": mode, "configured": False}
731
+
732
+ if mode == "generic":
733
+ report["http_url"] = self._redact_proxy_url(self._settings.transcript_http_proxy_url)
734
+ report["https_url"] = self._redact_proxy_url(self._settings.transcript_https_proxy_url)
735
+ missing_fields = [
736
+ field_name
737
+ for field_name, value in (
738
+ ("transcript_http_proxy_url", self._settings.transcript_http_proxy_url),
739
+ ("transcript_https_proxy_url", self._settings.transcript_https_proxy_url),
740
+ )
741
+ if not value
742
+ ]
743
+ report["configured"] = not missing_fields
744
+ if missing_fields:
745
+ report["missing_fields"] = missing_fields
746
+ elif mode == "webshare":
747
+ report["username"] = self._settings.webshare_proxy_username
748
+ report["regions"] = list(self._settings.webshare_proxy_regions or ())
749
+ missing_fields = [
750
+ field_name
751
+ for field_name, value in (
752
+ ("webshare_proxy_username", self._settings.webshare_proxy_username),
753
+ ("webshare_proxy_password", self._settings.webshare_proxy_password),
754
+ )
755
+ if not value
756
+ ]
757
+ report["configured"] = not missing_fields
758
+ if missing_fields:
759
+ report["missing_fields"] = missing_fields
760
+ else:
761
+ report["configured"] = False
762
+
763
+ return report
764
+
765
+ @staticmethod
766
+ def _run_version_command(command: Sequence[str]) -> str:
767
+ completed = subprocess.run(
768
+ list(command),
769
+ check=True,
770
+ capture_output=True,
771
+ text=True,
772
+ timeout=5,
773
+ )
774
+ return completed.stdout.strip() or completed.stderr.strip()
775
+
776
+ @staticmethod
777
+ def _redact_proxy_url(value: str | None) -> str | None:
778
+ if not value:
779
+ return value
780
+
781
+ try:
782
+ parts = urlsplit(value)
783
+ sanitized = urlsplit(redact_url(value))
784
+ port = f":{parts.port}" if parts.port is not None else ""
785
+ except ValueError:
786
+ return "<invalid proxy URL>"
787
+ if parts.username is None and parts.password is None:
788
+ return urlunsplit((parts.scheme, sanitized.netloc, parts.path, sanitized.query, ""))
789
+
790
+ hostname = parts.hostname or ""
791
+ if ":" in hostname:
792
+ hostname = f"[{hostname}]"
793
+ redacted_netloc = f"***:***@{hostname}{port}"
794
+ return urlunsplit((parts.scheme, redacted_netloc, parts.path, sanitized.query, ""))
795
+
796
+ @staticmethod
797
+ def _default_transcript_connectivity_checker(
798
+ settings: AppSettings,
799
+ *,
800
+ timeout_seconds: int,
801
+ ) -> dict[str, Any]:
802
+ if YouTubeTranscriptApi is None:
803
+ return {
804
+ "status": "missing",
805
+ "note": "youtube-transcript-api is not installed.",
806
+ "fix_hint": FIX_HINT_TRANSCRIPT_MISSING,
807
+ }
808
+
809
+ def _probe() -> None:
810
+ kwargs: dict[str, Any] = {}
811
+ proxy_config = TranscriptApiProvider.build_proxy_config(settings)
812
+ if proxy_config is not None:
813
+ kwargs["proxy_config"] = proxy_config
814
+ client = YouTubeTranscriptApi(**kwargs)
815
+ # Configurable probe id (§4.3): the hardcoded Rick Astley id causes a
816
+ # false "blocked" if that video is ever taken down. Override via env.
817
+ # `or` (not getenv default) so a set-but-EMPTY var still falls back.
818
+ probe_id = os.getenv("YTK_DOCTOR_PROBE_ID") or "dQw4w9WgXcQ"
819
+ transcript_list = client.list(probe_id)
820
+ try:
821
+ transcript = transcript_list.find_generated_transcript(["en"])
822
+ except Exception:
823
+ transcript = transcript_list.find_transcript(["en"])
824
+ transcript.fetch()
825
+
826
+ error_holder: list[BaseException] = []
827
+
828
+ def _run_probe() -> None:
829
+ try:
830
+ _probe()
831
+ except BaseException as exc: # pragma: no cover - exercised through return contract
832
+ error_holder.append(exc)
833
+
834
+ try:
835
+ thread = threading.Thread(target=_run_probe, daemon=True)
836
+ thread.start()
837
+ thread.join(timeout=timeout_seconds)
838
+ except Exception as exc:
839
+ return {
840
+ "status": "blocked",
841
+ "error": str(exc),
842
+ "note": "YouTube transcript requests appear blocked. Configure a proxy.",
843
+ "fix_hint": FIX_HINT_TRANSCRIPT_PROXY,
844
+ }
845
+
846
+ if thread.is_alive():
847
+ return {
848
+ "status": "timeout",
849
+ "note": "Connection timed out. Check network or proxy.",
850
+ "fix_hint": FIX_HINT_TRANSCRIPT_TIMEOUT,
851
+ }
852
+
853
+ if error_holder:
854
+ probe_error = error_holder[0]
855
+ return {
856
+ "status": "blocked",
857
+ "error": str(probe_error),
858
+ "note": "YouTube transcript requests appear blocked. Configure a proxy.",
859
+ "fix_hint": FIX_HINT_TRANSCRIPT_PROXY,
860
+ }
861
+
862
+ return {
863
+ "status": "ok",
864
+ "note": "Transcript API connectivity confirmed",
865
+ }
866
+
867
+
868
+ def format_doctor_report(report: dict[str, dict[str, Any]]) -> str:
869
+ return json.dumps(report, indent=2, ensure_ascii=False)
870
+
871
+
872
+ def doctor_report_failures(report: dict[str, dict[str, Any]]) -> list[str]:
873
+ """Section names whose local check failed in an offline report.
874
+
875
+ Intended for the CLI/install/CI chokepoint: ``doctor --offline`` should be
876
+ informative but must exit non-zero when a required local capability is
877
+ missing. Network-dependent sections are reported as ``skipped`` and are
878
+ never counted as failures; ``configured``/``not_configured`` describe
879
+ optional credentials and are not failures either.
880
+ """
881
+ failures: list[str] = []
882
+ for name, section in report.items():
883
+ status = section.get("status", "")
884
+ if status in {"missing", "partial"}:
885
+ failures.append(name)
886
+ selected = report.get("platforms", {}).get("platforms", {})
887
+ youtube_selected = not selected or bool(selected.get("youtube", {}).get("enabled"))
888
+ js_runtime = report.get("yt-dlp", {}).get("js_runtime")
889
+ if youtube_selected and isinstance(js_runtime, dict) and js_runtime.get("status") != "ok":
890
+ failures.append("yt-dlp.js_runtime")
891
+ return failures