kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
ytk/doctor.py
ADDED
|
@@ -0,0 +1,891 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import importlib.util
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import subprocess
|
|
8
|
+
import threading
|
|
9
|
+
from collections.abc import Callable, Sequence
|
|
10
|
+
from importlib.metadata import PackageNotFoundError, requires, version
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from shutil import which
|
|
13
|
+
from typing import Any
|
|
14
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
15
|
+
|
|
16
|
+
from ytk.config import AppSettings
|
|
17
|
+
from ytk.ingestion.providers.transcript_api import TranscriptApiProvider, YouTubeTranscriptApi
|
|
18
|
+
from ytk.logging import redact_url
|
|
19
|
+
|
|
20
|
+
BinaryResolver = Callable[[str], str | None]
|
|
21
|
+
VersionRunner = Callable[[Sequence[str]], str]
|
|
22
|
+
PackageVersionGetter = Callable[[str], str]
|
|
23
|
+
TranscriptConnectivityChecker = Callable[..., dict[str, Any]]
|
|
24
|
+
ModuleSpecFinder = Callable[[str], Any]
|
|
25
|
+
|
|
26
|
+
# Verification vocabulary used by the offline diagnostics (T43.2/T00.2).
|
|
27
|
+
# ``configured``/``configured_only`` means "the setting exists, nothing was
|
|
28
|
+
# contacted"; a platform is never reported as verified from configuration.
|
|
29
|
+
VERIFICATION_VERIFIED_LOCALLY = "verified_locally"
|
|
30
|
+
VERIFICATION_CONFIGURED_ONLY = "configured_only"
|
|
31
|
+
VERIFICATION_NOT_VERIFIED = "not_verified"
|
|
32
|
+
VERIFICATION_SKIPPED_OFFLINE = "skipped_offline"
|
|
33
|
+
|
|
34
|
+
FIX_HINT_OFFLINE_TRANSCRIPT = (
|
|
35
|
+
"Offline mode never probes YouTube; run `./scripts/doctor.sh` (online) "
|
|
36
|
+
"with a user-provided sample when connectivity must be verified."
|
|
37
|
+
)
|
|
38
|
+
FIX_HINT_ASR = (
|
|
39
|
+
"pip install 'kd-engine[asr]'"
|
|
40
|
+
)
|
|
41
|
+
FIX_HINT_OCR = (
|
|
42
|
+
"OCR 已随本商品移除:商品边界为视频流(字幕/ASR 文本),不处理图片内容"
|
|
43
|
+
)
|
|
44
|
+
FIX_HINT_FFPROBE = (
|
|
45
|
+
"Install ffprobe (ships with FFmpeg) | macOS: brew install ffmpeg | Ubuntu: sudo apt install ffmpeg"
|
|
46
|
+
)
|
|
47
|
+
FIX_HINT_JS_RUNTIME = (
|
|
48
|
+
"Explicitly prepare `python -m pip install -U 'yt-dlp[default]'` in the connector environment "
|
|
49
|
+
"and install Deno >=2.3.0 (recommended) or Node >=22.0.0. "
|
|
50
|
+
"Node must be enabled with --js-runtimes node; inference never downloads EJS components."
|
|
51
|
+
)
|
|
52
|
+
FIX_HINT_SIDECAR = (
|
|
53
|
+
"Configure the sidecar URL in .env or the optional TOML config; "
|
|
54
|
+
"offline diagnostics only report presence, never reachability"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# Sidecar configuration candidates. AppSettings fields win (when the v2
|
|
58
|
+
# config lands) and environment variables are the documented fallback; the
|
|
59
|
+
# offline doctor only reports presence, never reachability.
|
|
60
|
+
_SIDECAR_CONFIG: dict[str, dict[str, tuple[str, ...]]] = {
|
|
61
|
+
"dtk": {
|
|
62
|
+
"attributes": ("dtk_base_url", "douyin_base_url", "dtk_url"),
|
|
63
|
+
"env": ("YTK_DTK_BASE_URL", "DTK_BASE_URL"),
|
|
64
|
+
},
|
|
65
|
+
"xiaohongshu_mcp": {
|
|
66
|
+
"attributes": ("xhs_base_url",),
|
|
67
|
+
"env": ("YTK_XHS_BASE_URL", "YTK_XHS_MCP_BASE_URL"),
|
|
68
|
+
},
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
# fix_hint strings for transcript-connectivity problems. Single source of
|
|
72
|
+
# truth — referenced both by _fix_hint_for_connectivity (the chokepoint that
|
|
73
|
+
# decorates stub-injected results) and by the inline returns in
|
|
74
|
+
# _default_transcript_connectivity_checker (which is also called directly as a
|
|
75
|
+
# @staticmethod, bypassing the chokepoint). Keeping these as module constants
|
|
76
|
+
# prevents the two paths from drifting.
|
|
77
|
+
FIX_HINT_TRANSCRIPT_MISSING = (
|
|
78
|
+
"pip install youtube-transcript-api"
|
|
79
|
+
)
|
|
80
|
+
FIX_HINT_TRANSCRIPT_PROXY = (
|
|
81
|
+
"Set YTK_HTTP_PROXY_URL + YTK_HTTPS_PROXY_URL in .env "
|
|
82
|
+
"| or pass --proxy on the CLI"
|
|
83
|
+
)
|
|
84
|
+
FIX_HINT_TRANSCRIPT_TIMEOUT = (
|
|
85
|
+
"Check connectivity | if behind a firewall, set YTK_HTTP_PROXY_URL in .env"
|
|
86
|
+
)
|
|
87
|
+
FIX_HINT_TRANSCRIPT_PARTIAL = (
|
|
88
|
+
"Some transcript checks only partially succeeded | verify proxy/cookies and rerun ./scripts/doctor.sh"
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class DoctorService:
|
|
93
|
+
def __init__(
|
|
94
|
+
self,
|
|
95
|
+
settings: AppSettings | None = None,
|
|
96
|
+
*,
|
|
97
|
+
which: BinaryResolver = which,
|
|
98
|
+
version_runner: VersionRunner | None = None,
|
|
99
|
+
package_version_getter: PackageVersionGetter = version,
|
|
100
|
+
transcript_connectivity_checker: TranscriptConnectivityChecker | None = None,
|
|
101
|
+
module_spec_finder: ModuleSpecFinder | None = None,
|
|
102
|
+
offline: bool = False,
|
|
103
|
+
) -> None:
|
|
104
|
+
self._settings = settings or AppSettings()
|
|
105
|
+
self._which = which
|
|
106
|
+
self._version_runner = version_runner or self._run_version_command
|
|
107
|
+
self._package_version_getter = package_version_getter
|
|
108
|
+
self._module_spec_finder = module_spec_finder or importlib.util.find_spec
|
|
109
|
+
self._transcript_connectivity_checker = (
|
|
110
|
+
transcript_connectivity_checker or self._default_transcript_connectivity_checker
|
|
111
|
+
)
|
|
112
|
+
# Offline mode is a strict local-only diagnostic: no network probe, no
|
|
113
|
+
# sidecar contact, and every report states how strong its evidence is
|
|
114
|
+
# (verified locally vs merely configured).
|
|
115
|
+
self._offline = offline
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def offline(self) -> bool:
|
|
119
|
+
return self._offline
|
|
120
|
+
|
|
121
|
+
def run(self) -> dict[str, dict[str, Any]]:
|
|
122
|
+
transcript_api_report = self._check_distribution("youtube-transcript-api")
|
|
123
|
+
ytdlp_report = self._check_binary(
|
|
124
|
+
self._settings.yt_dlp_binary,
|
|
125
|
+
version_args=("--version",),
|
|
126
|
+
)
|
|
127
|
+
report = {
|
|
128
|
+
"ffmpeg": self._check_binary(self._settings.ffmpeg_binary, version_args=("-version",)),
|
|
129
|
+
"yt-dlp": ytdlp_report,
|
|
130
|
+
"youtube-transcript-api": transcript_api_report,
|
|
131
|
+
"transcript_cache": self._transcript_cache_report(),
|
|
132
|
+
}
|
|
133
|
+
if self._offline:
|
|
134
|
+
# Local-only probes added by T43.2. They are omitted in the
|
|
135
|
+
# default online report so its shape stays stable for callers.
|
|
136
|
+
ytdlp_report["python_module"] = self._check_ytdlp_module()
|
|
137
|
+
ytdlp_report["js_runtime"] = self._check_js_runtime()
|
|
138
|
+
report["ffprobe"] = self._check_ffprobe()
|
|
139
|
+
report["asr"] = self._check_asr_backends()
|
|
140
|
+
report["ocr"] = self._check_ocr_support()
|
|
141
|
+
report["sidecars"] = self._check_sidecar_configuration()
|
|
142
|
+
report["transcript_connectivity"] = {
|
|
143
|
+
"status": "skipped",
|
|
144
|
+
"mode": "offline",
|
|
145
|
+
"verification": VERIFICATION_SKIPPED_OFFLINE,
|
|
146
|
+
"note": (
|
|
147
|
+
"offline mode: no network probe was performed; YouTube "
|
|
148
|
+
"connectivity is NOT verified."
|
|
149
|
+
),
|
|
150
|
+
"fix_hint": FIX_HINT_OFFLINE_TRANSCRIPT,
|
|
151
|
+
}
|
|
152
|
+
elif transcript_api_report["status"] == "ok":
|
|
153
|
+
report["transcript_connectivity"] = self._check_transcript_connectivity(timeout_seconds=10)
|
|
154
|
+
else:
|
|
155
|
+
report["transcript_connectivity"] = {
|
|
156
|
+
"status": "missing",
|
|
157
|
+
"note": "youtube-transcript-api is not installed.",
|
|
158
|
+
"fix_hint": FIX_HINT_TRANSCRIPT_MISSING,
|
|
159
|
+
}
|
|
160
|
+
whisper_cpp_binary = self._settings.whisper_cpp_binary
|
|
161
|
+
if whisper_cpp_binary:
|
|
162
|
+
report["whisper-cpp"] = self._check_binary(whisper_cpp_binary, version_args=("--help",))
|
|
163
|
+
else:
|
|
164
|
+
report["whisper-cpp"] = {
|
|
165
|
+
"binary": None,
|
|
166
|
+
"status": "not_configured",
|
|
167
|
+
"note": "Set YTK_WHISPER_CPP_BINARY to enable ASR fallback.",
|
|
168
|
+
"fix_hint": "Build whisper.cpp, then set YTK_WHISPER_CPP_BINARY=/path/to/whisper-cli in .env",
|
|
169
|
+
}
|
|
170
|
+
minimax_api_key = self._settings.minimax_api_key
|
|
171
|
+
if minimax_api_key:
|
|
172
|
+
report["minimax"] = {
|
|
173
|
+
"status": "configured",
|
|
174
|
+
"verification": VERIFICATION_CONFIGURED_ONLY,
|
|
175
|
+
"region": self._settings.minimax_region,
|
|
176
|
+
"model": self._settings.minimax_model,
|
|
177
|
+
"max_tokens": self._settings.minimax_max_tokens,
|
|
178
|
+
"video_detail": self._settings.minimax_video_detail,
|
|
179
|
+
"video_fps": self._settings.minimax_video_fps,
|
|
180
|
+
"max_long_side_pixel": self._settings.minimax_max_long_side_pixel,
|
|
181
|
+
"max_video_mb": self._settings.minimax_max_video_mb,
|
|
182
|
+
"api_key_set": True,
|
|
183
|
+
}
|
|
184
|
+
else:
|
|
185
|
+
report["minimax"] = {
|
|
186
|
+
"status": "not_configured",
|
|
187
|
+
"note": "Set YTK_MINIMAX_API_KEY to enable MiniMax video understanding fallback.",
|
|
188
|
+
"fix_hint": "Add YTK_MINIMAX_API_KEY=your-key to .env | get one at https://platform.minimaxi.com",
|
|
189
|
+
}
|
|
190
|
+
# Summary renderer backend — distinct from the ingestion minimax
|
|
191
|
+
# provider. The renderer prefers the same MiniMax key (so a single
|
|
192
|
+
# YTK_MINIMAX_API_KEY covers both transcript extraction and summary
|
|
193
|
+
# generation) and falls back to the OpenAI-compatible endpoint.
|
|
194
|
+
openai_api_key = os.environ.get("OPENAI_API_KEY", "")
|
|
195
|
+
if minimax_api_key:
|
|
196
|
+
report["summary_llm"] = {
|
|
197
|
+
"status": "configured",
|
|
198
|
+
"verification": VERIFICATION_CONFIGURED_ONLY,
|
|
199
|
+
"backend": "minimax",
|
|
200
|
+
"region": self._settings.minimax_region,
|
|
201
|
+
"model": self._settings.minimax_model,
|
|
202
|
+
"openai_api_key_set": bool(openai_api_key),
|
|
203
|
+
"note": "Summary renderer will call MiniMax chat completions (same key as ingestion).",
|
|
204
|
+
}
|
|
205
|
+
elif openai_api_key:
|
|
206
|
+
report["summary_llm"] = {
|
|
207
|
+
"status": "configured",
|
|
208
|
+
"verification": VERIFICATION_CONFIGURED_ONLY,
|
|
209
|
+
"backend": "openai",
|
|
210
|
+
"base_url": redact_url(
|
|
211
|
+
os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1")
|
|
212
|
+
),
|
|
213
|
+
"model": os.environ.get("OPENAI_MODEL", "gpt-4o-mini"),
|
|
214
|
+
"openai_api_key_set": True,
|
|
215
|
+
"note": "Summary renderer will call OpenAI-compatible endpoint (no MiniMax key).",
|
|
216
|
+
}
|
|
217
|
+
else:
|
|
218
|
+
report["summary_llm"] = {
|
|
219
|
+
"status": "not_configured",
|
|
220
|
+
"backend": "none",
|
|
221
|
+
"note": "Set YTK_MINIMAX_API_KEY (preferred) or OPENAI_API_KEY to enable AI summaries.",
|
|
222
|
+
"fix_hint": "Add YTK_MINIMAX_API_KEY or OPENAI_API_KEY to .env",
|
|
223
|
+
}
|
|
224
|
+
return report
|
|
225
|
+
|
|
226
|
+
def _transcript_cache_report(self) -> dict[str, Any]:
|
|
227
|
+
"""Cache observability (tri-evolve-20 round-08): where it lives, how
|
|
228
|
+
big it is, whether it is on. Purely informational — never a failure."""
|
|
229
|
+
from ytk.ingestion.transcript_cache import TranscriptCache
|
|
230
|
+
|
|
231
|
+
cache = TranscriptCache(
|
|
232
|
+
cache_dir=self._settings.transcript_cache_dir,
|
|
233
|
+
enabled=self._settings.transcript_cache_enabled,
|
|
234
|
+
)
|
|
235
|
+
stats = cache.stats()
|
|
236
|
+
return {
|
|
237
|
+
"status": "ok" if self._settings.transcript_cache_enabled else "disabled",
|
|
238
|
+
"enabled": self._settings.transcript_cache_enabled,
|
|
239
|
+
"dir": stats["dir"],
|
|
240
|
+
"entries": stats["entries"],
|
|
241
|
+
"total_bytes": stats["total_bytes"],
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
def _check_binary(self, binary: str, *, version_args: tuple[str, ...]) -> dict[str, Any]:
|
|
245
|
+
# v2 answered "no yt-dlp console script" honestly with yt_dlp_binary="":
|
|
246
|
+
# that means "run sys.executable -m yt_dlp", not "binary missing".
|
|
247
|
+
if binary == "" and self._settings.yt_dlp_binary == "":
|
|
248
|
+
return self._check_python_module("yt_dlp", module="yt-dlp")
|
|
249
|
+
|
|
250
|
+
path = self._which(binary)
|
|
251
|
+
report: dict[str, Any] = {"binary": binary}
|
|
252
|
+
|
|
253
|
+
if path is None:
|
|
254
|
+
report["status"] = "missing"
|
|
255
|
+
report["fix_hint"] = self._fix_hint_for_binary(binary)
|
|
256
|
+
if binary == self._settings.yt_dlp_binary:
|
|
257
|
+
report["auth"] = self._yt_dlp_auth_report()
|
|
258
|
+
return report
|
|
259
|
+
|
|
260
|
+
report["status"] = "ok"
|
|
261
|
+
report["path"] = path
|
|
262
|
+
if binary == self._settings.yt_dlp_binary:
|
|
263
|
+
report["auth"] = self._yt_dlp_auth_report()
|
|
264
|
+
|
|
265
|
+
try:
|
|
266
|
+
report["version"] = self._version_runner([path, *version_args]).strip()
|
|
267
|
+
except Exception as exc:
|
|
268
|
+
report["version_error"] = str(exc)
|
|
269
|
+
|
|
270
|
+
return report
|
|
271
|
+
|
|
272
|
+
def _check_python_module(self, module_name: str, *, module: str) -> dict[str, Any]:
|
|
273
|
+
"""Report a dependency that is provided as a Python module (no CLI on PATH)."""
|
|
274
|
+
import importlib.util
|
|
275
|
+
|
|
276
|
+
report: dict[str, Any] = {"binary": "<python -m " + module_name + ">", "module": module_name}
|
|
277
|
+
spec = importlib.util.find_spec(module_name)
|
|
278
|
+
if spec is None:
|
|
279
|
+
report["status"] = "missing"
|
|
280
|
+
report["fix_hint"] = f"pip install {module}"
|
|
281
|
+
if module_name == "yt_dlp":
|
|
282
|
+
report["auth"] = self._yt_dlp_auth_report()
|
|
283
|
+
return report
|
|
284
|
+
report["status"] = "ok"
|
|
285
|
+
report["verification"] = "verified_locally"
|
|
286
|
+
report["path"] = "python -m " + module_name
|
|
287
|
+
try:
|
|
288
|
+
report["version"] = self._package_version_getter(module)
|
|
289
|
+
except PackageNotFoundError:
|
|
290
|
+
# Same contract as ``_check_distribution`` (doctor.py:319): absent
|
|
291
|
+
# metadata is a "missing" report, never an exception-shaped
|
|
292
|
+
# version_error — a bare PackageNotFoundError carries no message, so
|
|
293
|
+
# ``str(exc)`` raises ValueError on CPython 3.13.
|
|
294
|
+
report["status"] = "missing"
|
|
295
|
+
report["fix_hint"] = f"pip install {module}"
|
|
296
|
+
if module_name == "yt_dlp":
|
|
297
|
+
report["auth"] = self._yt_dlp_auth_report()
|
|
298
|
+
return report
|
|
299
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
300
|
+
report["version_error"] = str(exc)
|
|
301
|
+
if module_name == "yt_dlp":
|
|
302
|
+
report["auth"] = self._yt_dlp_auth_report()
|
|
303
|
+
return report
|
|
304
|
+
|
|
305
|
+
@staticmethod
|
|
306
|
+
def _fix_hint_for_binary(binary: str) -> str:
|
|
307
|
+
hints = {
|
|
308
|
+
"ffmpeg": "macOS: brew install ffmpeg | Ubuntu: sudo apt install ffmpeg | Windows: winget install ffmpeg",
|
|
309
|
+
"yt-dlp": "pip install -U yt-dlp | macOS: brew install yt-dlp",
|
|
310
|
+
"whisper-cli": "Build whisper.cpp and set YTK_WHISPER_CPP_BINARY | see https://github.com/ggerganov/whisper.cpp",
|
|
311
|
+
}
|
|
312
|
+
return hints.get(binary, f"Install '{binary}' or ensure it is on your PATH.")
|
|
313
|
+
|
|
314
|
+
def _yt_dlp_auth_report(self) -> dict[str, Any]:
|
|
315
|
+
cookiefile = self._settings.yt_dlp_cookies_file
|
|
316
|
+
cookiesfrombrowser = self._settings.yt_dlp_cookies_from_browser
|
|
317
|
+
return {
|
|
318
|
+
"configured": bool(cookiefile or cookiesfrombrowser),
|
|
319
|
+
"cookiefile": cookiefile,
|
|
320
|
+
"cookiefile_exists": bool(cookiefile and Path(cookiefile).exists()),
|
|
321
|
+
"cookiesfrombrowser": cookiesfrombrowser,
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
def _check_distribution(self, distribution_name: str) -> dict[str, Any]:
|
|
325
|
+
report: dict[str, Any] = {
|
|
326
|
+
"distribution": distribution_name,
|
|
327
|
+
"proxy": self._proxy_report(),
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
try:
|
|
331
|
+
report["version"] = self._package_version_getter(distribution_name)
|
|
332
|
+
except PackageNotFoundError:
|
|
333
|
+
report["status"] = "missing"
|
|
334
|
+
report["fix_hint"] = f"pip install {distribution_name}"
|
|
335
|
+
return report
|
|
336
|
+
|
|
337
|
+
report["status"] = "ok"
|
|
338
|
+
return report
|
|
339
|
+
|
|
340
|
+
# ------------------------------------------------------------------
|
|
341
|
+
# Offline diagnostics (T43.2/T00.2)
|
|
342
|
+
#
|
|
343
|
+
# Everything below is local-only: `which`, package metadata, file system
|
|
344
|
+
# checks and local version commands. Nothing here contacts a platform,
|
|
345
|
+
# downloads a model or opens the project .env. Each section carries a
|
|
346
|
+
# `verification` field so callers can distinguish "present on this
|
|
347
|
+
# machine" from "configuration exists" and never mistake config for a
|
|
348
|
+
# verified platform.
|
|
349
|
+
# ------------------------------------------------------------------
|
|
350
|
+
@staticmethod
|
|
351
|
+
def _aggregate_status(components: dict[str, dict[str, Any]]) -> str:
|
|
352
|
+
statuses = [component.get("status") for component in components.values()]
|
|
353
|
+
if any(status == "ok" for status in statuses):
|
|
354
|
+
return "ok"
|
|
355
|
+
if any(status == "missing" for status in statuses):
|
|
356
|
+
return "missing"
|
|
357
|
+
if statuses and all(status == "not_configured" for status in statuses):
|
|
358
|
+
return "not_configured"
|
|
359
|
+
return "partial"
|
|
360
|
+
|
|
361
|
+
@staticmethod
|
|
362
|
+
def _aggregate_verification(components: dict[str, dict[str, Any]]) -> str:
|
|
363
|
+
verified = [
|
|
364
|
+
component.get("verification")
|
|
365
|
+
for component in components.values()
|
|
366
|
+
if component.get("status") == "ok"
|
|
367
|
+
]
|
|
368
|
+
if not verified:
|
|
369
|
+
return VERIFICATION_NOT_VERIFIED
|
|
370
|
+
if all(item == VERIFICATION_VERIFIED_LOCALLY for item in verified):
|
|
371
|
+
return VERIFICATION_VERIFIED_LOCALLY
|
|
372
|
+
return VERIFICATION_CONFIGURED_ONLY
|
|
373
|
+
|
|
374
|
+
def _check_ffprobe(self) -> dict[str, Any]:
|
|
375
|
+
ffmpeg = self._settings.ffmpeg_binary
|
|
376
|
+
if os.sep in ffmpeg or (os.altsep is not None and os.altsep in ffmpeg):
|
|
377
|
+
probed_binary = str(Path(ffmpeg).with_name("ffprobe"))
|
|
378
|
+
else:
|
|
379
|
+
probed_binary = "ffprobe"
|
|
380
|
+
report = self._check_binary(probed_binary, version_args=("-version",))
|
|
381
|
+
if report.get("status") != "ok":
|
|
382
|
+
report.setdefault("fix_hint", FIX_HINT_FFPROBE)
|
|
383
|
+
report["verification"] = (
|
|
384
|
+
VERIFICATION_VERIFIED_LOCALLY
|
|
385
|
+
if report.get("status") == "ok"
|
|
386
|
+
else VERIFICATION_NOT_VERIFIED
|
|
387
|
+
)
|
|
388
|
+
return report
|
|
389
|
+
|
|
390
|
+
def _check_ytdlp_module(self) -> dict[str, Any]:
|
|
391
|
+
report: dict[str, Any] = {"distribution": "yt-dlp"}
|
|
392
|
+
try:
|
|
393
|
+
report["version"] = self._package_version_getter("yt-dlp")
|
|
394
|
+
except PackageNotFoundError:
|
|
395
|
+
report["status"] = "missing"
|
|
396
|
+
report["verification"] = VERIFICATION_NOT_VERIFIED
|
|
397
|
+
report["fix_hint"] = "pip install yt-dlp"
|
|
398
|
+
return report
|
|
399
|
+
report["status"] = "ok"
|
|
400
|
+
report["verification"] = VERIFICATION_VERIFIED_LOCALLY
|
|
401
|
+
return report
|
|
402
|
+
|
|
403
|
+
def _check_js_runtime(self) -> dict[str, Any]:
|
|
404
|
+
from ytk.ingestion.ytdlp_acquire import probe_javascript_runtimes
|
|
405
|
+
|
|
406
|
+
runtimes, selected = probe_javascript_runtimes(self._which, self._version_runner)
|
|
407
|
+
ejs = self._check_local_ejs()
|
|
408
|
+
report: dict[str, Any] = {
|
|
409
|
+
"component": "yt-dlp EJS JavaScript runtime",
|
|
410
|
+
"runtimes": runtimes,
|
|
411
|
+
"selected_runtime": selected,
|
|
412
|
+
"ejs": ejs,
|
|
413
|
+
"network_verified": False,
|
|
414
|
+
"note": (
|
|
415
|
+
"Supported runtime and matching local EJS scripts are checked locally only. "
|
|
416
|
+
"This does NOT verify YouTube access, authentication, or successful challenge solving."
|
|
417
|
+
),
|
|
418
|
+
}
|
|
419
|
+
if selected and ejs["status"] == "ok":
|
|
420
|
+
report["status"] = "ok"
|
|
421
|
+
report["verification"] = VERIFICATION_VERIFIED_LOCALLY
|
|
422
|
+
else:
|
|
423
|
+
report["status"] = "missing"
|
|
424
|
+
report["verification"] = VERIFICATION_NOT_VERIFIED
|
|
425
|
+
report["fix_hint"] = FIX_HINT_JS_RUNTIME
|
|
426
|
+
return report
|
|
427
|
+
|
|
428
|
+
def _check_local_ejs(self) -> dict[str, Any]:
|
|
429
|
+
report: dict[str, Any] = {"status": "missing", "verification": VERIFICATION_NOT_VERIFIED}
|
|
430
|
+
try:
|
|
431
|
+
installed = self._package_version_getter("yt-dlp-ejs")
|
|
432
|
+
report["version"] = installed
|
|
433
|
+
requirements = requires("yt-dlp") or []
|
|
434
|
+
expected = next((re.search(r"yt-dlp-ejs\s*==\s*([\w.+-]+)", item, re.I)
|
|
435
|
+
for item in requirements if re.match(r"yt-dlp-ejs\b", item, re.I)), None)
|
|
436
|
+
if expected is None or installed != expected[1]:
|
|
437
|
+
report["note"] = "EJS version is not verified against this yt-dlp distribution."
|
|
438
|
+
return report
|
|
439
|
+
report["required_version"] = expected[1]
|
|
440
|
+
spec = self._module_spec_finder("yt_dlp_ejs.yt.solver")
|
|
441
|
+
locations = getattr(spec, "submodule_search_locations", ()) or ()
|
|
442
|
+
if not any(all((Path(location) / name).is_file() and (Path(location) / name).stat().st_size > 0
|
|
443
|
+
for name in ("core.min.js", "lib.min.js")) for location in locations):
|
|
444
|
+
report["note"] = "Local EJS core/lib scripts are missing or empty."
|
|
445
|
+
return report
|
|
446
|
+
except (PackageNotFoundError, ImportError, OSError, ValueError):
|
|
447
|
+
return report
|
|
448
|
+
report.update(status="ok", verification=VERIFICATION_VERIFIED_LOCALLY)
|
|
449
|
+
return report
|
|
450
|
+
|
|
451
|
+
def _check_asr_backends(self) -> dict[str, Any]:
|
|
452
|
+
from ytk.errors import ModelUnavailableError
|
|
453
|
+
from ytk.pipeline.runners import resolve_asr_backend, resolve_asr_model
|
|
454
|
+
|
|
455
|
+
selected_backend = resolve_asr_backend(self._settings)
|
|
456
|
+
selected_model = self._settings.asr_model or (
|
|
457
|
+
self._settings.whisper_cpp_model
|
|
458
|
+
if selected_backend == "whisper_cpp"
|
|
459
|
+
else self._settings.faster_whisper_model
|
|
460
|
+
)
|
|
461
|
+
model_error = None
|
|
462
|
+
try:
|
|
463
|
+
selected_model = resolve_asr_model(self._settings)
|
|
464
|
+
except ModelUnavailableError as exc:
|
|
465
|
+
model_error = str(exc)
|
|
466
|
+
backends: dict[str, dict[str, Any]] = {}
|
|
467
|
+
|
|
468
|
+
faster_whisper: dict[str, Any] = {"distribution": "faster-whisper"}
|
|
469
|
+
try:
|
|
470
|
+
faster_whisper["version"] = self._package_version_getter("faster-whisper")
|
|
471
|
+
except PackageNotFoundError:
|
|
472
|
+
faster_whisper["status"] = "missing"
|
|
473
|
+
faster_whisper["verification"] = VERIFICATION_NOT_VERIFIED
|
|
474
|
+
faster_whisper["fix_hint"] = FIX_HINT_ASR
|
|
475
|
+
else:
|
|
476
|
+
faster_whisper["status"] = "ok"
|
|
477
|
+
faster_whisper["verification"] = VERIFICATION_CONFIGURED_ONLY
|
|
478
|
+
model_name = selected_model if selected_backend == "faster_whisper" else self._settings.faster_whisper_model
|
|
479
|
+
model_cached = self._faster_whisper_model_cached(model_name)
|
|
480
|
+
if selected_backend == "faster_whisper" and model_error is not None:
|
|
481
|
+
model_cached = False
|
|
482
|
+
faster_whisper["model"] = model_name
|
|
483
|
+
faster_whisper["model_available_locally"] = model_cached
|
|
484
|
+
if faster_whisper["status"] == "ok" and model_cached:
|
|
485
|
+
faster_whisper["verification"] = VERIFICATION_VERIFIED_LOCALLY
|
|
486
|
+
elif faster_whisper["status"] == "ok":
|
|
487
|
+
faster_whisper["status"] = "partial"
|
|
488
|
+
faster_whisper["fix_hint"] = "Provision nonempty model.bin, config.json and tokenizer.json in the selected local ASR directory."
|
|
489
|
+
backends["faster-whisper"] = faster_whisper
|
|
490
|
+
|
|
491
|
+
whisper_binary = self._settings.whisper_cpp_binary
|
|
492
|
+
whisper_cpp: dict[str, Any] = {"backend": "whisper.cpp", "binary": whisper_binary}
|
|
493
|
+
if not whisper_binary:
|
|
494
|
+
whisper_cpp["status"] = "not_configured"
|
|
495
|
+
whisper_cpp["verification"] = VERIFICATION_NOT_VERIFIED
|
|
496
|
+
whisper_cpp["fix_hint"] = (
|
|
497
|
+
"Build whisper.cpp, then set YTK_WHISPER_CPP_BINARY=/path/to/whisper-cli in .env"
|
|
498
|
+
)
|
|
499
|
+
else:
|
|
500
|
+
resolved = self._which(whisper_binary)
|
|
501
|
+
if resolved is None:
|
|
502
|
+
whisper_cpp["status"] = "missing"
|
|
503
|
+
whisper_cpp["verification"] = VERIFICATION_NOT_VERIFIED
|
|
504
|
+
whisper_cpp["fix_hint"] = FIX_HINT_ASR
|
|
505
|
+
else:
|
|
506
|
+
whisper_cpp["path"] = resolved
|
|
507
|
+
whisper_cpp["status"] = "ok"
|
|
508
|
+
whisper_cpp["verification"] = VERIFICATION_CONFIGURED_ONLY
|
|
509
|
+
whisper_model = selected_model if selected_backend == "whisper_cpp" else self._settings.whisper_cpp_model
|
|
510
|
+
whisper_cpp["model"] = whisper_model
|
|
511
|
+
whisper_cpp["model_exists"] = bool(
|
|
512
|
+
whisper_model and self._nonempty_file(Path(whisper_model).expanduser())
|
|
513
|
+
)
|
|
514
|
+
if selected_backend == "whisper_cpp" and model_error is not None:
|
|
515
|
+
whisper_cpp["model_exists"] = False
|
|
516
|
+
if whisper_cpp["status"] == "ok" and whisper_cpp["model_exists"]:
|
|
517
|
+
whisper_cpp["verification"] = VERIFICATION_VERIFIED_LOCALLY
|
|
518
|
+
elif whisper_cpp["status"] == "ok":
|
|
519
|
+
whisper_cpp["status"] = "partial"
|
|
520
|
+
whisper_cpp["fix_hint"] = "Provision a nonempty local whisper.cpp model file and set YTK_ASR_MODEL or YTK_WHISPER_CPP_MODEL."
|
|
521
|
+
backends["whisper-cpp"] = whisper_cpp
|
|
522
|
+
|
|
523
|
+
selected_report = backends["whisper-cpp" if selected_backend == "whisper_cpp" else "faster-whisper"]
|
|
524
|
+
status = selected_report["status"]
|
|
525
|
+
report: dict[str, Any] = {
|
|
526
|
+
"status": status,
|
|
527
|
+
"verification": selected_report["verification"],
|
|
528
|
+
"selected_backend": selected_backend,
|
|
529
|
+
"selected_model": str(Path(selected_model).expanduser()) if selected_model else None,
|
|
530
|
+
"backends": backends,
|
|
531
|
+
"note": (
|
|
532
|
+
"offline mode: ASR backend distribution/binary presence and "
|
|
533
|
+
"local model files are checked; model downloads are never "
|
|
534
|
+
"attempted and transcription is NOT verified."
|
|
535
|
+
),
|
|
536
|
+
}
|
|
537
|
+
if model_error is not None:
|
|
538
|
+
report["model_error"] = model_error
|
|
539
|
+
if status != "ok":
|
|
540
|
+
report["fix_hint"] = model_error or selected_report.get("fix_hint", FIX_HINT_ASR)
|
|
541
|
+
return report
|
|
542
|
+
|
|
543
|
+
@staticmethod
|
|
544
|
+
def _nonempty_file(path: Path) -> bool:
|
|
545
|
+
try:
|
|
546
|
+
return path.is_file() and path.stat().st_size > 0
|
|
547
|
+
except OSError:
|
|
548
|
+
return False
|
|
549
|
+
|
|
550
|
+
@staticmethod
|
|
551
|
+
def _hf_cache_roots() -> list[Path]:
|
|
552
|
+
roots: list[Path] = []
|
|
553
|
+
for env_name in ("HUGGINGFACE_HUB_CACHE", "HF_HOME"):
|
|
554
|
+
value = os.environ.get(env_name)
|
|
555
|
+
if not value:
|
|
556
|
+
continue
|
|
557
|
+
path = Path(value).expanduser()
|
|
558
|
+
roots.append(path / "hub" if env_name == "HF_HOME" else path)
|
|
559
|
+
if not roots:
|
|
560
|
+
roots.append(Path.home() / ".cache" / "huggingface" / "hub")
|
|
561
|
+
return roots
|
|
562
|
+
|
|
563
|
+
@classmethod
|
|
564
|
+
def _faster_whisper_model_cached(cls, model: str | None) -> bool:
|
|
565
|
+
if not model:
|
|
566
|
+
return False
|
|
567
|
+
root = Path(model).expanduser()
|
|
568
|
+
return all(cls._nonempty_file(root / name) for name in ("model.bin", "config.json", "tokenizer.json"))
|
|
569
|
+
|
|
570
|
+
def _check_ocr_support(self) -> dict[str, Any]:
|
|
571
|
+
backends: dict[str, dict[str, Any]] = {}
|
|
572
|
+
for distribution in ("rapidocr", "onnxruntime"):
|
|
573
|
+
entry: dict[str, Any] = {"distribution": distribution}
|
|
574
|
+
try:
|
|
575
|
+
entry["version"] = self._package_version_getter(distribution)
|
|
576
|
+
except PackageNotFoundError:
|
|
577
|
+
entry["status"] = "missing"
|
|
578
|
+
entry["verification"] = VERIFICATION_NOT_VERIFIED
|
|
579
|
+
entry["fix_hint"] = FIX_HINT_OCR
|
|
580
|
+
else:
|
|
581
|
+
entry["status"] = "ok"
|
|
582
|
+
entry["verification"] = VERIFICATION_VERIFIED_LOCALLY
|
|
583
|
+
backends[distribution] = entry
|
|
584
|
+
|
|
585
|
+
models = self._rapidocr_model_report()
|
|
586
|
+
status = self._aggregate_status(backends)
|
|
587
|
+
if status == "ok" and models["status"] != "ok":
|
|
588
|
+
status = "partial"
|
|
589
|
+
report: dict[str, Any] = {
|
|
590
|
+
"status": status,
|
|
591
|
+
"verification": (
|
|
592
|
+
VERIFICATION_VERIFIED_LOCALLY if status == "ok" else VERIFICATION_NOT_VERIFIED
|
|
593
|
+
),
|
|
594
|
+
"backends": backends,
|
|
595
|
+
"models": models,
|
|
596
|
+
"note": (
|
|
597
|
+
"offline mode: OCR runtime distributions and selected local ONNX "
|
|
598
|
+
"model files are checked; inference and model downloads are "
|
|
599
|
+
"NOT exercised."
|
|
600
|
+
),
|
|
601
|
+
}
|
|
602
|
+
if status != "ok":
|
|
603
|
+
report["fix_hint"] = FIX_HINT_OCR
|
|
604
|
+
return report
|
|
605
|
+
|
|
606
|
+
def _rapidocr_model_report(self) -> dict[str, Any]:
|
|
607
|
+
from ytk.ocr.engine import DEFAULT_MODEL_FILENAMES
|
|
608
|
+
|
|
609
|
+
root = self._settings.resolved_ocr_model_dir()
|
|
610
|
+
selected = [root / name for name in DEFAULT_MODEL_FILENAMES.values()]
|
|
611
|
+
sizes: dict[Path, int] = {}
|
|
612
|
+
for path in selected:
|
|
613
|
+
try:
|
|
614
|
+
size = path.stat().st_size if path.is_file() else 0
|
|
615
|
+
except OSError:
|
|
616
|
+
size = 0
|
|
617
|
+
if size > 0:
|
|
618
|
+
sizes[path] = size
|
|
619
|
+
missing = [path.name for path in selected if path not in sizes]
|
|
620
|
+
report: dict[str, Any] = {
|
|
621
|
+
"component": "RapidOCR ONNX models",
|
|
622
|
+
"model_dir": str(root),
|
|
623
|
+
"status": "missing" if missing else "ok",
|
|
624
|
+
"model_files": len(sizes),
|
|
625
|
+
"total_bytes": sum(sizes.values()),
|
|
626
|
+
"missing": missing,
|
|
627
|
+
"verification": VERIFICATION_NOT_VERIFIED if missing else VERIFICATION_VERIFIED_LOCALLY,
|
|
628
|
+
}
|
|
629
|
+
if missing:
|
|
630
|
+
report["fix_hint"] = f"Provision the three nonempty RapidOCR model files in {root}: {', '.join(missing)}."
|
|
631
|
+
return report
|
|
632
|
+
|
|
633
|
+
def _check_sidecar_configuration(self) -> dict[str, Any]:
|
|
634
|
+
services: dict[str, dict[str, Any]] = {}
|
|
635
|
+
configured_count = 0
|
|
636
|
+
for service_name, candidates in _SIDECAR_CONFIG.items():
|
|
637
|
+
value: str | None = None
|
|
638
|
+
source: str | None = None
|
|
639
|
+
for attribute in candidates["attributes"]:
|
|
640
|
+
candidate = getattr(self._settings, attribute, None)
|
|
641
|
+
if candidate:
|
|
642
|
+
value, source = str(candidate), f"settings.{attribute}"
|
|
643
|
+
break
|
|
644
|
+
if value is None:
|
|
645
|
+
for env_name in candidates["env"]:
|
|
646
|
+
candidate = os.environ.get(env_name)
|
|
647
|
+
if candidate:
|
|
648
|
+
value, source = candidate, f"env.{env_name}"
|
|
649
|
+
break
|
|
650
|
+
entry: dict[str, Any] = {
|
|
651
|
+
"configured": value is not None,
|
|
652
|
+
"required": service_name != "xiaohongshu_mcp",
|
|
653
|
+
"source": source,
|
|
654
|
+
"verification": (
|
|
655
|
+
VERIFICATION_CONFIGURED_ONLY if value else VERIFICATION_NOT_VERIFIED
|
|
656
|
+
),
|
|
657
|
+
}
|
|
658
|
+
if value is not None:
|
|
659
|
+
configured_count += int(entry["required"])
|
|
660
|
+
entry["url"] = self._redact_service_url(value)
|
|
661
|
+
if service_name == "xiaohongshu_mcp":
|
|
662
|
+
entry.update(native_path="yt-dlp", live_verified=False,
|
|
663
|
+
note="Optional sidecar; native video acquisition may be attempted. Actual platform access remains unverified.")
|
|
664
|
+
services[service_name] = entry
|
|
665
|
+
|
|
666
|
+
required_count = sum(bool(entry["required"]) for entry in services.values())
|
|
667
|
+
if configured_count == required_count:
|
|
668
|
+
status = "configured"
|
|
669
|
+
elif configured_count:
|
|
670
|
+
status = "partial"
|
|
671
|
+
else:
|
|
672
|
+
status = "not_configured"
|
|
673
|
+
report: dict[str, Any] = {
|
|
674
|
+
"status": status,
|
|
675
|
+
"verification": (
|
|
676
|
+
VERIFICATION_CONFIGURED_ONLY if configured_count else VERIFICATION_NOT_VERIFIED
|
|
677
|
+
),
|
|
678
|
+
"services": services,
|
|
679
|
+
"note": (
|
|
680
|
+
"offline mode: sidecar configuration presence only; DTK/"
|
|
681
|
+
"xiaohongshu-mcp reachability and platform access are NOT "
|
|
682
|
+
"verified."
|
|
683
|
+
),
|
|
684
|
+
}
|
|
685
|
+
if status != "configured":
|
|
686
|
+
report["fix_hint"] = FIX_HINT_SIDECAR
|
|
687
|
+
return report
|
|
688
|
+
|
|
689
|
+
@staticmethod
|
|
690
|
+
def _redact_service_url(value: str) -> str:
|
|
691
|
+
"""Keep scheme+host+port only; never echo credentials or query tokens."""
|
|
692
|
+
try:
|
|
693
|
+
parts = urlsplit(value)
|
|
694
|
+
host = parts.hostname or ""
|
|
695
|
+
if ":" in host:
|
|
696
|
+
host = f"[{host}]"
|
|
697
|
+
netloc = f"{host}:{parts.port}" if parts.port is not None else host
|
|
698
|
+
except ValueError:
|
|
699
|
+
return "***"
|
|
700
|
+
if not parts.scheme or not netloc:
|
|
701
|
+
return "***"
|
|
702
|
+
return f"{parts.scheme}://{netloc}"
|
|
703
|
+
|
|
704
|
+
def _check_transcript_connectivity(self, *, timeout_seconds: int) -> dict[str, Any]:
|
|
705
|
+
result = self._transcript_connectivity_checker(
|
|
706
|
+
self._settings,
|
|
707
|
+
timeout_seconds=timeout_seconds,
|
|
708
|
+
)
|
|
709
|
+
report = dict(result)
|
|
710
|
+
report.setdefault("status", "ok")
|
|
711
|
+
connectivity_hint = self._fix_hint_for_connectivity(report["status"])
|
|
712
|
+
if connectivity_hint is not None:
|
|
713
|
+
report.setdefault("fix_hint", connectivity_hint)
|
|
714
|
+
return report
|
|
715
|
+
|
|
716
|
+
@staticmethod
|
|
717
|
+
def _fix_hint_for_connectivity(status: str) -> str | None:
|
|
718
|
+
if status == "missing":
|
|
719
|
+
return FIX_HINT_TRANSCRIPT_MISSING
|
|
720
|
+
if status == "blocked":
|
|
721
|
+
return FIX_HINT_TRANSCRIPT_PROXY
|
|
722
|
+
if status == "timeout":
|
|
723
|
+
return FIX_HINT_TRANSCRIPT_TIMEOUT
|
|
724
|
+
if status == "partial":
|
|
725
|
+
return FIX_HINT_TRANSCRIPT_PARTIAL
|
|
726
|
+
return None
|
|
727
|
+
|
|
728
|
+
def _proxy_report(self) -> dict[str, Any]:
|
|
729
|
+
mode = self._settings.transcript_proxy_mode
|
|
730
|
+
report: dict[str, Any] = {"mode": mode, "configured": False}
|
|
731
|
+
|
|
732
|
+
if mode == "generic":
|
|
733
|
+
report["http_url"] = self._redact_proxy_url(self._settings.transcript_http_proxy_url)
|
|
734
|
+
report["https_url"] = self._redact_proxy_url(self._settings.transcript_https_proxy_url)
|
|
735
|
+
missing_fields = [
|
|
736
|
+
field_name
|
|
737
|
+
for field_name, value in (
|
|
738
|
+
("transcript_http_proxy_url", self._settings.transcript_http_proxy_url),
|
|
739
|
+
("transcript_https_proxy_url", self._settings.transcript_https_proxy_url),
|
|
740
|
+
)
|
|
741
|
+
if not value
|
|
742
|
+
]
|
|
743
|
+
report["configured"] = not missing_fields
|
|
744
|
+
if missing_fields:
|
|
745
|
+
report["missing_fields"] = missing_fields
|
|
746
|
+
elif mode == "webshare":
|
|
747
|
+
report["username"] = self._settings.webshare_proxy_username
|
|
748
|
+
report["regions"] = list(self._settings.webshare_proxy_regions or ())
|
|
749
|
+
missing_fields = [
|
|
750
|
+
field_name
|
|
751
|
+
for field_name, value in (
|
|
752
|
+
("webshare_proxy_username", self._settings.webshare_proxy_username),
|
|
753
|
+
("webshare_proxy_password", self._settings.webshare_proxy_password),
|
|
754
|
+
)
|
|
755
|
+
if not value
|
|
756
|
+
]
|
|
757
|
+
report["configured"] = not missing_fields
|
|
758
|
+
if missing_fields:
|
|
759
|
+
report["missing_fields"] = missing_fields
|
|
760
|
+
else:
|
|
761
|
+
report["configured"] = False
|
|
762
|
+
|
|
763
|
+
return report
|
|
764
|
+
|
|
765
|
+
@staticmethod
|
|
766
|
+
def _run_version_command(command: Sequence[str]) -> str:
|
|
767
|
+
completed = subprocess.run(
|
|
768
|
+
list(command),
|
|
769
|
+
check=True,
|
|
770
|
+
capture_output=True,
|
|
771
|
+
text=True,
|
|
772
|
+
timeout=5,
|
|
773
|
+
)
|
|
774
|
+
return completed.stdout.strip() or completed.stderr.strip()
|
|
775
|
+
|
|
776
|
+
@staticmethod
|
|
777
|
+
def _redact_proxy_url(value: str | None) -> str | None:
|
|
778
|
+
if not value:
|
|
779
|
+
return value
|
|
780
|
+
|
|
781
|
+
try:
|
|
782
|
+
parts = urlsplit(value)
|
|
783
|
+
sanitized = urlsplit(redact_url(value))
|
|
784
|
+
port = f":{parts.port}" if parts.port is not None else ""
|
|
785
|
+
except ValueError:
|
|
786
|
+
return "<invalid proxy URL>"
|
|
787
|
+
if parts.username is None and parts.password is None:
|
|
788
|
+
return urlunsplit((parts.scheme, sanitized.netloc, parts.path, sanitized.query, ""))
|
|
789
|
+
|
|
790
|
+
hostname = parts.hostname or ""
|
|
791
|
+
if ":" in hostname:
|
|
792
|
+
hostname = f"[{hostname}]"
|
|
793
|
+
redacted_netloc = f"***:***@{hostname}{port}"
|
|
794
|
+
return urlunsplit((parts.scheme, redacted_netloc, parts.path, sanitized.query, ""))
|
|
795
|
+
|
|
796
|
+
@staticmethod
|
|
797
|
+
def _default_transcript_connectivity_checker(
|
|
798
|
+
settings: AppSettings,
|
|
799
|
+
*,
|
|
800
|
+
timeout_seconds: int,
|
|
801
|
+
) -> dict[str, Any]:
|
|
802
|
+
if YouTubeTranscriptApi is None:
|
|
803
|
+
return {
|
|
804
|
+
"status": "missing",
|
|
805
|
+
"note": "youtube-transcript-api is not installed.",
|
|
806
|
+
"fix_hint": FIX_HINT_TRANSCRIPT_MISSING,
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
def _probe() -> None:
|
|
810
|
+
kwargs: dict[str, Any] = {}
|
|
811
|
+
proxy_config = TranscriptApiProvider.build_proxy_config(settings)
|
|
812
|
+
if proxy_config is not None:
|
|
813
|
+
kwargs["proxy_config"] = proxy_config
|
|
814
|
+
client = YouTubeTranscriptApi(**kwargs)
|
|
815
|
+
# Configurable probe id (§4.3): the hardcoded Rick Astley id causes a
|
|
816
|
+
# false "blocked" if that video is ever taken down. Override via env.
|
|
817
|
+
# `or` (not getenv default) so a set-but-EMPTY var still falls back.
|
|
818
|
+
probe_id = os.getenv("YTK_DOCTOR_PROBE_ID") or "dQw4w9WgXcQ"
|
|
819
|
+
transcript_list = client.list(probe_id)
|
|
820
|
+
try:
|
|
821
|
+
transcript = transcript_list.find_generated_transcript(["en"])
|
|
822
|
+
except Exception:
|
|
823
|
+
transcript = transcript_list.find_transcript(["en"])
|
|
824
|
+
transcript.fetch()
|
|
825
|
+
|
|
826
|
+
error_holder: list[BaseException] = []
|
|
827
|
+
|
|
828
|
+
def _run_probe() -> None:
|
|
829
|
+
try:
|
|
830
|
+
_probe()
|
|
831
|
+
except BaseException as exc: # pragma: no cover - exercised through return contract
|
|
832
|
+
error_holder.append(exc)
|
|
833
|
+
|
|
834
|
+
try:
|
|
835
|
+
thread = threading.Thread(target=_run_probe, daemon=True)
|
|
836
|
+
thread.start()
|
|
837
|
+
thread.join(timeout=timeout_seconds)
|
|
838
|
+
except Exception as exc:
|
|
839
|
+
return {
|
|
840
|
+
"status": "blocked",
|
|
841
|
+
"error": str(exc),
|
|
842
|
+
"note": "YouTube transcript requests appear blocked. Configure a proxy.",
|
|
843
|
+
"fix_hint": FIX_HINT_TRANSCRIPT_PROXY,
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
if thread.is_alive():
|
|
847
|
+
return {
|
|
848
|
+
"status": "timeout",
|
|
849
|
+
"note": "Connection timed out. Check network or proxy.",
|
|
850
|
+
"fix_hint": FIX_HINT_TRANSCRIPT_TIMEOUT,
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
if error_holder:
|
|
854
|
+
probe_error = error_holder[0]
|
|
855
|
+
return {
|
|
856
|
+
"status": "blocked",
|
|
857
|
+
"error": str(probe_error),
|
|
858
|
+
"note": "YouTube transcript requests appear blocked. Configure a proxy.",
|
|
859
|
+
"fix_hint": FIX_HINT_TRANSCRIPT_PROXY,
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
return {
|
|
863
|
+
"status": "ok",
|
|
864
|
+
"note": "Transcript API connectivity confirmed",
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def format_doctor_report(report: dict[str, dict[str, Any]]) -> str:
|
|
869
|
+
return json.dumps(report, indent=2, ensure_ascii=False)
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def doctor_report_failures(report: dict[str, dict[str, Any]]) -> list[str]:
|
|
873
|
+
"""Section names whose local check failed in an offline report.
|
|
874
|
+
|
|
875
|
+
Intended for the CLI/install/CI chokepoint: ``doctor --offline`` should be
|
|
876
|
+
informative but must exit non-zero when a required local capability is
|
|
877
|
+
missing. Network-dependent sections are reported as ``skipped`` and are
|
|
878
|
+
never counted as failures; ``configured``/``not_configured`` describe
|
|
879
|
+
optional credentials and are not failures either.
|
|
880
|
+
"""
|
|
881
|
+
failures: list[str] = []
|
|
882
|
+
for name, section in report.items():
|
|
883
|
+
status = section.get("status", "")
|
|
884
|
+
if status in {"missing", "partial"}:
|
|
885
|
+
failures.append(name)
|
|
886
|
+
selected = report.get("platforms", {}).get("platforms", {})
|
|
887
|
+
youtube_selected = not selected or bool(selected.get("youtube", {}).get("enabled"))
|
|
888
|
+
js_runtime = report.get("yt-dlp", {}).get("js_runtime")
|
|
889
|
+
if youtube_selected and isinstance(js_runtime, dict) and js_runtime.get("status") != "ok":
|
|
890
|
+
failures.append("yt-dlp.js_runtime")
|
|
891
|
+
return failures
|