kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
ytk/config.py ADDED
@@ -0,0 +1,757 @@
1
+ """Application settings (T43.1) — legacy-compatible plus v2 platform config.
2
+
3
+ Priority (highest first), as frozen in contracts-v2 §1.7:
4
+
5
+ 1. explicit CLI arguments (``overrides=`` on :meth:`AppSettings.load`),
6
+ 2. platform-level / v2 environment variables (``YTK_DOUYIN_COOKIES_FILE``,
7
+ ``YTK_ASR_BACKEND``, ``YTK_MAX_MEDIA_MB`` …),
8
+ 3. an explicit TOML config file (``--config``, or ``$YTK_CONFIG``),
9
+ 4. the legacy global environment variables (``YTK_MINIMAX_API_KEY`` …),
10
+ 5. field defaults.
11
+
12
+ Only keys that are actually present in a higher layer are applied, so an
13
+ object default can never overwrite a user-provided value (explicit-only
14
+ mapping). ``AppSettings()`` keeps its historical behaviour: field defaults
15
+ overridden by the environment.
16
+
17
+ Platform isolation is deliberate: the legacy global YouTube cookie
18
+ (``YTK_YT_DLP_COOKIES_FILE``) and transcript proxy are only inherited by the
19
+ ``youtube`` platform. TikTok/Douyin/Bilibili/Xiaohongshu must configure their
20
+ own ``[platforms.<name>] cookies_file`` / ``proxy`` (or platform env vars).
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import logging
26
+ import os
27
+ import re
28
+ import shutil
29
+ import sys
30
+ from collections.abc import Mapping
31
+ from pathlib import Path
32
+ from typing import Any, ClassVar, Literal
33
+ from urllib.parse import urlsplit
34
+
35
+ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
36
+
37
+ from ytk.errors import UserFacingError
38
+ from ytk.ingestion.providers.transcript_api import DEFAULT_LANG_PRIORITY
39
+ from ytk.models.source import SUPPORTED_PLATFORMS, Platform
40
+
41
+ LOGGER = logging.getLogger(__name__)
42
+
43
+ TranscriptProxyMode = Literal["generic", "webshare"]
44
+ AsrBackend = Literal["auto", "whisper_cpp", "faster_whisper"]
45
+ CloudKind = Literal["text", "media"]
46
+
47
+ NO_DOTENV_ENV_VAR = "YTK_NO_DOTENV"
48
+ CONFIG_PATH_ENV_VAR = "YTK_CONFIG"
49
+ DEFAULT_OCR_MODEL_DIR = str(Path.home() / ".cache" / "ytk-models" / "rapidocr" / "models")
50
+ _TRUE_VALUES = frozenset({"1", "true", "yes", "on"})
51
+
52
+
53
+ class ConfigFileError(UserFacingError):
54
+ """The explicit TOML config file is missing, malformed or has unknown keys."""
55
+
56
+
57
+ def resolve_credential_ref(
58
+ reference: str | None, *, environ: Mapping[str, str] | None = None
59
+ ) -> str | None:
60
+ """Resolve an explicit ``env:NAME`` reference without reading any file.
61
+
62
+ Error messages deliberately exclude both the reference and its value.
63
+ Resolve only the chosen credential, after per-request overrides.
64
+ """
65
+ if reference is None:
66
+ return None
67
+ if not isinstance(reference, str) or re.fullmatch(r"env:[A-Za-z_][A-Za-z0-9_]*", reference) is None:
68
+ raise ConfigFileError("credential_ref must name an environment variable using the env scheme")
69
+ source = os.environ if environ is None else environ
70
+ raw = source.get(reference[4:])
71
+ if not isinstance(raw, str) or not raw.strip():
72
+ raise ConfigFileError("credential_ref environment variable is missing or empty")
73
+ value = raw.strip()
74
+ if any(ord(char) < 33 or ord(char) > 126 for char in value) or "\r" in raw or "\n" in raw:
75
+ raise ConfigFileError("credential_ref must resolve to a single HTTP token")
76
+ return value
77
+
78
+
79
+ def _env_flag(raw: str | None) -> bool:
80
+ return str(raw or "").strip().lower() in _TRUE_VALUES
81
+
82
+
83
+ def _comma_tuple(raw: str | None) -> tuple[str, ...]:
84
+ return tuple(item.strip() for item in str(raw or "").split(",") if item.strip())
85
+
86
+
87
+ def _safe_int(raw: str | None, default: int) -> int:
88
+ try:
89
+ return int(str(raw).strip()) if raw not in (None, "") else default
90
+ except (TypeError, ValueError):
91
+ LOGGER.warning("ignoring invalid integer environment value %r", raw)
92
+ return default
93
+
94
+
95
+ def _safe_float(raw: str | None, default: float) -> float:
96
+ try:
97
+ return float(str(raw).strip()) if raw not in (None, "") else default
98
+ except (TypeError, ValueError):
99
+ LOGGER.warning("ignoring invalid float environment value %r", raw)
100
+ return default
101
+
102
+
103
+ def dotenv_disabled(environ: dict[str, str] | os._Environ[str] | None = None) -> bool:
104
+ """True when ``YTK_NO_DOTENV`` disables reading the project ``.env``."""
105
+ source = os.environ if environ is None else environ
106
+ return _env_flag(source.get(NO_DOTENV_ENV_VAR))
107
+
108
+
109
+ class AppSettings(BaseModel):
110
+ model_config = ConfigDict(extra="forbid")
111
+
112
+ # ------------------------------------------------------------------ legacy
113
+ _ENV_MAP: ClassVar[dict[str, str]] = {
114
+ "YTK_DEBUG": "debug",
115
+ "YTK_PROXY_MODE": "transcript_proxy_mode",
116
+ "YTK_HTTP_PROXY_URL": "transcript_http_proxy_url",
117
+ "YTK_HTTPS_PROXY_URL": "transcript_https_proxy_url",
118
+ "YTK_WEBSHARE_USERNAME": "webshare_proxy_username",
119
+ "YTK_WEBSHARE_PASSWORD": "webshare_proxy_password",
120
+ "YTK_WEBSHARE_REGIONS": "webshare_proxy_regions",
121
+ "YTK_WHISPER_CPP_BINARY": "whisper_cpp_binary",
122
+ "YTK_WHISPER_CPP_MODEL": "whisper_cpp_model",
123
+ "YTK_WHISPER_CPP_LANGUAGE": "whisper_cpp_language",
124
+ "YTK_FASTER_WHISPER_MODEL": "faster_whisper_model",
125
+ "YTK_FFMPEG_BINARY": "ffmpeg_binary",
126
+ "YTK_YT_DLP_BINARY": "yt_dlp_binary",
127
+ "YTK_YT_DLP_COOKIES_FILE": "yt_dlp_cookies_file",
128
+ "YTK_YT_DLP_COOKIES_FROM_BROWSER": "yt_dlp_cookies_from_browser",
129
+ "YTK_TRANSCRIPT_LANGUAGES": "transcript_languages",
130
+ "YTK_SENTENCE_TRANSFORMER_MODEL": "sentence_transformer_model",
131
+ "YTK_MINIMAX_API_KEY": "minimax_api_key",
132
+ "YTK_MINIMAX_REGION": "minimax_region",
133
+ "YTK_MINIMAX_MODEL": "minimax_model",
134
+ "YTK_MINIMAX_MAX_TOKENS": "minimax_max_tokens",
135
+ "YTK_MINIMAX_VIDEO_DETAIL": "minimax_video_detail",
136
+ "YTK_MINIMAX_VIDEO_FPS": "minimax_video_fps",
137
+ "YTK_MINIMAX_MAX_LONG_SIDE_PIXEL": "minimax_max_long_side_pixel",
138
+ "YTK_MINIMAX_MAX_VIDEO_MB": "minimax_max_video_mb",
139
+ "YTK_TRANSCRIPT_CACHE_ENABLED": "transcript_cache_enabled",
140
+ "YTK_TRANSCRIPT_CACHE_DIR": "transcript_cache_dir",
141
+ "YTK_DOC_LANGUAGE": "doc_language",
142
+ "YTK_KNOWLEDGE_DOC_TIMEOUT": "knowledge_doc_timeout",
143
+ "YTK_KNOWLEDGE_DOC_MAX_TOKENS": "knowledge_doc_max_tokens",
144
+ }
145
+ _BOOL_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
146
+ {"debug", "transcript_cache_enabled"}
147
+ )
148
+ _TUPLE_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
149
+ {"webshare_proxy_regions", "transcript_languages"}
150
+ )
151
+ _INT_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
152
+ {
153
+ "minimax_max_tokens",
154
+ "minimax_max_long_side_pixel",
155
+ "minimax_max_video_mb",
156
+ "knowledge_doc_max_tokens",
157
+ }
158
+ )
159
+ _FLOAT_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
160
+ {"minimax_video_fps", "knowledge_doc_timeout"}
161
+ )
162
+
163
+ # ---------------------------------------------------------------- defaults
164
+ debug: bool = False
165
+ transcript_proxy_mode: TranscriptProxyMode | None = None
166
+ transcript_http_proxy_url: str | None = None
167
+ transcript_https_proxy_url: str | None = None
168
+ webshare_proxy_username: str | None = None
169
+ webshare_proxy_password: str | None = None
170
+ webshare_proxy_regions: tuple[str, ...] | None = None
171
+ whisper_cpp_binary: str | None = None
172
+ whisper_cpp_model: str | None = None
173
+ whisper_cpp_language: str = "auto"
174
+ faster_whisper_model: str | None = "base.en"
175
+ ffmpeg_binary: str = "ffmpeg"
176
+ #: Empty means "no standalone CLI": consumers run ``<sys.executable> -m yt_dlp``.
177
+ yt_dlp_binary: str = ""
178
+ yt_dlp_cookies_file: str | None = None
179
+ yt_dlp_cookies_from_browser: str | None = None
180
+ transcript_languages: tuple[str, ...] = tuple(DEFAULT_LANG_PRIORITY)
181
+ transcript_cache_enabled: bool = True
182
+ transcript_cache_dir: str | None = None
183
+ sentence_transformer_model: str = "sentence-transformers/all-MiniLM-L6-v2"
184
+ minimax_api_key: str | None = None
185
+ minimax_region: Literal["cn", "global"] = "cn"
186
+ minimax_model: str = "MiniMax-M3"
187
+ minimax_max_tokens: int = 16384
188
+ minimax_video_detail: Literal["low", "default", "high"] = "low"
189
+ minimax_video_fps: float = 0.2
190
+ minimax_max_long_side_pixel: int = 640
191
+ minimax_max_video_mb: int = 512
192
+ doc_language: Literal["zh", "en", "auto"] | None = None
193
+ knowledge_doc_timeout: float = 300.0
194
+ knowledge_doc_max_tokens: int = 16384
195
+
196
+ # --------------------------------------------------------------- platforms
197
+ #: Platforms enabled for this machine. Everything defaults to enabled; a
198
+ #: missing sidecar still reports as ``not_configured`` (never as verified).
199
+ platforms_enabled: tuple[Platform, ...] = (
200
+ "youtube",
201
+ "tiktok",
202
+ "douyin",
203
+ "bilibili",
204
+ "xiaohongshu",
205
+ )
206
+ dtk_base_url: str | None = None
207
+ dtk_token: str | None = None
208
+ dtk_credential_ref: str | None = None
209
+ xhs_base_url: str | None = None
210
+ xhs_credential_ref: str | None = None
211
+
212
+ # Per-platform credentials/network. The legacy global YouTube cookie and
213
+ # transcript proxy are only inherited by YouTube (see helpers below).
214
+ youtube_cookies_file: str | None = None
215
+ tiktok_cookies_file: str | None = None
216
+ douyin_cookies_file: str | None = None
217
+ bilibili_cookies_file: str | None = None
218
+ xiaohongshu_cookies_file: str | None = None
219
+ youtube_proxy: str | None = None
220
+ tiktok_proxy: str | None = None
221
+ douyin_proxy: str | None = None
222
+ bilibili_proxy: str | None = None
223
+ xiaohongshu_proxy: str | None = None
224
+
225
+ # --------------------------------------------------------- ASR / OCR models
226
+ asr_backend: AsrBackend = "auto"
227
+ asr_model: str | None = None
228
+ asr_language: str | None = None
229
+ asr_beam_size: int = Field(default=5, ge=1, le=20)
230
+ asr_initial_prompt: str | None = Field(default=None, max_length=2000)
231
+ asr_hotwords: str | None = Field(default=None, max_length=1000)
232
+ asr_word_timestamps: bool = True
233
+ ocr_enabled: bool = False
234
+ ocr_model_dir: str = Field(default_factory=lambda: DEFAULT_OCR_MODEL_DIR)
235
+
236
+ # ----------------------------------------------------------------- budgets
237
+ max_media_mb: int = 512
238
+ max_duration_s: float | None = Field(default=None, gt=0, allow_inf_nan=False)
239
+ frame_budget: int = 24
240
+ #: Wall-clock budget for one task, local ASR included. Local recognition
241
+ #: scales with audio length (a 46-minute recording needs ~45 minutes of
242
+ #: ASR on this class of machine), so a 15-minute default killed real long
243
+ #: videos mid-run. This is a floor that fits ordinary media, not a promise
244
+ #: about very long recordings: raise YTK_TASK_DEADLINE_S for those.
245
+ task_deadline_s: float = Field(default=3600.0, gt=0, allow_inf_nan=False)
246
+ concurrency: int = 2
247
+
248
+ # ------------------------------------------------------------------- cloud
249
+ cloud_text_enabled: bool = False
250
+ cloud_media_enabled: bool = False
251
+
252
+ # ------------------------------------------------------------------- cache
253
+ cache_dir: str | None = None
254
+
255
+ # -------------------------------------------------------------- validation
256
+ @field_validator("asr_language", mode="before")
257
+ @classmethod
258
+ def _normalize_asr_language(cls, value: Any) -> str | None:
259
+ if value is None:
260
+ return None
261
+ if not isinstance(value, str):
262
+ raise ValueError("asr_language must be a language code such as zh or en, or auto")
263
+ language = value.strip().lower()
264
+ if language in {"", "auto"}:
265
+ return None
266
+ if re.fullmatch(r"[a-z]{2,3}", language) is None:
267
+ raise ValueError("asr_language must be a language code such as zh or en, or auto")
268
+ return language
269
+
270
+ @model_validator(mode="after")
271
+ def _validate_v2_fields(self) -> AppSettings:
272
+ unknown = [value for value in self.platforms_enabled if value not in SUPPORTED_PLATFORMS]
273
+ if unknown:
274
+ raise ValueError(f"unknown enabled platform(s): {sorted(set(unknown))}")
275
+ # de-duplicate but keep the canonical platform order
276
+ self.platforms_enabled = tuple(
277
+ value for value in SUPPORTED_PLATFORMS if value in set(self.platforms_enabled)
278
+ )
279
+ if self.max_media_mb < 0:
280
+ raise ValueError("max_media_mb must be >= 0")
281
+ if self.frame_budget < 0:
282
+ raise ValueError("frame_budget must be >= 0")
283
+ if self.concurrency < 1:
284
+ raise ValueError("concurrency must be >= 1")
285
+ if self.task_deadline_s <= 0:
286
+ raise ValueError("task_deadline_s must be > 0")
287
+ return self
288
+
289
+ # ------------------------------------------------------------- env loading
290
+ @model_validator(mode="before")
291
+ @classmethod
292
+ def _from_env(cls, values: Any, info: ValidationInfo) -> Any:
293
+ """Historical behaviour: ``AppSettings()`` reads the legacy env only.
294
+
295
+ v2 platform/backend env variables and TOML are applied by
296
+ :meth:`load`; instantiating ``AppSettings()`` directly keeps the exact
297
+ pre-v2 semantics (defaults + the historical ``YTK_*`` variables) so
298
+ existing call sites and diagnostics do not change meaning.
299
+ """
300
+ merged: dict[str, Any] = {} if not isinstance(values, dict) else dict(values)
301
+ if info.context and info.context.get("environment_loaded"):
302
+ return merged
303
+ for field_name, value in _legacy_env_values(os.environ).items():
304
+ merged.setdefault(field_name, value)
305
+ return merged
306
+
307
+ # ---------------------------------------------------------------- loading
308
+ @classmethod
309
+ def load(
310
+ cls,
311
+ *,
312
+ config_path: Path | str | None = None,
313
+ overrides: dict[str, Any] | None = None,
314
+ environ: dict[str, str] | os._Environ[str] | None = None,
315
+ ) -> AppSettings:
316
+ """Build settings with the frozen priority order.
317
+
318
+ Only explicitly provided keys are applied at each layer; defaults never
319
+ overwrite a user-provided value.
320
+ """
321
+ env = os.environ if environ is None else environ
322
+ explicit = {key: value for key, value in (overrides or {}).items() if value is not None}
323
+ values: dict[str, Any] = {}
324
+ values.update(_legacy_env_values(env)) # tier 4
325
+
326
+ toml_platform_enable: dict[str, bool] = {}
327
+ resolved_config = config_path if config_path is not None else env.get(CONFIG_PATH_ENV_VAR)
328
+ if resolved_config:
329
+ toml_values, toml_platform_enable = load_config_file(resolved_config)
330
+ values.update(toml_values) # tier 3
331
+
332
+ v2_values, v2_platform_enable = _v2_env_values_split(env)
333
+ values.update(v2_values) # tier 2
334
+
335
+ values.update(explicit) # tier 1
336
+
337
+ values["platforms_enabled"] = _resolve_platforms_enabled(
338
+ explicit.get("platforms_enabled"),
339
+ values.get("platforms_enabled"),
340
+ env,
341
+ toml_enable=toml_platform_enable,
342
+ env_enable=v2_platform_enable,
343
+ )
344
+ return cls.model_validate(values, context={"environment_loaded": True})
345
+
346
+ # -------------------------------------------------------------- helpers
347
+ @property
348
+ def platforms_enabled_set(self) -> frozenset[Platform]:
349
+ return frozenset(self.platforms_enabled)
350
+
351
+ def is_platform_enabled(self, platform: Platform) -> bool:
352
+ return platform in self.platforms_enabled_set
353
+
354
+ def cookies_file_for(self, platform: Platform) -> str | None:
355
+ """Per-platform cookie jar; the legacy global cookie is YouTube-only."""
356
+ if platform == "youtube":
357
+ return self.youtube_cookies_file or self.yt_dlp_cookies_file
358
+ return getattr(self, f"{platform}_cookies_file", None)
359
+
360
+ def cookies_from_browser_for(self, platform: Platform) -> str | None:
361
+ if platform == "youtube":
362
+ return self.yt_dlp_cookies_from_browser
363
+ return None
364
+
365
+ def proxy_for(self, platform: Platform) -> str | None:
366
+ """Per-platform proxy; legacy transcript proxy is YouTube-only."""
367
+ if platform == "youtube":
368
+ return (
369
+ self.youtube_proxy
370
+ or self.transcript_https_proxy_url
371
+ or self.transcript_http_proxy_url
372
+ )
373
+ return getattr(self, f"{platform}_proxy", None)
374
+
375
+ def base_url_for(self, platform: Platform) -> str | None:
376
+ if platform == "douyin":
377
+ return self.dtk_base_url
378
+ if platform == "xiaohongshu":
379
+ return self.xhs_base_url
380
+ return None
381
+
382
+ def credential_ref_for(self, platform: Platform) -> str | None:
383
+ if platform == "douyin":
384
+ return self.dtk_credential_ref
385
+ if platform == "xiaohongshu":
386
+ return self.xhs_credential_ref
387
+ return None
388
+
389
+ def resolved_ocr_model_dir(self) -> Path:
390
+ return Path(self.ocr_model_dir).expanduser()
391
+
392
+ def ytdlp_command(self) -> list[str]:
393
+ """Command prefix for yt-dlp: explicit binary, else ``python -m yt_dlp``."""
394
+ if self.yt_dlp_binary:
395
+ resolved = shutil.which(self.yt_dlp_binary) or self.yt_dlp_binary
396
+ return [resolved]
397
+ return [sys.executable, "-m", "yt_dlp"]
398
+
399
+ def cloud_allowed(self, platform: Platform, kind: CloudKind) -> bool:
400
+ """Cloud use is opt-in; legacy YouTube consent is honoured for YouTube only."""
401
+ explicit = (
402
+ self.cloud_text_enabled if kind == "text" else self.cloud_media_enabled
403
+ )
404
+ if f"cloud_{kind}_enabled" in self.model_fields_set:
405
+ return bool(explicit)
406
+ legacy_youtube_consent = platform == "youtube" and bool(self.minimax_api_key)
407
+ return bool(explicit or legacy_youtube_consent)
408
+
409
+ def redacted(self) -> dict[str, Any]:
410
+ """Credential-free settings view safe for logs/diagnostics/JSON output."""
411
+ from ytk.logging import redact_text
412
+
413
+ credential_fields = {
414
+ "minimax_api_key",
415
+ "dtk_token",
416
+ "webshare_proxy_username",
417
+ "webshare_proxy_password",
418
+ }
419
+ cookie_fields = {name for name in _field_names() if name.endswith("_cookies_file")}
420
+ proxy_fields = {
421
+ name for name in _field_names() if name.endswith("_proxy") or "proxy_url" in name
422
+ }
423
+ view: dict[str, Any] = {}
424
+ for name in _field_names():
425
+ value = getattr(self, name)
426
+ if name in credential_fields or name in cookie_fields or name in {"asr_initial_prompt", "asr_hotwords"}:
427
+ view[name] = "<configured>" if value else None
428
+ elif name in proxy_fields:
429
+ view[name] = (urlsplit(str(value)).hostname or "<configured>") if value else None
430
+ elif isinstance(value, tuple):
431
+ view[name] = list(value)
432
+ elif isinstance(value, str):
433
+ view[name] = redact_text(value)
434
+ else:
435
+ view[name] = value
436
+ return view
437
+
438
+
439
+ def _field_names() -> tuple[str, ...]:
440
+ return tuple(AppSettings.model_fields)
441
+
442
+
443
+ # --------------------------------------------------------------------------- #
444
+ # TOML config file
445
+ # --------------------------------------------------------------------------- #
446
+ _PLATFORM_FIELD_TEMPLATES: dict[str, str] = {
447
+ "cookies_file": "{platform}_cookies_file",
448
+ "proxy": "{platform}_proxy",
449
+ }
450
+ _SERVICE_BASE_URL_FIELD: dict[str, str] = {
451
+ "douyin": "dtk_base_url",
452
+ "xiaohongshu": "xhs_base_url",
453
+ }
454
+ _SERVICE_TOKEN_FIELD: dict[str, str | None] = {
455
+ "douyin": "dtk_token",
456
+ "xiaohongshu": None,
457
+ }
458
+ _SERVICE_CRED_FIELD: dict[str, str] = {
459
+ "douyin": "dtk_credential_ref",
460
+ "xiaohongshu": "xhs_credential_ref",
461
+ }
462
+ _TOP_LEVEL_GROUPS: dict[str, dict[str, str]] = {
463
+ "asr": {
464
+ "backend": "asr_backend", "model": "asr_model", "language": "asr_language",
465
+ "beam_size": "asr_beam_size", "initial_prompt": "asr_initial_prompt",
466
+ "hotwords": "asr_hotwords", "word_timestamps": "asr_word_timestamps",
467
+ },
468
+ "ocr": {"enabled": "ocr_enabled", "model_dir": "ocr_model_dir"},
469
+ "budgets": {
470
+ "max_media_mb": "max_media_mb",
471
+ "max_duration_s": "max_duration_s",
472
+ "frame_budget": "frame_budget",
473
+ "task_deadline_s": "task_deadline_s",
474
+ "concurrency": "concurrency",
475
+ },
476
+ "cloud": {"text_enabled": "cloud_text_enabled", "media_enabled": "cloud_media_enabled"},
477
+ "cache": {"dir": "cache_dir"},
478
+ }
479
+ _PLATFORM_ENABLE_KEY = "enabled"
480
+
481
+
482
+ def load_config_file(
483
+ path: Path | str,
484
+ ) -> tuple[dict[str, Any], dict[str, bool]]:
485
+ """Load an explicit TOML config file into field overrides + platform flags.
486
+
487
+ Only explicitly present keys are returned. Unknown keys/sections are a
488
+ clear error instead of being silently ignored (config typos must not look
489
+ like working settings).
490
+ """
491
+ import tomllib
492
+
493
+ target = Path(path).expanduser()
494
+ try:
495
+ payload = tomllib.loads(target.read_text(encoding="utf-8"))
496
+ except FileNotFoundError as exc:
497
+ raise ConfigFileError(f"config file not found: {target}") from exc
498
+ except OSError as exc:
499
+ raise ConfigFileError(f"cannot read config file {target}: {exc}") from exc
500
+ except tomllib.TOMLDecodeError as exc:
501
+ raise ConfigFileError(f"invalid TOML in {target}: {exc}") from exc
502
+
503
+ known_fields = set(AppSettings.model_fields)
504
+ values: dict[str, Any] = {}
505
+ platform_enable: dict[str, bool] = {}
506
+ unknown: list[str] = []
507
+
508
+ def _assign(key: str, value: Any, context: str) -> None:
509
+ if key == "platforms_enabled":
510
+ values[key] = _coerce_platforms(value, context)
511
+ return
512
+ if key not in known_fields:
513
+ unknown.append(context)
514
+ return
515
+ values[key] = value
516
+
517
+ for key, value in payload.items():
518
+ if key == "platforms":
519
+ if not isinstance(value, dict):
520
+ raise ConfigFileError("[platforms] must be a table")
521
+ for platform, section in value.items():
522
+ if platform not in SUPPORTED_PLATFORMS:
523
+ raise ConfigFileError(f"unknown platform in config: [platforms.{platform}]")
524
+ if not isinstance(section, dict):
525
+ raise ConfigFileError(f"[platforms.{platform}] must be a table")
526
+ for section_key, section_value in section.items():
527
+ if section_key == _PLATFORM_ENABLE_KEY:
528
+ if not isinstance(section_value, bool):
529
+ raise ConfigFileError(
530
+ f"platforms.{platform}.enabled must be a boolean"
531
+ )
532
+ platform_enable[platform] = section_value
533
+ continue
534
+ template = _PLATFORM_FIELD_TEMPLATES.get(section_key)
535
+ if template is not None:
536
+ _assign(
537
+ template.format(platform=platform),
538
+ section_value,
539
+ f"platforms.{platform}.{section_key}",
540
+ )
541
+ continue
542
+ if section_key == "base_url" and platform in _SERVICE_BASE_URL_FIELD:
543
+ _assign(
544
+ _SERVICE_BASE_URL_FIELD[platform],
545
+ section_value,
546
+ f"platforms.{platform}.base_url",
547
+ )
548
+ continue
549
+ token_field = _SERVICE_TOKEN_FIELD.get(platform)
550
+ if section_key == "token" and token_field:
551
+ _assign(token_field, section_value, f"platforms.{platform}.token")
552
+ continue
553
+ if section_key == "credential_ref" and platform in _SERVICE_CRED_FIELD:
554
+ _assign(
555
+ _SERVICE_CRED_FIELD[platform],
556
+ section_value,
557
+ f"platforms.{platform}.credential_ref",
558
+ )
559
+ continue
560
+ unknown.append(f"platforms.{platform}.{section_key}")
561
+ continue
562
+ group = _TOP_LEVEL_GROUPS.get(key)
563
+ if group is not None:
564
+ if not isinstance(value, dict):
565
+ raise ConfigFileError(f"[{key}] must be a table")
566
+ for section_key, section_value in value.items():
567
+ field = group.get(section_key)
568
+ if field is None:
569
+ unknown.append(f"{key}.{section_key}")
570
+ continue
571
+ _assign(field, section_value, f"{key}.{section_key}")
572
+ continue
573
+ _assign(key, value, key)
574
+
575
+ if unknown:
576
+ raise ConfigFileError(
577
+ "unknown config key(s): "
578
+ + ", ".join(sorted(unknown))
579
+ + ". See docs/implementation/five-platform/infra-cli.md for the accepted schema."
580
+ )
581
+ return values, platform_enable
582
+
583
+
584
+ def _coerce_platforms(value: Any, context: str) -> tuple[str, ...]:
585
+ if isinstance(value, str):
586
+ if value.strip().lower() in {"all", "*"}:
587
+ return tuple(SUPPORTED_PLATFORMS)
588
+ items = _comma_tuple(value)
589
+ elif isinstance(value, (list, tuple)):
590
+ items = tuple(str(item).strip() for item in value if str(item).strip())
591
+ else:
592
+ raise ConfigFileError(f"{context} must be a list of platform names")
593
+ unknown = [item for item in items if item not in SUPPORTED_PLATFORMS]
594
+ if unknown:
595
+ raise ConfigFileError(f"{context} contains unknown platform(s): {sorted(set(unknown))}")
596
+ return tuple(item for item in SUPPORTED_PLATFORMS if item in set(items))
597
+
598
+
599
+ # --------------------------------------------------------------------------- #
600
+ # Environment parsing
601
+ # --------------------------------------------------------------------------- #
602
+ #: v2 environment variables (tier 2). Applied in order, so platform-specific
603
+ #: aliases can override the generic names for their platform.
604
+ _V2_ENV_MAP: tuple[tuple[str, str], ...] = (
605
+ ("YTK_DTK_BASE_URL", "dtk_base_url"),
606
+ ("YTK_DTK_TOKEN", "dtk_token"),
607
+ ("YTK_DTK_CREDENTIAL_REF", "dtk_credential_ref"),
608
+ ("YTK_XHS_BASE_URL", "xhs_base_url"),
609
+ ("YTK_XHS_CREDENTIAL_REF", "xhs_credential_ref"),
610
+ ("YTK_ASR_BACKEND", "asr_backend"),
611
+ ("YTK_ASR_MODEL", "asr_model"),
612
+ ("YTK_ASR_LANGUAGE", "asr_language"),
613
+ ("YTK_ASR_BEAM_SIZE", "asr_beam_size"),
614
+ ("YTK_ASR_INITIAL_PROMPT", "asr_initial_prompt"),
615
+ ("YTK_ASR_HOTWORDS", "asr_hotwords"),
616
+ ("YTK_ASR_WORD_TIMESTAMPS", "asr_word_timestamps"),
617
+ ("YTK_OCR_ENABLED", "ocr_enabled"),
618
+ ("YTK_OCR_MODEL_DIR", "ocr_model_dir"),
619
+ ("YTK_MAX_MEDIA_MB", "max_media_mb"),
620
+ ("YTK_MAX_DURATION_S", "max_duration_s"),
621
+ ("YTK_FRAME_BUDGET", "frame_budget"),
622
+ ("YTK_TASK_DEADLINE_S", "task_deadline_s"),
623
+ ("YTK_CONCURRENCY", "concurrency"),
624
+ ("YTK_CLOUD_TEXT_ENABLED", "cloud_text_enabled"),
625
+ ("YTK_CLOUD_MEDIA_ENABLED", "cloud_media_enabled"),
626
+ ("YTK_CACHE_DIR", "cache_dir"),
627
+ ("YTK_V2_CACHE_DIR", "cache_dir"),
628
+ # platform-level aliases (contracts-v2 §1.7)
629
+ ("YTK_YOUTUBE_COOKIES_FILE", "youtube_cookies_file"),
630
+ ("YTK_TIKTOK_COOKIES_FILE", "tiktok_cookies_file"),
631
+ ("YTK_DOUYIN_COOKIES_FILE", "douyin_cookies_file"),
632
+ ("YTK_BILIBILI_COOKIES_FILE", "bilibili_cookies_file"),
633
+ ("YTK_XIAOHONGSHU_COOKIES_FILE", "xiaohongshu_cookies_file"),
634
+ ("YTK_YOUTUBE_PROXY", "youtube_proxy"),
635
+ ("YTK_TIKTOK_PROXY", "tiktok_proxy"),
636
+ ("YTK_DOUYIN_PROXY", "douyin_proxy"),
637
+ ("YTK_BILIBILI_PROXY", "bilibili_proxy"),
638
+ ("YTK_XIAOHONGSHU_PROXY", "xiaohongshu_proxy"),
639
+ ("YTK_DOUYIN_BASE_URL", "dtk_base_url"),
640
+ ("YTK_DOUYIN_TOKEN", "dtk_token"),
641
+ )
642
+
643
+ _V2_BOOL_FIELDS: frozenset[str] = frozenset(
644
+ {"ocr_enabled", "cloud_text_enabled", "cloud_media_enabled", "asr_word_timestamps"}
645
+ )
646
+ _V2_INT_FIELDS: frozenset[str] = frozenset({"max_media_mb", "frame_budget", "concurrency", "asr_beam_size"})
647
+ _V2_FLOAT_FIELDS: frozenset[str] = frozenset({"max_duration_s", "task_deadline_s"})
648
+
649
+
650
+ def _legacy_env_values(environ: dict[str, str] | os._Environ[str]) -> dict[str, Any]:
651
+ """Tier-4: the historical global env variables."""
652
+ values: dict[str, Any] = {}
653
+ for env_var, field_name in AppSettings._ENV_MAP.items():
654
+ raw = environ.get(env_var)
655
+ if raw is None:
656
+ continue
657
+ if field_name in AppSettings._BOOL_ENV_FIELDS:
658
+ values[field_name] = _env_flag(raw)
659
+ elif field_name in AppSettings._TUPLE_ENV_FIELDS:
660
+ values[field_name] = _comma_tuple(raw)
661
+ elif field_name in AppSettings._INT_ENV_FIELDS:
662
+ values[field_name] = _safe_int(raw, 0)
663
+ elif field_name in AppSettings._FLOAT_ENV_FIELDS:
664
+ values[field_name] = _safe_float(raw, 0.0)
665
+ else:
666
+ values[field_name] = raw
667
+ return values
668
+
669
+
670
+ def _v2_env_values_split(
671
+ environ: dict[str, str] | os._Environ[str],
672
+ ) -> tuple[dict[str, Any], dict[str, bool]]:
673
+ """Tier-2: v2 platform/backend environment variables + platform enable flags."""
674
+ values: dict[str, Any] = {}
675
+ for env_var, field_name in _V2_ENV_MAP:
676
+ raw = environ.get(env_var)
677
+ if raw is None:
678
+ continue
679
+ if field_name in _V2_BOOL_FIELDS:
680
+ values[field_name] = _env_flag(raw)
681
+ elif field_name in _V2_INT_FIELDS:
682
+ values[field_name] = _safe_int(raw, 0)
683
+ elif field_name in _V2_FLOAT_FIELDS:
684
+ values[field_name] = _safe_float(raw, 0.0)
685
+ else:
686
+ values[field_name] = raw
687
+ return values, _platform_env_flags(environ)
688
+
689
+
690
+ def _platform_env_flags(environ: dict[str, str] | os._Environ[str]) -> dict[str, bool]:
691
+ flags: dict[str, bool] = {}
692
+ for platform in SUPPORTED_PLATFORMS:
693
+ raw = environ.get(f"YTK_{platform.upper()}_ENABLED")
694
+ if raw is not None:
695
+ flags[platform] = _env_flag(raw)
696
+ return flags
697
+
698
+
699
+ def _resolve_platforms_enabled(
700
+ explicit: Any,
701
+ merged_value: Any,
702
+ environ: dict[str, str] | os._Environ[str],
703
+ *,
704
+ toml_enable: dict[str, bool] | None = None,
705
+ env_enable: dict[str, bool] | None = None,
706
+ ) -> tuple[str, ...]:
707
+ """Compute the effective enabled platform tuple (explicit > env > lower tiers)."""
708
+ if explicit is not None:
709
+ # An explicit whitelist is final: no lower layer may add a platform back.
710
+ return tuple(_normalise_platforms(explicit))
711
+ env_list = environ.get("YTK_PLATFORMS_ENABLED")
712
+ if env_list not in (None, ""):
713
+ base = _normalise_platforms(env_list)
714
+ adjustments = dict(env_enable or {}) # env whitelist > TOML enabled flags
715
+ else:
716
+ base = (
717
+ _normalise_platforms(merged_value)
718
+ if merged_value is not None
719
+ else list(SUPPORTED_PLATFORMS)
720
+ )
721
+ adjustments = {**(toml_enable or {}), **(env_enable or {})}
722
+ for platform, enabled in adjustments.items():
723
+ if platform not in SUPPORTED_PLATFORMS:
724
+ continue
725
+ if enabled and platform not in base:
726
+ base.append(platform)
727
+ elif not enabled and platform in base:
728
+ base.remove(platform)
729
+ return tuple(_normalise_platforms(base))
730
+
731
+
732
+ def _normalise_platforms(value: Any) -> list[str]:
733
+ if isinstance(value, str):
734
+ if value.strip().lower() in {"all", "*"}:
735
+ return list(SUPPORTED_PLATFORMS)
736
+ items = list(_comma_tuple(value))
737
+ elif isinstance(value, (list, tuple, set, frozenset)):
738
+ items = [str(item).strip() for item in value if str(item).strip()]
739
+ else:
740
+ raise ValueError("platforms_enabled must be a list of platform names")
741
+ unknown = [item for item in items if item not in SUPPORTED_PLATFORMS]
742
+ if unknown:
743
+ raise ValueError(f"unknown platform(s): {sorted(set(unknown))}")
744
+ return [platform for platform in SUPPORTED_PLATFORMS if platform in set(items)]
745
+
746
+
747
+ __all__ = [
748
+ "AppSettings",
749
+ "AsrBackend",
750
+ "CONFIG_PATH_ENV_VAR",
751
+ "ConfigFileError",
752
+ "DEFAULT_OCR_MODEL_DIR",
753
+ "NO_DOTENV_ENV_VAR",
754
+ "TranscriptProxyMode",
755
+ "dotenv_disabled",
756
+ "load_config_file",
757
+ ]