kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
ytk/config.py
ADDED
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
"""Application settings (T43.1) — legacy-compatible plus v2 platform config.
|
|
2
|
+
|
|
3
|
+
Priority (highest first), as frozen in contracts-v2 §1.7:
|
|
4
|
+
|
|
5
|
+
1. explicit CLI arguments (``overrides=`` on :meth:`AppSettings.load`),
|
|
6
|
+
2. platform-level / v2 environment variables (``YTK_DOUYIN_COOKIES_FILE``,
|
|
7
|
+
``YTK_ASR_BACKEND``, ``YTK_MAX_MEDIA_MB`` …),
|
|
8
|
+
3. an explicit TOML config file (``--config``, or ``$YTK_CONFIG``),
|
|
9
|
+
4. the legacy global environment variables (``YTK_MINIMAX_API_KEY`` …),
|
|
10
|
+
5. field defaults.
|
|
11
|
+
|
|
12
|
+
Only keys that are actually present in a higher layer are applied, so an
|
|
13
|
+
object default can never overwrite a user-provided value (explicit-only
|
|
14
|
+
mapping). ``AppSettings()`` keeps its historical behaviour: field defaults
|
|
15
|
+
overridden by the environment.
|
|
16
|
+
|
|
17
|
+
Platform isolation is deliberate: the legacy global YouTube cookie
|
|
18
|
+
(``YTK_YT_DLP_COOKIES_FILE``) and transcript proxy are only inherited by the
|
|
19
|
+
``youtube`` platform. TikTok/Douyin/Bilibili/Xiaohongshu must configure their
|
|
20
|
+
own ``[platforms.<name>] cookies_file`` / ``proxy`` (or platform env vars).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import logging
|
|
26
|
+
import os
|
|
27
|
+
import re
|
|
28
|
+
import shutil
|
|
29
|
+
import sys
|
|
30
|
+
from collections.abc import Mapping
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any, ClassVar, Literal
|
|
33
|
+
from urllib.parse import urlsplit
|
|
34
|
+
|
|
35
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
|
|
36
|
+
|
|
37
|
+
from ytk.errors import UserFacingError
|
|
38
|
+
from ytk.ingestion.providers.transcript_api import DEFAULT_LANG_PRIORITY
|
|
39
|
+
from ytk.models.source import SUPPORTED_PLATFORMS, Platform
|
|
40
|
+
|
|
41
|
+
LOGGER = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
TranscriptProxyMode = Literal["generic", "webshare"]
|
|
44
|
+
AsrBackend = Literal["auto", "whisper_cpp", "faster_whisper"]
|
|
45
|
+
CloudKind = Literal["text", "media"]
|
|
46
|
+
|
|
47
|
+
NO_DOTENV_ENV_VAR = "YTK_NO_DOTENV"
|
|
48
|
+
CONFIG_PATH_ENV_VAR = "YTK_CONFIG"
|
|
49
|
+
DEFAULT_OCR_MODEL_DIR = str(Path.home() / ".cache" / "ytk-models" / "rapidocr" / "models")
|
|
50
|
+
_TRUE_VALUES = frozenset({"1", "true", "yes", "on"})
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ConfigFileError(UserFacingError):
|
|
54
|
+
"""The explicit TOML config file is missing, malformed or has unknown keys."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def resolve_credential_ref(
|
|
58
|
+
reference: str | None, *, environ: Mapping[str, str] | None = None
|
|
59
|
+
) -> str | None:
|
|
60
|
+
"""Resolve an explicit ``env:NAME`` reference without reading any file.
|
|
61
|
+
|
|
62
|
+
Error messages deliberately exclude both the reference and its value.
|
|
63
|
+
Resolve only the chosen credential, after per-request overrides.
|
|
64
|
+
"""
|
|
65
|
+
if reference is None:
|
|
66
|
+
return None
|
|
67
|
+
if not isinstance(reference, str) or re.fullmatch(r"env:[A-Za-z_][A-Za-z0-9_]*", reference) is None:
|
|
68
|
+
raise ConfigFileError("credential_ref must name an environment variable using the env scheme")
|
|
69
|
+
source = os.environ if environ is None else environ
|
|
70
|
+
raw = source.get(reference[4:])
|
|
71
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
72
|
+
raise ConfigFileError("credential_ref environment variable is missing or empty")
|
|
73
|
+
value = raw.strip()
|
|
74
|
+
if any(ord(char) < 33 or ord(char) > 126 for char in value) or "\r" in raw or "\n" in raw:
|
|
75
|
+
raise ConfigFileError("credential_ref must resolve to a single HTTP token")
|
|
76
|
+
return value
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _env_flag(raw: str | None) -> bool:
|
|
80
|
+
return str(raw or "").strip().lower() in _TRUE_VALUES
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _comma_tuple(raw: str | None) -> tuple[str, ...]:
|
|
84
|
+
return tuple(item.strip() for item in str(raw or "").split(",") if item.strip())
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _safe_int(raw: str | None, default: int) -> int:
|
|
88
|
+
try:
|
|
89
|
+
return int(str(raw).strip()) if raw not in (None, "") else default
|
|
90
|
+
except (TypeError, ValueError):
|
|
91
|
+
LOGGER.warning("ignoring invalid integer environment value %r", raw)
|
|
92
|
+
return default
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _safe_float(raw: str | None, default: float) -> float:
|
|
96
|
+
try:
|
|
97
|
+
return float(str(raw).strip()) if raw not in (None, "") else default
|
|
98
|
+
except (TypeError, ValueError):
|
|
99
|
+
LOGGER.warning("ignoring invalid float environment value %r", raw)
|
|
100
|
+
return default
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def dotenv_disabled(environ: dict[str, str] | os._Environ[str] | None = None) -> bool:
|
|
104
|
+
"""True when ``YTK_NO_DOTENV`` disables reading the project ``.env``."""
|
|
105
|
+
source = os.environ if environ is None else environ
|
|
106
|
+
return _env_flag(source.get(NO_DOTENV_ENV_VAR))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class AppSettings(BaseModel):
|
|
110
|
+
model_config = ConfigDict(extra="forbid")
|
|
111
|
+
|
|
112
|
+
# ------------------------------------------------------------------ legacy
|
|
113
|
+
_ENV_MAP: ClassVar[dict[str, str]] = {
|
|
114
|
+
"YTK_DEBUG": "debug",
|
|
115
|
+
"YTK_PROXY_MODE": "transcript_proxy_mode",
|
|
116
|
+
"YTK_HTTP_PROXY_URL": "transcript_http_proxy_url",
|
|
117
|
+
"YTK_HTTPS_PROXY_URL": "transcript_https_proxy_url",
|
|
118
|
+
"YTK_WEBSHARE_USERNAME": "webshare_proxy_username",
|
|
119
|
+
"YTK_WEBSHARE_PASSWORD": "webshare_proxy_password",
|
|
120
|
+
"YTK_WEBSHARE_REGIONS": "webshare_proxy_regions",
|
|
121
|
+
"YTK_WHISPER_CPP_BINARY": "whisper_cpp_binary",
|
|
122
|
+
"YTK_WHISPER_CPP_MODEL": "whisper_cpp_model",
|
|
123
|
+
"YTK_WHISPER_CPP_LANGUAGE": "whisper_cpp_language",
|
|
124
|
+
"YTK_FASTER_WHISPER_MODEL": "faster_whisper_model",
|
|
125
|
+
"YTK_FFMPEG_BINARY": "ffmpeg_binary",
|
|
126
|
+
"YTK_YT_DLP_BINARY": "yt_dlp_binary",
|
|
127
|
+
"YTK_YT_DLP_COOKIES_FILE": "yt_dlp_cookies_file",
|
|
128
|
+
"YTK_YT_DLP_COOKIES_FROM_BROWSER": "yt_dlp_cookies_from_browser",
|
|
129
|
+
"YTK_TRANSCRIPT_LANGUAGES": "transcript_languages",
|
|
130
|
+
"YTK_SENTENCE_TRANSFORMER_MODEL": "sentence_transformer_model",
|
|
131
|
+
"YTK_MINIMAX_API_KEY": "minimax_api_key",
|
|
132
|
+
"YTK_MINIMAX_REGION": "minimax_region",
|
|
133
|
+
"YTK_MINIMAX_MODEL": "minimax_model",
|
|
134
|
+
"YTK_MINIMAX_MAX_TOKENS": "minimax_max_tokens",
|
|
135
|
+
"YTK_MINIMAX_VIDEO_DETAIL": "minimax_video_detail",
|
|
136
|
+
"YTK_MINIMAX_VIDEO_FPS": "minimax_video_fps",
|
|
137
|
+
"YTK_MINIMAX_MAX_LONG_SIDE_PIXEL": "minimax_max_long_side_pixel",
|
|
138
|
+
"YTK_MINIMAX_MAX_VIDEO_MB": "minimax_max_video_mb",
|
|
139
|
+
"YTK_TRANSCRIPT_CACHE_ENABLED": "transcript_cache_enabled",
|
|
140
|
+
"YTK_TRANSCRIPT_CACHE_DIR": "transcript_cache_dir",
|
|
141
|
+
"YTK_DOC_LANGUAGE": "doc_language",
|
|
142
|
+
"YTK_KNOWLEDGE_DOC_TIMEOUT": "knowledge_doc_timeout",
|
|
143
|
+
"YTK_KNOWLEDGE_DOC_MAX_TOKENS": "knowledge_doc_max_tokens",
|
|
144
|
+
}
|
|
145
|
+
_BOOL_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
|
|
146
|
+
{"debug", "transcript_cache_enabled"}
|
|
147
|
+
)
|
|
148
|
+
_TUPLE_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
|
|
149
|
+
{"webshare_proxy_regions", "transcript_languages"}
|
|
150
|
+
)
|
|
151
|
+
_INT_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
|
|
152
|
+
{
|
|
153
|
+
"minimax_max_tokens",
|
|
154
|
+
"minimax_max_long_side_pixel",
|
|
155
|
+
"minimax_max_video_mb",
|
|
156
|
+
"knowledge_doc_max_tokens",
|
|
157
|
+
}
|
|
158
|
+
)
|
|
159
|
+
_FLOAT_ENV_FIELDS: ClassVar[frozenset[str]] = frozenset(
|
|
160
|
+
{"minimax_video_fps", "knowledge_doc_timeout"}
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
# ---------------------------------------------------------------- defaults
|
|
164
|
+
debug: bool = False
|
|
165
|
+
transcript_proxy_mode: TranscriptProxyMode | None = None
|
|
166
|
+
transcript_http_proxy_url: str | None = None
|
|
167
|
+
transcript_https_proxy_url: str | None = None
|
|
168
|
+
webshare_proxy_username: str | None = None
|
|
169
|
+
webshare_proxy_password: str | None = None
|
|
170
|
+
webshare_proxy_regions: tuple[str, ...] | None = None
|
|
171
|
+
whisper_cpp_binary: str | None = None
|
|
172
|
+
whisper_cpp_model: str | None = None
|
|
173
|
+
whisper_cpp_language: str = "auto"
|
|
174
|
+
faster_whisper_model: str | None = "base.en"
|
|
175
|
+
ffmpeg_binary: str = "ffmpeg"
|
|
176
|
+
#: Empty means "no standalone CLI": consumers run ``<sys.executable> -m yt_dlp``.
|
|
177
|
+
yt_dlp_binary: str = ""
|
|
178
|
+
yt_dlp_cookies_file: str | None = None
|
|
179
|
+
yt_dlp_cookies_from_browser: str | None = None
|
|
180
|
+
transcript_languages: tuple[str, ...] = tuple(DEFAULT_LANG_PRIORITY)
|
|
181
|
+
transcript_cache_enabled: bool = True
|
|
182
|
+
transcript_cache_dir: str | None = None
|
|
183
|
+
sentence_transformer_model: str = "sentence-transformers/all-MiniLM-L6-v2"
|
|
184
|
+
minimax_api_key: str | None = None
|
|
185
|
+
minimax_region: Literal["cn", "global"] = "cn"
|
|
186
|
+
minimax_model: str = "MiniMax-M3"
|
|
187
|
+
minimax_max_tokens: int = 16384
|
|
188
|
+
minimax_video_detail: Literal["low", "default", "high"] = "low"
|
|
189
|
+
minimax_video_fps: float = 0.2
|
|
190
|
+
minimax_max_long_side_pixel: int = 640
|
|
191
|
+
minimax_max_video_mb: int = 512
|
|
192
|
+
doc_language: Literal["zh", "en", "auto"] | None = None
|
|
193
|
+
knowledge_doc_timeout: float = 300.0
|
|
194
|
+
knowledge_doc_max_tokens: int = 16384
|
|
195
|
+
|
|
196
|
+
# --------------------------------------------------------------- platforms
|
|
197
|
+
#: Platforms enabled for this machine. Everything defaults to enabled; a
|
|
198
|
+
#: missing sidecar still reports as ``not_configured`` (never as verified).
|
|
199
|
+
platforms_enabled: tuple[Platform, ...] = (
|
|
200
|
+
"youtube",
|
|
201
|
+
"tiktok",
|
|
202
|
+
"douyin",
|
|
203
|
+
"bilibili",
|
|
204
|
+
"xiaohongshu",
|
|
205
|
+
)
|
|
206
|
+
dtk_base_url: str | None = None
|
|
207
|
+
dtk_token: str | None = None
|
|
208
|
+
dtk_credential_ref: str | None = None
|
|
209
|
+
xhs_base_url: str | None = None
|
|
210
|
+
xhs_credential_ref: str | None = None
|
|
211
|
+
|
|
212
|
+
# Per-platform credentials/network. The legacy global YouTube cookie and
|
|
213
|
+
# transcript proxy are only inherited by YouTube (see helpers below).
|
|
214
|
+
youtube_cookies_file: str | None = None
|
|
215
|
+
tiktok_cookies_file: str | None = None
|
|
216
|
+
douyin_cookies_file: str | None = None
|
|
217
|
+
bilibili_cookies_file: str | None = None
|
|
218
|
+
xiaohongshu_cookies_file: str | None = None
|
|
219
|
+
youtube_proxy: str | None = None
|
|
220
|
+
tiktok_proxy: str | None = None
|
|
221
|
+
douyin_proxy: str | None = None
|
|
222
|
+
bilibili_proxy: str | None = None
|
|
223
|
+
xiaohongshu_proxy: str | None = None
|
|
224
|
+
|
|
225
|
+
# --------------------------------------------------------- ASR / OCR models
|
|
226
|
+
asr_backend: AsrBackend = "auto"
|
|
227
|
+
asr_model: str | None = None
|
|
228
|
+
asr_language: str | None = None
|
|
229
|
+
asr_beam_size: int = Field(default=5, ge=1, le=20)
|
|
230
|
+
asr_initial_prompt: str | None = Field(default=None, max_length=2000)
|
|
231
|
+
asr_hotwords: str | None = Field(default=None, max_length=1000)
|
|
232
|
+
asr_word_timestamps: bool = True
|
|
233
|
+
ocr_enabled: bool = False
|
|
234
|
+
ocr_model_dir: str = Field(default_factory=lambda: DEFAULT_OCR_MODEL_DIR)
|
|
235
|
+
|
|
236
|
+
# ----------------------------------------------------------------- budgets
|
|
237
|
+
max_media_mb: int = 512
|
|
238
|
+
max_duration_s: float | None = Field(default=None, gt=0, allow_inf_nan=False)
|
|
239
|
+
frame_budget: int = 24
|
|
240
|
+
#: Wall-clock budget for one task, local ASR included. Local recognition
|
|
241
|
+
#: scales with audio length (a 46-minute recording needs ~45 minutes of
|
|
242
|
+
#: ASR on this class of machine), so a 15-minute default killed real long
|
|
243
|
+
#: videos mid-run. This is a floor that fits ordinary media, not a promise
|
|
244
|
+
#: about very long recordings: raise YTK_TASK_DEADLINE_S for those.
|
|
245
|
+
task_deadline_s: float = Field(default=3600.0, gt=0, allow_inf_nan=False)
|
|
246
|
+
concurrency: int = 2
|
|
247
|
+
|
|
248
|
+
# ------------------------------------------------------------------- cloud
|
|
249
|
+
cloud_text_enabled: bool = False
|
|
250
|
+
cloud_media_enabled: bool = False
|
|
251
|
+
|
|
252
|
+
# ------------------------------------------------------------------- cache
|
|
253
|
+
cache_dir: str | None = None
|
|
254
|
+
|
|
255
|
+
# -------------------------------------------------------------- validation
|
|
256
|
+
@field_validator("asr_language", mode="before")
|
|
257
|
+
@classmethod
|
|
258
|
+
def _normalize_asr_language(cls, value: Any) -> str | None:
|
|
259
|
+
if value is None:
|
|
260
|
+
return None
|
|
261
|
+
if not isinstance(value, str):
|
|
262
|
+
raise ValueError("asr_language must be a language code such as zh or en, or auto")
|
|
263
|
+
language = value.strip().lower()
|
|
264
|
+
if language in {"", "auto"}:
|
|
265
|
+
return None
|
|
266
|
+
if re.fullmatch(r"[a-z]{2,3}", language) is None:
|
|
267
|
+
raise ValueError("asr_language must be a language code such as zh or en, or auto")
|
|
268
|
+
return language
|
|
269
|
+
|
|
270
|
+
@model_validator(mode="after")
|
|
271
|
+
def _validate_v2_fields(self) -> AppSettings:
|
|
272
|
+
unknown = [value for value in self.platforms_enabled if value not in SUPPORTED_PLATFORMS]
|
|
273
|
+
if unknown:
|
|
274
|
+
raise ValueError(f"unknown enabled platform(s): {sorted(set(unknown))}")
|
|
275
|
+
# de-duplicate but keep the canonical platform order
|
|
276
|
+
self.platforms_enabled = tuple(
|
|
277
|
+
value for value in SUPPORTED_PLATFORMS if value in set(self.platforms_enabled)
|
|
278
|
+
)
|
|
279
|
+
if self.max_media_mb < 0:
|
|
280
|
+
raise ValueError("max_media_mb must be >= 0")
|
|
281
|
+
if self.frame_budget < 0:
|
|
282
|
+
raise ValueError("frame_budget must be >= 0")
|
|
283
|
+
if self.concurrency < 1:
|
|
284
|
+
raise ValueError("concurrency must be >= 1")
|
|
285
|
+
if self.task_deadline_s <= 0:
|
|
286
|
+
raise ValueError("task_deadline_s must be > 0")
|
|
287
|
+
return self
|
|
288
|
+
|
|
289
|
+
# ------------------------------------------------------------- env loading
|
|
290
|
+
@model_validator(mode="before")
|
|
291
|
+
@classmethod
|
|
292
|
+
def _from_env(cls, values: Any, info: ValidationInfo) -> Any:
|
|
293
|
+
"""Historical behaviour: ``AppSettings()`` reads the legacy env only.
|
|
294
|
+
|
|
295
|
+
v2 platform/backend env variables and TOML are applied by
|
|
296
|
+
:meth:`load`; instantiating ``AppSettings()`` directly keeps the exact
|
|
297
|
+
pre-v2 semantics (defaults + the historical ``YTK_*`` variables) so
|
|
298
|
+
existing call sites and diagnostics do not change meaning.
|
|
299
|
+
"""
|
|
300
|
+
merged: dict[str, Any] = {} if not isinstance(values, dict) else dict(values)
|
|
301
|
+
if info.context and info.context.get("environment_loaded"):
|
|
302
|
+
return merged
|
|
303
|
+
for field_name, value in _legacy_env_values(os.environ).items():
|
|
304
|
+
merged.setdefault(field_name, value)
|
|
305
|
+
return merged
|
|
306
|
+
|
|
307
|
+
# ---------------------------------------------------------------- loading
|
|
308
|
+
@classmethod
|
|
309
|
+
def load(
|
|
310
|
+
cls,
|
|
311
|
+
*,
|
|
312
|
+
config_path: Path | str | None = None,
|
|
313
|
+
overrides: dict[str, Any] | None = None,
|
|
314
|
+
environ: dict[str, str] | os._Environ[str] | None = None,
|
|
315
|
+
) -> AppSettings:
|
|
316
|
+
"""Build settings with the frozen priority order.
|
|
317
|
+
|
|
318
|
+
Only explicitly provided keys are applied at each layer; defaults never
|
|
319
|
+
overwrite a user-provided value.
|
|
320
|
+
"""
|
|
321
|
+
env = os.environ if environ is None else environ
|
|
322
|
+
explicit = {key: value for key, value in (overrides or {}).items() if value is not None}
|
|
323
|
+
values: dict[str, Any] = {}
|
|
324
|
+
values.update(_legacy_env_values(env)) # tier 4
|
|
325
|
+
|
|
326
|
+
toml_platform_enable: dict[str, bool] = {}
|
|
327
|
+
resolved_config = config_path if config_path is not None else env.get(CONFIG_PATH_ENV_VAR)
|
|
328
|
+
if resolved_config:
|
|
329
|
+
toml_values, toml_platform_enable = load_config_file(resolved_config)
|
|
330
|
+
values.update(toml_values) # tier 3
|
|
331
|
+
|
|
332
|
+
v2_values, v2_platform_enable = _v2_env_values_split(env)
|
|
333
|
+
values.update(v2_values) # tier 2
|
|
334
|
+
|
|
335
|
+
values.update(explicit) # tier 1
|
|
336
|
+
|
|
337
|
+
values["platforms_enabled"] = _resolve_platforms_enabled(
|
|
338
|
+
explicit.get("platforms_enabled"),
|
|
339
|
+
values.get("platforms_enabled"),
|
|
340
|
+
env,
|
|
341
|
+
toml_enable=toml_platform_enable,
|
|
342
|
+
env_enable=v2_platform_enable,
|
|
343
|
+
)
|
|
344
|
+
return cls.model_validate(values, context={"environment_loaded": True})
|
|
345
|
+
|
|
346
|
+
# -------------------------------------------------------------- helpers
|
|
347
|
+
@property
|
|
348
|
+
def platforms_enabled_set(self) -> frozenset[Platform]:
|
|
349
|
+
return frozenset(self.platforms_enabled)
|
|
350
|
+
|
|
351
|
+
def is_platform_enabled(self, platform: Platform) -> bool:
|
|
352
|
+
return platform in self.platforms_enabled_set
|
|
353
|
+
|
|
354
|
+
def cookies_file_for(self, platform: Platform) -> str | None:
|
|
355
|
+
"""Per-platform cookie jar; the legacy global cookie is YouTube-only."""
|
|
356
|
+
if platform == "youtube":
|
|
357
|
+
return self.youtube_cookies_file or self.yt_dlp_cookies_file
|
|
358
|
+
return getattr(self, f"{platform}_cookies_file", None)
|
|
359
|
+
|
|
360
|
+
def cookies_from_browser_for(self, platform: Platform) -> str | None:
|
|
361
|
+
if platform == "youtube":
|
|
362
|
+
return self.yt_dlp_cookies_from_browser
|
|
363
|
+
return None
|
|
364
|
+
|
|
365
|
+
def proxy_for(self, platform: Platform) -> str | None:
|
|
366
|
+
"""Per-platform proxy; legacy transcript proxy is YouTube-only."""
|
|
367
|
+
if platform == "youtube":
|
|
368
|
+
return (
|
|
369
|
+
self.youtube_proxy
|
|
370
|
+
or self.transcript_https_proxy_url
|
|
371
|
+
or self.transcript_http_proxy_url
|
|
372
|
+
)
|
|
373
|
+
return getattr(self, f"{platform}_proxy", None)
|
|
374
|
+
|
|
375
|
+
def base_url_for(self, platform: Platform) -> str | None:
|
|
376
|
+
if platform == "douyin":
|
|
377
|
+
return self.dtk_base_url
|
|
378
|
+
if platform == "xiaohongshu":
|
|
379
|
+
return self.xhs_base_url
|
|
380
|
+
return None
|
|
381
|
+
|
|
382
|
+
def credential_ref_for(self, platform: Platform) -> str | None:
|
|
383
|
+
if platform == "douyin":
|
|
384
|
+
return self.dtk_credential_ref
|
|
385
|
+
if platform == "xiaohongshu":
|
|
386
|
+
return self.xhs_credential_ref
|
|
387
|
+
return None
|
|
388
|
+
|
|
389
|
+
def resolved_ocr_model_dir(self) -> Path:
|
|
390
|
+
return Path(self.ocr_model_dir).expanduser()
|
|
391
|
+
|
|
392
|
+
def ytdlp_command(self) -> list[str]:
|
|
393
|
+
"""Command prefix for yt-dlp: explicit binary, else ``python -m yt_dlp``."""
|
|
394
|
+
if self.yt_dlp_binary:
|
|
395
|
+
resolved = shutil.which(self.yt_dlp_binary) or self.yt_dlp_binary
|
|
396
|
+
return [resolved]
|
|
397
|
+
return [sys.executable, "-m", "yt_dlp"]
|
|
398
|
+
|
|
399
|
+
def cloud_allowed(self, platform: Platform, kind: CloudKind) -> bool:
|
|
400
|
+
"""Cloud use is opt-in; legacy YouTube consent is honoured for YouTube only."""
|
|
401
|
+
explicit = (
|
|
402
|
+
self.cloud_text_enabled if kind == "text" else self.cloud_media_enabled
|
|
403
|
+
)
|
|
404
|
+
if f"cloud_{kind}_enabled" in self.model_fields_set:
|
|
405
|
+
return bool(explicit)
|
|
406
|
+
legacy_youtube_consent = platform == "youtube" and bool(self.minimax_api_key)
|
|
407
|
+
return bool(explicit or legacy_youtube_consent)
|
|
408
|
+
|
|
409
|
+
def redacted(self) -> dict[str, Any]:
|
|
410
|
+
"""Credential-free settings view safe for logs/diagnostics/JSON output."""
|
|
411
|
+
from ytk.logging import redact_text
|
|
412
|
+
|
|
413
|
+
credential_fields = {
|
|
414
|
+
"minimax_api_key",
|
|
415
|
+
"dtk_token",
|
|
416
|
+
"webshare_proxy_username",
|
|
417
|
+
"webshare_proxy_password",
|
|
418
|
+
}
|
|
419
|
+
cookie_fields = {name for name in _field_names() if name.endswith("_cookies_file")}
|
|
420
|
+
proxy_fields = {
|
|
421
|
+
name for name in _field_names() if name.endswith("_proxy") or "proxy_url" in name
|
|
422
|
+
}
|
|
423
|
+
view: dict[str, Any] = {}
|
|
424
|
+
for name in _field_names():
|
|
425
|
+
value = getattr(self, name)
|
|
426
|
+
if name in credential_fields or name in cookie_fields or name in {"asr_initial_prompt", "asr_hotwords"}:
|
|
427
|
+
view[name] = "<configured>" if value else None
|
|
428
|
+
elif name in proxy_fields:
|
|
429
|
+
view[name] = (urlsplit(str(value)).hostname or "<configured>") if value else None
|
|
430
|
+
elif isinstance(value, tuple):
|
|
431
|
+
view[name] = list(value)
|
|
432
|
+
elif isinstance(value, str):
|
|
433
|
+
view[name] = redact_text(value)
|
|
434
|
+
else:
|
|
435
|
+
view[name] = value
|
|
436
|
+
return view
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _field_names() -> tuple[str, ...]:
|
|
440
|
+
return tuple(AppSettings.model_fields)
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
# --------------------------------------------------------------------------- #
|
|
444
|
+
# TOML config file
|
|
445
|
+
# --------------------------------------------------------------------------- #
|
|
446
|
+
_PLATFORM_FIELD_TEMPLATES: dict[str, str] = {
|
|
447
|
+
"cookies_file": "{platform}_cookies_file",
|
|
448
|
+
"proxy": "{platform}_proxy",
|
|
449
|
+
}
|
|
450
|
+
_SERVICE_BASE_URL_FIELD: dict[str, str] = {
|
|
451
|
+
"douyin": "dtk_base_url",
|
|
452
|
+
"xiaohongshu": "xhs_base_url",
|
|
453
|
+
}
|
|
454
|
+
_SERVICE_TOKEN_FIELD: dict[str, str | None] = {
|
|
455
|
+
"douyin": "dtk_token",
|
|
456
|
+
"xiaohongshu": None,
|
|
457
|
+
}
|
|
458
|
+
_SERVICE_CRED_FIELD: dict[str, str] = {
|
|
459
|
+
"douyin": "dtk_credential_ref",
|
|
460
|
+
"xiaohongshu": "xhs_credential_ref",
|
|
461
|
+
}
|
|
462
|
+
_TOP_LEVEL_GROUPS: dict[str, dict[str, str]] = {
|
|
463
|
+
"asr": {
|
|
464
|
+
"backend": "asr_backend", "model": "asr_model", "language": "asr_language",
|
|
465
|
+
"beam_size": "asr_beam_size", "initial_prompt": "asr_initial_prompt",
|
|
466
|
+
"hotwords": "asr_hotwords", "word_timestamps": "asr_word_timestamps",
|
|
467
|
+
},
|
|
468
|
+
"ocr": {"enabled": "ocr_enabled", "model_dir": "ocr_model_dir"},
|
|
469
|
+
"budgets": {
|
|
470
|
+
"max_media_mb": "max_media_mb",
|
|
471
|
+
"max_duration_s": "max_duration_s",
|
|
472
|
+
"frame_budget": "frame_budget",
|
|
473
|
+
"task_deadline_s": "task_deadline_s",
|
|
474
|
+
"concurrency": "concurrency",
|
|
475
|
+
},
|
|
476
|
+
"cloud": {"text_enabled": "cloud_text_enabled", "media_enabled": "cloud_media_enabled"},
|
|
477
|
+
"cache": {"dir": "cache_dir"},
|
|
478
|
+
}
|
|
479
|
+
_PLATFORM_ENABLE_KEY = "enabled"
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def load_config_file(
|
|
483
|
+
path: Path | str,
|
|
484
|
+
) -> tuple[dict[str, Any], dict[str, bool]]:
|
|
485
|
+
"""Load an explicit TOML config file into field overrides + platform flags.
|
|
486
|
+
|
|
487
|
+
Only explicitly present keys are returned. Unknown keys/sections are a
|
|
488
|
+
clear error instead of being silently ignored (config typos must not look
|
|
489
|
+
like working settings).
|
|
490
|
+
"""
|
|
491
|
+
import tomllib
|
|
492
|
+
|
|
493
|
+
target = Path(path).expanduser()
|
|
494
|
+
try:
|
|
495
|
+
payload = tomllib.loads(target.read_text(encoding="utf-8"))
|
|
496
|
+
except FileNotFoundError as exc:
|
|
497
|
+
raise ConfigFileError(f"config file not found: {target}") from exc
|
|
498
|
+
except OSError as exc:
|
|
499
|
+
raise ConfigFileError(f"cannot read config file {target}: {exc}") from exc
|
|
500
|
+
except tomllib.TOMLDecodeError as exc:
|
|
501
|
+
raise ConfigFileError(f"invalid TOML in {target}: {exc}") from exc
|
|
502
|
+
|
|
503
|
+
known_fields = set(AppSettings.model_fields)
|
|
504
|
+
values: dict[str, Any] = {}
|
|
505
|
+
platform_enable: dict[str, bool] = {}
|
|
506
|
+
unknown: list[str] = []
|
|
507
|
+
|
|
508
|
+
def _assign(key: str, value: Any, context: str) -> None:
|
|
509
|
+
if key == "platforms_enabled":
|
|
510
|
+
values[key] = _coerce_platforms(value, context)
|
|
511
|
+
return
|
|
512
|
+
if key not in known_fields:
|
|
513
|
+
unknown.append(context)
|
|
514
|
+
return
|
|
515
|
+
values[key] = value
|
|
516
|
+
|
|
517
|
+
for key, value in payload.items():
|
|
518
|
+
if key == "platforms":
|
|
519
|
+
if not isinstance(value, dict):
|
|
520
|
+
raise ConfigFileError("[platforms] must be a table")
|
|
521
|
+
for platform, section in value.items():
|
|
522
|
+
if platform not in SUPPORTED_PLATFORMS:
|
|
523
|
+
raise ConfigFileError(f"unknown platform in config: [platforms.{platform}]")
|
|
524
|
+
if not isinstance(section, dict):
|
|
525
|
+
raise ConfigFileError(f"[platforms.{platform}] must be a table")
|
|
526
|
+
for section_key, section_value in section.items():
|
|
527
|
+
if section_key == _PLATFORM_ENABLE_KEY:
|
|
528
|
+
if not isinstance(section_value, bool):
|
|
529
|
+
raise ConfigFileError(
|
|
530
|
+
f"platforms.{platform}.enabled must be a boolean"
|
|
531
|
+
)
|
|
532
|
+
platform_enable[platform] = section_value
|
|
533
|
+
continue
|
|
534
|
+
template = _PLATFORM_FIELD_TEMPLATES.get(section_key)
|
|
535
|
+
if template is not None:
|
|
536
|
+
_assign(
|
|
537
|
+
template.format(platform=platform),
|
|
538
|
+
section_value,
|
|
539
|
+
f"platforms.{platform}.{section_key}",
|
|
540
|
+
)
|
|
541
|
+
continue
|
|
542
|
+
if section_key == "base_url" and platform in _SERVICE_BASE_URL_FIELD:
|
|
543
|
+
_assign(
|
|
544
|
+
_SERVICE_BASE_URL_FIELD[platform],
|
|
545
|
+
section_value,
|
|
546
|
+
f"platforms.{platform}.base_url",
|
|
547
|
+
)
|
|
548
|
+
continue
|
|
549
|
+
token_field = _SERVICE_TOKEN_FIELD.get(platform)
|
|
550
|
+
if section_key == "token" and token_field:
|
|
551
|
+
_assign(token_field, section_value, f"platforms.{platform}.token")
|
|
552
|
+
continue
|
|
553
|
+
if section_key == "credential_ref" and platform in _SERVICE_CRED_FIELD:
|
|
554
|
+
_assign(
|
|
555
|
+
_SERVICE_CRED_FIELD[platform],
|
|
556
|
+
section_value,
|
|
557
|
+
f"platforms.{platform}.credential_ref",
|
|
558
|
+
)
|
|
559
|
+
continue
|
|
560
|
+
unknown.append(f"platforms.{platform}.{section_key}")
|
|
561
|
+
continue
|
|
562
|
+
group = _TOP_LEVEL_GROUPS.get(key)
|
|
563
|
+
if group is not None:
|
|
564
|
+
if not isinstance(value, dict):
|
|
565
|
+
raise ConfigFileError(f"[{key}] must be a table")
|
|
566
|
+
for section_key, section_value in value.items():
|
|
567
|
+
field = group.get(section_key)
|
|
568
|
+
if field is None:
|
|
569
|
+
unknown.append(f"{key}.{section_key}")
|
|
570
|
+
continue
|
|
571
|
+
_assign(field, section_value, f"{key}.{section_key}")
|
|
572
|
+
continue
|
|
573
|
+
_assign(key, value, key)
|
|
574
|
+
|
|
575
|
+
if unknown:
|
|
576
|
+
raise ConfigFileError(
|
|
577
|
+
"unknown config key(s): "
|
|
578
|
+
+ ", ".join(sorted(unknown))
|
|
579
|
+
+ ". See docs/implementation/five-platform/infra-cli.md for the accepted schema."
|
|
580
|
+
)
|
|
581
|
+
return values, platform_enable
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _coerce_platforms(value: Any, context: str) -> tuple[str, ...]:
|
|
585
|
+
if isinstance(value, str):
|
|
586
|
+
if value.strip().lower() in {"all", "*"}:
|
|
587
|
+
return tuple(SUPPORTED_PLATFORMS)
|
|
588
|
+
items = _comma_tuple(value)
|
|
589
|
+
elif isinstance(value, (list, tuple)):
|
|
590
|
+
items = tuple(str(item).strip() for item in value if str(item).strip())
|
|
591
|
+
else:
|
|
592
|
+
raise ConfigFileError(f"{context} must be a list of platform names")
|
|
593
|
+
unknown = [item for item in items if item not in SUPPORTED_PLATFORMS]
|
|
594
|
+
if unknown:
|
|
595
|
+
raise ConfigFileError(f"{context} contains unknown platform(s): {sorted(set(unknown))}")
|
|
596
|
+
return tuple(item for item in SUPPORTED_PLATFORMS if item in set(items))
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
# --------------------------------------------------------------------------- #
|
|
600
|
+
# Environment parsing
|
|
601
|
+
# --------------------------------------------------------------------------- #
|
|
602
|
+
#: v2 environment variables (tier 2). Applied in order, so platform-specific
|
|
603
|
+
#: aliases can override the generic names for their platform.
|
|
604
|
+
_V2_ENV_MAP: tuple[tuple[str, str], ...] = (
|
|
605
|
+
("YTK_DTK_BASE_URL", "dtk_base_url"),
|
|
606
|
+
("YTK_DTK_TOKEN", "dtk_token"),
|
|
607
|
+
("YTK_DTK_CREDENTIAL_REF", "dtk_credential_ref"),
|
|
608
|
+
("YTK_XHS_BASE_URL", "xhs_base_url"),
|
|
609
|
+
("YTK_XHS_CREDENTIAL_REF", "xhs_credential_ref"),
|
|
610
|
+
("YTK_ASR_BACKEND", "asr_backend"),
|
|
611
|
+
("YTK_ASR_MODEL", "asr_model"),
|
|
612
|
+
("YTK_ASR_LANGUAGE", "asr_language"),
|
|
613
|
+
("YTK_ASR_BEAM_SIZE", "asr_beam_size"),
|
|
614
|
+
("YTK_ASR_INITIAL_PROMPT", "asr_initial_prompt"),
|
|
615
|
+
("YTK_ASR_HOTWORDS", "asr_hotwords"),
|
|
616
|
+
("YTK_ASR_WORD_TIMESTAMPS", "asr_word_timestamps"),
|
|
617
|
+
("YTK_OCR_ENABLED", "ocr_enabled"),
|
|
618
|
+
("YTK_OCR_MODEL_DIR", "ocr_model_dir"),
|
|
619
|
+
("YTK_MAX_MEDIA_MB", "max_media_mb"),
|
|
620
|
+
("YTK_MAX_DURATION_S", "max_duration_s"),
|
|
621
|
+
("YTK_FRAME_BUDGET", "frame_budget"),
|
|
622
|
+
("YTK_TASK_DEADLINE_S", "task_deadline_s"),
|
|
623
|
+
("YTK_CONCURRENCY", "concurrency"),
|
|
624
|
+
("YTK_CLOUD_TEXT_ENABLED", "cloud_text_enabled"),
|
|
625
|
+
("YTK_CLOUD_MEDIA_ENABLED", "cloud_media_enabled"),
|
|
626
|
+
("YTK_CACHE_DIR", "cache_dir"),
|
|
627
|
+
("YTK_V2_CACHE_DIR", "cache_dir"),
|
|
628
|
+
# platform-level aliases (contracts-v2 §1.7)
|
|
629
|
+
("YTK_YOUTUBE_COOKIES_FILE", "youtube_cookies_file"),
|
|
630
|
+
("YTK_TIKTOK_COOKIES_FILE", "tiktok_cookies_file"),
|
|
631
|
+
("YTK_DOUYIN_COOKIES_FILE", "douyin_cookies_file"),
|
|
632
|
+
("YTK_BILIBILI_COOKIES_FILE", "bilibili_cookies_file"),
|
|
633
|
+
("YTK_XIAOHONGSHU_COOKIES_FILE", "xiaohongshu_cookies_file"),
|
|
634
|
+
("YTK_YOUTUBE_PROXY", "youtube_proxy"),
|
|
635
|
+
("YTK_TIKTOK_PROXY", "tiktok_proxy"),
|
|
636
|
+
("YTK_DOUYIN_PROXY", "douyin_proxy"),
|
|
637
|
+
("YTK_BILIBILI_PROXY", "bilibili_proxy"),
|
|
638
|
+
("YTK_XIAOHONGSHU_PROXY", "xiaohongshu_proxy"),
|
|
639
|
+
("YTK_DOUYIN_BASE_URL", "dtk_base_url"),
|
|
640
|
+
("YTK_DOUYIN_TOKEN", "dtk_token"),
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
_V2_BOOL_FIELDS: frozenset[str] = frozenset(
|
|
644
|
+
{"ocr_enabled", "cloud_text_enabled", "cloud_media_enabled", "asr_word_timestamps"}
|
|
645
|
+
)
|
|
646
|
+
_V2_INT_FIELDS: frozenset[str] = frozenset({"max_media_mb", "frame_budget", "concurrency", "asr_beam_size"})
|
|
647
|
+
_V2_FLOAT_FIELDS: frozenset[str] = frozenset({"max_duration_s", "task_deadline_s"})
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def _legacy_env_values(environ: dict[str, str] | os._Environ[str]) -> dict[str, Any]:
|
|
651
|
+
"""Tier-4: the historical global env variables."""
|
|
652
|
+
values: dict[str, Any] = {}
|
|
653
|
+
for env_var, field_name in AppSettings._ENV_MAP.items():
|
|
654
|
+
raw = environ.get(env_var)
|
|
655
|
+
if raw is None:
|
|
656
|
+
continue
|
|
657
|
+
if field_name in AppSettings._BOOL_ENV_FIELDS:
|
|
658
|
+
values[field_name] = _env_flag(raw)
|
|
659
|
+
elif field_name in AppSettings._TUPLE_ENV_FIELDS:
|
|
660
|
+
values[field_name] = _comma_tuple(raw)
|
|
661
|
+
elif field_name in AppSettings._INT_ENV_FIELDS:
|
|
662
|
+
values[field_name] = _safe_int(raw, 0)
|
|
663
|
+
elif field_name in AppSettings._FLOAT_ENV_FIELDS:
|
|
664
|
+
values[field_name] = _safe_float(raw, 0.0)
|
|
665
|
+
else:
|
|
666
|
+
values[field_name] = raw
|
|
667
|
+
return values
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _v2_env_values_split(
|
|
671
|
+
environ: dict[str, str] | os._Environ[str],
|
|
672
|
+
) -> tuple[dict[str, Any], dict[str, bool]]:
|
|
673
|
+
"""Tier-2: v2 platform/backend environment variables + platform enable flags."""
|
|
674
|
+
values: dict[str, Any] = {}
|
|
675
|
+
for env_var, field_name in _V2_ENV_MAP:
|
|
676
|
+
raw = environ.get(env_var)
|
|
677
|
+
if raw is None:
|
|
678
|
+
continue
|
|
679
|
+
if field_name in _V2_BOOL_FIELDS:
|
|
680
|
+
values[field_name] = _env_flag(raw)
|
|
681
|
+
elif field_name in _V2_INT_FIELDS:
|
|
682
|
+
values[field_name] = _safe_int(raw, 0)
|
|
683
|
+
elif field_name in _V2_FLOAT_FIELDS:
|
|
684
|
+
values[field_name] = _safe_float(raw, 0.0)
|
|
685
|
+
else:
|
|
686
|
+
values[field_name] = raw
|
|
687
|
+
return values, _platform_env_flags(environ)
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _platform_env_flags(environ: dict[str, str] | os._Environ[str]) -> dict[str, bool]:
|
|
691
|
+
flags: dict[str, bool] = {}
|
|
692
|
+
for platform in SUPPORTED_PLATFORMS:
|
|
693
|
+
raw = environ.get(f"YTK_{platform.upper()}_ENABLED")
|
|
694
|
+
if raw is not None:
|
|
695
|
+
flags[platform] = _env_flag(raw)
|
|
696
|
+
return flags
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def _resolve_platforms_enabled(
|
|
700
|
+
explicit: Any,
|
|
701
|
+
merged_value: Any,
|
|
702
|
+
environ: dict[str, str] | os._Environ[str],
|
|
703
|
+
*,
|
|
704
|
+
toml_enable: dict[str, bool] | None = None,
|
|
705
|
+
env_enable: dict[str, bool] | None = None,
|
|
706
|
+
) -> tuple[str, ...]:
|
|
707
|
+
"""Compute the effective enabled platform tuple (explicit > env > lower tiers)."""
|
|
708
|
+
if explicit is not None:
|
|
709
|
+
# An explicit whitelist is final: no lower layer may add a platform back.
|
|
710
|
+
return tuple(_normalise_platforms(explicit))
|
|
711
|
+
env_list = environ.get("YTK_PLATFORMS_ENABLED")
|
|
712
|
+
if env_list not in (None, ""):
|
|
713
|
+
base = _normalise_platforms(env_list)
|
|
714
|
+
adjustments = dict(env_enable or {}) # env whitelist > TOML enabled flags
|
|
715
|
+
else:
|
|
716
|
+
base = (
|
|
717
|
+
_normalise_platforms(merged_value)
|
|
718
|
+
if merged_value is not None
|
|
719
|
+
else list(SUPPORTED_PLATFORMS)
|
|
720
|
+
)
|
|
721
|
+
adjustments = {**(toml_enable or {}), **(env_enable or {})}
|
|
722
|
+
for platform, enabled in adjustments.items():
|
|
723
|
+
if platform not in SUPPORTED_PLATFORMS:
|
|
724
|
+
continue
|
|
725
|
+
if enabled and platform not in base:
|
|
726
|
+
base.append(platform)
|
|
727
|
+
elif not enabled and platform in base:
|
|
728
|
+
base.remove(platform)
|
|
729
|
+
return tuple(_normalise_platforms(base))
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def _normalise_platforms(value: Any) -> list[str]:
|
|
733
|
+
if isinstance(value, str):
|
|
734
|
+
if value.strip().lower() in {"all", "*"}:
|
|
735
|
+
return list(SUPPORTED_PLATFORMS)
|
|
736
|
+
items = list(_comma_tuple(value))
|
|
737
|
+
elif isinstance(value, (list, tuple, set, frozenset)):
|
|
738
|
+
items = [str(item).strip() for item in value if str(item).strip()]
|
|
739
|
+
else:
|
|
740
|
+
raise ValueError("platforms_enabled must be a list of platform names")
|
|
741
|
+
unknown = [item for item in items if item not in SUPPORTED_PLATFORMS]
|
|
742
|
+
if unknown:
|
|
743
|
+
raise ValueError(f"unknown platform(s): {sorted(set(unknown))}")
|
|
744
|
+
return [platform for platform in SUPPORTED_PLATFORMS if platform in set(items)]
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
__all__ = [
|
|
748
|
+
"AppSettings",
|
|
749
|
+
"AsrBackend",
|
|
750
|
+
"CONFIG_PATH_ENV_VAR",
|
|
751
|
+
"ConfigFileError",
|
|
752
|
+
"DEFAULT_OCR_MODEL_DIR",
|
|
753
|
+
"NO_DOTENV_ENV_VAR",
|
|
754
|
+
"TranscriptProxyMode",
|
|
755
|
+
"dotenv_disabled",
|
|
756
|
+
"load_config_file",
|
|
757
|
+
]
|