kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,754 @@
|
|
|
1
|
+
"""T20.1 — generic yt-dlp acquisition: one info call feeds subtitles, audio, media.
|
|
2
|
+
|
|
3
|
+
Design rules from the frozen contract (§1.2) and acceptance items A34/A38:
|
|
4
|
+
|
|
5
|
+
* **One metadata call per task.** ``info(url)`` is cached, so subtitle mapping,
|
|
6
|
+
identity (Bilibili ``cid``) and media decisions all reuse the same payload.
|
|
7
|
+
* **No directory scavenging (A34).** Downloads name the exact output template
|
|
8
|
+
and resolve the produced file from ``--print after_move:filepath`` (plus a
|
|
9
|
+
bounded same-stem fallback). There is no "read every ``*.json3``/``*`` file in
|
|
10
|
+
the directory" fallback: residual files from another work are never read.
|
|
11
|
+
* **Soft metadata (A38).** :func:`metadata_from_info` never raises; a broken
|
|
12
|
+
chapter list or non-numeric duration loses only that field. Callers keep the
|
|
13
|
+
already-acquired subtitle blocks.
|
|
14
|
+
* Cookies/proxy come from explicit arguments first, then the platform settings;
|
|
15
|
+
every subprocess gets a timeout (a hung download is killed, not wedged).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import html
|
|
21
|
+
import json
|
|
22
|
+
import re
|
|
23
|
+
import shutil
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
import warnings
|
|
27
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
28
|
+
from http.cookiejar import DefaultCookiePolicy, LoadError, MozillaCookieJar
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
from urllib.parse import urljoin, urlsplit
|
|
32
|
+
from urllib.request import Request
|
|
33
|
+
|
|
34
|
+
import httpx
|
|
35
|
+
|
|
36
|
+
from ytk.errors import AnalysisBlockedError, ContentIntegrityError
|
|
37
|
+
from ytk.ingestion.resolver import _validate_redirect_target
|
|
38
|
+
from ytk.ingestion.subtitles import EXT_TO_FORMAT, TrackInfo, is_narration_track
|
|
39
|
+
from ytk.models.source import sanitize_public_url
|
|
40
|
+
|
|
41
|
+
Runner = Callable[..., subprocess.CompletedProcess[str]]
|
|
42
|
+
HttpGetter = Callable[[str, Mapping[str, str], bool], Any]
|
|
43
|
+
|
|
44
|
+
DEFAULT_TIMEOUT = 600.0
|
|
45
|
+
DEFAULT_IMAGE_MAX_MB = 64
|
|
46
|
+
_USER_AGENT = "knowledge-distiller/2.0 (+https://github.com/Shockang/knowledge-distiller)"
|
|
47
|
+
|
|
48
|
+
_URL_IN_TEXT_RE = re.compile(r"https?://[^\s\"'<>]+")
|
|
49
|
+
_SRT_HINT_RE = re.compile(r"^\s*\d+\s*\n\s*\d{1,2}:\d{2}:\d{2}[,.]\d{3}\s*-->")
|
|
50
|
+
_HTTP_HEADER_NAMES = frozenset({"user-agent", "referer", "origin", "accept", "accept-language", "authorization"})
|
|
51
|
+
|
|
52
|
+
#: ``(markers, reason_code, blocked)`` — first match wins.
|
|
53
|
+
_FAILURE_CLASSES: tuple[tuple[tuple[str, ...], str, bool], ...] = (
|
|
54
|
+
(
|
|
55
|
+
(
|
|
56
|
+
"sign in to confirm",
|
|
57
|
+
"not a bot",
|
|
58
|
+
"login required",
|
|
59
|
+
"log in",
|
|
60
|
+
"private video",
|
|
61
|
+
"members-only",
|
|
62
|
+
"requires authentication",
|
|
63
|
+
"use --cookies",
|
|
64
|
+
"cookies are no longer valid",
|
|
65
|
+
"account authentication",
|
|
66
|
+
),
|
|
67
|
+
"AUTH_REQUIRED",
|
|
68
|
+
True,
|
|
69
|
+
),
|
|
70
|
+
(("captcha",), "CAPTCHA_REQUIRED", True),
|
|
71
|
+
(("too many requests", "rate limit", "rate-limit"), "RATE_LIMITED", True),
|
|
72
|
+
(
|
|
73
|
+
(
|
|
74
|
+
"not available in your country",
|
|
75
|
+
"geo-restricted",
|
|
76
|
+
"geo restricted",
|
|
77
|
+
"blocked it in your country",
|
|
78
|
+
),
|
|
79
|
+
"GEO_BLOCKED",
|
|
80
|
+
True,
|
|
81
|
+
),
|
|
82
|
+
(
|
|
83
|
+
(
|
|
84
|
+
"video unavailable",
|
|
85
|
+
"has been removed",
|
|
86
|
+
"removed by the uploader",
|
|
87
|
+
"does not exist",
|
|
88
|
+
"this content is not available",
|
|
89
|
+
),
|
|
90
|
+
"CONTENT_UNAVAILABLE",
|
|
91
|
+
True,
|
|
92
|
+
),
|
|
93
|
+
(
|
|
94
|
+
(
|
|
95
|
+
"unable to download",
|
|
96
|
+
"getaddrinfo",
|
|
97
|
+
"connection refused",
|
|
98
|
+
"connection reset",
|
|
99
|
+
"timed out",
|
|
100
|
+
"timeout",
|
|
101
|
+
"ssl",
|
|
102
|
+
"temporary failure",
|
|
103
|
+
"unable to connect",
|
|
104
|
+
"http error 5",
|
|
105
|
+
"network is unreachable",
|
|
106
|
+
),
|
|
107
|
+
"NETWORK_ERROR",
|
|
108
|
+
True,
|
|
109
|
+
),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# ---------------------------------------------------------------------------
|
|
114
|
+
# URL / failure helpers (module level so adapters can reuse them)
|
|
115
|
+
# ---------------------------------------------------------------------------
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def normalize_http_url(url: object) -> str | None:
|
|
119
|
+
"""Normalise a provider URL: HTML-unescape it, add https, reject unsafe shapes."""
|
|
120
|
+
if not isinstance(url, str) or not url.strip():
|
|
121
|
+
return None
|
|
122
|
+
candidate = html.unescape(url.strip())
|
|
123
|
+
if candidate.startswith("//"):
|
|
124
|
+
candidate = "https:" + candidate
|
|
125
|
+
try:
|
|
126
|
+
parts = urlsplit(candidate)
|
|
127
|
+
except ValueError:
|
|
128
|
+
return None
|
|
129
|
+
if parts.scheme not in {"http", "https"} or not parts.hostname:
|
|
130
|
+
return None
|
|
131
|
+
if parts.username or parts.password:
|
|
132
|
+
return None
|
|
133
|
+
return candidate
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def redact_urls(text: str) -> str:
|
|
137
|
+
"""Strip query strings/userinfo from URLs so signed links never reach logs."""
|
|
138
|
+
|
|
139
|
+
def _replace(match: re.Match[str]) -> str:
|
|
140
|
+
try:
|
|
141
|
+
return sanitize_public_url(match.group(0))
|
|
142
|
+
except ValueError:
|
|
143
|
+
return "<redacted-url>"
|
|
144
|
+
|
|
145
|
+
return _URL_IN_TEXT_RE.sub(_replace, text)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def classify_failure(stderr: str) -> tuple[bool, str]:
|
|
149
|
+
"""Classify a yt-dlp failure into ``(blocked, reason_code)``.
|
|
150
|
+
|
|
151
|
+
The bare status code ``429`` is anchored with word boundaries: a URL path
|
|
152
|
+
containing those digits must not classify as rate limiting.
|
|
153
|
+
"""
|
|
154
|
+
haystack = (stderr or "").casefold()
|
|
155
|
+
for markers, reason_code, blocked in _FAILURE_CLASSES:
|
|
156
|
+
if any(marker in haystack for marker in markers):
|
|
157
|
+
return blocked, reason_code
|
|
158
|
+
if re.search(r"\b429\b", haystack):
|
|
159
|
+
return True, "RATE_LIMITED"
|
|
160
|
+
return False, "PROVIDER_FAILED"
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def provider_error(stderr: str, *, platform: str | None = None) -> Exception:
|
|
164
|
+
"""Build the typed exception for a failed yt-dlp call, with ``reason_code``."""
|
|
165
|
+
blocked, reason_code = classify_failure(stderr)
|
|
166
|
+
xhs_no_video = (
|
|
167
|
+
platform == "xiaohongshu"
|
|
168
|
+
and reason_code == "PROVIDER_FAILED"
|
|
169
|
+
and "no video formats found" in (stderr or "").casefold()
|
|
170
|
+
)
|
|
171
|
+
if xhs_no_video:
|
|
172
|
+
blocked, reason_code = True, "XHS_VIDEO_UNAVAILABLE"
|
|
173
|
+
tail = redact_urls((stderr or "").strip())[:400]
|
|
174
|
+
message = f"yt-dlp failed ({reason_code}): {tail or 'no diagnostic output'}"
|
|
175
|
+
if xhs_no_video:
|
|
176
|
+
message += (
|
|
177
|
+
" The page did not expose a playable video. Re-copy a share link that plays "
|
|
178
|
+
"in your browser; the link may have expired or access may be restricted."
|
|
179
|
+
)
|
|
180
|
+
error: Exception = AnalysisBlockedError(message) if blocked else RuntimeError(message)
|
|
181
|
+
error.reason_code = reason_code # type: ignore[attr-defined]
|
|
182
|
+
return error
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def metadata_from_info(info: object) -> dict[str, object]:
|
|
186
|
+
"""Extract title/channel/duration/chapters; any malformed field is skipped.
|
|
187
|
+
|
|
188
|
+
This is the A38 soft-failure boundary: enrichment problems must never raise
|
|
189
|
+
and never invalidate already acquired subtitles/blocks.
|
|
190
|
+
"""
|
|
191
|
+
if not isinstance(info, Mapping):
|
|
192
|
+
return {}
|
|
193
|
+
metadata: dict[str, object] = {}
|
|
194
|
+
try:
|
|
195
|
+
title = _clean_text(info.get("title"))
|
|
196
|
+
if title:
|
|
197
|
+
metadata["title"] = title
|
|
198
|
+
channel = _clean_text(info.get("uploader")) or _clean_text(info.get("channel"))
|
|
199
|
+
if channel:
|
|
200
|
+
metadata["channel"] = channel
|
|
201
|
+
description = _clean_text(info.get("description"))
|
|
202
|
+
if description:
|
|
203
|
+
metadata["description"] = description
|
|
204
|
+
upload_date = _clean_text(info.get("upload_date"))
|
|
205
|
+
if upload_date:
|
|
206
|
+
metadata["upload_date"] = upload_date
|
|
207
|
+
duration = _to_float(info.get("duration"))
|
|
208
|
+
if duration is not None and duration >= 0:
|
|
209
|
+
metadata["duration"] = duration
|
|
210
|
+
chapters = _normalize_chapters(info.get("chapters"))
|
|
211
|
+
if chapters:
|
|
212
|
+
metadata["chapters"] = chapters
|
|
213
|
+
except Exception: # noqa: BLE001 - enrichment is deliberately soft (A38)
|
|
214
|
+
return metadata
|
|
215
|
+
return metadata
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _normalize_chapters(value: object) -> list[dict[str, object]]:
|
|
219
|
+
if not isinstance(value, Sequence) or isinstance(value, str | bytes):
|
|
220
|
+
return []
|
|
221
|
+
chapters: list[dict[str, object]] = []
|
|
222
|
+
for entry in value:
|
|
223
|
+
if not isinstance(entry, Mapping):
|
|
224
|
+
continue
|
|
225
|
+
start = _to_float(entry.get("start_time"))
|
|
226
|
+
end = _to_float(entry.get("end_time"))
|
|
227
|
+
if start is None or end is None or end < start:
|
|
228
|
+
continue
|
|
229
|
+
chapters.append(
|
|
230
|
+
{
|
|
231
|
+
"start_time": start,
|
|
232
|
+
"end_time": end,
|
|
233
|
+
"title": _clean_text(entry.get("title")) or "",
|
|
234
|
+
}
|
|
235
|
+
)
|
|
236
|
+
return chapters
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _clean_text(value: object) -> str | None:
|
|
240
|
+
if not isinstance(value, str):
|
|
241
|
+
return None
|
|
242
|
+
cleaned = value.strip()
|
|
243
|
+
return cleaned or None
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _to_float(value: object) -> float | None:
|
|
247
|
+
if value is None or isinstance(value, bool):
|
|
248
|
+
return None
|
|
249
|
+
try:
|
|
250
|
+
return float(value) # type: ignore[arg-type]
|
|
251
|
+
except (TypeError, ValueError):
|
|
252
|
+
return None
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _settings_attr(settings: object | None, name: str) -> Any:
|
|
256
|
+
if settings is None:
|
|
257
|
+
return None
|
|
258
|
+
value = getattr(settings, name, None)
|
|
259
|
+
if isinstance(value, str) and not value.strip():
|
|
260
|
+
return None
|
|
261
|
+
return value
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def probe_javascript_runtimes(
|
|
265
|
+
resolver: Callable[[str], str | None] = shutil.which,
|
|
266
|
+
version_runner: Callable[[Sequence[str]], str] | None = None,
|
|
267
|
+
) -> tuple[dict[str, dict[str, Any]], str | None]:
|
|
268
|
+
"""Local-only EJS prerequisites; executable presence is not compatibility."""
|
|
269
|
+
runtimes: dict[str, dict[str, Any]] = {}
|
|
270
|
+
for name, minimum in (("deno", (2, 3, 0)), ("node", (22, 0, 0)), ("bun", None)):
|
|
271
|
+
binary = resolver(name)
|
|
272
|
+
if not binary:
|
|
273
|
+
continue
|
|
274
|
+
entry: dict[str, Any] = {"path": binary, "supported": False}
|
|
275
|
+
try:
|
|
276
|
+
if version_runner is None:
|
|
277
|
+
result = subprocess.run([binary, "--version"], capture_output=True,
|
|
278
|
+
text=True, check=True, timeout=5)
|
|
279
|
+
output = result.stdout.strip() or result.stderr.strip()
|
|
280
|
+
else:
|
|
281
|
+
output = version_runner([binary, "--version"]).strip()
|
|
282
|
+
entry["version"] = output
|
|
283
|
+
match = re.search(r"(?:^|\s)v?(\d+)\.(\d+)\.(\d+)(?![\d.-])", output)
|
|
284
|
+
entry["supported"] = bool(minimum and match and tuple(map(int, match.groups())) >= minimum)
|
|
285
|
+
if name == "bun":
|
|
286
|
+
entry["note"] = "Bun support is deprecated; use supported Deno or Node instead."
|
|
287
|
+
except Exception: # local version checks must fail closed, never run downloads
|
|
288
|
+
entry["note"] = "Could not verify the runtime version locally."
|
|
289
|
+
runtimes[name] = entry
|
|
290
|
+
selected = next((name for name in ("deno", "node") if runtimes.get(name, {}).get("supported")), None)
|
|
291
|
+
return runtimes, selected
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# ---------------------------------------------------------------------------
|
|
295
|
+
# Acquirer
|
|
296
|
+
# ---------------------------------------------------------------------------
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
class YtDlpAcquirer:
|
|
300
|
+
"""Thin, injectable wrapper around the yt-dlp CLI used by all adapters."""
|
|
301
|
+
|
|
302
|
+
def __init__(
|
|
303
|
+
self,
|
|
304
|
+
*,
|
|
305
|
+
binary: str | None = None,
|
|
306
|
+
settings: object | None = None,
|
|
307
|
+
platform: str = "youtube",
|
|
308
|
+
runner: Runner | None = None,
|
|
309
|
+
cookies_file: str | None = None,
|
|
310
|
+
cookies_from_browser: str | None = None,
|
|
311
|
+
proxy: str | None = None,
|
|
312
|
+
allow_network: bool = True,
|
|
313
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
314
|
+
http_client: object | None = None,
|
|
315
|
+
http_timeout: float = 30.0,
|
|
316
|
+
user_agent: str | None = None,
|
|
317
|
+
) -> None:
|
|
318
|
+
if timeout <= 0:
|
|
319
|
+
raise ValueError("timeout must be > 0")
|
|
320
|
+
self._binary = binary
|
|
321
|
+
self._platform = platform
|
|
322
|
+
self._js_runtime_args: list[str] | None = None
|
|
323
|
+
self._settings = settings
|
|
324
|
+
self._runner = runner or _default_runner
|
|
325
|
+
self._cookies_file = cookies_file or _settings_attr(settings, f"{platform}_cookies_file")
|
|
326
|
+
self._cookies_from_browser = cookies_from_browser
|
|
327
|
+
if platform == "youtube":
|
|
328
|
+
self._cookies_file = self._cookies_file or _settings_attr(settings, "yt_dlp_cookies_file")
|
|
329
|
+
self._cookies_from_browser = self._cookies_from_browser or _settings_attr(settings, "yt_dlp_cookies_from_browser")
|
|
330
|
+
self._proxy = proxy or _settings_attr(settings, "proxy")
|
|
331
|
+
self._allow_network = allow_network
|
|
332
|
+
self._timeout = float(timeout)
|
|
333
|
+
self._http_client = http_client
|
|
334
|
+
self._http_timeout = float(http_timeout)
|
|
335
|
+
self._user_agent = user_agent or _USER_AGENT
|
|
336
|
+
self._info_cache: dict[str, dict[str, Any]] = {}
|
|
337
|
+
self._track_headers: dict[TrackInfo, dict[str, str]] = {}
|
|
338
|
+
self._cookie_jar: MozillaCookieJar | None = None
|
|
339
|
+
|
|
340
|
+
# -- command construction ------------------------------------------------
|
|
341
|
+
def _binary_command(self) -> list[str]:
|
|
342
|
+
if self._binary:
|
|
343
|
+
return [self._binary]
|
|
344
|
+
configured = _settings_attr(self._settings, "yt_dlp_binary")
|
|
345
|
+
if configured and configured != "yt-dlp":
|
|
346
|
+
return [configured]
|
|
347
|
+
if configured and shutil.which(configured):
|
|
348
|
+
return [shutil.which(configured)]
|
|
349
|
+
# The Python package is a hard dependency; running it through the same
|
|
350
|
+
# interpreter works even when the CLI script is not on PATH.
|
|
351
|
+
return [sys.executable, "-m", "yt_dlp"]
|
|
352
|
+
|
|
353
|
+
def _auth_args(self) -> list[str]:
|
|
354
|
+
args: list[str] = []
|
|
355
|
+
if self._cookies_file:
|
|
356
|
+
args.extend(["--cookies", str(self._cookies_file)])
|
|
357
|
+
if self._cookies_from_browser:
|
|
358
|
+
args.extend(["--cookies-from-browser", str(self._cookies_from_browser)])
|
|
359
|
+
if self._proxy:
|
|
360
|
+
args.extend(["--proxy", str(self._proxy)])
|
|
361
|
+
return args
|
|
362
|
+
|
|
363
|
+
def _command(self, extra: Sequence[str], *, target: str | None = None) -> list[str]:
|
|
364
|
+
runtime_args: list[str] = []
|
|
365
|
+
if self._platform == "youtube":
|
|
366
|
+
if self._js_runtime_args is None:
|
|
367
|
+
self._js_runtime_args = []
|
|
368
|
+
if self._supports_js_runtime_flags():
|
|
369
|
+
self._js_runtime_args = ["--no-remote-components"]
|
|
370
|
+
runtimes, selected = probe_javascript_runtimes()
|
|
371
|
+
if selected:
|
|
372
|
+
self._js_runtime_args += ["--js-runtimes", f"{selected}:{runtimes[selected]['path']}"]
|
|
373
|
+
runtime_args = self._js_runtime_args
|
|
374
|
+
command = [*self._binary_command(), "--ignore-config", *runtime_args, *extra, *self._auth_args()]
|
|
375
|
+
if target is not None:
|
|
376
|
+
command.append(target)
|
|
377
|
+
return command
|
|
378
|
+
|
|
379
|
+
def _supports_js_runtime_flags(self) -> bool:
|
|
380
|
+
"""Never send modern options to an older explicitly configured yt-dlp."""
|
|
381
|
+
try:
|
|
382
|
+
binary_command = self._binary_command()
|
|
383
|
+
if binary_command != [sys.executable, "-m", "yt_dlp"]:
|
|
384
|
+
result = subprocess.run([*binary_command, "--ignore-config", "--help"],
|
|
385
|
+
capture_output=True, text=True, timeout=5, check=True)
|
|
386
|
+
return all(option in result.stdout for option in ("--js-runtimes", "--no-remote-components"))
|
|
387
|
+
from yt_dlp.options import create_parser
|
|
388
|
+
|
|
389
|
+
parser = create_parser()
|
|
390
|
+
return all(parser.has_option(option) for option in ("--js-runtimes", "--no-remote-components"))
|
|
391
|
+
except Exception:
|
|
392
|
+
return False
|
|
393
|
+
|
|
394
|
+
def _require_network(self) -> None:
|
|
395
|
+
if not self._allow_network:
|
|
396
|
+
error = AnalysisBlockedError(
|
|
397
|
+
"outbound network access is disabled for this acquisition task (offline mode)"
|
|
398
|
+
)
|
|
399
|
+
error.reason_code = "NETWORK_DISABLED" # type: ignore[attr-defined]
|
|
400
|
+
raise error
|
|
401
|
+
|
|
402
|
+
def _run(self, extra: Sequence[str], *, target: str | None = None) -> subprocess.CompletedProcess[str]:
|
|
403
|
+
self._require_network()
|
|
404
|
+
command = self._command(extra, target=target)
|
|
405
|
+
try:
|
|
406
|
+
return self._runner(
|
|
407
|
+
command,
|
|
408
|
+
timeout=self._timeout,
|
|
409
|
+
capture_output=True,
|
|
410
|
+
text=True,
|
|
411
|
+
)
|
|
412
|
+
except FileNotFoundError as exc:
|
|
413
|
+
raise RuntimeError(
|
|
414
|
+
"yt-dlp is not installed; install the project dependencies "
|
|
415
|
+
"(`uv pip install yt-dlp`) or configure YTK_YT_DLP_BINARY"
|
|
416
|
+
) from exc
|
|
417
|
+
except subprocess.TimeoutExpired as exc:
|
|
418
|
+
error = AnalysisBlockedError(f"yt-dlp timed out after {self._timeout:g}s and was killed")
|
|
419
|
+
error.reason_code = "PROVIDER_TIMEOUT" # type: ignore[attr-defined]
|
|
420
|
+
raise error from exc
|
|
421
|
+
|
|
422
|
+
# -- metadata ------------------------------------------------------------
|
|
423
|
+
def info(self, url: str) -> dict[str, Any]:
|
|
424
|
+
"""Return the single ``--dump-single-json`` payload for this task."""
|
|
425
|
+
if not isinstance(url, str) or not url.strip():
|
|
426
|
+
raise ValueError("url must be a non-empty string")
|
|
427
|
+
cached = self._info_cache.get(url)
|
|
428
|
+
if cached is not None:
|
|
429
|
+
return cached
|
|
430
|
+
result = self._run(
|
|
431
|
+
["--dump-single-json", "--skip-download", "--no-playlist", "--no-warnings", "--write-subs", "--write-auto-subs"],
|
|
432
|
+
target=url,
|
|
433
|
+
)
|
|
434
|
+
if result.returncode != 0:
|
|
435
|
+
raise provider_error(result.stderr or "", platform=self._platform)
|
|
436
|
+
stdout = (result.stdout or "").strip()
|
|
437
|
+
if not stdout:
|
|
438
|
+
raise RuntimeError("yt-dlp returned no metadata")
|
|
439
|
+
try:
|
|
440
|
+
payload = json.loads(stdout)
|
|
441
|
+
except json.JSONDecodeError as exc:
|
|
442
|
+
raise ContentIntegrityError(f"yt-dlp metadata is not valid JSON: {exc.msg}") from exc
|
|
443
|
+
if not isinstance(payload, Mapping):
|
|
444
|
+
raise ContentIntegrityError("yt-dlp metadata payload is not an object")
|
|
445
|
+
info = dict(payload)
|
|
446
|
+
self._info_cache[url] = info
|
|
447
|
+
return info
|
|
448
|
+
|
|
449
|
+
# -- subtitles -----------------------------------------------------------
|
|
450
|
+
def subtitle_map(self, info: Mapping[str, Any]) -> list[TrackInfo]:
|
|
451
|
+
"""All narration-capable tracks, manual/inline first, danmaku excluded."""
|
|
452
|
+
tracks: list[TrackInfo] = []
|
|
453
|
+
for field_name, origin in (("subtitles", "manual"), ("automatic_captions", "automatic")):
|
|
454
|
+
groups = info.get(field_name) if isinstance(info, Mapping) else None
|
|
455
|
+
if not isinstance(groups, Mapping):
|
|
456
|
+
continue
|
|
457
|
+
for language, entries in groups.items():
|
|
458
|
+
language_text = str(language)
|
|
459
|
+
if not is_narration_track(language_text, ""):
|
|
460
|
+
continue
|
|
461
|
+
if not isinstance(entries, Sequence) or isinstance(entries, str | bytes):
|
|
462
|
+
continue
|
|
463
|
+
for entry in entries:
|
|
464
|
+
track = self._track_from_entry(entry, origin=origin, language=language_text)
|
|
465
|
+
if track is not None:
|
|
466
|
+
headers = _provider_http_headers(info.get("http_headers"))
|
|
467
|
+
if isinstance(entry, Mapping):
|
|
468
|
+
headers.update(_provider_http_headers(entry.get("http_headers")))
|
|
469
|
+
self._track_headers[track] = headers
|
|
470
|
+
tracks.append(track)
|
|
471
|
+
# Manual/live inline tracks before automatic ones, stable otherwise.
|
|
472
|
+
tracks.sort(key=lambda track: track.kind == "automatic")
|
|
473
|
+
return tracks
|
|
474
|
+
|
|
475
|
+
def subtitle_tracks(self, info: Mapping[str, Any]) -> list[TrackInfo]:
|
|
476
|
+
"""Alias used by the adapters; identical to :meth:`subtitle_map`."""
|
|
477
|
+
return self.subtitle_map(info)
|
|
478
|
+
|
|
479
|
+
def _track_from_entry(self, entry: object, *, origin: str, language: str) -> TrackInfo | None:
|
|
480
|
+
if not isinstance(entry, Mapping):
|
|
481
|
+
return None
|
|
482
|
+
ext = str(entry.get("ext") or "").strip().casefold()
|
|
483
|
+
name = _clean_text(entry.get("name")) or language
|
|
484
|
+
if not is_narration_track(name, ext) or not is_narration_track(language, ext):
|
|
485
|
+
return None
|
|
486
|
+
|
|
487
|
+
inline = entry.get("data")
|
|
488
|
+
if isinstance(inline, str) and inline.strip():
|
|
489
|
+
fmt = EXT_TO_FORMAT.get(ext)
|
|
490
|
+
if fmt is None:
|
|
491
|
+
fmt = "srt" if _SRT_HINT_RE.search(inline[:200]) else "plain"
|
|
492
|
+
return TrackInfo(
|
|
493
|
+
name=name,
|
|
494
|
+
language=language,
|
|
495
|
+
kind="inline",
|
|
496
|
+
format=fmt,
|
|
497
|
+
inline_data=inline,
|
|
498
|
+
ext=ext or fmt,
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
url = normalize_http_url(entry.get("url"))
|
|
502
|
+
fmt = EXT_TO_FORMAT.get(ext)
|
|
503
|
+
if fmt is None or url is None:
|
|
504
|
+
return None
|
|
505
|
+
return TrackInfo(
|
|
506
|
+
name=name,
|
|
507
|
+
language=language,
|
|
508
|
+
kind=origin, # type: ignore[arg-type]
|
|
509
|
+
format=fmt,
|
|
510
|
+
url=url,
|
|
511
|
+
ext=ext,
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
def fetch_subtitle(self, track: TrackInfo | str) -> str:
|
|
515
|
+
"""Return the raw subtitle text for an inline track or a subtitle URL."""
|
|
516
|
+
if isinstance(track, TrackInfo):
|
|
517
|
+
if track.inline_data is not None:
|
|
518
|
+
return track.inline_data
|
|
519
|
+
if not track.url:
|
|
520
|
+
raise ContentIntegrityError(f"subtitle track {track.name!r} has neither data nor a url")
|
|
521
|
+
url = normalize_http_url(track.url)
|
|
522
|
+
else:
|
|
523
|
+
url = normalize_http_url(track)
|
|
524
|
+
if url is None:
|
|
525
|
+
raise ContentIntegrityError("subtitle url is not a safe http(s) URL")
|
|
526
|
+
self._require_network()
|
|
527
|
+
try:
|
|
528
|
+
response = self._http_get(url, headers=self._track_headers.get(track, {}) if isinstance(track, TrackInfo) else None)
|
|
529
|
+
response.raise_for_status()
|
|
530
|
+
except httpx.HTTPError as exc:
|
|
531
|
+
status = exc.response.status_code if isinstance(exc, httpx.HTTPStatusError) else None
|
|
532
|
+
if status in {404, 410}:
|
|
533
|
+
raise ContentIntegrityError(f"subtitle track is unavailable (HTTP {status})") from exc
|
|
534
|
+
reason = "RATE_LIMITED" if status == 429 else "AUTH_REQUIRED" if status in {401, 403} else "NETWORK_ERROR"
|
|
535
|
+
error = AnalysisBlockedError(f"subtitle fetch blocked ({reason})")
|
|
536
|
+
error.reason_code = reason # type: ignore[attr-defined]
|
|
537
|
+
raise error from exc
|
|
538
|
+
text = getattr(response, "text", "")
|
|
539
|
+
if not isinstance(text, str) or not text.strip():
|
|
540
|
+
raise ContentIntegrityError("subtitle response is empty")
|
|
541
|
+
return text
|
|
542
|
+
|
|
543
|
+
# -- downloads -----------------------------------------------------------
|
|
544
|
+
def download_audio(self, url: str, dest: Path | str) -> Path:
|
|
545
|
+
"""Download the audio track only (never the full high-resolution video)."""
|
|
546
|
+
return self._download(
|
|
547
|
+
url,
|
|
548
|
+
dest,
|
|
549
|
+
format_selector="bestaudio[ext=m4a]/bestaudio/best",
|
|
550
|
+
merge_video=False,
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
def download_media(self, url: str, dest: Path | str) -> Path:
|
|
554
|
+
"""Download and merge a video+audio rendition when picture evidence is needed."""
|
|
555
|
+
return self._download(
|
|
556
|
+
url,
|
|
557
|
+
dest,
|
|
558
|
+
format_selector="bv*+ba/b",
|
|
559
|
+
merge_video=True,
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
def _download(self, url: str, dest: Path | str, *, format_selector: str, merge_video: bool) -> Path:
|
|
563
|
+
if not isinstance(url, str) or not url.strip():
|
|
564
|
+
raise ValueError("url must be a non-empty string")
|
|
565
|
+
target = Path(dest)
|
|
566
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
567
|
+
stem = target.with_suffix("") if target.suffix else target
|
|
568
|
+
extra: list[str] = [
|
|
569
|
+
"-f",
|
|
570
|
+
format_selector,
|
|
571
|
+
"--no-playlist",
|
|
572
|
+
"--no-warnings",
|
|
573
|
+
"--no-progress",
|
|
574
|
+
"-o",
|
|
575
|
+
f"{stem}.%(ext)s",
|
|
576
|
+
"--print",
|
|
577
|
+
"after_move:filepath",
|
|
578
|
+
]
|
|
579
|
+
if merge_video:
|
|
580
|
+
extra.extend(["--merge-output-format", "mp4"])
|
|
581
|
+
result = self._run(extra, target=url)
|
|
582
|
+
if result.returncode != 0:
|
|
583
|
+
raise provider_error(result.stderr or "", platform=self._platform)
|
|
584
|
+
return self._resolve_download(result.stdout or "", target)
|
|
585
|
+
|
|
586
|
+
def _resolve_download(self, stdout: str, dest: Path) -> Path:
|
|
587
|
+
"""Resolve the file this run produced; never scavenge unrelated files."""
|
|
588
|
+
for line in reversed([line.strip() for line in stdout.splitlines() if line.strip()]):
|
|
589
|
+
candidate = Path(line)
|
|
590
|
+
if candidate.is_file() and candidate.stat().st_size > 0:
|
|
591
|
+
return candidate
|
|
592
|
+
stem = dest.with_suffix("") if dest.suffix else dest
|
|
593
|
+
produced = [
|
|
594
|
+
candidate
|
|
595
|
+
for candidate in sorted(stem.parent.glob(f"{stem.name}.*"))
|
|
596
|
+
if candidate.is_file()
|
|
597
|
+
and candidate.suffix.casefold() not in {".part", ".ytdl", ".temp"}
|
|
598
|
+
and candidate.stat().st_size > 0
|
|
599
|
+
]
|
|
600
|
+
if len(produced) == 1:
|
|
601
|
+
return produced[0]
|
|
602
|
+
if dest.is_file() and dest.stat().st_size > 0:
|
|
603
|
+
return dest
|
|
604
|
+
raise ContentIntegrityError(
|
|
605
|
+
f"yt-dlp reported success but no media file was produced at {dest}; "
|
|
606
|
+
"existing directory residue is never reused"
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
def download_image(self, url: str, dest: Path | str) -> Path:
|
|
610
|
+
"""Download a still image (cover/illustration) with a size and type guard."""
|
|
611
|
+
normalized = normalize_http_url(url)
|
|
612
|
+
if normalized is None:
|
|
613
|
+
raise ContentIntegrityError("image url is not a safe http(s) URL")
|
|
614
|
+
self._require_network()
|
|
615
|
+
response = self._http_get(normalized)
|
|
616
|
+
response.raise_for_status()
|
|
617
|
+
content = getattr(response, "content", b"")
|
|
618
|
+
if not isinstance(content, bytes | bytearray) or not content:
|
|
619
|
+
raise ContentIntegrityError("image response is empty")
|
|
620
|
+
payload = bytes(content)
|
|
621
|
+
max_bytes = self._max_image_bytes()
|
|
622
|
+
if len(payload) > max_bytes:
|
|
623
|
+
raise ContentIntegrityError(
|
|
624
|
+
f"image exceeds max_media_mb ({max_bytes // (1024 * 1024)} MB)"
|
|
625
|
+
)
|
|
626
|
+
content_type = _response_header(response, "content-type")
|
|
627
|
+
if content_type:
|
|
628
|
+
if not content_type.casefold().startswith("image/"):
|
|
629
|
+
raise ContentIntegrityError(f"downloaded payload is not an image ({content_type})")
|
|
630
|
+
elif not _looks_like_image(payload):
|
|
631
|
+
raise ContentIntegrityError("downloaded payload is not an image")
|
|
632
|
+
|
|
633
|
+
target = Path(dest)
|
|
634
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
635
|
+
temporary = target.with_name(target.name + ".part")
|
|
636
|
+
temporary.write_bytes(payload)
|
|
637
|
+
temporary.replace(target)
|
|
638
|
+
return target
|
|
639
|
+
|
|
640
|
+
def _max_image_bytes(self) -> int:
|
|
641
|
+
value = _to_float(_settings_attr(self._settings, "max_media_mb"))
|
|
642
|
+
megabytes = DEFAULT_IMAGE_MAX_MB if value is None or value <= 0 else value
|
|
643
|
+
return int(megabytes * 1024 * 1024)
|
|
644
|
+
|
|
645
|
+
# -- http ----------------------------------------------------------------
|
|
646
|
+
def _http_get(self, url: str, *, headers: Mapping[str, str] | None = None) -> Any:
|
|
647
|
+
headers = _provider_http_headers(headers)
|
|
648
|
+
if self._http_client is not None:
|
|
649
|
+
return self._http_get_with_client(self._http_client, url, headers)
|
|
650
|
+
from ytk.ingestion.http_client import make_client
|
|
651
|
+
|
|
652
|
+
with make_client(url=url, timeout=self._http_timeout, proxy=self._proxy) as client:
|
|
653
|
+
return self._http_get_with_client(client, url, headers)
|
|
654
|
+
|
|
655
|
+
def _http_get_with_client(self, client: Any, url: str, headers: Mapping[str, str] | None) -> Any:
|
|
656
|
+
current = url
|
|
657
|
+
original_origin = (urlsplit(url).scheme, urlsplit(url).netloc)
|
|
658
|
+
for _ in range(6):
|
|
659
|
+
_validate_redirect_target(current)
|
|
660
|
+
request_headers = {"user-agent": self._user_agent, **dict(headers or {})}
|
|
661
|
+
if (urlsplit(current).scheme, urlsplit(current).netloc) != original_origin:
|
|
662
|
+
for key in ("authorization", "referer", "origin"):
|
|
663
|
+
request_headers.pop(key, None)
|
|
664
|
+
if self._cookies_file:
|
|
665
|
+
# An empty override also prevents a client's unrelated cookie
|
|
666
|
+
# state from bypassing the task jar's strict domain policy.
|
|
667
|
+
request_headers["Cookie"] = self._cookie_header(current)
|
|
668
|
+
response = client.get(current, headers=request_headers, follow_redirects=False)
|
|
669
|
+
if self._cookie_jar is not None and isinstance(response, httpx.Response):
|
|
670
|
+
httpx.Cookies(self._cookie_jar).extract_cookies(response)
|
|
671
|
+
if getattr(response, "status_code", 200) not in {301, 302, 303, 307, 308}:
|
|
672
|
+
return response
|
|
673
|
+
location = getattr(response, "headers", {}).get("location")
|
|
674
|
+
if not location:
|
|
675
|
+
raise ContentIntegrityError("provider redirect has no Location header")
|
|
676
|
+
current = urljoin(current, location)
|
|
677
|
+
raise ContentIntegrityError("provider redirect limit exceeded")
|
|
678
|
+
|
|
679
|
+
def _cookie_header(self, url: str) -> str:
|
|
680
|
+
if self._cookie_jar is None:
|
|
681
|
+
path = Path(str(self._cookies_file)).expanduser().resolve()
|
|
682
|
+
if path.name == ".env" or path.name.startswith(".env."):
|
|
683
|
+
raise ContentIntegrityError("dotenv files cannot be used as cookie jars")
|
|
684
|
+
jar = MozillaCookieJar(str(path), policy=DefaultCookiePolicy(strict_ns_domain=DefaultCookiePolicy.DomainStrictNonDomain))
|
|
685
|
+
try:
|
|
686
|
+
# stdlib parsing avoids yt-dlp's malformed-line stderr echo.
|
|
687
|
+
# Even the stdlib diagnostic warning must stay in memory.
|
|
688
|
+
with warnings.catch_warnings(record=True):
|
|
689
|
+
warnings.simplefilter("always")
|
|
690
|
+
jar.load(ignore_discard=True, ignore_expires=True)
|
|
691
|
+
for cookie in jar:
|
|
692
|
+
if cookie.expires == 0: # yt-dlp's Netscape session convention
|
|
693
|
+
cookie.expires = None
|
|
694
|
+
cookie.discard = True
|
|
695
|
+
except (OSError, ValueError, LoadError):
|
|
696
|
+
error = AnalysisBlockedError("configured cookie jar is missing or invalid")
|
|
697
|
+
error.reason_code = "AUTH_REQUIRED" # type: ignore[attr-defined]
|
|
698
|
+
raise error from None
|
|
699
|
+
self._cookie_jar = jar
|
|
700
|
+
request = Request(url)
|
|
701
|
+
self._cookie_jar.add_cookie_header(request)
|
|
702
|
+
return request.get_header("Cookie") or ""
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _provider_http_headers(value: object) -> dict[str, str]:
|
|
706
|
+
if not isinstance(value, Mapping):
|
|
707
|
+
return {}
|
|
708
|
+
headers: dict[str, str] = {}
|
|
709
|
+
for key, item in value.items():
|
|
710
|
+
if str(key).lower() not in _HTTP_HEADER_NAMES or not isinstance(item, str):
|
|
711
|
+
continue
|
|
712
|
+
if any(ord(char) < 32 or ord(char) > 126 for char in item):
|
|
713
|
+
raise ContentIntegrityError("provider HTTP header contains invalid characters")
|
|
714
|
+
headers[str(key).lower()] = item
|
|
715
|
+
return headers
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def _default_runner(command: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
|
|
719
|
+
return subprocess.run(list(command), **kwargs)
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def _response_header(response: object, name: str) -> str | None:
|
|
723
|
+
headers = getattr(response, "headers", None)
|
|
724
|
+
if headers is None:
|
|
725
|
+
return None
|
|
726
|
+
try:
|
|
727
|
+
value = headers.get(name) if isinstance(headers, Mapping) else None
|
|
728
|
+
except Exception: # noqa: BLE001 - defensive: odd httpx header objects
|
|
729
|
+
return None
|
|
730
|
+
return value if isinstance(value, str) and value.strip() else None
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _looks_like_image(payload: bytes) -> bool:
|
|
734
|
+
signatures = (
|
|
735
|
+
b"\x89PNG\r\n\x1a\n",
|
|
736
|
+
b"\xff\xd8\xff",
|
|
737
|
+
b"GIF87a",
|
|
738
|
+
b"GIF89a",
|
|
739
|
+
b"BM",
|
|
740
|
+
)
|
|
741
|
+
if payload.startswith(signatures):
|
|
742
|
+
return True
|
|
743
|
+
return payload[:4] == b"RIFF" and payload[8:12] == b"WEBP"
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
__all__ = [
|
|
747
|
+
"DEFAULT_TIMEOUT",
|
|
748
|
+
"YtDlpAcquirer",
|
|
749
|
+
"classify_failure",
|
|
750
|
+
"metadata_from_info",
|
|
751
|
+
"normalize_http_url",
|
|
752
|
+
"provider_error",
|
|
753
|
+
"redact_urls",
|
|
754
|
+
]
|