textflowkit 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- textflowkit/__init__.py +8 -0
- textflowkit/adapters/__init__.py +11 -0
- textflowkit/adapters/http_server.py +465 -0
- textflowkit/adapters/mcp_server.py +554 -0
- textflowkit/cli.py +473 -0
- textflowkit/core/__init__.py +5 -0
- textflowkit/core/batch.py +186 -0
- textflowkit/core/bind.py +66 -0
- textflowkit/core/cancel.py +19 -0
- textflowkit/core/checkpoint.py +389 -0
- textflowkit/core/diarize.py +229 -0
- textflowkit/core/engine.py +127 -0
- textflowkit/core/executor.py +301 -0
- textflowkit/core/jobs.py +241 -0
- textflowkit/core/model.py +98 -0
- textflowkit/core/paths.py +226 -0
- textflowkit/core/pipeline.py +368 -0
- textflowkit/core/retrieval.py +148 -0
- textflowkit/core/runner.py +146 -0
- textflowkit/core/service.py +146 -0
- textflowkit/core/sqlite_store.py +233 -0
- textflowkit/core/submission.py +220 -0
- textflowkit/core/timeutil.py +20 -0
- textflowkit/core/translate.py +244 -0
- textflowkit/render/__init__.py +191 -0
- textflowkit/render/docx.py +71 -0
- textflowkit/render/fonts/NotoSans.ttf +0 -0
- textflowkit/render/fonts/NotoSansArabic.ttf +0 -0
- textflowkit/render/fonts/NotoSansSC.ttf +0 -0
- textflowkit/render/fonts/OFL-NotoSans.txt +94 -0
- textflowkit/render/fonts/OFL-NotoSansSC.txt +93 -0
- textflowkit/render/fonts/README.md +19 -0
- textflowkit/render/markdown.py +33 -0
- textflowkit/render/pdf.py +136 -0
- textflowkit/render/srt.py +22 -0
- textflowkit/render/txt.py +19 -0
- textflowkit/render/vtt.py +20 -0
- textflowkit/sources/__init__.py +16 -0
- textflowkit/sources/acquire.py +437 -0
- textflowkit/sources/detect.py +200 -0
- textflowkit/sources/scratch.py +32 -0
- textflowkit-0.1.3.dist-info/METADATA +266 -0
- textflowkit-0.1.3.dist-info/RECORD +46 -0
- textflowkit-0.1.3.dist-info/WHEEL +4 -0
- textflowkit-0.1.3.dist-info/entry_points.txt +4 -0
- textflowkit-0.1.3.dist-info/licenses/LICENSE +203 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
"""Media acquisition: local files pass through, URLs go through yt-dlp."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
import wave
|
|
12
|
+
from collections.abc import Callable
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from textflowkit.core.cancel import CancelledError
|
|
16
|
+
from textflowkit.core.paths import UnsafeInputPathError, opened_file_path
|
|
17
|
+
from textflowkit.core.service import (
|
|
18
|
+
ENV_EGRESS_PROXY,
|
|
19
|
+
ENV_FFMPEG_TIMEOUT_SECONDS,
|
|
20
|
+
ENV_MAX_DURATION_SECONDS,
|
|
21
|
+
ENV_MAX_MEDIA_BYTES,
|
|
22
|
+
positive_limit,
|
|
23
|
+
production_enabled,
|
|
24
|
+
)
|
|
25
|
+
from textflowkit.sources.detect import SourceRef, UnsafeUrlError, assert_url_is_fetchable
|
|
26
|
+
from textflowkit.sources.scratch import ScratchPaths
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AcquisitionError(RuntimeError):
|
|
30
|
+
"""Raised when media cannot be obtained."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def stage_confined_local_media(
|
|
34
|
+
source: str | Path, *, work_dir: Path, input_root: str | Path
|
|
35
|
+
) -> Path:
|
|
36
|
+
"""Copy a handle-verified local input into isolated scratch before ffmpeg."""
|
|
37
|
+
base = Path(input_root).expanduser().resolve()
|
|
38
|
+
path = Path(source)
|
|
39
|
+
out = ScratchPaths(Path(work_dir)).staged_local(path.suffix)
|
|
40
|
+
maximum = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024) if production_enabled() else None
|
|
41
|
+
with path.open("rb") as opened:
|
|
42
|
+
actual = opened_file_path(opened.fileno(), path)
|
|
43
|
+
if actual != base and base not in actual.parents:
|
|
44
|
+
raise UnsafeInputPathError(
|
|
45
|
+
f"opened input file '{actual}' is outside the allowed root '{base}'"
|
|
46
|
+
)
|
|
47
|
+
written = 0
|
|
48
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
with out.open("xb") as destination:
|
|
50
|
+
while chunk := opened.read(1024 * 1024):
|
|
51
|
+
written += len(chunk)
|
|
52
|
+
if maximum is not None and written > maximum:
|
|
53
|
+
raise AcquisitionError("media exceeds the configured size limit")
|
|
54
|
+
destination.write(chunk)
|
|
55
|
+
return out
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def require_tool(name: str, *, module: str | None = None) -> str | None:
|
|
59
|
+
"""Locate an external tool.
|
|
60
|
+
|
|
61
|
+
Checks PATH first, then the interpreter's own Scripts/bin directory. The
|
|
62
|
+
second check matters for the common case where a dependency was installed
|
|
63
|
+
into the active environment (or a venv) but the environment is not
|
|
64
|
+
activated, so its entry point is not on PATH.
|
|
65
|
+
|
|
66
|
+
Returns None when the tool is absent and a viable in-process module exists;
|
|
67
|
+
callers then use the module instead of shelling out.
|
|
68
|
+
"""
|
|
69
|
+
found = shutil.which(name)
|
|
70
|
+
if found:
|
|
71
|
+
return found
|
|
72
|
+
|
|
73
|
+
# Look beside the running interpreter (venv/Scripts, venv/bin, ...).
|
|
74
|
+
exe = name + (".exe" if os.name == "nt" else "")
|
|
75
|
+
scripts_dir = Path(sys.executable).parent
|
|
76
|
+
candidate = scripts_dir / exe
|
|
77
|
+
if candidate.exists():
|
|
78
|
+
return str(candidate)
|
|
79
|
+
|
|
80
|
+
if module:
|
|
81
|
+
try:
|
|
82
|
+
__import__(module)
|
|
83
|
+
except ImportError:
|
|
84
|
+
pass
|
|
85
|
+
else:
|
|
86
|
+
return None # module usable in-process
|
|
87
|
+
|
|
88
|
+
raise AcquisitionError(
|
|
89
|
+
f"required tool '{name}' not found. Install it (pip install {module or name}) and retry."
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# JavaScript runtimes yt-dlp can use, in its own priority order. yt-dlp enables
|
|
94
|
+
# only "deno" by default, so a machine that has Node (or bun/quickjs) still emits
|
|
95
|
+
# "No supported JavaScript runtime could be found". Detecting what is actually
|
|
96
|
+
# present and enabling it explicitly avoids requiring a specific runtime.
|
|
97
|
+
JS_RUNTIMES = ("deno", "node", "bun", "quickjs")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def detect_js_runtime() -> str | None:
|
|
101
|
+
"""Return the highest-priority JavaScript runtime available, or None.
|
|
102
|
+
|
|
103
|
+
Mirrors yt-dlp's own priority order. Returning None is safe: yt-dlp falls
|
|
104
|
+
back to non-JS extraction, which works for many videos but can leave some
|
|
105
|
+
formats unavailable.
|
|
106
|
+
"""
|
|
107
|
+
for name in JS_RUNTIMES:
|
|
108
|
+
if shutil.which(name):
|
|
109
|
+
return name
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _js_runtime_args() -> list[str]:
|
|
114
|
+
"""Build yt-dlp JS-runtime flags for whatever runtime is installed."""
|
|
115
|
+
runtime = detect_js_runtime()
|
|
116
|
+
if not runtime:
|
|
117
|
+
return []
|
|
118
|
+
# yt-dlp enables deno by default; enabling another runtime requires clearing
|
|
119
|
+
# the defaults first so the detected runtime is the one actually used.
|
|
120
|
+
if runtime == "deno":
|
|
121
|
+
return []
|
|
122
|
+
return ["--no-js-runtimes", "--js-runtimes", runtime]
|
|
123
|
+
|
|
124
|
+
def _fetch_with_module(
|
|
125
|
+
url: str,
|
|
126
|
+
*,
|
|
127
|
+
work_dir: Path,
|
|
128
|
+
cookies_from_browser: str | None,
|
|
129
|
+
check_cancel: Callable[[], None] | None = None,
|
|
130
|
+
) -> Path:
|
|
131
|
+
"""Download using the yt_dlp Python API so URL checks cover its requests."""
|
|
132
|
+
from yt_dlp import YoutubeDL
|
|
133
|
+
|
|
134
|
+
layout = ScratchPaths(work_dir)
|
|
135
|
+
layout.media_dir.mkdir(parents=True, exist_ok=True)
|
|
136
|
+
outtmpl = layout.download_template
|
|
137
|
+
hooks: list[Path] = []
|
|
138
|
+
maximum = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024) if production_enabled() else None
|
|
139
|
+
|
|
140
|
+
def _check_download_size(status: dict) -> None:
|
|
141
|
+
if maximum is None:
|
|
142
|
+
return
|
|
143
|
+
for key in ("downloaded_bytes", "total_bytes", "total_bytes_estimate"):
|
|
144
|
+
value = status.get(key)
|
|
145
|
+
if isinstance(value, (int, float)) and value > maximum:
|
|
146
|
+
raise AcquisitionError("download exceeds the configured size limit")
|
|
147
|
+
# Aggregate fragments and temporary files as well as the final output;
|
|
148
|
+
# a per-fragment counter alone would reset below the cap each time.
|
|
149
|
+
total = sum(p.stat().st_size for p in layout.media_dir.rglob("*") if p.is_file())
|
|
150
|
+
if total > maximum:
|
|
151
|
+
raise AcquisitionError("download exceeds the configured size limit")
|
|
152
|
+
|
|
153
|
+
def _hook(status: dict) -> None:
|
|
154
|
+
# yt-dlp calls this frequently during a download. Raising here aborts
|
|
155
|
+
# the download, which is what makes cancellation responsive for the
|
|
156
|
+
# slowest common case instead of waiting for the whole fetch to finish.
|
|
157
|
+
if check_cancel is not None:
|
|
158
|
+
check_cancel()
|
|
159
|
+
_check_download_size(status)
|
|
160
|
+
if status.get("status") == "finished":
|
|
161
|
+
path = status.get("filename") or status.get("_filename")
|
|
162
|
+
if path:
|
|
163
|
+
hooks.append(Path(path))
|
|
164
|
+
|
|
165
|
+
runtime = detect_js_runtime()
|
|
166
|
+
opts: dict = {
|
|
167
|
+
"js_runtimes": {runtime: {}} if runtime else {},
|
|
168
|
+
"outtmpl": outtmpl,
|
|
169
|
+
"noplaylist": True,
|
|
170
|
+
"quiet": True,
|
|
171
|
+
"no_warnings": True,
|
|
172
|
+
"format": "bestaudio/best",
|
|
173
|
+
"restrictfilenames": True,
|
|
174
|
+
"progress_hooks": [_hook],
|
|
175
|
+
}
|
|
176
|
+
if maximum is not None:
|
|
177
|
+
opts["max_filesize"] = maximum
|
|
178
|
+
if cookies_from_browser:
|
|
179
|
+
opts["cookiesfrombrowser"] = (cookies_from_browser,)
|
|
180
|
+
if production_enabled():
|
|
181
|
+
proxy = os.environ.get(ENV_EGRESS_PROXY)
|
|
182
|
+
if not proxy:
|
|
183
|
+
raise AcquisitionError(
|
|
184
|
+
f"production URL acquisition requires an SSRF-filtering {ENV_EGRESS_PROXY}"
|
|
185
|
+
)
|
|
186
|
+
opts["proxy"] = proxy
|
|
187
|
+
# External JS runtimes are separate processes and are not guaranteed to
|
|
188
|
+
# honor yt-dlp's proxy option. Do not let them create an egress bypass.
|
|
189
|
+
opts["js_runtimes"] = {}
|
|
190
|
+
|
|
191
|
+
try:
|
|
192
|
+
with YoutubeDL(opts) as ydl:
|
|
193
|
+
original_open = ydl.urlopen
|
|
194
|
+
|
|
195
|
+
def checked_open(request):
|
|
196
|
+
requested = request if isinstance(request, str) else request.url
|
|
197
|
+
_validate_fetch_url(requested)
|
|
198
|
+
response = original_open(request)
|
|
199
|
+
if maximum is not None:
|
|
200
|
+
headers = getattr(response, "headers", None)
|
|
201
|
+
length = headers.get("Content-Length") if headers is not None else None
|
|
202
|
+
if length is not None:
|
|
203
|
+
try:
|
|
204
|
+
too_large = int(length) > maximum
|
|
205
|
+
except (TypeError, ValueError):
|
|
206
|
+
too_large = False
|
|
207
|
+
if too_large:
|
|
208
|
+
close = getattr(response, "close", None)
|
|
209
|
+
if callable(close):
|
|
210
|
+
close()
|
|
211
|
+
raise AcquisitionError("download Content-Length exceeds the configured size limit")
|
|
212
|
+
final = getattr(response, "url", None)
|
|
213
|
+
if final:
|
|
214
|
+
_validate_fetch_url(final)
|
|
215
|
+
return response
|
|
216
|
+
|
|
217
|
+
ydl.urlopen = checked_open
|
|
218
|
+
_validate_fetch_url(url)
|
|
219
|
+
info = ydl.extract_info(url, download=False)
|
|
220
|
+
_validate_download_info(info, maximum=maximum)
|
|
221
|
+
ydl.process_info(info)
|
|
222
|
+
except CancelledError:
|
|
223
|
+
raise # an orderly stop, not a fetch failure
|
|
224
|
+
except AcquisitionError:
|
|
225
|
+
raise
|
|
226
|
+
except Exception as exc:
|
|
227
|
+
raise AcquisitionError(f"yt-dlp failed: {exc}") from exc
|
|
228
|
+
|
|
229
|
+
if hooks and hooks[-1].exists():
|
|
230
|
+
if maximum is not None and hooks[-1].stat().st_size > maximum:
|
|
231
|
+
raise AcquisitionError("download exceeds the configured size limit")
|
|
232
|
+
return hooks[-1]
|
|
233
|
+
|
|
234
|
+
requested = info.get("requested_downloads") or []
|
|
235
|
+
for item in requested:
|
|
236
|
+
candidate = Path(item.get("filepath", ""))
|
|
237
|
+
if candidate.exists():
|
|
238
|
+
if maximum is not None and candidate.stat().st_size > maximum:
|
|
239
|
+
raise AcquisitionError("download exceeds the configured size limit")
|
|
240
|
+
return candidate
|
|
241
|
+
|
|
242
|
+
vid = info.get("id")
|
|
243
|
+
if vid:
|
|
244
|
+
for candidate in sorted(work_dir.glob(f"{vid}.*"), key=lambda p: p.stat().st_mtime, reverse=True):
|
|
245
|
+
if candidate.is_file():
|
|
246
|
+
return candidate
|
|
247
|
+
|
|
248
|
+
files = [f for f in sorted(work_dir.glob("*"), key=lambda p: p.stat().st_mtime, reverse=True) if f.is_file()]
|
|
249
|
+
if files:
|
|
250
|
+
return files[0]
|
|
251
|
+
raise AcquisitionError("yt-dlp reported success but no output file was found")
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _validate_fetch_url(url: str) -> None:
|
|
255
|
+
if not url.startswith(("http://", "https://")):
|
|
256
|
+
raise AcquisitionError(f"refusing non-HTTP download destination: {url[:100]}")
|
|
257
|
+
try:
|
|
258
|
+
assert_url_is_fetchable(url)
|
|
259
|
+
except UnsafeUrlError as exc:
|
|
260
|
+
raise AcquisitionError(f"unsafe download destination: {exc}") from exc
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _validate_download_info(info: dict | None, *, maximum: int | None = None) -> None:
|
|
264
|
+
"""Recheck final media/fragment URLs selected after yt-dlp extraction."""
|
|
265
|
+
if not isinstance(info, dict) or info.get("_type", "video") != "video":
|
|
266
|
+
raise AcquisitionError("yt-dlp did not resolve a single video")
|
|
267
|
+
selected = [info]
|
|
268
|
+
selected.extend(item for item in info.get("requested_formats") or [] if isinstance(item, dict))
|
|
269
|
+
for item in selected:
|
|
270
|
+
if maximum is not None:
|
|
271
|
+
size = item.get("filesize") or item.get("filesize_approx")
|
|
272
|
+
if isinstance(size, (int, float)) and size > maximum:
|
|
273
|
+
raise AcquisitionError("reported download size exceeds the configured limit")
|
|
274
|
+
for key in ("url", "manifest_url", "fragment_base_url"):
|
|
275
|
+
value = item.get(key)
|
|
276
|
+
if isinstance(value, str) and value.startswith(("http://", "https://")):
|
|
277
|
+
_validate_fetch_url(value)
|
|
278
|
+
for fragment in item.get("fragments") or []:
|
|
279
|
+
value = fragment.get("url") if isinstance(fragment, dict) else None
|
|
280
|
+
if isinstance(value, str) and value.startswith(("http://", "https://")):
|
|
281
|
+
_validate_fetch_url(value)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def fetch_media(
|
|
285
|
+
source: SourceRef,
|
|
286
|
+
*,
|
|
287
|
+
work_dir: str | Path,
|
|
288
|
+
cookies_from_browser: str | None = None,
|
|
289
|
+
check_cancel: Callable[[], None] | None = None,
|
|
290
|
+
) -> Path:
|
|
291
|
+
"""Return a local path to the media.
|
|
292
|
+
|
|
293
|
+
Local files are returned unchanged. URLs use yt-dlp's in-process API so
|
|
294
|
+
requested and selected media URLs can be checked before download.
|
|
295
|
+
"""
|
|
296
|
+
if source.kind == "file":
|
|
297
|
+
return Path(source.location)
|
|
298
|
+
|
|
299
|
+
if source.kind != "url":
|
|
300
|
+
raise AcquisitionError(f"unsupported source kind: {source.kind}")
|
|
301
|
+
|
|
302
|
+
work = Path(work_dir)
|
|
303
|
+
work.mkdir(parents=True, exist_ok=True)
|
|
304
|
+
|
|
305
|
+
return _fetch_with_module(
|
|
306
|
+
source.location,
|
|
307
|
+
work_dir=work,
|
|
308
|
+
cookies_from_browser=cookies_from_browser,
|
|
309
|
+
check_cancel=check_cancel,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
def extract_audio(
|
|
313
|
+
media_path: str | Path,
|
|
314
|
+
*,
|
|
315
|
+
work_dir: str | Path,
|
|
316
|
+
sample_rate: int = 16000,
|
|
317
|
+
check_cancel: Callable[[], None] | None = None,
|
|
318
|
+
) -> Path:
|
|
319
|
+
"""Decode through a bounded pipe, never an unbounded ffmpeg output file."""
|
|
320
|
+
ffmpeg = require_tool("ffmpeg")
|
|
321
|
+
media = Path(media_path)
|
|
322
|
+
if not media.exists():
|
|
323
|
+
raise AcquisitionError(f"media not found: {media}")
|
|
324
|
+
|
|
325
|
+
work = Path(work_dir)
|
|
326
|
+
work.mkdir(parents=True, exist_ok=True)
|
|
327
|
+
out = ScratchPaths(work).decoded_audio
|
|
328
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
329
|
+
if media.resolve() == out.resolve():
|
|
330
|
+
raise AcquisitionError("decoded audio must not overwrite source media")
|
|
331
|
+
|
|
332
|
+
production = production_enabled()
|
|
333
|
+
timeout = positive_limit(ENV_FFMPEG_TIMEOUT_SECONDS, 600)
|
|
334
|
+
max_media = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024) if production else None
|
|
335
|
+
max_duration = positive_limit(ENV_MAX_DURATION_SECONDS, 4 * 3600) if production else None
|
|
336
|
+
max_pcm = max_media - 44 if max_media is not None else None
|
|
337
|
+
duration_pcm = max_duration * sample_rate * 2 if max_duration is not None else None
|
|
338
|
+
cmd = [
|
|
339
|
+
ffmpeg, "-hide_banner", "-loglevel", "error", "-nostdin",
|
|
340
|
+
"-i", str(media),
|
|
341
|
+
"-vn", "-ac", "1", "-ar", str(sample_rate),
|
|
342
|
+
"-c:a", "pcm_s16le",
|
|
343
|
+
]
|
|
344
|
+
if max_duration is not None:
|
|
345
|
+
# The extra second lets the pipe reader distinguish an overlong input
|
|
346
|
+
# from one that ends exactly at the allowed duration. It never lands on
|
|
347
|
+
# disk beyond the byte cap below.
|
|
348
|
+
cmd.extend(["-t", str(max_duration + 1)])
|
|
349
|
+
cmd.extend(["-f", "s16le", "pipe:1"])
|
|
350
|
+
if check_cancel is not None:
|
|
351
|
+
check_cancel()
|
|
352
|
+
try:
|
|
353
|
+
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
354
|
+
except OSError as exc:
|
|
355
|
+
raise AcquisitionError(f"ffmpeg could not start: {exc}") from exc
|
|
356
|
+
|
|
357
|
+
failures: list[str] = []
|
|
358
|
+
stderr_tail = bytearray()
|
|
359
|
+
|
|
360
|
+
def _read_stdout() -> None:
|
|
361
|
+
written = 0
|
|
362
|
+
try:
|
|
363
|
+
assert proc.stdout is not None
|
|
364
|
+
with wave.open(str(out), "wb") as wav:
|
|
365
|
+
wav.setnchannels(1)
|
|
366
|
+
wav.setsampwidth(2)
|
|
367
|
+
wav.setframerate(sample_rate)
|
|
368
|
+
while chunk := proc.stdout.read(64 * 1024):
|
|
369
|
+
if duration_pcm is not None and written + len(chunk) > duration_pcm:
|
|
370
|
+
failures.append("source duration exceeds the configured limit")
|
|
371
|
+
return
|
|
372
|
+
if max_pcm is not None and written + len(chunk) > max_pcm:
|
|
373
|
+
failures.append("decoded output exceeds the configured size limit")
|
|
374
|
+
return
|
|
375
|
+
wav.writeframesraw(chunk)
|
|
376
|
+
written += len(chunk)
|
|
377
|
+
except (OSError, wave.Error) as exc:
|
|
378
|
+
failures.append(f"ffmpeg output failed: {exc}")
|
|
379
|
+
|
|
380
|
+
def _read_stderr() -> None:
|
|
381
|
+
assert proc.stderr is not None
|
|
382
|
+
while chunk := proc.stderr.read(4096):
|
|
383
|
+
stderr_tail.extend(chunk)
|
|
384
|
+
if len(stderr_tail) > 8192:
|
|
385
|
+
del stderr_tail[:-8192]
|
|
386
|
+
|
|
387
|
+
output_thread = threading.Thread(target=_read_stdout, daemon=True)
|
|
388
|
+
error_thread = threading.Thread(target=_read_stderr, daemon=True)
|
|
389
|
+
output_thread.start()
|
|
390
|
+
error_thread.start()
|
|
391
|
+
deadline = time.monotonic() + timeout
|
|
392
|
+
try:
|
|
393
|
+
while proc.poll() is None or output_thread.is_alive():
|
|
394
|
+
if check_cancel is not None:
|
|
395
|
+
check_cancel()
|
|
396
|
+
if failures:
|
|
397
|
+
raise AcquisitionError(failures[0])
|
|
398
|
+
if time.monotonic() >= deadline:
|
|
399
|
+
raise AcquisitionError("ffmpeg timed out during decode")
|
|
400
|
+
time.sleep(0.05)
|
|
401
|
+
output_thread.join(timeout=5)
|
|
402
|
+
error_thread.join(timeout=5)
|
|
403
|
+
if failures:
|
|
404
|
+
raise AcquisitionError(failures[0])
|
|
405
|
+
if proc.returncode != 0 or not out.exists():
|
|
406
|
+
detail = stderr_tail.decode("utf-8", errors="replace").strip()
|
|
407
|
+
raise AcquisitionError(f"ffmpeg failed: {detail[-1000:] or 'unknown error'}")
|
|
408
|
+
if max_media is not None and out.stat().st_size > max_media:
|
|
409
|
+
raise AcquisitionError("decoded output exceeds the configured size limit")
|
|
410
|
+
except BaseException:
|
|
411
|
+
if os.name == "nt":
|
|
412
|
+
# Windows package-manager shims can launch the actual ffmpeg as a
|
|
413
|
+
# child. Killing only the shim leaves that child holding staged
|
|
414
|
+
# media open and decoding after cancellation.
|
|
415
|
+
try:
|
|
416
|
+
subprocess.run(
|
|
417
|
+
["taskkill", "/F", "/T", "/PID", str(proc.pid)],
|
|
418
|
+
capture_output=True, timeout=10, check=False,
|
|
419
|
+
)
|
|
420
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
421
|
+
pass
|
|
422
|
+
if proc.poll() is None:
|
|
423
|
+
proc.kill()
|
|
424
|
+
proc.wait(timeout=5)
|
|
425
|
+
output_thread.join(timeout=5)
|
|
426
|
+
error_thread.join(timeout=5)
|
|
427
|
+
if proc.stdout is not None:
|
|
428
|
+
proc.stdout.close()
|
|
429
|
+
if proc.stderr is not None:
|
|
430
|
+
proc.stderr.close()
|
|
431
|
+
out.unlink(missing_ok=True)
|
|
432
|
+
raise
|
|
433
|
+
if proc.stdout is not None:
|
|
434
|
+
proc.stdout.close()
|
|
435
|
+
if proc.stderr is not None:
|
|
436
|
+
proc.stderr.close()
|
|
437
|
+
return out
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""Platform detection and URL normalization.
|
|
2
|
+
|
|
3
|
+
The platform layer has exactly one job: figure out where media lives and how to
|
|
4
|
+
fetch it. Transcription never varies by platform.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import ipaddress
|
|
10
|
+
import socket
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from urllib.parse import urlparse
|
|
14
|
+
|
|
15
|
+
PLATFORMS: dict[str, tuple[str, ...]] = {
|
|
16
|
+
"youtube": ("youtube.com", "youtu.be", "m.youtube.com", "music.youtube.com"),
|
|
17
|
+
"tiktok": ("tiktok.com", "vm.tiktok.com", "vt.tiktok.com"),
|
|
18
|
+
"facebook": ("facebook.com", "fb.watch", "fb.com", "m.facebook.com"),
|
|
19
|
+
"instagram": ("instagram.com", "instagr.am"),
|
|
20
|
+
"vimeo": ("vimeo.com", "player.vimeo.com"),
|
|
21
|
+
"twitch": ("twitch.tv", "clips.twitch.tv"),
|
|
22
|
+
"bilibili": ("bilibili.com", "b23.tv"),
|
|
23
|
+
"rumble": ("rumble.com",),
|
|
24
|
+
"kick": ("kick.com",),
|
|
25
|
+
"zoom": ("zoom.us", "zoom.com"),
|
|
26
|
+
"medal": ("medal.tv",),
|
|
27
|
+
"loom": ("loom.com",),
|
|
28
|
+
"dropbox": ("dropbox.com", "dropboxusercontent.com"),
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
DIRECT_MEDIA_SUFFIXES = (".mp4", ".mkv", ".webm", ".mov", ".m4a", ".mp3", ".wav", ".ogg", ".flac")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class UnsafeUrlError(ValueError):
|
|
35
|
+
"""Raised when a URL targets a loopback, link-local, or private address.
|
|
36
|
+
|
|
37
|
+
This is an SSRF guard, not a content policy: transcription is meant for
|
|
38
|
+
public media, and letting a caller aim the fetcher at the local network or a
|
|
39
|
+
cloud metadata endpoint turns a tool call into a network probe.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
BLOCKED_HOSTNAMES = frozenset(
|
|
44
|
+
{
|
|
45
|
+
"localhost",
|
|
46
|
+
"localhost.localdomain",
|
|
47
|
+
"ip6-localhost",
|
|
48
|
+
"ip6-loopback",
|
|
49
|
+
"metadata",
|
|
50
|
+
"metadata.google.internal",
|
|
51
|
+
}
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# Explicitly blocked networks - the real SSRF targets. Deliberately narrower than
|
|
56
|
+
# `ipaddress.is_private`, which also covers the RFC 2544 benchmarking range
|
|
57
|
+
# (198.18.0.0/15); some local DNS filters resolve public hostnames there, and
|
|
58
|
+
# treating that as private would reject legitimate sites.
|
|
59
|
+
BLOCKED_NETWORKS = tuple(
|
|
60
|
+
ipaddress.ip_network(n)
|
|
61
|
+
for n in (
|
|
62
|
+
"0.0.0.0/8", # this-network
|
|
63
|
+
"10.0.0.0/8", # RFC 1918
|
|
64
|
+
"100.64.0.0/10", # CGNAT
|
|
65
|
+
"127.0.0.0/8", # loopback
|
|
66
|
+
"169.254.0.0/16", # link-local (incl. 169.254.169.254 metadata)
|
|
67
|
+
"172.16.0.0/12", # RFC 1918
|
|
68
|
+
"192.0.0.0/24", # IETF protocol assignments
|
|
69
|
+
"192.168.0.0/16", # RFC 1918
|
|
70
|
+
"224.0.0.0/4", # multicast
|
|
71
|
+
"240.0.0.0/4", # reserved
|
|
72
|
+
"::1/128", # IPv6 loopback
|
|
73
|
+
"fc00::/7", # IPv6 unique local
|
|
74
|
+
"fe80::/10", # IPv6 link-local
|
|
75
|
+
"ff00::/8", # IPv6 multicast
|
|
76
|
+
)
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
|
81
|
+
"""True for addresses that must never be reachable from a user-supplied URL."""
|
|
82
|
+
if ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_unspecified:
|
|
83
|
+
return True
|
|
84
|
+
return any(ip in net for net in BLOCKED_NETWORKS if net.version == ip.version)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _host_of(url: str) -> str:
|
|
88
|
+
"""Lowercased host from a URL, handling bracketed IPv6 literals.
|
|
89
|
+
|
|
90
|
+
`netloc` for `http://[::1]:8080/x` is `[::1]:8080`; naive splitting on ":"
|
|
91
|
+
yields "[", which silently bypasses any IP check.
|
|
92
|
+
"""
|
|
93
|
+
netloc = (urlparse(url).netloc or "").lower()
|
|
94
|
+
netloc = netloc.split("@")[-1]
|
|
95
|
+
if netloc.startswith("["):
|
|
96
|
+
end = netloc.find("]")
|
|
97
|
+
if end != -1:
|
|
98
|
+
return netloc[1:end]
|
|
99
|
+
return netloc.split(":")[0]
|
|
100
|
+
|
|
101
|
+
def assert_url_is_fetchable(url: str) -> None:
|
|
102
|
+
"""Reject URLs aimed at local/private infrastructure.
|
|
103
|
+
|
|
104
|
+
Blocks by literal IP and by hostname. Hostname blocking resolves first, so a
|
|
105
|
+
name pointing at 127.0.0.1 or 169.254.169.254 is caught, and it also catches
|
|
106
|
+
the well-known metadata hostnames directly.
|
|
107
|
+
|
|
108
|
+
Raises `UnsafeUrlError` with an actionable message.
|
|
109
|
+
"""
|
|
110
|
+
if not is_url(url):
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
host = _host_of(url)
|
|
114
|
+
if not host:
|
|
115
|
+
raise UnsafeUrlError(f"URL has no host: {url}")
|
|
116
|
+
|
|
117
|
+
if host in BLOCKED_HOSTNAMES or host.endswith(".localhost"):
|
|
118
|
+
raise UnsafeUrlError(
|
|
119
|
+
f"refusing to fetch a local address ('{host}'). textflowkit only "
|
|
120
|
+
"fetches publicly reachable media URLs."
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
# Literal IP? Check it directly.
|
|
124
|
+
try:
|
|
125
|
+
literal = ipaddress.ip_address(host)
|
|
126
|
+
except ValueError:
|
|
127
|
+
literal = None
|
|
128
|
+
if literal is not None:
|
|
129
|
+
if _is_blocked_ip(literal):
|
|
130
|
+
raise UnsafeUrlError(
|
|
131
|
+
f"refusing to fetch a non-public IP address ('{host}'). "
|
|
132
|
+
"Loopback, link-local, and private ranges are blocked."
|
|
133
|
+
)
|
|
134
|
+
return
|
|
135
|
+
|
|
136
|
+
# Hostname: resolve and check every address it maps to.
|
|
137
|
+
try:
|
|
138
|
+
infos = socket.getaddrinfo(host, None)
|
|
139
|
+
except socket.gaierror:
|
|
140
|
+
# Unresolvable hosts are left to the downloader to report; they are not
|
|
141
|
+
# an SSRF path because they cannot be connected to.
|
|
142
|
+
return
|
|
143
|
+
|
|
144
|
+
for info in infos:
|
|
145
|
+
addr = info[4][0]
|
|
146
|
+
try:
|
|
147
|
+
resolved = ipaddress.ip_address(addr.split("%")[0])
|
|
148
|
+
except ValueError:
|
|
149
|
+
continue
|
|
150
|
+
if _is_blocked_ip(resolved):
|
|
151
|
+
raise UnsafeUrlError(
|
|
152
|
+
f"refusing to fetch '{host}': it resolves to a non-public address "
|
|
153
|
+
f"({addr}). Loopback, link-local, and private ranges are blocked."
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
@dataclass(slots=True)
|
|
157
|
+
class SourceRef:
|
|
158
|
+
"""A resolved input: either a local file or a remote URL."""
|
|
159
|
+
|
|
160
|
+
kind: str # "file" | "url"
|
|
161
|
+
location: str # absolute path or URL
|
|
162
|
+
platform: str # e.g. "youtube", "local", "direct"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def is_url(value: str) -> bool:
|
|
166
|
+
parsed = urlparse(value)
|
|
167
|
+
return parsed.scheme in ("http", "https") and bool(parsed.netloc)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def detect_platform(value: str) -> str:
|
|
171
|
+
"""Identify the platform for a URL, or 'local' for a filesystem path."""
|
|
172
|
+
if not is_url(value):
|
|
173
|
+
return "local"
|
|
174
|
+
host = (urlparse(value).netloc or "").lower().split("@")[-1].split(":")[0]
|
|
175
|
+
host = host.removeprefix("www.")
|
|
176
|
+
for platform, domains in PLATFORMS.items():
|
|
177
|
+
for domain in domains:
|
|
178
|
+
if host == domain or host.endswith("." + domain):
|
|
179
|
+
return platform
|
|
180
|
+
if host and host.split("/")[-1].lower().endswith(DIRECT_MEDIA_SUFFIXES):
|
|
181
|
+
return "direct"
|
|
182
|
+
path = urlparse(value).path.lower()
|
|
183
|
+
if path.endswith(DIRECT_MEDIA_SUFFIXES):
|
|
184
|
+
return "direct"
|
|
185
|
+
return "direct"
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def resolve_source(value: str) -> SourceRef:
|
|
189
|
+
"""Turn user input into a SourceRef, validating local paths."""
|
|
190
|
+
if is_url(value):
|
|
191
|
+
assert_url_is_fetchable(value)
|
|
192
|
+
return SourceRef(kind="url", location=value, platform=detect_platform(value))
|
|
193
|
+
p = Path(value).expanduser()
|
|
194
|
+
if not p.exists():
|
|
195
|
+
raise FileNotFoundError(f"no such file: {value}")
|
|
196
|
+
if not p.is_file():
|
|
197
|
+
raise ValueError(f"not a file: {value}")
|
|
198
|
+
return SourceRef(kind="file", location=str(p.resolve()), platform="local")
|
|
199
|
+
|
|
200
|
+
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""The one layout for temporary source media and decoded audio.
|
|
2
|
+
|
|
3
|
+
Never derive decoded output from the media basename: a staged WAV and its
|
|
4
|
+
decoded WAV would otherwise be the same path. Every pipeline attempt gets its
|
|
5
|
+
own temporary root, and both normal and diarization-reacquisition paths use
|
|
6
|
+
this layout through the acquisition helpers.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True, slots=True)
|
|
16
|
+
class ScratchPaths:
|
|
17
|
+
root: Path
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def media_dir(self) -> Path:
|
|
21
|
+
return self.root / "media"
|
|
22
|
+
|
|
23
|
+
def staged_local(self, suffix: str) -> Path:
|
|
24
|
+
return self.media_dir / f"input{suffix.lower()}"
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def download_template(self) -> str:
|
|
28
|
+
return str(self.media_dir / "%(id)s.%(ext)s")
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def decoded_audio(self) -> Path:
|
|
32
|
+
return self.root / "audio" / "decoded.wav"
|