textflowkit 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. textflowkit/__init__.py +8 -0
  2. textflowkit/adapters/__init__.py +11 -0
  3. textflowkit/adapters/http_server.py +465 -0
  4. textflowkit/adapters/mcp_server.py +554 -0
  5. textflowkit/cli.py +473 -0
  6. textflowkit/core/__init__.py +5 -0
  7. textflowkit/core/batch.py +186 -0
  8. textflowkit/core/bind.py +66 -0
  9. textflowkit/core/cancel.py +19 -0
  10. textflowkit/core/checkpoint.py +389 -0
  11. textflowkit/core/diarize.py +229 -0
  12. textflowkit/core/engine.py +127 -0
  13. textflowkit/core/executor.py +301 -0
  14. textflowkit/core/jobs.py +241 -0
  15. textflowkit/core/model.py +98 -0
  16. textflowkit/core/paths.py +226 -0
  17. textflowkit/core/pipeline.py +368 -0
  18. textflowkit/core/retrieval.py +148 -0
  19. textflowkit/core/runner.py +146 -0
  20. textflowkit/core/service.py +146 -0
  21. textflowkit/core/sqlite_store.py +233 -0
  22. textflowkit/core/submission.py +220 -0
  23. textflowkit/core/timeutil.py +20 -0
  24. textflowkit/core/translate.py +244 -0
  25. textflowkit/render/__init__.py +191 -0
  26. textflowkit/render/docx.py +71 -0
  27. textflowkit/render/fonts/NotoSans.ttf +0 -0
  28. textflowkit/render/fonts/NotoSansArabic.ttf +0 -0
  29. textflowkit/render/fonts/NotoSansSC.ttf +0 -0
  30. textflowkit/render/fonts/OFL-NotoSans.txt +94 -0
  31. textflowkit/render/fonts/OFL-NotoSansSC.txt +93 -0
  32. textflowkit/render/fonts/README.md +19 -0
  33. textflowkit/render/markdown.py +33 -0
  34. textflowkit/render/pdf.py +136 -0
  35. textflowkit/render/srt.py +22 -0
  36. textflowkit/render/txt.py +19 -0
  37. textflowkit/render/vtt.py +20 -0
  38. textflowkit/sources/__init__.py +16 -0
  39. textflowkit/sources/acquire.py +437 -0
  40. textflowkit/sources/detect.py +200 -0
  41. textflowkit/sources/scratch.py +32 -0
  42. textflowkit-0.1.3.dist-info/METADATA +266 -0
  43. textflowkit-0.1.3.dist-info/RECORD +46 -0
  44. textflowkit-0.1.3.dist-info/WHEEL +4 -0
  45. textflowkit-0.1.3.dist-info/entry_points.txt +4 -0
  46. textflowkit-0.1.3.dist-info/licenses/LICENSE +203 -0
@@ -0,0 +1,437 @@
1
+ """Media acquisition: local files pass through, URLs go through yt-dlp."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import shutil
7
+ import subprocess
8
+ import sys
9
+ import threading
10
+ import time
11
+ import wave
12
+ from collections.abc import Callable
13
+ from pathlib import Path
14
+
15
+ from textflowkit.core.cancel import CancelledError
16
+ from textflowkit.core.paths import UnsafeInputPathError, opened_file_path
17
+ from textflowkit.core.service import (
18
+ ENV_EGRESS_PROXY,
19
+ ENV_FFMPEG_TIMEOUT_SECONDS,
20
+ ENV_MAX_DURATION_SECONDS,
21
+ ENV_MAX_MEDIA_BYTES,
22
+ positive_limit,
23
+ production_enabled,
24
+ )
25
+ from textflowkit.sources.detect import SourceRef, UnsafeUrlError, assert_url_is_fetchable
26
+ from textflowkit.sources.scratch import ScratchPaths
27
+
28
+
29
+ class AcquisitionError(RuntimeError):
30
+ """Raised when media cannot be obtained."""
31
+
32
+
33
+ def stage_confined_local_media(
34
+ source: str | Path, *, work_dir: Path, input_root: str | Path
35
+ ) -> Path:
36
+ """Copy a handle-verified local input into isolated scratch before ffmpeg."""
37
+ base = Path(input_root).expanduser().resolve()
38
+ path = Path(source)
39
+ out = ScratchPaths(Path(work_dir)).staged_local(path.suffix)
40
+ maximum = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024) if production_enabled() else None
41
+ with path.open("rb") as opened:
42
+ actual = opened_file_path(opened.fileno(), path)
43
+ if actual != base and base not in actual.parents:
44
+ raise UnsafeInputPathError(
45
+ f"opened input file '{actual}' is outside the allowed root '{base}'"
46
+ )
47
+ written = 0
48
+ out.parent.mkdir(parents=True, exist_ok=True)
49
+ with out.open("xb") as destination:
50
+ while chunk := opened.read(1024 * 1024):
51
+ written += len(chunk)
52
+ if maximum is not None and written > maximum:
53
+ raise AcquisitionError("media exceeds the configured size limit")
54
+ destination.write(chunk)
55
+ return out
56
+
57
+
58
+ def require_tool(name: str, *, module: str | None = None) -> str | None:
59
+ """Locate an external tool.
60
+
61
+ Checks PATH first, then the interpreter's own Scripts/bin directory. The
62
+ second check matters for the common case where a dependency was installed
63
+ into the active environment (or a venv) but the environment is not
64
+ activated, so its entry point is not on PATH.
65
+
66
+ Returns None when the tool is absent and a viable in-process module exists;
67
+ callers then use the module instead of shelling out.
68
+ """
69
+ found = shutil.which(name)
70
+ if found:
71
+ return found
72
+
73
+ # Look beside the running interpreter (venv/Scripts, venv/bin, ...).
74
+ exe = name + (".exe" if os.name == "nt" else "")
75
+ scripts_dir = Path(sys.executable).parent
76
+ candidate = scripts_dir / exe
77
+ if candidate.exists():
78
+ return str(candidate)
79
+
80
+ if module:
81
+ try:
82
+ __import__(module)
83
+ except ImportError:
84
+ pass
85
+ else:
86
+ return None # module usable in-process
87
+
88
+ raise AcquisitionError(
89
+ f"required tool '{name}' not found. Install it (pip install {module or name}) and retry."
90
+ )
91
+
92
+
93
+ # JavaScript runtimes yt-dlp can use, in its own priority order. yt-dlp enables
94
+ # only "deno" by default, so a machine that has Node (or bun/quickjs) still emits
95
+ # "No supported JavaScript runtime could be found". Detecting what is actually
96
+ # present and enabling it explicitly avoids requiring a specific runtime.
97
+ JS_RUNTIMES = ("deno", "node", "bun", "quickjs")
98
+
99
+
100
+ def detect_js_runtime() -> str | None:
101
+ """Return the highest-priority JavaScript runtime available, or None.
102
+
103
+ Mirrors yt-dlp's own priority order. Returning None is safe: yt-dlp falls
104
+ back to non-JS extraction, which works for many videos but can leave some
105
+ formats unavailable.
106
+ """
107
+ for name in JS_RUNTIMES:
108
+ if shutil.which(name):
109
+ return name
110
+ return None
111
+
112
+
113
+ def _js_runtime_args() -> list[str]:
114
+ """Build yt-dlp JS-runtime flags for whatever runtime is installed."""
115
+ runtime = detect_js_runtime()
116
+ if not runtime:
117
+ return []
118
+ # yt-dlp enables deno by default; enabling another runtime requires clearing
119
+ # the defaults first so the detected runtime is the one actually used.
120
+ if runtime == "deno":
121
+ return []
122
+ return ["--no-js-runtimes", "--js-runtimes", runtime]
123
+
124
+ def _fetch_with_module(
125
+ url: str,
126
+ *,
127
+ work_dir: Path,
128
+ cookies_from_browser: str | None,
129
+ check_cancel: Callable[[], None] | None = None,
130
+ ) -> Path:
131
+ """Download using the yt_dlp Python API so URL checks cover its requests."""
132
+ from yt_dlp import YoutubeDL
133
+
134
+ layout = ScratchPaths(work_dir)
135
+ layout.media_dir.mkdir(parents=True, exist_ok=True)
136
+ outtmpl = layout.download_template
137
+ hooks: list[Path] = []
138
+ maximum = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024) if production_enabled() else None
139
+
140
+ def _check_download_size(status: dict) -> None:
141
+ if maximum is None:
142
+ return
143
+ for key in ("downloaded_bytes", "total_bytes", "total_bytes_estimate"):
144
+ value = status.get(key)
145
+ if isinstance(value, (int, float)) and value > maximum:
146
+ raise AcquisitionError("download exceeds the configured size limit")
147
+ # Aggregate fragments and temporary files as well as the final output;
148
+ # a per-fragment counter alone would reset below the cap each time.
149
+ total = sum(p.stat().st_size for p in layout.media_dir.rglob("*") if p.is_file())
150
+ if total > maximum:
151
+ raise AcquisitionError("download exceeds the configured size limit")
152
+
153
+ def _hook(status: dict) -> None:
154
+ # yt-dlp calls this frequently during a download. Raising here aborts
155
+ # the download, which is what makes cancellation responsive for the
156
+ # slowest common case instead of waiting for the whole fetch to finish.
157
+ if check_cancel is not None:
158
+ check_cancel()
159
+ _check_download_size(status)
160
+ if status.get("status") == "finished":
161
+ path = status.get("filename") or status.get("_filename")
162
+ if path:
163
+ hooks.append(Path(path))
164
+
165
+ runtime = detect_js_runtime()
166
+ opts: dict = {
167
+ "js_runtimes": {runtime: {}} if runtime else {},
168
+ "outtmpl": outtmpl,
169
+ "noplaylist": True,
170
+ "quiet": True,
171
+ "no_warnings": True,
172
+ "format": "bestaudio/best",
173
+ "restrictfilenames": True,
174
+ "progress_hooks": [_hook],
175
+ }
176
+ if maximum is not None:
177
+ opts["max_filesize"] = maximum
178
+ if cookies_from_browser:
179
+ opts["cookiesfrombrowser"] = (cookies_from_browser,)
180
+ if production_enabled():
181
+ proxy = os.environ.get(ENV_EGRESS_PROXY)
182
+ if not proxy:
183
+ raise AcquisitionError(
184
+ f"production URL acquisition requires an SSRF-filtering {ENV_EGRESS_PROXY}"
185
+ )
186
+ opts["proxy"] = proxy
187
+ # External JS runtimes are separate processes and are not guaranteed to
188
+ # honor yt-dlp's proxy option. Do not let them create an egress bypass.
189
+ opts["js_runtimes"] = {}
190
+
191
+ try:
192
+ with YoutubeDL(opts) as ydl:
193
+ original_open = ydl.urlopen
194
+
195
+ def checked_open(request):
196
+ requested = request if isinstance(request, str) else request.url
197
+ _validate_fetch_url(requested)
198
+ response = original_open(request)
199
+ if maximum is not None:
200
+ headers = getattr(response, "headers", None)
201
+ length = headers.get("Content-Length") if headers is not None else None
202
+ if length is not None:
203
+ try:
204
+ too_large = int(length) > maximum
205
+ except (TypeError, ValueError):
206
+ too_large = False
207
+ if too_large:
208
+ close = getattr(response, "close", None)
209
+ if callable(close):
210
+ close()
211
+ raise AcquisitionError("download Content-Length exceeds the configured size limit")
212
+ final = getattr(response, "url", None)
213
+ if final:
214
+ _validate_fetch_url(final)
215
+ return response
216
+
217
+ ydl.urlopen = checked_open
218
+ _validate_fetch_url(url)
219
+ info = ydl.extract_info(url, download=False)
220
+ _validate_download_info(info, maximum=maximum)
221
+ ydl.process_info(info)
222
+ except CancelledError:
223
+ raise # an orderly stop, not a fetch failure
224
+ except AcquisitionError:
225
+ raise
226
+ except Exception as exc:
227
+ raise AcquisitionError(f"yt-dlp failed: {exc}") from exc
228
+
229
+ if hooks and hooks[-1].exists():
230
+ if maximum is not None and hooks[-1].stat().st_size > maximum:
231
+ raise AcquisitionError("download exceeds the configured size limit")
232
+ return hooks[-1]
233
+
234
+ requested = info.get("requested_downloads") or []
235
+ for item in requested:
236
+ candidate = Path(item.get("filepath", ""))
237
+ if candidate.exists():
238
+ if maximum is not None and candidate.stat().st_size > maximum:
239
+ raise AcquisitionError("download exceeds the configured size limit")
240
+ return candidate
241
+
242
+ vid = info.get("id")
243
+ if vid:
244
+ for candidate in sorted(work_dir.glob(f"{vid}.*"), key=lambda p: p.stat().st_mtime, reverse=True):
245
+ if candidate.is_file():
246
+ return candidate
247
+
248
+ files = [f for f in sorted(work_dir.glob("*"), key=lambda p: p.stat().st_mtime, reverse=True) if f.is_file()]
249
+ if files:
250
+ return files[0]
251
+ raise AcquisitionError("yt-dlp reported success but no output file was found")
252
+
253
+
254
+ def _validate_fetch_url(url: str) -> None:
255
+ if not url.startswith(("http://", "https://")):
256
+ raise AcquisitionError(f"refusing non-HTTP download destination: {url[:100]}")
257
+ try:
258
+ assert_url_is_fetchable(url)
259
+ except UnsafeUrlError as exc:
260
+ raise AcquisitionError(f"unsafe download destination: {exc}") from exc
261
+
262
+
263
+ def _validate_download_info(info: dict | None, *, maximum: int | None = None) -> None:
264
+ """Recheck final media/fragment URLs selected after yt-dlp extraction."""
265
+ if not isinstance(info, dict) or info.get("_type", "video") != "video":
266
+ raise AcquisitionError("yt-dlp did not resolve a single video")
267
+ selected = [info]
268
+ selected.extend(item for item in info.get("requested_formats") or [] if isinstance(item, dict))
269
+ for item in selected:
270
+ if maximum is not None:
271
+ size = item.get("filesize") or item.get("filesize_approx")
272
+ if isinstance(size, (int, float)) and size > maximum:
273
+ raise AcquisitionError("reported download size exceeds the configured limit")
274
+ for key in ("url", "manifest_url", "fragment_base_url"):
275
+ value = item.get(key)
276
+ if isinstance(value, str) and value.startswith(("http://", "https://")):
277
+ _validate_fetch_url(value)
278
+ for fragment in item.get("fragments") or []:
279
+ value = fragment.get("url") if isinstance(fragment, dict) else None
280
+ if isinstance(value, str) and value.startswith(("http://", "https://")):
281
+ _validate_fetch_url(value)
282
+
283
+
284
+ def fetch_media(
285
+ source: SourceRef,
286
+ *,
287
+ work_dir: str | Path,
288
+ cookies_from_browser: str | None = None,
289
+ check_cancel: Callable[[], None] | None = None,
290
+ ) -> Path:
291
+ """Return a local path to the media.
292
+
293
+ Local files are returned unchanged. URLs use yt-dlp's in-process API so
294
+ requested and selected media URLs can be checked before download.
295
+ """
296
+ if source.kind == "file":
297
+ return Path(source.location)
298
+
299
+ if source.kind != "url":
300
+ raise AcquisitionError(f"unsupported source kind: {source.kind}")
301
+
302
+ work = Path(work_dir)
303
+ work.mkdir(parents=True, exist_ok=True)
304
+
305
+ return _fetch_with_module(
306
+ source.location,
307
+ work_dir=work,
308
+ cookies_from_browser=cookies_from_browser,
309
+ check_cancel=check_cancel,
310
+ )
311
+
312
+ def extract_audio(
313
+ media_path: str | Path,
314
+ *,
315
+ work_dir: str | Path,
316
+ sample_rate: int = 16000,
317
+ check_cancel: Callable[[], None] | None = None,
318
+ ) -> Path:
319
+ """Decode through a bounded pipe, never an unbounded ffmpeg output file."""
320
+ ffmpeg = require_tool("ffmpeg")
321
+ media = Path(media_path)
322
+ if not media.exists():
323
+ raise AcquisitionError(f"media not found: {media}")
324
+
325
+ work = Path(work_dir)
326
+ work.mkdir(parents=True, exist_ok=True)
327
+ out = ScratchPaths(work).decoded_audio
328
+ out.parent.mkdir(parents=True, exist_ok=True)
329
+ if media.resolve() == out.resolve():
330
+ raise AcquisitionError("decoded audio must not overwrite source media")
331
+
332
+ production = production_enabled()
333
+ timeout = positive_limit(ENV_FFMPEG_TIMEOUT_SECONDS, 600)
334
+ max_media = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024) if production else None
335
+ max_duration = positive_limit(ENV_MAX_DURATION_SECONDS, 4 * 3600) if production else None
336
+ max_pcm = max_media - 44 if max_media is not None else None
337
+ duration_pcm = max_duration * sample_rate * 2 if max_duration is not None else None
338
+ cmd = [
339
+ ffmpeg, "-hide_banner", "-loglevel", "error", "-nostdin",
340
+ "-i", str(media),
341
+ "-vn", "-ac", "1", "-ar", str(sample_rate),
342
+ "-c:a", "pcm_s16le",
343
+ ]
344
+ if max_duration is not None:
345
+ # The extra second lets the pipe reader distinguish an overlong input
346
+ # from one that ends exactly at the allowed duration. It never lands on
347
+ # disk beyond the byte cap below.
348
+ cmd.extend(["-t", str(max_duration + 1)])
349
+ cmd.extend(["-f", "s16le", "pipe:1"])
350
+ if check_cancel is not None:
351
+ check_cancel()
352
+ try:
353
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
354
+ except OSError as exc:
355
+ raise AcquisitionError(f"ffmpeg could not start: {exc}") from exc
356
+
357
+ failures: list[str] = []
358
+ stderr_tail = bytearray()
359
+
360
+ def _read_stdout() -> None:
361
+ written = 0
362
+ try:
363
+ assert proc.stdout is not None
364
+ with wave.open(str(out), "wb") as wav:
365
+ wav.setnchannels(1)
366
+ wav.setsampwidth(2)
367
+ wav.setframerate(sample_rate)
368
+ while chunk := proc.stdout.read(64 * 1024):
369
+ if duration_pcm is not None and written + len(chunk) > duration_pcm:
370
+ failures.append("source duration exceeds the configured limit")
371
+ return
372
+ if max_pcm is not None and written + len(chunk) > max_pcm:
373
+ failures.append("decoded output exceeds the configured size limit")
374
+ return
375
+ wav.writeframesraw(chunk)
376
+ written += len(chunk)
377
+ except (OSError, wave.Error) as exc:
378
+ failures.append(f"ffmpeg output failed: {exc}")
379
+
380
+ def _read_stderr() -> None:
381
+ assert proc.stderr is not None
382
+ while chunk := proc.stderr.read(4096):
383
+ stderr_tail.extend(chunk)
384
+ if len(stderr_tail) > 8192:
385
+ del stderr_tail[:-8192]
386
+
387
+ output_thread = threading.Thread(target=_read_stdout, daemon=True)
388
+ error_thread = threading.Thread(target=_read_stderr, daemon=True)
389
+ output_thread.start()
390
+ error_thread.start()
391
+ deadline = time.monotonic() + timeout
392
+ try:
393
+ while proc.poll() is None or output_thread.is_alive():
394
+ if check_cancel is not None:
395
+ check_cancel()
396
+ if failures:
397
+ raise AcquisitionError(failures[0])
398
+ if time.monotonic() >= deadline:
399
+ raise AcquisitionError("ffmpeg timed out during decode")
400
+ time.sleep(0.05)
401
+ output_thread.join(timeout=5)
402
+ error_thread.join(timeout=5)
403
+ if failures:
404
+ raise AcquisitionError(failures[0])
405
+ if proc.returncode != 0 or not out.exists():
406
+ detail = stderr_tail.decode("utf-8", errors="replace").strip()
407
+ raise AcquisitionError(f"ffmpeg failed: {detail[-1000:] or 'unknown error'}")
408
+ if max_media is not None and out.stat().st_size > max_media:
409
+ raise AcquisitionError("decoded output exceeds the configured size limit")
410
+ except BaseException:
411
+ if os.name == "nt":
412
+ # Windows package-manager shims can launch the actual ffmpeg as a
413
+ # child. Killing only the shim leaves that child holding staged
414
+ # media open and decoding after cancellation.
415
+ try:
416
+ subprocess.run(
417
+ ["taskkill", "/F", "/T", "/PID", str(proc.pid)],
418
+ capture_output=True, timeout=10, check=False,
419
+ )
420
+ except (OSError, subprocess.TimeoutExpired):
421
+ pass
422
+ if proc.poll() is None:
423
+ proc.kill()
424
+ proc.wait(timeout=5)
425
+ output_thread.join(timeout=5)
426
+ error_thread.join(timeout=5)
427
+ if proc.stdout is not None:
428
+ proc.stdout.close()
429
+ if proc.stderr is not None:
430
+ proc.stderr.close()
431
+ out.unlink(missing_ok=True)
432
+ raise
433
+ if proc.stdout is not None:
434
+ proc.stdout.close()
435
+ if proc.stderr is not None:
436
+ proc.stderr.close()
437
+ return out
@@ -0,0 +1,200 @@
1
+ """Platform detection and URL normalization.
2
+
3
+ The platform layer has exactly one job: figure out where media lives and how to
4
+ fetch it. Transcription never varies by platform.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import ipaddress
10
+ import socket
11
+ from dataclasses import dataclass
12
+ from pathlib import Path
13
+ from urllib.parse import urlparse
14
+
15
+ PLATFORMS: dict[str, tuple[str, ...]] = {
16
+ "youtube": ("youtube.com", "youtu.be", "m.youtube.com", "music.youtube.com"),
17
+ "tiktok": ("tiktok.com", "vm.tiktok.com", "vt.tiktok.com"),
18
+ "facebook": ("facebook.com", "fb.watch", "fb.com", "m.facebook.com"),
19
+ "instagram": ("instagram.com", "instagr.am"),
20
+ "vimeo": ("vimeo.com", "player.vimeo.com"),
21
+ "twitch": ("twitch.tv", "clips.twitch.tv"),
22
+ "bilibili": ("bilibili.com", "b23.tv"),
23
+ "rumble": ("rumble.com",),
24
+ "kick": ("kick.com",),
25
+ "zoom": ("zoom.us", "zoom.com"),
26
+ "medal": ("medal.tv",),
27
+ "loom": ("loom.com",),
28
+ "dropbox": ("dropbox.com", "dropboxusercontent.com"),
29
+ }
30
+
31
+ DIRECT_MEDIA_SUFFIXES = (".mp4", ".mkv", ".webm", ".mov", ".m4a", ".mp3", ".wav", ".ogg", ".flac")
32
+
33
+
34
+ class UnsafeUrlError(ValueError):
35
+ """Raised when a URL targets a loopback, link-local, or private address.
36
+
37
+ This is an SSRF guard, not a content policy: transcription is meant for
38
+ public media, and letting a caller aim the fetcher at the local network or a
39
+ cloud metadata endpoint turns a tool call into a network probe.
40
+ """
41
+
42
+
43
+ BLOCKED_HOSTNAMES = frozenset(
44
+ {
45
+ "localhost",
46
+ "localhost.localdomain",
47
+ "ip6-localhost",
48
+ "ip6-loopback",
49
+ "metadata",
50
+ "metadata.google.internal",
51
+ }
52
+ )
53
+
54
+
55
+ # Explicitly blocked networks - the real SSRF targets. Deliberately narrower than
56
+ # `ipaddress.is_private`, which also covers the RFC 2544 benchmarking range
57
+ # (198.18.0.0/15); some local DNS filters resolve public hostnames there, and
58
+ # treating that as private would reject legitimate sites.
59
+ BLOCKED_NETWORKS = tuple(
60
+ ipaddress.ip_network(n)
61
+ for n in (
62
+ "0.0.0.0/8", # this-network
63
+ "10.0.0.0/8", # RFC 1918
64
+ "100.64.0.0/10", # CGNAT
65
+ "127.0.0.0/8", # loopback
66
+ "169.254.0.0/16", # link-local (incl. 169.254.169.254 metadata)
67
+ "172.16.0.0/12", # RFC 1918
68
+ "192.0.0.0/24", # IETF protocol assignments
69
+ "192.168.0.0/16", # RFC 1918
70
+ "224.0.0.0/4", # multicast
71
+ "240.0.0.0/4", # reserved
72
+ "::1/128", # IPv6 loopback
73
+ "fc00::/7", # IPv6 unique local
74
+ "fe80::/10", # IPv6 link-local
75
+ "ff00::/8", # IPv6 multicast
76
+ )
77
+ )
78
+
79
+
80
+ def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
81
+ """True for addresses that must never be reachable from a user-supplied URL."""
82
+ if ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_unspecified:
83
+ return True
84
+ return any(ip in net for net in BLOCKED_NETWORKS if net.version == ip.version)
85
+
86
+
87
+ def _host_of(url: str) -> str:
88
+ """Lowercased host from a URL, handling bracketed IPv6 literals.
89
+
90
+ `netloc` for `http://[::1]:8080/x` is `[::1]:8080`; naive splitting on ":"
91
+ yields "[", which silently bypasses any IP check.
92
+ """
93
+ netloc = (urlparse(url).netloc or "").lower()
94
+ netloc = netloc.split("@")[-1]
95
+ if netloc.startswith("["):
96
+ end = netloc.find("]")
97
+ if end != -1:
98
+ return netloc[1:end]
99
+ return netloc.split(":")[0]
100
+
101
+ def assert_url_is_fetchable(url: str) -> None:
102
+ """Reject URLs aimed at local/private infrastructure.
103
+
104
+ Blocks by literal IP and by hostname. Hostname blocking resolves first, so a
105
+ name pointing at 127.0.0.1 or 169.254.169.254 is caught, and it also catches
106
+ the well-known metadata hostnames directly.
107
+
108
+ Raises `UnsafeUrlError` with an actionable message.
109
+ """
110
+ if not is_url(url):
111
+ return
112
+
113
+ host = _host_of(url)
114
+ if not host:
115
+ raise UnsafeUrlError(f"URL has no host: {url}")
116
+
117
+ if host in BLOCKED_HOSTNAMES or host.endswith(".localhost"):
118
+ raise UnsafeUrlError(
119
+ f"refusing to fetch a local address ('{host}'). textflowkit only "
120
+ "fetches publicly reachable media URLs."
121
+ )
122
+
123
+ # Literal IP? Check it directly.
124
+ try:
125
+ literal = ipaddress.ip_address(host)
126
+ except ValueError:
127
+ literal = None
128
+ if literal is not None:
129
+ if _is_blocked_ip(literal):
130
+ raise UnsafeUrlError(
131
+ f"refusing to fetch a non-public IP address ('{host}'). "
132
+ "Loopback, link-local, and private ranges are blocked."
133
+ )
134
+ return
135
+
136
+ # Hostname: resolve and check every address it maps to.
137
+ try:
138
+ infos = socket.getaddrinfo(host, None)
139
+ except socket.gaierror:
140
+ # Unresolvable hosts are left to the downloader to report; they are not
141
+ # an SSRF path because they cannot be connected to.
142
+ return
143
+
144
+ for info in infos:
145
+ addr = info[4][0]
146
+ try:
147
+ resolved = ipaddress.ip_address(addr.split("%")[0])
148
+ except ValueError:
149
+ continue
150
+ if _is_blocked_ip(resolved):
151
+ raise UnsafeUrlError(
152
+ f"refusing to fetch '{host}': it resolves to a non-public address "
153
+ f"({addr}). Loopback, link-local, and private ranges are blocked."
154
+ )
155
+
156
+ @dataclass(slots=True)
157
+ class SourceRef:
158
+ """A resolved input: either a local file or a remote URL."""
159
+
160
+ kind: str # "file" | "url"
161
+ location: str # absolute path or URL
162
+ platform: str # e.g. "youtube", "local", "direct"
163
+
164
+
165
+ def is_url(value: str) -> bool:
166
+ parsed = urlparse(value)
167
+ return parsed.scheme in ("http", "https") and bool(parsed.netloc)
168
+
169
+
170
+ def detect_platform(value: str) -> str:
171
+ """Identify the platform for a URL, or 'local' for a filesystem path."""
172
+ if not is_url(value):
173
+ return "local"
174
+ host = (urlparse(value).netloc or "").lower().split("@")[-1].split(":")[0]
175
+ host = host.removeprefix("www.")
176
+ for platform, domains in PLATFORMS.items():
177
+ for domain in domains:
178
+ if host == domain or host.endswith("." + domain):
179
+ return platform
180
+ if host and host.split("/")[-1].lower().endswith(DIRECT_MEDIA_SUFFIXES):
181
+ return "direct"
182
+ path = urlparse(value).path.lower()
183
+ if path.endswith(DIRECT_MEDIA_SUFFIXES):
184
+ return "direct"
185
+ return "direct"
186
+
187
+
188
+ def resolve_source(value: str) -> SourceRef:
189
+ """Turn user input into a SourceRef, validating local paths."""
190
+ if is_url(value):
191
+ assert_url_is_fetchable(value)
192
+ return SourceRef(kind="url", location=value, platform=detect_platform(value))
193
+ p = Path(value).expanduser()
194
+ if not p.exists():
195
+ raise FileNotFoundError(f"no such file: {value}")
196
+ if not p.is_file():
197
+ raise ValueError(f"not a file: {value}")
198
+ return SourceRef(kind="file", location=str(p.resolve()), platform="local")
199
+
200
+
@@ -0,0 +1,32 @@
1
+ """The one layout for temporary source media and decoded audio.
2
+
3
+ Never derive decoded output from the media basename: a staged WAV and its
4
+ decoded WAV would otherwise be the same path. Every pipeline attempt gets its
5
+ own temporary root, and both normal and diarization-reacquisition paths use
6
+ this layout through the acquisition helpers.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass
12
+ from pathlib import Path
13
+
14
+
15
+ @dataclass(frozen=True, slots=True)
16
+ class ScratchPaths:
17
+ root: Path
18
+
19
+ @property
20
+ def media_dir(self) -> Path:
21
+ return self.root / "media"
22
+
23
+ def staged_local(self, suffix: str) -> Path:
24
+ return self.media_dir / f"input{suffix.lower()}"
25
+
26
+ @property
27
+ def download_template(self) -> str:
28
+ return str(self.media_dir / "%(id)s.%(ext)s")
29
+
30
+ @property
31
+ def decoded_audio(self) -> Path:
32
+ return self.root / "audio" / "decoded.wav"