u-transcript-max 0.1.0a1__py3-none-any.whl

utmax/adapters/files.py

@@ -0,0 +1,130 @@
+"""Filesystem helpers: atomic writes that tolerate Windows file locks, and muxer output."""
+
+from __future__ import annotations
+
+import io
+import os
+import secrets
+import time
+from collections.abc import Callable, Iterator, Sequence
+from pathlib import Path
+from typing import Self
+
+from utmax.core.media.boxes import ByteSource, read_exact
+from utmax.core.media.mux import Blob, CopyOp, MuxPlan
+
+__all__ = [
+    "COPY_BLOCK_SIZE",
+    "REPLACE_RETRY_DELAYS",
+    "FileByteSource",
+    "replace_with_retry",
+    "write_mux_plan",
+    "write_text_atomic",
+]
+
+REPLACE_RETRY_DELAYS = (0.1, 0.2, 0.4, 0.8, 1.6, 3.2)
+COPY_BLOCK_SIZE = 1 << 20
+
+
+def write_text_atomic(path: str | os.PathLike[str], text: str) -> Path:
+    """Write ``text`` as UTF-8 with ``\\n`` newlines; readers never see a half-written file."""
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    temporary = target.with_name(f".{target.name}.{secrets.token_hex(4)}.tmp")
+    try:
+        with temporary.open("x", encoding="utf-8", newline="\n") as handle:
+            handle.write(text)
+            handle.flush()
+            os.fsync(handle.fileno())
+        replace_with_retry(temporary, target)
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+    return target
+
+
+def replace_with_retry(
+    source: Path,
+    target: Path,
+    *,
+    replace: Callable[[Path, Path], None] = os.replace,
+    sleep: Callable[[float], None] = time.sleep,
+) -> None:
+    """``os.replace`` that retries while antivirus or indexers briefly hold the file (Windows)."""
+    for delay in REPLACE_RETRY_DELAYS:
+        try:
+            replace(source, target)
+        except PermissionError:
+            sleep(delay)
+        else:
+            return
+    replace(source, target)
+
+
+class FileByteSource:
+    """A :class:`~utmax.core.media.boxes.ByteSource` over a file on disk; not thread-safe."""
+
+    def __init__(self, path: str | os.PathLike[str]) -> None:
+        self.path = Path(path)
+        self._handle: io.BufferedReader = self.path.open("rb")
+        self.size = os.fstat(self._handle.fileno()).st_size
+
+    def read(self, offset: int, n: int) -> bytes:
+        self._handle.seek(offset)
+        return self._handle.read(n)
+
+    def close(self) -> None:
+        self._handle.close()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+
+def write_mux_plan(
+    plan: MuxPlan,
+    sources: Sequence[ByteSource],
+    path: str | os.PathLike[str],
+    *,
+    progress: Callable[[int, int], None] | None = None,
+) -> Path:
+    """Write the file ``plan`` describes to ``path`` and return ``path``.
+
+    Copies stream in 1 MiB blocks into a temporary file next to ``path``, which is flushed to
+    disk and then atomically replaces ``path`` (retrying while Windows holds a lock); on any
+    error the temporary file is removed. ``progress(written, total)`` runs after each block.
+
+    Raises:
+        MuxError: an input is shorter than the plan expects.
+    """
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    temporary = target.with_name(f".{target.name}.{secrets.token_hex(4)}.tmp")
+    total = plan.size
+    written = 0
+    try:
+        with temporary.open("xb") as handle:
+            for op in plan.ops:
+                for block in _blocks(op, sources):
+                    handle.write(block)
+                    written += len(block)
+                    if progress is not None:
+                        progress(written, total)
+            handle.flush()
+            os.fsync(handle.fileno())
+        replace_with_retry(temporary, target)
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+    return target
+
+
+def _blocks(op: CopyOp | Blob, sources: Sequence[ByteSource]) -> Iterator[bytes]:
+    if isinstance(op, Blob):
+        yield op.data
+        return
+    end = op.offset + op.size
+    for start in range(op.offset, end, COPY_BLOCK_SIZE):
+        yield read_exact(sources[op.source], start, min(COPY_BLOCK_SIZE, end - start))

utmax/adapters/http.py

@@ -0,0 +1,285 @@
+"""The only module that talks to the network: a urllib transport plus a retry wrapper."""
+
+from __future__ import annotations
+
+import gzip
+import http.client
+import io
+import logging
+import random
+import socket
+import ssl
+import time
+import urllib.error
+import urllib.request
+import zlib
+from collections.abc import Callable, Mapping
+from datetime import UTC, datetime
+from typing import IO
+from urllib.parse import urlsplit
+
+from utmax.core.retry import backoff_delay, is_transient_status, parse_retry_after
+from utmax.errors import InvalidOption, NetworkError
+from utmax.transport import HttpRequest, HttpResponse, HttpStream, StreamingTransport, Transport
+
+__all__ = ["BufferedStream", "RetryingTransport", "UrllibTransport", "open_stream", "redact"]
+
+log = logging.getLogger("utmax.http")
+
+_NETWORK_ERRORS = (OSError, http.client.HTTPException, EOFError, zlib.error)
+
+# Binding outgoing sockets here makes every IPv6 address fail, so only IPv4 is tried.
+_IPV4_ONLY = ("0.0.0.0", 0)
+
+
+class UrllibTransport:
+    """Sends requests with the standard library; every request uses a fresh connection.
+
+    ``force_ipv4=True`` connects over IPv4 only. Stream URLs are bound to the address that
+    requested them, so machines whose IPv4 and IPv6 addresses differ may need it.
+    """
+
+    def __init__(
+        self, *, proxy: str | None = None, timeout: float = 30.0, force_ipv4: bool = False
+    ) -> None:
+        context = ssl.create_default_context()
+        handlers: list[urllib.request.BaseHandler] = (
+            [_IPv4HTTPHandler(), _IPv4HTTPSHandler(context)]
+            if force_ipv4
+            else [urllib.request.HTTPSHandler(context=context)]
+        )
+        if proxy is not None:
+            _check_proxy(proxy)
+            handlers.append(urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
+        self._opener = urllib.request.build_opener(*handlers)
+        self._timeout = timeout
+
+    def send(self, request: HttpRequest) -> HttpResponse:
+        status: int
+        headers: dict[str, str]
+        body: bytes
+        try:
+            with self._opener.open(_prepare(request), timeout=self._timeout) as response:
+                status = response.status
+                headers = dict(response.headers.items())
+                body = response.read()
+        except urllib.error.HTTPError as error:
+            with error:
+                status = error.code
+                headers = dict(error.headers.items()) if error.headers else {}
+                body = error.read()
+        if _header(headers, "content-encoding").lower() == "gzip":
+            body = gzip.decompress(body)
+        log.debug("%s %s -> %d", request.method, redact(request.url), status)
+        return HttpResponse(status=status, url=request.url, headers=headers, body=body)
+
+    def stream(self, request: HttpRequest) -> HttpStream:
+        """Open ``request`` and hand back its body unread and never decompressed.
+
+        Raises:
+            NetworkError: the connection could not be made.
+        """
+        try:
+            response = self._opener.open(_prepare(request), timeout=self._timeout)
+        except urllib.error.HTTPError as error:
+            log.debug("%s %s -> %d (stream)", request.method, redact(request.url), error.code)
+            headers = dict(error.headers.items()) if error.headers else {}
+            return _UrllibStream(error, error.code, headers, request)
+        except _NETWORK_ERRORS as error:
+            raise _network_error(error, request) from error
+        log.debug("%s %s -> %d (stream)", request.method, redact(request.url), response.status)
+        return _UrllibStream(response, response.status, dict(response.headers.items()), request)
+
+
+class RetryingTransport:
+    """Retries timeouts, connection resets and HTTP 408/5xx with exponential backoff."""
+
+    def __init__(
+        self,
+        inner: Transport,
+        *,
+        retries: int = 2,
+        sleep: Callable[[float], None] = time.sleep,
+        rng: random.Random | None = None,
+        clock: Callable[[], datetime] | None = None,
+    ) -> None:
+        self._inner = inner
+        self._retries = retries
+        self._sleep = sleep
+        self._rng = rng or random.Random()
+        self._clock = clock or _utc_now
+
+    def send(self, request: HttpRequest) -> HttpResponse:
+        attempt = 0
+        while True:
+            try:
+                response = self._inner.send(request)
+            except _NETWORK_ERRORS as error:
+                if attempt >= self._retries or not _is_transient(error):
+                    raise _network_error(error, request) from error
+                delay = backoff_delay(attempt, self._rng)
+                log.info(
+                    "%s %s failed (%s); retrying in %.1fs",
+                    request.method,
+                    redact(request.url),
+                    error,
+                    delay,
+                )
+            else:
+                if attempt >= self._retries or not is_transient_status(response.status):
+                    return response
+                retry_after = parse_retry_after(response.header("retry-after"), now=self._clock())
+                delay = backoff_delay(attempt, self._rng) if retry_after is None else retry_after
+                log.info(
+                    "%s %s answered %d; retrying in %.1fs",
+                    request.method,
+                    redact(request.url),
+                    response.status,
+                    delay,
+                )
+            self._sleep(delay)
+            attempt += 1
+
+    def stream(self, request: HttpRequest) -> HttpStream:
+        """Stream through the inner transport; the downloader retries byte ranges itself."""
+        return open_stream(self._inner, request)
+
+
+def redact(url: str) -> str:
+    """``url`` without credentials or query string, safe for logs and error messages."""
+    try:
+        parts = urlsplit(url)
+        port = f":{parts.port}" if parts.port else ""
+    except ValueError:
+        return "<invalid url>"
+    return f"{parts.scheme}://{parts.hostname or ''}{port}{parts.path}"
+
+
+def open_stream(transport: Transport, request: HttpRequest) -> HttpStream:
+    """Stream ``request`` when ``transport`` can; otherwise send it and serve the body from memory."""
+    if isinstance(transport, StreamingTransport):
+        return transport.stream(request)
+    return BufferedStream(transport.send(request))
+
+
+class BufferedStream:
+    """An :class:`~utmax.transport.HttpStream` over a response that is already in memory."""
+
+    def __init__(self, response: HttpResponse) -> None:
+        self.status = response.status
+        self._response = response
+        self._body = io.BytesIO(response.body)
+
+    def header(self, name: str) -> str | None:
+        return self._response.header(name)
+
+    def read(self, n: int) -> bytes:
+        return self._body.read(n)
+
+    def close(self) -> None:
+        self._body.close()
+
+
+class _UrllibStream:
+    """A urllib response (or HTTP error response) whose body is read in pieces."""
+
+    def __init__(
+        self, body: IO[bytes], status: int, headers: Mapping[str, str], request: HttpRequest
+    ) -> None:
+        self.status = status
+        self._body = body
+        self._headers = {key.lower(): value for key, value in headers.items()}
+        self._request = request
+
+    def header(self, name: str) -> str | None:
+        return self._headers.get(name.lower())
+
+    def read(self, n: int) -> bytes:
+        try:
+            return self._body.read(n)
+        except _NETWORK_ERRORS as error:
+            raise _network_error(error, self._request) from error
+
+    def close(self) -> None:
+        self._body.close()
+
+
+class _IPv4HTTPHandler(urllib.request.HTTPHandler):
+    def http_open(self, req: urllib.request.Request) -> http.client.HTTPResponse:
+        return self.do_open(http.client.HTTPConnection, req, source_address=_IPV4_ONLY)
+
+
+class _IPv4HTTPSHandler(urllib.request.HTTPSHandler):
+    def __init__(self, context: ssl.SSLContext) -> None:
+        super().__init__(context=context)
+        self._tls_context = context
+
+    def https_open(self, req: urllib.request.Request) -> http.client.HTTPResponse:
+        return self.do_open(
+            http.client.HTTPSConnection,
+            req,
+            context=self._tls_context,
+            source_address=_IPV4_ONLY,
+        )
+
+
+def _prepare(request: HttpRequest) -> urllib.request.Request:
+    return urllib.request.Request(
+        request.url, data=request.body, headers=dict(request.headers), method=request.method
+    )
+
+
+def _check_proxy(proxy: str) -> None:
+    try:
+        parts = urlsplit(proxy)
+        host = parts.hostname
+    except ValueError:
+        host = None
+    if not proxy.startswith("http://") or not host:
+        raise InvalidOption(
+            f"Unsupported proxy URL {redact(proxy)!r}: use http://[user:password@]host:port "
+            "(HTTPS traffic is tunnelled through it).",
+            suggestion="For SOCKS or https:// proxies, pass your own transport= to utmax.Client.",
+        )
+
+
+def _header(headers: Mapping[str, str], name: str) -> str:
+    return next((value for key, value in headers.items() if key.lower() == name), "")
+
+
+def _is_transient(error: BaseException) -> bool:
+    reason: object = error.reason if isinstance(error, urllib.error.URLError) else error
+    if isinstance(reason, ssl.SSLCertVerificationError):
+        return False
+    if isinstance(reason, socket.gaierror):
+        return reason.errno == socket.EAI_AGAIN
+    return isinstance(
+        reason,
+        (
+            TimeoutError,
+            ConnectionError,
+            ssl.SSLError,
+            http.client.HTTPException,
+            EOFError,
+            gzip.BadGzipFile,
+            zlib.error,
+        ),
+    )
+
+
+def _network_error(error: BaseException, request: HttpRequest) -> NetworkError:
+    reason = error.reason if isinstance(error, urllib.error.URLError) else error
+    message = f"Could not complete {request.method} {redact(request.url)}: {reason}"
+    if isinstance(reason, ssl.SSLCertVerificationError):
+        return NetworkError(
+            message,
+            suggestion=(
+                "TLS certificate verification failed; a proxy or antivirus may be "
+                "intercepting HTTPS traffic."
+            ),
+        )
+    return NetworkError(message)
+
+
+def _utc_now() -> datetime:
+    return datetime.now(UTC)

utmax/adapters/innertube.py

@@ -0,0 +1,207 @@
+"""InnerTube (YouTube's internal API): player requests with client fallback, captions, and
+the ``browse`` and ``navigation/resolve_url`` requests behind playlists and channels."""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Callable
+from functools import partial
+from typing import Any, TypeVar
+
+from utmax.adapters.watch_page import fetch_api_key
+from utmax.core.captions import caption_url, check_caption_url
+from utmax.core.clients import ANDROID, DESKTOP_USER_AGENT, ORDER, ClientProfile, Purpose
+from utmax.core.playability import check_playability
+from utmax.core.player import PlayerData, parse_player_response
+from utmax.errors import (
+    IpBlocked,
+    RequestBlocked,
+    VideoUnplayable,
+    YouTubeDataUnparsable,
+    YouTubeError,
+    YouTubeRequestFailed,
+)
+from utmax.transport import HttpRequest, HttpResponse, Transport
+
+__all__ = ["API_BASE", "InnerTubeClient"]
+
+API_BASE = "https://www.youtube.com/youtubei/v1"
+log = logging.getLogger("utmax.youtube")
+T = TypeVar("T")
+
+_HTTP_LEVEL_FAILURES = (YouTubeRequestFailed, YouTubeDataUnparsable)
+
+
+class InnerTubeClient:
+    """Talks to InnerTube with a chain of client profiles (see ``utmax.core.clients``)."""
+
+    def __init__(self, transport: Transport, *, block_retries: int = 0) -> None:
+        self._transport = transport
+        self._block_retries = block_retries
+
+    def player(self, video_id: str, *, purpose: Purpose = "captions") -> PlayerData:
+        """A playable player response, trying each profile for ``purpose`` in order.
+
+        For ``"streams"``, a response without a direct MP4 stream URL counts as a failed profile.
+        """
+        profiles = ORDER[purpose]
+        failures: list[YouTubeError] = []
+        http_failures = 0
+        for profile in profiles:
+            try:
+                return self._with_block_retries(
+                    partial(self._playable, profile, video_id, None, purpose=purpose)
+                )
+            except _HTTP_LEVEL_FAILURES as error:
+                http_failures += 1
+                failures.append(error)
+            except (VideoUnplayable, RequestBlocked) as error:
+                if isinstance(error, IpBlocked):
+                    raise
+                failures.append(error)
+            log.info("InnerTube client %s failed for %s: %s", profile.name, video_id, failures[-1])
+        if http_failures == len(profiles):
+            log.info("every InnerTube client failed at the HTTP level; trying the watch page")
+            api_key = fetch_api_key(self._transport, video_id)
+            return self._with_block_retries(
+                partial(self._playable, ANDROID, video_id, api_key, purpose=purpose)
+            )
+        raise failures[0]
+
+    def player_json(
+        self, profile: ClientProfile, video_id: str, *, api_key: str | None = None
+    ) -> dict[str, Any]:
+        """The raw player response of one profile (low level; used by the fixture recorder)."""
+        url = f"{API_BASE}/player?prettyPrint=false"
+        if api_key is not None:
+            url += f"&key={api_key}"
+        payload = {
+            "context": profile.context_payload(),
+            "videoId": video_id,
+            "contentCheckOk": True,
+            "racyCheckOk": True,
+        }
+        body = json.dumps(payload).encode("utf-8")
+        response = self._transport.send(HttpRequest("POST", url, profile.request_headers(), body))
+        return _json_object(response, video_id=video_id)
+
+    def browse(
+        self,
+        profile: ClientProfile,
+        *,
+        browse_id: str | None = None,
+        continuation: str | None = None,
+    ) -> dict[str, Any]:
+        """One raw ``browse`` answer of ``profile``: the first page of ``browse_id`` (``"VL"``
+        plus a playlist ID), or the page behind a ``continuation`` token."""
+        payload: dict[str, Any] = {"context": profile.context_payload()}
+        if browse_id is not None:
+            payload["browseId"] = browse_id
+        if continuation is not None:
+            payload["continuation"] = continuation
+        return self._with_block_retries(partial(self._post, profile, "browse", payload))
+
+    def resolve_url(self, profile: ClientProfile, url: str) -> dict[str, Any]:
+        """The raw ``navigation/resolve_url`` answer of ``profile`` for a YouTube ``url``, such as
+        a channel's ``https://www.youtube.com/@handle``."""
+        payload = {"context": profile.context_payload(), "url": url}
+        return self._with_block_retries(
+            partial(self._post, profile, "navigation/resolve_url", payload)
+        )
+
+    def fetch_captions(
+        self, base_url: str, *, video_id: str, fmt: str | None = "json3"
+    ) -> HttpResponse:
+        """Download a caption track as ``fmt``; ``None`` asks for YouTube's legacy XML."""
+        check_caption_url(base_url, video_id=video_id)
+        url = caption_url(base_url, fmt=fmt)
+        return self._with_block_retries(partial(self._caption_response, url, video_id))
+
+    def _post(
+        self, profile: ClientProfile, endpoint: str, payload: dict[str, Any]
+    ) -> dict[str, Any]:
+        url = f"{API_BASE}/{endpoint}?prettyPrint=false"
+        body = json.dumps(payload).encode("utf-8")
+        response = self._transport.send(HttpRequest("POST", url, profile.request_headers(), body))
+        return _json_object(response)
+
+    def _playable(
+        self, profile: ClientProfile, video_id: str, api_key: str | None, *, purpose: Purpose
+    ) -> PlayerData:
+        data = self.player_json(profile, video_id, api_key=api_key)
+        player = parse_player_response(data, video_id=video_id)
+        check_playability(player.playability, video_id=video_id)
+        if purpose == "streams" and not any(
+            stream.url
+            and stream.format.container == "mp4"
+            and not stream.drm
+            and not stream.progressive
+            for stream in player.streams
+        ):
+            raise YouTubeDataUnparsable(
+                f"The {profile.name} client returned no direct MP4 stream URLs for {video_id}.",
+                video_id=video_id,
+            )
+        return player
+
+    def _caption_response(self, url: str, video_id: str) -> HttpResponse:
+        headers = {
+            "User-Agent": DESKTOP_USER_AGENT,
+            "Accept-Language": "en-US,en;q=0.9",
+            "Accept-Encoding": "gzip",
+        }
+        response = self._transport.send(HttpRequest("GET", url, headers))
+        if response.status == 429:
+            hint = (
+                "YouTube's own translation is heavily rate-limited; use AI translation or a proxy."
+                if "tlang=" in url
+                else None
+            )
+            raise IpBlocked(
+                "YouTube rate-limited the caption download (HTTP 429).",
+                video_id=video_id,
+                suggestion=hint,
+            )
+        if response.status != 200:
+            raise YouTubeRequestFailed(
+                f"The caption download answered HTTP {response.status}.",
+                status_code=response.status,
+                video_id=video_id,
+            )
+        return response
+
+    def _with_block_retries(self, action: Callable[[], T]) -> T:
+        attempt = 0
+        while True:
+            try:
+                return action()
+            except RequestBlocked as error:
+                if attempt >= self._block_retries:
+                    raise
+                attempt += 1
+                log.info(
+                    "blocked by YouTube (%s); retry %d/%d", error, attempt, self._block_retries
+                )
+
+
+def _json_object(response: HttpResponse, *, video_id: str | None = None) -> dict[str, Any]:
+    if response.status == 429:
+        raise IpBlocked("YouTube rate-limited this IP address (HTTP 429).", video_id=video_id)
+    if response.status != 200:
+        raise YouTubeRequestFailed(
+            f"InnerTube answered HTTP {response.status}.",
+            status_code=response.status,
+            video_id=video_id,
+        )
+    try:
+        data = response.json()
+    except ValueError:
+        raise YouTubeDataUnparsable(
+            "InnerTube returned a response that is not JSON.", video_id=video_id
+        ) from None
+    if not isinstance(data, dict):
+        raise YouTubeDataUnparsable(
+            "InnerTube returned an unexpected JSON value.", video_id=video_id
+        )
+    return data

utmax/adapters/providers/__init__.py

@@ -0,0 +1,45 @@
+"""AI translation providers: the Translator base class and one adapter per provider SDK."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from utmax.adapters.providers.base import EngineOptions, SDKTranslator, Translator
+from utmax.adapters.providers.claude import ClaudeTranslator
+from utmax.adapters.providers.gemini import GeminiTranslator
+from utmax.adapters.providers.openai import OpenAITranslator
+from utmax.adapters.providers.openrouter import OpenRouterTranslator
+from utmax.core.translate.spec import parse_model_spec
+from utmax.errors import InvalidOption
+
+__all__ = [
+    "TRANSLATORS",
+    "ClaudeTranslator",
+    "EngineOptions",
+    "GeminiTranslator",
+    "OpenAITranslator",
+    "OpenRouterTranslator",
+    "Translator",
+    "create_translator",
+]
+
+TRANSLATORS: dict[str, type[SDKTranslator]] = {
+    "claude": ClaudeTranslator,
+    "openai": OpenAITranslator,
+    "gemini": GeminiTranslator,
+    "openrouter": OpenRouterTranslator,
+}
+
+
+def create_translator(model: str, **options: Any) -> Translator:
+    """The built-in translator for ``model`` (``"provider=model-id"``); see :func:`utmax.translator`."""
+    spec = parse_model_spec(model)
+    if "base_url" in options and spec.provider != "openai":
+        raise InvalidOption(
+            f"base_url only works with the openai provider, not {spec.provider}.",
+            suggestion=(
+                'Use "openai=<model-id>" with base_url=... for Ollama, LM Studio and other '
+                "OpenAI-compatible servers."
+            ),
+        )
+    return TRANSLATORS[spec.provider](spec.model, **options)