u-transcript-max 0.1.0a1__py3-none-any.whl

utmax/adapters/watch_page.py

@@ -0,0 +1,73 @@
+"""Last resort: read the InnerTube API key from the watch page, passing the EU consent wall."""
+
+from __future__ import annotations
+
+import re
+
+from utmax.core.clients import DESKTOP_USER_AGENT
+from utmax.errors import (
+    FailedToCreateConsentCookie,
+    IpBlocked,
+    YouTubeDataUnparsable,
+    YouTubeRequestFailed,
+)
+from utmax.transport import HttpRequest, Transport
+
+__all__ = ["extract_api_key", "fetch_api_key"]
+
+_WATCH_URL = "https://www.youtube.com/watch?v={video_id}"
+_API_KEY = re.compile(r'"INNERTUBE_API_KEY":\s*"([a-zA-Z0-9_-]+)"')
+_CONSENT_FORM = 'action="https://consent.youtube.com/s"'
+_CONSENT_VALUE = re.compile(r'name="v" value="(.*?)"')
+_HEADERS = {
+    "User-Agent": DESKTOP_USER_AGENT,
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip",
+}
+
+
+def fetch_api_key(transport: Transport, video_id: str) -> str:
+    """Download the watch page (accepting cookies for this call only) and read the API key."""
+    page = _watch_page(transport, video_id, cookie=None)
+    if _CONSENT_FORM in page:
+        match = _CONSENT_VALUE.search(page)
+        if match is None:
+            raise FailedToCreateConsentCookie(
+                "Could not read YouTube's cookie-consent form.", video_id=video_id
+            )
+        page = _watch_page(transport, video_id, cookie=f"CONSENT=YES+{match.group(1)}")
+        if _CONSENT_FORM in page:
+            raise FailedToCreateConsentCookie(
+                "YouTube kept asking for cookie consent.", video_id=video_id
+            )
+    return extract_api_key(page, video_id=video_id)
+
+
+def extract_api_key(page: str, *, video_id: str) -> str:
+    """The ``INNERTUBE_API_KEY`` embedded in a watch page."""
+    match = _API_KEY.search(page)
+    if match is not None:
+        return match.group(1)
+    if 'class="g-recaptcha"' in page:
+        raise IpBlocked("YouTube answered with a CAPTCHA page.", video_id=video_id)
+    raise YouTubeDataUnparsable(
+        "Could not find the InnerTube API key on the watch page.", video_id=video_id
+    )
+
+
+def _watch_page(transport: Transport, video_id: str, *, cookie: str | None) -> str:
+    headers = dict(_HEADERS)
+    if cookie is not None:
+        headers["Cookie"] = cookie
+    response = transport.send(HttpRequest("GET", _WATCH_URL.format(video_id=video_id), headers))
+    if response.status == 429:
+        raise IpBlocked(
+            "YouTube rate-limited the watch page request (HTTP 429).", video_id=video_id
+        )
+    if response.status != 200:
+        raise YouTubeRequestFailed(
+            f"The watch page answered HTTP {response.status}.",
+            status_code=response.status,
+            video_id=video_id,
+        )
+    return response.text

utmax/client.py

@@ -0,0 +1,300 @@
+"""The Client: configuration plus one connection pipeline that every call goes through."""
+
+from __future__ import annotations
+
+import os
+import threading
+from collections.abc import Callable, Iterable, Sequence
+from functools import partial
+from typing import Any, Self
+
+from utmax.adapters.http import RetryingTransport, UrllibTransport, open_stream
+from utmax.adapters.innertube import InnerTubeClient
+from utmax.adapters.providers import Translator, create_translator
+from utmax.core.bilingual import bilingual
+from utmax.core.downloads import DEFAULT_CHUNK_SIZE
+from utmax.errors import InvalidOption
+from utmax.models import (
+    BulkReport,
+    BulkResult,
+    CollectionKind,
+    Container,
+    DownloadResult,
+    FormatName,
+    Progress,
+    Quality,
+    SubtitleMode,
+    TrackList,
+    Transcript,
+    VideoEntry,
+    VideoInfo,
+    VideoList,
+)
+from utmax.services.bulk import DEFAULT_DOWNLOAD_NAME, DEFAULT_TRANSCRIPT_NAME, BulkService
+from utmax.services.collections import CollectionService
+from utmax.services.download import DownloadOptions, DownloadService
+from utmax.services.transcripts import TranscriptService
+from utmax.services.translation import translate
+from utmax.transport import Transport
+
+__all__ = ["Client"]
+
+
+class Client:
+    """A configured, thread-safe connection to YouTube.
+
+    Module-level functions such as :func:`utmax.fetch` share a default ``Client``. Create your
+    own for a proxy, timeouts or a retry policy::
+
+        with utmax.Client(proxy="http://user:pass@host:8080", timeout=20) as client:
+            transcript = client.fetch("dQw4w9WgXcQ")
+
+    Args:
+        proxy: ``http://[user:password@]host:port`` used for all YouTube traffic.
+        timeout: seconds before a request is abandoned.
+        retries: extra attempts for timeouts, connection resets and HTTP 408/5xx.
+        block_retries: extra attempts, each on a new connection, when YouTube blocks the IP
+            (useful with rotating proxies).
+        force_ipv4: connect over IPv4 only; try it when downloads fail with StreamForbidden on a
+            machine with both IPv4 and IPv6, because stream URLs are bound to one address.
+        transport: a custom :class:`utmax.transport.Transport` replacing the whole HTTP stack;
+            ``timeout`` and ``retries`` are then ignored and ``proxy`` must not be set.
+    """
+
+    def __init__(
+        self,
+        *,
+        proxy: str | None = None,
+        timeout: float = 30.0,
+        retries: int = 2,
+        block_retries: int = 0,
+        force_ipv4: bool = False,
+        transport: Transport | None = None,
+    ) -> None:
+        if timeout <= 0:
+            raise InvalidOption("timeout must be a positive number of seconds.")
+        if retries < 0 or block_retries < 0:
+            raise InvalidOption("retries and block_retries cannot be negative.")
+        if transport is not None and proxy is not None:
+            raise InvalidOption(
+                "Pass either proxy= or transport=, not both; configure the proxy in your transport."
+            )
+        if transport is not None and force_ipv4:
+            raise InvalidOption(
+                "force_ipv4 applies to utmax's own HTTP stack; configure IPv4 in your transport."
+            )
+        if transport is None:
+            transport = RetryingTransport(
+                UrllibTransport(proxy=proxy, timeout=timeout, force_ipv4=force_ipv4),
+                retries=retries,
+            )
+        self._transport = transport
+        innertube = InnerTubeClient(transport, block_retries=block_retries)
+        self._transcripts = TranscriptService(innertube)
+        self._downloads = DownloadService(
+            innertube, self._transcripts, partial(open_stream, transport)
+        )
+        self._collections = CollectionService(innertube)
+        self._bulk = BulkService(self._transcripts, self._downloads)
+
+    def fetch(
+        self,
+        video: str,
+        languages: Sequence[str] | str | None = None,
+        *,
+        include_manual: bool = True,
+        include_generated: bool = True,
+        preserve_formatting: bool = False,
+        youtube_translation: str | None = None,
+    ) -> Transcript:
+        """Fetch the best subtitle track of ``video``; see :func:`utmax.fetch`."""
+        return self._transcripts.fetch(
+            video,
+            languages,
+            include_manual=include_manual,
+            include_generated=include_generated,
+            preserve_formatting=preserve_formatting,
+            youtube_translation=youtube_translation,
+        )
+
+    def list_tracks(self, video: str) -> TrackList:
+        """Every subtitle track of ``video``, in YouTube's order."""
+        return self._transcripts.list_tracks(video)
+
+    def video_info(self, video: str) -> VideoInfo:
+        """Title, channel and duration of ``video``."""
+        return self._transcripts.video_info(video)
+
+    def translator(self, model: str, **options: Any) -> Translator:
+        """The translator for ``"provider=model-id"``; see :func:`utmax.translator`."""
+        return create_translator(model, **options)
+
+    def translate(
+        self,
+        transcript: Transcript,
+        to: str,
+        *,
+        model: str | Translator,
+        instructions: str | None = None,
+        resegment: bool | None = None,
+        **options: Any,
+    ) -> Transcript:
+        """Translate ``transcript`` with an AI model; see :func:`utmax.translate`."""
+        return translate(
+            transcript,
+            to,
+            model=model,
+            instructions=instructions,
+            resegment=resegment,
+            **options,
+        )
+
+    def bilingual(
+        self, original: Transcript, translation: Transcript, *, translation_first: bool = False
+    ) -> Transcript:
+        """Combine a transcript and its translation; see :func:`utmax.bilingual`."""
+        return bilingual(original, translation, translation_first=translation_first)
+
+    def download(
+        self,
+        video: str,
+        path: str | os.PathLike[str],
+        *,
+        format: Container | None = None,
+        quality: Quality = "compat",
+        subtitles: Sequence[str | Transcript] | None = None,
+        subtitle_mode: SubtitleMode = "embed",
+        default_subtitle: str | None = None,
+        connections: int = 4,
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
+        resume: bool = True,
+        overwrite: bool = False,
+        ffmpeg: str | os.PathLike[str] | None = None,
+        progress: Callable[[Progress], None] | None = None,
+        cancel: threading.Event | None = None,
+    ) -> DownloadResult:
+        """Download a video or its audio; see :func:`utmax.download`."""
+        options = DownloadOptions(
+            format=format,
+            quality=quality,
+            subtitles=subtitles,
+            subtitle_mode=subtitle_mode,
+            default_subtitle=default_subtitle,
+            connections=connections,
+            chunk_size=chunk_size,
+            resume=resume,
+            overwrite=overwrite,
+            ffmpeg=ffmpeg,
+            progress=progress,
+            cancel=cancel,
+        )
+        return self._downloads.download(video, path, options)
+
+    def list_videos(
+        self, source: str, *, kind: CollectionKind | None = None, limit: int | None = None
+    ) -> VideoList:
+        """The videos of a playlist or channel; see :func:`utmax.list_videos`."""
+        return self._collections.list_videos(source, kind=kind, limit=limit)
+
+    def fetch_many(
+        self,
+        videos: Iterable[str | VideoEntry],
+        *,
+        out_dir: str | os.PathLike[str] | None = None,
+        format: FormatName = "srt",
+        languages: Sequence[str] | str | None = None,
+        include_manual: bool = True,
+        include_generated: bool = True,
+        concurrency: int = 4,
+        skip_existing: bool = True,
+        filename: str = DEFAULT_TRANSCRIPT_NAME,
+        progress: Callable[[BulkResult[Transcript]], None] | None = None,
+    ) -> BulkReport[Transcript]:
+        """Fetch the transcripts of many videos; see :func:`utmax.fetch_many`."""
+        return self._bulk.fetch_many(
+            videos,
+            out_dir=out_dir,
+            format=format,
+            languages=languages,
+            include_manual=include_manual,
+            include_generated=include_generated,
+            concurrency=concurrency,
+            skip_existing=skip_existing,
+            filename=filename,
+            progress=progress,
+        )
+
+    def translate_many(
+        self,
+        videos: Iterable[str | VideoEntry],
+        to: str,
+        *,
+        model: str | Translator,
+        out_dir: str | os.PathLike[str] | None = None,
+        format: FormatName = "srt",
+        languages: Sequence[str] | str | None = None,
+        bilingual: bool = False,
+        instructions: str | None = None,
+        resegment: bool | None = None,
+        concurrency: int = 2,
+        skip_existing: bool = True,
+        filename: str = DEFAULT_TRANSCRIPT_NAME,
+        progress: Callable[[BulkResult[Transcript]], None] | None = None,
+        **options: Any,
+    ) -> BulkReport[Transcript]:
+        """Fetch and translate many transcripts; see :func:`utmax.translate_many`."""
+        return self._bulk.translate_many(
+            videos,
+            to,
+            model=model,
+            out_dir=out_dir,
+            format=format,
+            languages=languages,
+            bilingual=bilingual,
+            instructions=instructions,
+            resegment=resegment,
+            concurrency=concurrency,
+            skip_existing=skip_existing,
+            filename=filename,
+            progress=progress,
+            **options,
+        )
+
+    def download_many(
+        self,
+        videos: Iterable[str | VideoEntry],
+        out_dir: str | os.PathLike[str],
+        *,
+        format: Container = "mp4",
+        quality: Quality = "compat",
+        subtitles: Sequence[str] | None = None,
+        subtitle_mode: SubtitleMode = "embed",
+        concurrency: int = 2,
+        skip_existing: bool = True,
+        filename: str = DEFAULT_DOWNLOAD_NAME,
+        ffmpeg: str | os.PathLike[str] | None = None,
+        progress: Callable[[BulkResult[DownloadResult]], None] | None = None,
+    ) -> BulkReport[DownloadResult]:
+        """Download many videos into a folder; see :func:`utmax.download_many`."""
+        return self._bulk.download_many(
+            videos,
+            out_dir,
+            format=format,
+            quality=quality,
+            subtitles=subtitles,
+            subtitle_mode=subtitle_mode,
+            concurrency=concurrency,
+            skip_existing=skip_existing,
+            filename=filename,
+            ffmpeg=ffmpeg,
+            progress=progress,
+        )
+
+    def close(self) -> None:
+        """Release resources; utmax keeps no open connections today, so this does nothing yet."""
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()

utmax/compat/__init__.py

@@ -0,0 +1,134 @@
+"""A drop-in replacement for youtube-transcript-api 1.2.4 that runs on utmax.
+
+Change the import and the rest of the code stays as it is::
+
+    from utmax.compat import YouTubeTranscriptApi  # was: from youtube_transcript_api import ...
+
+    transcript = YouTubeTranscriptApi().fetch("dQw4w9WgXcQ", languages=["de", "en"])
+
+The modules mirror youtube-transcript-api's (``utmax.compat.formatters``,
+``utmax.compat.proxies``, ...) with the same classes, signatures, attributes and messages. The
+class methods of version 0.6 (``get_transcript``, ``get_transcripts``, ``list_transcripts``) and
+its exception names work too. Unlike the original, ``YouTubeTranscriptApi`` is thread-safe,
+accepts video URLs, and needs no third-party packages.
+
+For libraries that import ``youtube_transcript_api`` themselves, call :func:`install` once,
+before they import it.
+"""
+
+# utmax.compat reproduces the interface, messages and output formats of youtube-transcript-api
+# (https://github.com/jdepoix/youtube-transcript-api), which is distributed under this license:
+#
+# MIT License
+#
+# Copyright (c) 2018 Jonas Depoix
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import annotations
+
+import sys
+
+from utmax.compat import _api, _errors, _settings, _transcripts, formatters, proxies
+from utmax.compat._api import YouTubeTranscriptApi
+from utmax.compat._errors import (
+    AgeRestricted,
+    CookieError,
+    CookieInvalid,
+    CookiePathInvalid,
+    CookiesInvalid,
+    CouldNotRetrieveTranscript,
+    FailedToCreateConsentCookie,
+    InvalidVideoId,
+    IpBlocked,
+    NoTranscriptAvailable,
+    NoTranscriptFound,
+    NotTranslatable,
+    PoTokenRequired,
+    RequestBlocked,
+    TooManyRequests,
+    TranscriptsDisabled,
+    TranslationLanguageNotAvailable,
+    VideoUnavailable,
+    VideoUnplayable,
+    YouTubeDataUnparsable,
+    YouTubeRequestFailed,
+    YouTubeTranscriptApiException,
+)
+from utmax.compat._transcripts import (
+    FetchedTranscript,
+    FetchedTranscriptSnippet,
+    Transcript,
+    TranscriptList,
+)
+
+__all__ = [
+    "AgeRestricted",
+    "CookieError",
+    "CookieInvalid",
+    "CookiePathInvalid",
+    "CookiesInvalid",
+    "CouldNotRetrieveTranscript",
+    "FailedToCreateConsentCookie",
+    "FetchedTranscript",
+    "FetchedTranscriptSnippet",
+    "InvalidVideoId",
+    "IpBlocked",
+    "NoTranscriptAvailable",
+    "NoTranscriptFound",
+    "NotTranslatable",
+    "PoTokenRequired",
+    "RequestBlocked",
+    "TooManyRequests",
+    "Transcript",
+    "TranscriptList",
+    "TranscriptsDisabled",
+    "TranslationLanguageNotAvailable",
+    "VideoUnavailable",
+    "VideoUnplayable",
+    "YouTubeDataUnparsable",
+    "YouTubeRequestFailed",
+    "YouTubeTranscriptApi",
+    "YouTubeTranscriptApiException",
+    "install",
+]
+
+_MODULE = "youtube_transcript_api"
+
+
+def install() -> None:
+    """Make ``import youtube_transcript_api`` load utmax.compat.
+
+    For libraries that import youtube-transcript-api themselves (such as LangChain's YouTube
+    loader): call it once, before they import it. It registers ``youtube_transcript_api`` and
+    its modules in ``sys.modules``, replacing a real installation for every import that
+    follows; modules that imported it earlier keep what they have. Calling it again changes
+    nothing.
+    """
+    package = sys.modules[__name__]
+    modules = {
+        _MODULE: package,
+        f"{_MODULE}._api": _api,
+        f"{_MODULE}._errors": _errors,
+        f"{_MODULE}._settings": _settings,
+        f"{_MODULE}._transcripts": _transcripts,
+        f"{_MODULE}.formatters": formatters,
+        f"{_MODULE}.proxies": proxies,
+    }
+    sys.modules.update(modules)