u-transcript-max 0.1.0a1__py3-none-any.whl

utmax/compat/_api.py

@@ -0,0 +1,203 @@
+"""``YouTubeTranscriptApi``, compatible with youtube-transcript-api 1.2.4 and the class methods
+of 0.6."""
+
+from __future__ import annotations
+
+import builtins
+import os
+import warnings
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+from utmax.compat._transcripts import FetchedTranscript, TranscriptList, TranscriptListFetcher
+from utmax.compat.proxies import GenericProxyConfig, ProxyConfig
+
+__all__ = [
+    "FetchedTranscript",
+    "ProxyConfig",
+    "TranscriptList",
+    "TranscriptListFetcher",
+    "YouTubeTranscriptApi",
+]
+
+_LegacyProxies = ProxyConfig | Mapping[str, str] | None
+_RawTranscript = builtins.list[dict[str, Any]]
+
+
+class YouTubeTranscriptApi:
+    """Fetches transcripts with utmax; a drop-in replacement for youtube-transcript-api's class.
+
+    Unlike youtube-transcript-api, an instance is thread-safe (unless ``http_client`` is not),
+    and every method also accepts YouTube video URLs.
+
+    Args:
+        proxy_config: proxies for all requests, such as ``GenericProxyConfig`` or
+            ``WebshareProxyConfig``. Without ``http_client`` only ``http://`` proxies work.
+        http_client: a ``requests.Session``-like object that sends every request instead of
+            utmax's own HTTP code: use it for SOCKS or HTTPS proxies, custom certificates or
+            shared cookies.
+    """
+
+    def __init__(
+        self,
+        proxy_config: ProxyConfig | None = None,
+        http_client: Any = None,
+    ) -> None:
+        if http_client is not None:
+            http_client.headers.update({"Accept-Language": "en-US"})
+            if proxy_config is not None:
+                http_client.proxies = proxy_config.to_requests_dict()
+                if proxy_config.prevent_keeping_connections_alive:
+                    http_client.headers.update({"Connection": "close"})
+        self._fetcher = TranscriptListFetcher(http_client, proxy_config=proxy_config)
+
+    def fetch(
+        self,
+        video_id: str,
+        languages: Iterable[str] = ("en",),
+        preserve_formatting: bool = False,
+    ) -> FetchedTranscript:
+        """
+        Retrieves the transcript for a single video. This is just a shortcut for
+        calling:
+        `YouTubeTranscriptApi().list(video_id).find_transcript(languages).fetch(preserve_formatting=preserve_formatting)`
+
+        :param video_id: the ID or URL of the video you want to retrieve the transcript for.
+        :param languages: A list of language codes in a descending priority. For
+            example, if this is set to ["de", "en"] it will first try to fetch the
+            german transcript (de) and then fetch the english transcript (en) if
+            it fails to do so. This defaults to ["en"].
+        :param preserve_formatting: whether to keep select HTML text formatting
+        """
+        return (
+            self.list(video_id)
+            .find_transcript(languages)
+            .fetch(preserve_formatting=preserve_formatting)
+        )
+
+    def list(
+        self,
+        video_id: str,
+    ) -> TranscriptList:
+        """
+        Retrieves the list of transcripts which are available for a given video. It
+        returns a `TranscriptList` object which is iterable and provides methods to
+        filter the list of transcripts for specific languages. While iterating over
+        the `TranscriptList` the individual transcripts are represented by
+        `Transcript` objects, which provide metadata and can either be fetched by
+        calling `transcript.fetch()` or translated by calling `transcript.translate(
+        'en')`.
+
+        :param video_id: the ID or URL of the video you want to retrieve the transcript for.
+        """
+        return self._fetcher.fetch(video_id)
+
+    @classmethod
+    def list_transcripts(
+        cls,
+        video_id: str,
+        proxies: _LegacyProxies = None,
+        cookies: str | os.PathLike[str] | None = None,
+    ) -> TranscriptList:
+        """
+        DEPRECATED: use the `list` method instead!
+
+        The class method of youtube-transcript-api 0.6. ``proxies`` is a ``ProxyConfig`` or a
+        requests-style ``{"http": url, "https": url}`` mapping; ``cookies`` is ignored with a
+        warning, because cookie authentication is not supported.
+        """
+        warnings.warn(
+            "`list_transcripts` is deprecated and will be removed in a future version. "
+            "Use the `list` method instead!",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if cookies:
+            warnings.warn(
+                "Cookie authentication is not supported; the cookies argument is ignored.",
+                UserWarning,
+                stacklevel=2,
+            )
+        return cls(proxy_config=_legacy_proxy_config(proxies)).list(video_id)
+
+    @classmethod
+    def get_transcripts(  # noqa: PLR0917
+        cls,
+        video_ids: builtins.list[str],
+        languages: Iterable[str] = ("en",),
+        continue_after_error: bool = False,
+        proxies: _LegacyProxies = None,
+        cookies: str | os.PathLike[str] | None = None,
+        preserve_formatting: bool = False,
+    ) -> tuple[dict[str, _RawTranscript], builtins.list[str]]:
+        """
+        DEPRECATED: use the `fetch` method instead!
+
+        Retrieves the transcripts for a list of videos.
+
+        :return: a tuple containing a dictionary mapping video ids onto their corresponding
+            transcripts (lists of ``{"text", "start", "duration"}`` dictionaries), and a list of
+            video ids, which could not be retrieved
+        """
+        warnings.warn(
+            "`get_transcripts` is deprecated and will be removed in a future version. "
+            "Use the `fetch` method instead!",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if not isinstance(video_ids, builtins.list):
+            raise AssertionError("`video_ids` must be a list of strings")
+
+        data: dict[str, _RawTranscript] = {}
+        unretrievable_videos: builtins.list[str] = []
+
+        for video_id in video_ids:
+            try:
+                data[video_id] = cls.get_transcript(
+                    video_id, languages, proxies, cookies, preserve_formatting
+                )
+            except Exception:
+                if not continue_after_error:
+                    raise
+
+                unretrievable_videos.append(video_id)
+
+        return data, unretrievable_videos
+
+    @classmethod
+    def get_transcript(
+        cls,
+        video_id: str,
+        languages: Iterable[str] = ("en",),
+        proxies: _LegacyProxies = None,
+        cookies: str | os.PathLike[str] | None = None,
+        preserve_formatting: bool = False,
+    ) -> _RawTranscript:
+        """
+        DEPRECATED: use the `fetch` method instead!
+
+        Retrieves the transcript for a single video as a list of ``{"text", "start",
+        "duration"}`` dictionaries.
+        """
+        warnings.warn(
+            "`get_transcript` is deprecated and will be removed in a future version. "
+            "Use the `fetch` method instead!",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if not isinstance(video_id, str):
+            raise AssertionError("`video_id` must be a string")
+        return (
+            cls.list_transcripts(video_id, proxies, cookies)
+            .find_transcript(languages)
+            .fetch(preserve_formatting=preserve_formatting)
+            .to_raw_data()
+        )
+
+
+def _legacy_proxy_config(proxies: _LegacyProxies) -> ProxyConfig | None:
+    if not proxies:
+        return None
+    if isinstance(proxies, ProxyConfig):
+        return proxies
+    return GenericProxyConfig(http_url=proxies.get("http"), https_url=proxies.get("https"))

utmax/compat/_bridge.py

@@ -0,0 +1,212 @@
+"""How utmax.compat reaches YouTube: utmax's transports and InnerTube client, with utmax errors
+turned into the exceptions of youtube-transcript-api."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from utmax import errors
+from utmax.adapters.http import RetryingTransport, UrllibTransport, redact
+from utmax.adapters.innertube import InnerTubeClient
+from utmax.compat._errors import (
+    AgeRestricted,
+    CouldNotRetrieveTranscript,
+    FailedToCreateConsentCookie,
+    InvalidVideoId,
+    IpBlocked,
+    PoTokenRequired,
+    RequestBlocked,
+    TranscriptsDisabled,
+    VideoUnavailable,
+    VideoUnplayable,
+    YouTubeDataUnparsable,
+    YouTubeRequestFailed,
+)
+from utmax.compat.proxies import InvalidProxyConfig, ProxyConfig
+from utmax.core.ids import parse_video_id
+from utmax.core.player import PlayerData
+from utmax.transport import HttpRequest, HttpResponse, Transport
+
+__all__ = [
+    "TIMEOUT",
+    "Connection",
+    "SessionTransport",
+    "compat_error",
+    "connection_for",
+    "make_transport",
+    "video_id_of",
+]
+
+TIMEOUT = 30.0
+"""Seconds before a request is abandoned (also passed to an ``http_client``)."""
+
+_SIMPLE_ERRORS: tuple[tuple[type[errors.UTMaxError], type[CouldNotRetrieveTranscript]], ...] = (
+    (errors.VideoUnavailable, VideoUnavailable),
+    (errors.AgeRestricted, AgeRestricted),
+    (errors.PoTokenRequired, PoTokenRequired),
+    (errors.FailedToCreateConsentCookie, FailedToCreateConsentCookie),
+    (errors.YouTubeDataUnparsable, YouTubeDataUnparsable),
+    (errors.TranscriptsDisabled, TranscriptsDisabled),
+    (errors.InvalidVideoId, InvalidVideoId),
+)
+_DECODED_HEADERS = frozenset({"content-encoding", "content-length", "transfer-encoding"})
+_QUERY_RUN = re.compile(r"\?[^\s)'\"]+")
+"""A ``?`` and what follows it up to whitespace, a closing parenthesis or a quote."""
+
+
+class SessionTransport:
+    """Sends utmax's requests through a ``requests.Session``-like ``http_client``.
+
+    The client needs ``request(method, url, headers=..., data=..., timeout=...)`` returning an
+    object with ``status_code``, ``headers`` and ``content`` (the decoded body); that is how
+    SOCKS proxies, custom certificates or shared cookies reach utmax without utmax depending on
+    requests.
+    """
+
+    def __init__(self, session: Any, *, timeout: float = TIMEOUT) -> None:
+        self._session = session
+        self._timeout = timeout
+
+    def send(self, request: HttpRequest) -> HttpResponse:
+        try:
+            response = self._session.request(
+                request.method,
+                request.url,
+                headers=dict(request.headers),
+                data=request.body,
+                timeout=self._timeout,
+            )
+        except Exception as error:  # the session's own exception types are unknown here
+            reason = _without_query(str(error), request.url)
+            raise errors.NetworkError(
+                f"Could not complete {request.method} {redact(request.url)}: {reason}"
+            ) from error
+        headers = {
+            str(name): str(value)
+            for name, value in response.headers.items()
+            if str(name).lower() not in _DECODED_HEADERS
+        }
+        return HttpResponse(
+            status=int(response.status_code),
+            url=request.url,
+            headers=headers,
+            body=bytes(response.content),
+        )
+
+
+def _without_query(text: str, url: str) -> str:
+    """``text`` without the query string of ``url`` and without any other ``?...`` run.
+
+    A session's own error text repeats the request (``Max retries exceeded with url:
+    /api/timedtext?...&sig=...``), and the query of a caption URL holds the client's IP address,
+    an expiry time and a signature, none of which belongs in a message (spec section 7).
+    """
+    query = url.partition("?")[2].partition("#")[0]
+    if query:
+        text = text.replace(f"?{query}", "").replace(query, "")
+    return _QUERY_RUN.sub("", text)
+
+
+def make_transport(http_client: Any, proxy_config: ProxyConfig | None) -> Transport:
+    """The transport for an ``http_client`` (used as given) or for utmax's own HTTP stack.
+
+    Raises:
+        InvalidProxyConfig: utmax's own stack cannot use the proxy of ``proxy_config``.
+    """
+    if http_client is not None:
+        return SessionTransport(http_client)
+    proxy = None
+    if proxy_config is not None:
+        proxies = proxy_config.to_requests_dict()
+        proxy = proxies.get("https") or proxies.get("http")
+        if not proxy:
+            raise InvalidProxyConfig(
+                f"{type(proxy_config).__name__}.to_requests_dict() names no proxy URL."
+            )
+    try:
+        inner = UrllibTransport(proxy=proxy, timeout=TIMEOUT)
+    except errors.InvalidOption as error:
+        raise InvalidProxyConfig(
+            f"{error} Without an http_client only http:// proxies work; for https:// or "
+            "SOCKS proxies pass http_client=requests.Session() to YouTubeTranscriptApi."
+        ) from error
+    return RetryingTransport(inner)
+
+
+class Connection:
+    """utmax's InnerTube client behind one ``YouTubeTranscriptApi`` and its transcripts."""
+
+    def __init__(self, transport: Transport, *, proxy_config: ProxyConfig | None = None) -> None:
+        retries = proxy_config.retries_when_blocked if proxy_config is not None else 0
+        self._innertube = InnerTubeClient(transport, block_retries=max(0, retries))
+        self.proxy_config = proxy_config
+
+    def player(self, video: str) -> tuple[str, PlayerData]:
+        """The video ID in ``video`` and its player response, which lists subtitle tracks.
+
+        Raises:
+            CouldNotRetrieveTranscript: the subclass matching what went wrong.
+        """
+        video_id = video_id_of(video)
+        try:
+            player = self._innertube.player(video_id, purpose="captions")
+        except errors.UTMaxError as error:
+            raise compat_error(error, video_id, self.proxy_config) from error
+        if not player.caption_tracks:
+            raise TranscriptsDisabled(video_id)
+        return video_id, player
+
+    def caption_text(self, url: str, video_id: str) -> str:
+        """The legacy XML of the caption track at ``url``.
+
+        Raises:
+            CouldNotRetrieveTranscript: the subclass matching what went wrong.
+        """
+        try:
+            response = self._innertube.fetch_captions(url, video_id=video_id, fmt=None)
+        except errors.UTMaxError as error:
+            raise compat_error(error, video_id) from error
+        return response.text
+
+
+def connection_for(http_client: Any) -> Connection:
+    """The connection of a ``Transcript``: the one utmax made, or one around ``http_client``."""
+    if isinstance(http_client, Connection):
+        return http_client
+    return Connection(make_transport(http_client, None))
+
+
+def video_id_of(video: str) -> str:
+    """The ID in ``video``, a bare video ID or any YouTube video URL.
+
+    Raises:
+        InvalidVideoId: ``video`` is a URL without a video ID.
+        VideoUnavailable: ``video`` cannot be a video ID (what YouTube would answer).
+    """
+    try:
+        return parse_video_id(video)
+    except errors.InvalidVideoId as error:
+        if video.strip().startswith(("http://", "https://")):
+            raise InvalidVideoId(video) from error
+        raise VideoUnavailable(video) from error
+
+
+def compat_error(
+    error: errors.UTMaxError, video_id: str, proxy_config: ProxyConfig | None = None
+) -> CouldNotRetrieveTranscript:
+    """The youtube-transcript-api exception for a utmax error.
+
+    Blocks carry ``proxy_config`` (their message depends on it). Network failures and any other
+    error become ``YouTubeRequestFailed`` with the utmax message as the reason.
+    """
+    if isinstance(error, errors.IpBlocked):
+        return IpBlocked(video_id).with_proxy_config(proxy_config)
+    if isinstance(error, errors.RequestBlocked):
+        return RequestBlocked(video_id).with_proxy_config(proxy_config)
+    if isinstance(error, errors.VideoUnplayable):
+        return VideoUnplayable(video_id, error.reason or None, list(error.sub_reasons))
+    for source, target in _SIMPLE_ERRORS:
+        if isinstance(error, source):
+            return target(video_id)
+    return YouTubeRequestFailed(video_id, error)