ytextract 0.0.1__py3-none-any.whl

ytextract/__init__.py

@@ -0,0 +1,32 @@
+# flake8: noqa: F401
+# noreorder
+"""
+ytextract: a very serious Python library for downloading YouTube Videos.
+"""
+from importlib.metadata import version, PackageNotFoundError
+
+__title__ = "ytextract"
+__author__ = "Josh-XT"
+__license__ = "The Unlicense (Unlicense)"
+__js__ = None
+__js_url__ = None
+
+try:
+    __version__ = version("ytextract")
+except PackageNotFoundError:
+    __version__ = "0.0.1"  # fallback for development
+
+from ytextract.streams import Stream
+from ytextract.captions import Caption
+from ytextract.query import CaptionQuery, StreamQuery
+from ytextract.__main__ import YouTube
+from ytextract.innertube import InnerTube
+from ytextract.download_helper import (
+    download_video,
+    download_videos_from_channels,
+    download_videos_from_list,
+    download_captions,
+    get_videos_from_channel,
+)
+
+download = download_video

ytextract/__main__.py

@@ -0,0 +1,491 @@
+"""
+This module implements the core developer interface for ytextract.
+
+The problem domain of the :class:`YouTube <YouTube> class focuses almost
+exclusively on the developer interface. ytextract offloads the heavy lifting to
+smaller peripheral modules and functions.
+
+"""
+
+import logging
+from typing import Any, Callable, Dict, List, Optional
+
+import ytextract
+import ytextract.exceptions as exceptions
+from ytextract import extract, request
+from ytextract import Stream, StreamQuery
+from ytextract.helpers import install_proxy
+from ytextract.innertube import InnerTube, _default_clients
+from ytextract.metadata import YouTubeMetadata
+from ytextract.monostate import Monostate
+
+logger = logging.getLogger(__name__)
+
+
+class YouTube:
+    """Core developer interface for ytextract."""
+
+    def __init__(
+        self,
+        url: str,
+        on_progress_callback: Optional[Callable[[Any, bytes, int], None]] = None,
+        on_complete_callback: Optional[Callable[[Any, Optional[str]], None]] = None,
+        proxies: Dict[str, str] = None,
+        use_oauth: bool = False,
+        allow_oauth_cache: bool = True,
+    ):
+        """Construct a :class:`YouTube <YouTube>`.
+
+        :param str url:
+            A valid YouTube watch URL.
+        :param func on_progress_callback:
+            (Optional) User defined callback function for stream download
+            progress events.
+        :param func on_complete_callback:
+            (Optional) User defined callback function for stream download
+            complete events.
+        :param dict proxies:
+            (Optional) A dict mapping protocol to proxy address which will be used by ytextract.
+        :param bool use_oauth:
+            (Optional) Prompt the user to authenticate to YouTube.
+            If allow_oauth_cache is set to True, the user should only be prompted once.
+        :param bool allow_oauth_cache:
+            (Optional) Cache OAuth tokens locally on the machine. Defaults to True.
+            These tokens are only generated if use_oauth is set to True as well.
+        """
+        self._js: Optional[str] = None  # js fetched by js_url
+        self._js_url: Optional[str] = None  # the url to the js, parsed from watch html
+
+        self._vid_info: Optional[Dict] = None  # content fetched from innertube/player
+
+        self._watch_html: Optional[str] = None  # the html of /watch?v=<video_id>
+        self._embed_html: Optional[str] = None
+        self._player_config_args: Optional[Dict] = (
+            None  # inline js in the html containing
+        )
+        self._age_restricted: Optional[bool] = None
+
+        self._fmt_streams: Optional[List[Stream]] = None
+
+        self._initial_data = None
+        self._metadata: Optional[YouTubeMetadata] = None
+
+        # video_id part of /watch?v=<video_id>
+        self.video_id = extract.video_id(url)
+
+        self.watch_url = f"https://youtube.com/watch?v={self.video_id}"
+        self.embed_url = f"https://www.youtube.com/embed/{self.video_id}"
+
+        # Shared between all instances of `Stream` (Borg pattern).
+        self.stream_monostate = Monostate(
+            on_progress=on_progress_callback, on_complete=on_complete_callback
+        )
+
+        if proxies:
+            install_proxy(proxies)
+
+        self._author = None
+        self._title = None
+        self._publish_date = None
+
+        self.use_oauth = use_oauth
+        self.allow_oauth_cache = allow_oauth_cache
+
+    def __repr__(self):
+        return f"<ytextract.__main__.YouTube object: videoId={self.video_id}>"
+
+    def __eq__(self, o: object) -> bool:
+        # Compare types and urls, if they're same return true, else return false.
+        return type(o) == type(self) and o.watch_url == self.watch_url
+
+    @property
+    def watch_html(self):
+        if self._watch_html:
+            return self._watch_html
+        self._watch_html = request.get(url=self.watch_url)
+        return self._watch_html
+
+    @property
+    def embed_html(self):
+        if self._embed_html:
+            return self._embed_html
+        self._embed_html = request.get(url=self.embed_url)
+        return self._embed_html
+
+    @property
+    def age_restricted(self):
+        if self._age_restricted:
+            return self._age_restricted
+        self._age_restricted = extract.is_age_restricted(self.watch_html)
+        return self._age_restricted
+
+    @property
+    def js_url(self):
+        if self._js_url:
+            return self._js_url
+
+        if self.age_restricted:
+            self._js_url = extract.js_url(self.embed_html)
+        else:
+            self._js_url = extract.js_url(self.watch_html)
+
+        return self._js_url
+
+    @property
+    def js(self):
+        if self._js:
+            return self._js
+
+        # If the js_url doesn't match the cached url, fetch the new js and update
+        #  the cache; otherwise, load the cache.
+        if ytextract.__js_url__ != self.js_url:
+            self._js = request.get(self.js_url)
+            ytextract.__js__ = self._js
+            ytextract.__js_url__ = self.js_url
+        else:
+            self._js = ytextract.__js__
+
+        return self._js
+
+    @property
+    def initial_data(self):
+        if self._initial_data:
+            return self._initial_data
+        self._initial_data = extract.initial_data(self.watch_html)
+        return self._initial_data
+
+    @property
+    def streaming_data(self):
+        """Return streamingData from video info."""
+        if "streamingData" in self.vid_info:
+            return self.vid_info["streamingData"]
+        else:
+            self.bypass_age_gate()
+            return self.vid_info["streamingData"]
+
+    @property
+    def fmt_streams(self):
+        """Returns a list of streams if they have been initialized.
+
+        If the streams have not been initialized, finds all relevant
+        streams and initializes them.
+        """
+        self.check_availability()
+        if self._fmt_streams:
+            return self._fmt_streams
+
+        self._fmt_streams = []
+
+        stream_manifest = extract.apply_descrambler(self.streaming_data)
+
+        # If the cached js doesn't work, try fetching a new js file
+        # https://github.com/Josh-XT/ytextract/issues/1054
+        try:
+            extract.apply_signature(stream_manifest, self.vid_info, self.js)
+        except exceptions.ExtractError:
+            # To force an update to the js file, we clear the cache and retry
+            self._js = None
+            self._js_url = None
+            ytextract.__js__ = None
+            ytextract.__js_url__ = None
+            extract.apply_signature(stream_manifest, self.vid_info, self.js)
+
+        # build instances of :class:`Stream <Stream>`
+        # Initialize stream objects
+        for stream in stream_manifest:
+            video = Stream(
+                stream=stream,
+                monostate=self.stream_monostate,
+            )
+            self._fmt_streams.append(video)
+
+        self.stream_monostate.title = self.title
+        self.stream_monostate.duration = self.length
+
+        return self._fmt_streams
+
+    def check_availability(self):
+        """Check whether the video is available.
+
+        Raises different exceptions based on why the video is unavailable,
+        otherwise does nothing.
+        """
+        status, messages = extract.playability_status(self.watch_html)
+
+        for reason in messages:
+            if status == "UNPLAYABLE":
+                if reason == (
+                    "Join this channel to get access to members-only content "
+                    "like this video, and other exclusive perks."
+                ):
+                    raise exceptions.MembersOnly(video_id=self.video_id)
+                elif reason == "This live stream recording is not available.":
+                    raise exceptions.RecordingUnavailable(video_id=self.video_id)
+                else:
+                    raise exceptions.VideoUnavailable(video_id=self.video_id)
+            elif status == "LOGIN_REQUIRED":
+                if reason == (
+                    "This is a private video. "
+                    "Please sign in to verify that you may see it."
+                ):
+                    raise exceptions.VideoPrivate(video_id=self.video_id)
+            elif status == "ERROR":
+                if reason == "Video unavailable":
+                    raise exceptions.VideoUnavailable(video_id=self.video_id)
+            elif status == "LIVE_STREAM":
+                raise exceptions.LiveStreamError(video_id=self.video_id)
+
+    @property
+    def vid_info(self):
+        """Parse the raw vid info and return the parsed result.
+
+        :rtype: Dict[Any, Any]
+        """
+        if self._vid_info:
+            return self._vid_info
+
+        innertube = InnerTube(
+            use_oauth=self.use_oauth, allow_cache=self.allow_oauth_cache
+        )
+
+        innertube_response = innertube.player(self.video_id)
+        self._vid_info = innertube_response
+        return self._vid_info
+
+    def bypass_age_gate(self):
+        """Attempt to update the vid_info by bypassing the age gate."""
+        try:
+            innertube = InnerTube(
+                client="ANDROID_EMBED",
+                use_oauth=self.use_oauth,
+                allow_cache=self.allow_oauth_cache,
+            )
+        except:
+            for client in _default_clients:
+                try:
+                    innertube = InnerTube(
+                        client=client,
+                        use_oauth=self.use_oauth,
+                        allow_cache=self.allow_oauth_cache,
+                    )
+                    break
+                except:
+                    continue
+        innertube_response = innertube.player(self.video_id)
+        playability_status = innertube_response["playabilityStatus"].get("status", None)
+
+        # If we still can't access the video, raise an exception
+        # (tier 3 age restriction)
+        if playability_status == "UNPLAYABLE":
+            raise exceptions.AgeRestrictedError(self.video_id)
+
+        self._vid_info = innertube_response
+
+    @property
+    def caption_tracks(self) -> List[ytextract.Caption]:
+        """Get a list of :class:`Caption <Caption>`.
+
+        :rtype: List[Caption]
+        """
+        raw_tracks = (
+            self.vid_info.get("captions", {})
+            .get("playerCaptionsTracklistRenderer", {})
+            .get("captionTracks", [])
+        )
+        return [ytextract.Caption(track) for track in raw_tracks]
+
+    @property
+    def captions(self) -> ytextract.CaptionQuery:
+        """Interface to query caption tracks.
+
+        :rtype: :class:`CaptionQuery <CaptionQuery>`.
+        """
+        return ytextract.CaptionQuery(self.caption_tracks)
+
+    @property
+    def streams(self) -> StreamQuery:
+        """Interface to query both adaptive (DASH) and progressive streams.
+
+        :rtype: :class:`StreamQuery <StreamQuery>`.
+        """
+        self.check_availability()
+        return StreamQuery(self.fmt_streams)
+
+    @property
+    def thumbnail_url(self) -> str:
+        """Get the thumbnail url image.
+
+        :rtype: str
+        """
+        thumbnail_details = (
+            self.vid_info.get("videoDetails", {}).get("thumbnail", {}).get("thumbnails")
+        )
+        if thumbnail_details:
+            thumbnail_details = thumbnail_details[-1]  # last item has max size
+            return thumbnail_details["url"]
+
+        return f"https://img.youtube.com/vi/{self.video_id}/maxresdefault.jpg"
+
+    @property
+    def publish_date(self):
+        """Get the publish date.
+
+        :rtype: datetime
+        """
+        if self._publish_date:
+            return self._publish_date
+        self._publish_date = extract.publish_date(self.watch_html)
+        return self._publish_date
+
+    @publish_date.setter
+    def publish_date(self, value):
+        """Sets the publish date."""
+        self._publish_date = value
+
+    @property
+    def title(self) -> str:
+        """Get the video title.
+
+        :rtype: str
+        """
+        if self._title:
+            return self._title
+
+        try:
+            self._title = self.vid_info["videoDetails"]["title"]
+        except KeyError:
+            # Check_availability will raise the correct exception in most cases
+            #  if it doesn't, ask for a report.
+            self.check_availability()
+            raise exceptions.YtextractError(
+                (
+                    f"Exception while accessing title of {self.watch_url}. "
+                    "Please file a bug report at https://github.com/Josh-XT/ytextract"
+                )
+            )
+
+        return self._title
+
+    @title.setter
+    def title(self, value):
+        """Sets the title value."""
+        self._title = value
+
+    @property
+    def description(self) -> str:
+        """Get the video description.
+
+        :rtype: str
+        """
+        return self.vid_info.get("videoDetails", {}).get("shortDescription")
+
+    @property
+    def rating(self) -> float:
+        """Get the video average rating.
+
+        :rtype: float
+
+        """
+        return self.vid_info.get("videoDetails", {}).get("averageRating")
+
+    @property
+    def length(self) -> int:
+        """Get the video length in seconds.
+
+        :rtype: int
+        """
+        return int(self.vid_info.get("videoDetails", {}).get("lengthSeconds"))
+
+    @property
+    def views(self) -> int:
+        """Get the number of the times the video has been viewed.
+
+        :rtype: int
+        """
+        return int(self.vid_info.get("videoDetails", {}).get("viewCount"))
+
+    @property
+    def author(self) -> str:
+        """Get the video author.
+        :rtype: str
+        """
+        if self._author:
+            return self._author
+        self._author = self.vid_info.get("videoDetails", {}).get("author", "unknown")
+        return self._author
+
+    @author.setter
+    def author(self, value):
+        """Set the video author."""
+        self._author = value
+
+    @property
+    def keywords(self) -> List[str]:
+        """Get the video keywords.
+
+        :rtype: List[str]
+        """
+        return self.vid_info.get("videoDetails", {}).get("keywords", [])
+
+    @property
+    def channel_id(self) -> str:
+        """Get the video poster's channel id.
+
+        :rtype: str
+        """
+        return self.vid_info.get("videoDetails", {}).get("channelId", None)
+
+    @property
+    def channel_url(self) -> str:
+        """Construct the channel url for the video's poster from the channel id.
+
+        :rtype: str
+        """
+        return f"https://www.youtube.com/channel/{self.channel_id}"
+
+    @property
+    def metadata(self) -> Optional[YouTubeMetadata]:
+        """Get the metadata for the video.
+
+        :rtype: YouTubeMetadata
+        """
+        if self._metadata:
+            return self._metadata
+        else:
+            self._metadata = extract.metadata(self.initial_data)
+            return self._metadata
+
+    def register_on_progress_callback(self, func: Callable[[Any, bytes, int], None]):
+        """Register a download progress callback function post initialization.
+
+        :param callable func:
+            A callback function that takes ``stream``, ``chunk``,
+             and ``bytes_remaining`` as parameters.
+
+        :rtype: None
+
+        """
+        self.stream_monostate.on_progress = func
+
+    def register_on_complete_callback(self, func: Callable[[Any, Optional[str]], None]):
+        """Register a download complete callback function post initialization.
+
+        :param callable func:
+            A callback function that takes ``stream`` and  ``file_path``.
+
+        :rtype: None
+
+        """
+        self.stream_monostate.on_complete = func
+
+    @staticmethod
+    def from_id(video_id: str) -> "YouTube":
+        """Construct a :class:`YouTube <YouTube>` object from a video id.
+
+        :param str video_id:
+            The video id of the YouTube video.
+
+        :rtype: :class:`YouTube <YouTube>`
+
+        """
+        return YouTube(f"https://www.youtube.com/watch?v={video_id}")

ytextract/captions.py

@@ -0,0 +1,184 @@
+import math
+import os
+import time
+import json
+import xml.etree.ElementTree as ElementTree
+from html import unescape
+from typing import Dict, Optional
+
+from ytextract import request
+from ytextract.helpers import safe_filename, target_directory
+
+
+class Caption:
+    """Container for caption tracks."""
+
+    def __init__(self, caption_track: Dict):
+        """Construct a :class:`Caption <Caption>`.
+
+        :param dict caption_track:
+            Caption track data extracted from ``watch_html``.
+        """
+        self.url = caption_track.get("baseUrl")
+
+        # Certain videos have runs instead of simpleText
+        #  this handles that edge case
+        name_dict = caption_track["name"]
+        if "simpleText" in name_dict:
+            self.name = name_dict["simpleText"]
+        else:
+            for el in name_dict["runs"]:
+                if "text" in el:
+                    self.name = el["text"]
+
+        # Use "vssId" instead of "languageCode", fix issue #779
+        self.code = caption_track["vssId"]
+        # Remove preceding '.' for backwards compatibility, e.g.:
+        # English -> vssId: .en, languageCode: en
+        # English (auto-generated) -> vssId: a.en, languageCode: en
+        self.code = self.code.strip(".")
+
+    @property
+    def xml_captions(self) -> str:
+        """Download the xml caption tracks."""
+        return request.get(self.url)
+
+    @property
+    def json_captions(self) -> dict:
+        """Download and parse the json caption tracks."""
+        json_captions_url = self.url.replace("fmt=srv3", "fmt=json3")
+        text = request.get(json_captions_url)
+        parsed = json.loads(text)
+        assert parsed["wireMagic"] == "pb3", "Unexpected captions format"
+        return parsed
+
+    def generate_srt_captions(self) -> str:
+        """Generate "SubRip Subtitle" captions.
+
+        Takes the xml captions from :meth:`~ytextract.Caption.xml_captions` and
+        recompiles them into the "SubRip Subtitle" format.
+        """
+        return self.xml_caption_to_srt(self.xml_captions)
+
+    @staticmethod
+    def float_to_srt_time_format(d: float) -> str:
+        """Convert decimal durations into proper srt format.
+
+        :rtype: str
+        :returns:
+            SubRip Subtitle (str) formatted time duration.
+
+        float_to_srt_time_format(3.89) -> '00:00:03,890'
+        """
+        fraction, whole = math.modf(d)
+        time_fmt = time.strftime("%H:%M:%S,", time.gmtime(whole))
+        ms = f"{fraction:.3f}".replace("0.", "")
+        return time_fmt + ms
+
+    def xml_caption_to_srt(self, xml_captions: str) -> str:
+        """Convert xml caption tracks to "SubRip Subtitle (srt)".
+        :param str xml_captions:
+            XML formatted caption tracks.
+        """
+        segments = []
+        try:
+            root = ElementTree.fromstring(xml_captions)
+        except ElementTree.ParseError as e:
+            print(f"Warning: Failed to parse the XML captions. Error: {e}")
+            return ""  # Return an empty string if parsing fails
+
+        try:
+            for i, child in enumerate(
+                list(root[0])
+            ):  # Assuming the first child is the correct element
+                text = child.text or ""
+                caption = unescape(
+                    text.replace("\n", " ").replace("  ", " "),
+                )
+                try:
+                    duration = float(child.attrib["d"]) / 1000.0
+                except KeyError:
+                    duration = 0.0
+                try:
+                    start = float(child.attrib["t"]) / 1000.0
+                except KeyError:
+                    start = 0.0
+                end = start + duration
+                sequence_number = i + 1  # convert from 0-indexed to 1.
+                line = "{seq}\n{start} --> {end}\n{text}\n".format(
+                    seq=sequence_number,
+                    start=self.float_to_srt_time_format(start),
+                    end=self.float_to_srt_time_format(end),
+                    text=caption,
+                )
+                segments.append(line)
+
+        except IndexError as e:
+            print(
+                f"Warning: The XML structure does not contain the expected elements. Error: {e}"
+            )
+        except Exception as e:
+            print(f"An unexpected error occurred: {e}")
+
+        return "\n".join(segments).strip()
+
+    def download(
+        self,
+        title: str,
+        srt: bool = True,
+        output_path: Optional[str] = None,
+        filename_prefix: Optional[str] = None,
+    ) -> str:
+        """Write the media stream to disk.
+
+        :param title:
+            Output filename (stem only) for writing media file.
+            If one is not specified, the default filename is used.
+        :type title: str
+        :param srt:
+            Set to True to download srt, false to download xml. Defaults to True.
+        :type srt bool
+        :param output_path:
+            (optional) Output path for writing media file. If one is not
+            specified, defaults to the current working directory.
+        :type output_path: str or None
+        :param filename_prefix:
+            (optional) A string that will be prepended to the filename.
+            For example a number in a playlist or the name of a series.
+            If one is not specified, nothing will be prepended
+            This is separate from filename so you can use the default
+            filename but still add a prefix.
+        :type filename_prefix: str or None
+
+        :rtype: str
+        """
+        if title.endswith(".srt") or title.endswith(".xml"):
+            filename = ".".join(title.split(".")[:-1])
+        else:
+            filename = title
+
+        if filename_prefix:
+            filename = f"{safe_filename(filename_prefix)}{filename}"
+
+        filename = safe_filename(filename)
+
+        filename += f" ({self.code})"
+
+        if srt:
+            filename += ".srt"
+        else:
+            filename += ".xml"
+
+        file_path = os.path.join(target_directory(output_path), filename)
+
+        with open(file_path, "w", encoding="utf-8") as file_handle:
+            if srt:
+                file_handle.write(self.generate_srt_captions())
+            else:
+                file_handle.write(self.xml_captions)
+
+        return file_path
+
+    def __repr__(self):
+        """Printable object representation."""
+        return '<Caption lang="{s.name}" code="{s.code}">'.format(s=self)