webscout 6.4__py3-none-any.whl → 6.5__py3-none-any.whl

webscout/Extra/YTToolkit/__init__.py

@@ -0,0 +1,3 @@
+from .YTdownloader import *
+from .transcriber import *
+from .ytapi import *

webscout/{transcriber.py → Extra/YTToolkit/transcriber.py}

@@ -1,6 +1,6 @@
 """Wassup fam! 🔥 This module is your go-to for getting those YouTube transcripts! 
 
->>> from webscout.transcriber import YTTranscriber
+>>> from webscout import YTTranscriber
 >>> transcript = YTTranscriber.get_transcript('https://www.youtube.com/watch?v=dQw4w9WgXcQ')
 >>> print(transcript)
 {'text': 'Never gonna give you up', 'start': 0.0, 'duration': 4.5}

webscout/Extra/YTToolkit/ytapi/__init__.py

@@ -0,0 +1,6 @@
+from .errors import *
+from .video import Video
+from .query import Search
+from .extras import Extras
+from .channel import Channel
+from .playlist import Playlist

webscout/Extra/YTToolkit/ytapi/channel.py

@@ -0,0 +1,307 @@
+import json
+import re
+
+from .https import (
+    channel_about,
+    streams_data,
+    uploads_data,
+    channel_playlists,
+    upcoming_videos
+)
+from .video import Video
+from .pool import collect
+from .utils import dup_filter
+from urllib.parse import unquote
+from typing import List, Optional, Dict
+from .patterns import _ChannelPatterns as Patterns
+
+
+class Channel:
+
+    _HEAD = 'https://www.youtube.com/channel/'
+    _CUSTOM = 'https://www.youtube.com/c/'
+    _USER = 'https://www.youtube.com/'
+
+    def __init__(self, channel_id: str):
+        """
+        Represents a YouTube channel
+
+        Parameters
+        ----------
+        channel_id : str
+            The id or url or custom url or user id of the channel
+        """
+        pattern = re.compile("UC(.+)|c/(.+)|@(.+)")
+        results = pattern.findall(channel_id)
+        if not results:
+            self._usable_id = channel_id
+            self._target_url = self._CUSTOM + channel_id
+        elif results[0][0]:
+            self._usable_id = results[0][0]
+            self._target_url = self._HEAD + 'UC' + results[0][0]
+        elif results[0][1]:
+            self._usable_id = results[0][1]
+            self._target_url = self._CUSTOM + results[0][1]
+        elif results[0][2]:
+            self._usable_id = results[0][2]
+            self._target_url = self._USER + '@' + results[0][2]
+        self.id = None
+        self.name = None
+        self.subscribers = None
+        self.views = None
+        self.country = None
+        self.custom_url = None
+        self.avatar = None
+        self.banner = None
+        self.url = None
+        self.description = None
+        self.socials = None
+        self.__meta = None
+        self._about_page = channel_about(self._target_url)
+        self.__populate()
+
+    def __populate(self):
+        self.__meta = self.__prepare_metadata()
+        for k, v in self.__meta.items():
+            setattr(self, k, v)
+
+    def __repr__(self):
+        return f'<Channel `{self._target_url}`>'
+
+    def __prepare_metadata(self) -> Optional[Dict[str, any]]:
+        """
+        Returns channel metadata in a dict format
+
+        Returns
+        -------
+        Dict
+            Channel metadata containing the following keys:
+            id, name, subscribers, views, country, custom_url, avatar, banner, url, description, socials
+        """
+        patterns = [
+            Patterns.name,
+            Patterns.avatar,
+            Patterns.banner,
+            Patterns.verified,
+            Patterns.socials
+        ]
+        extracted = collect(lambda x: x.findall(self._about_page) or None, patterns)
+        name, avatar, banner, verified, socials = [e[0] if e else None for e in extracted]
+        
+        # Add robust error handling for info extraction
+        info_pattern = re.compile("\\[{\"aboutChannelRenderer\":(.*?)],")
+        info_match = info_pattern.search(self._about_page)
+        
+        if not info_match:
+            # Fallback metadata for search results or incomplete channel data
+            return {
+                "id": self._usable_id,
+                "name": name,
+                "url": self._target_url,
+                "description": None,
+                "country": None,
+                "custom_url": None,
+                "subscribers": None,
+                "views": None,
+                "created_at": None,
+                "video_count": None,
+                "avatar": avatar,
+                "banner": banner,
+                "verified": bool(verified),
+                "socials": unquote(socials) if socials is not None else None
+            }
+        
+        try:
+            info_str = info_match.group(1) + "]}}}}"
+            info = json.loads(info_str)["metadata"]["aboutChannelViewModel"]
+            
+            return {
+                "id": info.get("channelId", self._usable_id),
+                "name": name,
+                "url": "https://www.youtube.com/channel/" + info.get("channelId", self._usable_id),
+                "description": info.get("description"),
+                "country": info.get("country"),
+                "custom_url": info.get("canonicalChannelUrl"),
+                "subscribers": info.get("subscriberCountText", "").split(' ')[0] if "subscriberCountText" in info else None,
+                "views": info.get("viewCountText", "").replace(' views', '') if "viewCountText" in info else None,
+                "created_at": info.get("joinedDateText", {}).get("content", "").replace('Joined ', '') if "joinedDateText" in info else None,
+                "video_count": info.get("videoCountText", "").split(' ')[0] if "videoCountText" in info else None,
+                "avatar": avatar,
+                "banner": banner,
+                "verified": bool(verified),
+                "socials": unquote(socials) if socials is not None else None
+            }
+        except (KeyError, json.JSONDecodeError):
+            # Fallback if JSON parsing fails
+            return {
+                "id": self._usable_id,
+                "name": name,
+                "url": self._target_url,
+                "description": None,
+                "country": None,
+                "custom_url": None,
+                "subscribers": None,
+                "views": None,
+                "created_at": None,
+                "video_count": None,
+                "avatar": avatar,
+                "banner": banner,
+                "verified": bool(verified),
+                "socials": unquote(socials) if socials is not None else None
+            }
+
+    @property
+    def metadata(self) -> Optional[Dict[str, any]]:
+        """
+        Returns channel metadata in a dict format
+
+        Returns
+        -------
+        Dict
+            Channel metadata containing the following keys:
+            id, name, subscribers, views, country, custom_url, avatar, banner, url, description, socials etc.
+        """
+        return self.__meta
+
+    @property
+    def live(self) -> bool:
+        """
+        Checks if the channel is live
+
+        Returns
+        -------
+        bool
+            True if the channel is live
+        """
+        return bool(self.current_streams)
+
+    @property
+    def streaming_now(self) -> Optional[str]:
+        """
+        Fetches the id of currently streaming video
+
+        Returns
+        -------
+        str | None
+            The id of the currently streaming video or None
+        """
+        streams = self.current_streams
+        return streams[0] if streams else None
+
+    @property
+    def current_streams(self) -> Optional[List[str]]:
+        """
+        Fetches the ids of all ongoing streams
+
+        Returns
+        -------
+        List[str] | None
+            The ids of all ongoing streams or None
+        """
+        raw = streams_data(self._target_url)
+        filtered_ids = dup_filter(Patterns.stream_ids.findall(raw))
+        if not filtered_ids:
+            return None
+        return [id_ for id_ in filtered_ids if f"vi/{id_}/hqdefault_live.jpg" in raw]
+
+    @property
+    def old_streams(self) -> Optional[List[str]]:
+        """
+        Fetches the ids of all old or completed streams
+
+        Returns
+        -------
+        List[str] | None
+            The ids of all old or completed streams or None
+        """
+        raw = streams_data(self._target_url)
+        filtered_ids = dup_filter(Patterns.stream_ids.findall(raw))
+        if not filtered_ids:
+            return None
+        return [id_ for id_ in filtered_ids if f"vi/{id_}/hqdefault_live.jpg" not in raw]
+
+    @property
+    def last_streamed(self) -> Optional[str]:
+        """
+        Fetches the id of the last completed livestream
+
+        Returns
+        -------
+        str | None
+            The id of the last livestreamed video or None
+        """
+        ids = self.old_streams
+        return ids[0] if ids else None
+    
+    def uploads(self, limit: int = 20) -> Optional[List[str]]:
+        """
+        Fetches the ids of all uploaded videos
+
+        Parameters
+        ----------
+        limit : int
+            The number of videos to fetch, defaults to 20
+
+        Returns
+        -------
+        List[str] | None
+            The ids of uploaded videos or None
+        """
+        return dup_filter(Patterns.upload_ids.findall(uploads_data(self._target_url)), limit)
+
+    @property
+    def last_uploaded(self) -> Optional[str]:
+        """
+        Fetches the id of the last uploaded video
+
+        Returns
+        -------
+        str | None
+            The id of the last uploaded video or None
+        """
+        ids = self.uploads()
+        return ids[0] if ids else None
+
+    @property
+    def upcoming(self) -> Optional[Video]:
+        """
+        Fetches the upcoming video
+
+        Returns
+        -------
+        Video | None
+            The upcoming video or None
+        """
+        raw = upcoming_videos(self._target_url)
+        if not Patterns.upcoming_check.search(raw):
+            return None
+        upcoming = Patterns.upcoming.findall(raw)
+        return Video(upcoming[0]) if upcoming else None
+
+    @property
+    def upcomings(self) -> Optional[List[str]]:
+        """
+        Fetches the upcoming videos
+
+        Returns
+        -------
+        List[str] | None
+            The ids of upcoming videos or None
+        """
+        raw = upcoming_videos(self._target_url)
+        if not Patterns.upcoming_check.search(raw):
+            return None
+        video_ids = Patterns.upcoming.findall(raw)
+        return video_ids
+
+    @property
+    def playlists(self) -> Optional[List[str]]:
+        """
+        Fetches the ids of all playlists
+
+        Returns
+        -------
+        List[str] | None
+            The ids of all playlists or None
+        """
+        return dup_filter(Patterns.playlists.findall(channel_playlists(self._target_url)))

webscout/Extra/YTToolkit/ytapi/errors.py

@@ -0,0 +1,13 @@
+class TooManyRequests(Exception):
+    def __init__(self, message):
+        self.message = message
+
+
+class InvalidURL(Exception):
+    def __init__(self, message):
+        self.message = message
+
+
+class RequestError(Exception):
+    def __init__(self, message):
+        self.message = message

webscout/Extra/YTToolkit/ytapi/extras.py

@@ -0,0 +1,45 @@
+from .https import (
+    trending_videos,
+    trending_songs,
+    trending_games,
+    trending_feeds,
+    trending_streams,
+    _get_trending_learning_videos,
+    trending_sports
+)
+from .utils import dup_filter
+from .patterns import _ExtraPatterns as Patterns
+from typing import Optional, List
+
+
+class Extras:
+
+    @staticmethod
+    def trending_videos() -> Optional[List[str]]:
+        data = Patterns.video_id.findall(trending_videos())
+        return dup_filter(data) if data else None
+
+    @staticmethod
+    def music_videos() -> Optional[List[str]]:
+        data = Patterns.video_id.findall(trending_songs())
+        return dup_filter(data) if data else None
+
+    @staticmethod
+    def gaming_videos() -> Optional[List[str]]:
+        return dup_filter(Patterns.video_id.findall(trending_games()))
+
+    @staticmethod
+    def news_videos() -> Optional[List[str]]:
+        return dup_filter(Patterns.video_id.findall(trending_feeds()))
+
+    @staticmethod
+    def live_videos() -> Optional[List[str]]:
+        return dup_filter(Patterns.video_id.findall(trending_streams()))
+
+    @staticmethod
+    def educational_videos() -> Optional[List[str]]:
+        return dup_filter(Patterns.video_id.findall(_get_trending_learning_videos()))
+
+    @staticmethod
+    def sport_videos() -> Optional[List[str]]:
+        return dup_filter(Patterns.video_id.findall(trending_sports()))

webscout/Extra/YTToolkit/ytapi/https.py

@@ -0,0 +1,88 @@
+from urllib.parse import quote
+from .utils import request
+
+
+def channel_about(head: str) -> str:
+    return request(head + '/about')
+
+
+def video_count(channel_id: str) -> str:
+    head = 'https://www.youtube.com/results?search_query='
+    tail = '&sp=EgIQAg%253D%253D'
+    return request(head + channel_id + tail)
+
+
+def uploads_data(head: str) -> str:
+    url = head + '/videos'
+    return request(url)
+
+
+def streams_data(head: str) -> str:
+    url = head + '/streams'
+    return request(url)
+
+
+def channel_playlists(head: str) -> str:
+    url = head + '/playlists'
+    return request(url)
+
+
+def upcoming_videos(head: str) -> str:
+    url = head + '/videos?view=2&live_view=502'
+    return request(url)
+
+
+def video_data(video_id: str) -> str:
+    url = f'https://www.youtube.com/watch?v={video_id}'
+    return request(url)
+
+
+def playlist_data(playlist_id: str) -> str:
+    url = 'https://www.youtube.com/playlist?list=' + playlist_id
+    return request(url)
+
+
+def trending_videos() -> str:
+    return request('https://www.youtube.com/feed/trending')
+
+
+def trending_songs() -> str:
+    return request('https://www.youtube.com/feed/music')
+
+
+def trending_games() -> str:
+    return request('https://www.youtube.com/gaming')
+
+
+def trending_feeds() -> str:
+    return request('https://www.youtube.com/news')
+
+
+def trending_streams() -> str:
+    return request('https://www.youtube.com/live')
+
+
+def _get_trending_learning_videos() -> str:
+    return request('https://www.youtube.com/learning')
+
+
+def trending_sports() -> str:
+    return request('https://www.youtube.com/sports')
+
+
+def find_videos(query: str) -> str:
+    head = 'https://www.youtube.com/results?search_query='
+    tail = '&sp=EgIQAQ%253D%253D'
+    return request(head + quote(query) + tail)
+
+
+def find_channels(query: str) -> str:
+    head = 'https://www.youtube.com/results?search_query='
+    tail = '&sp=EgIQAg%253D%253D'
+    return request(head + quote(query) + tail)
+
+
+def find_playlists(query: str) -> str:
+    head = 'https://www.youtube.com/results?search_query='
+    tail = '&sp=EgIQAw%253D%253D'
+    return request(head + quote(query) + tail)

webscout/Extra/YTToolkit/ytapi/patterns.py

@@ -0,0 +1,61 @@
+import re
+
+
+class _ChannelPatterns:
+    name = re.compile('channelMetadataRenderer\":{\"title\":\"(.*?)\"')
+    id = re.compile('channelId\":\"(.*?)\"')
+    verified = re.compile('"label":"Verified"')
+    check_live = re.compile('{"text":"LIVE"}')
+    live = re.compile("thumbnailOverlays\":\[(.*?)]")
+    video_id = re.compile('videoId\":\"(.*?)\"')
+    uploads = re.compile("gridVideoRenderer\":{\"videoId\":\"(.*?)\"")
+    subscribers = re.compile("\"subscriberCountText\":{\"accessibility\":(.*?),")
+    views = re.compile("viewCountText\":{\"simpleText\":\"(.*?)\"}")
+    creation = re.compile("{\"text\":\"Joined \"},{\"text\":\"(.*?)\"}")
+    country = re.compile("country\":{\"simpleText\":\"(.*?)\"}")
+    custom_url = re.compile("canonicalChannelUrl\":\"(.*?)\"")
+    description = re.compile("{\"description\":{\"simpleText\":\"(.*?)\"}")
+    avatar = re.compile("height\":88},{\"url\":\"(.*?)\"")
+    banner = re.compile("width\":1280,\"height\":351},{\"url\":\"(.*?)\"")
+    playlists = re.compile("{\"url\":\"/playlist\\?list=(.*?)\"")
+    video_count = re.compile("videoCountText\":{\"runs\":\[{\"text\":(.*?)}")
+    socials = re.compile("q=https%3A%2F%2F(.*?)\"")
+    upload_ids = re.compile("videoId\":\"(.*?)\"")
+    stream_ids = re.compile("videoId\":\"(.*?)\"")
+    upload_chunk = re.compile("gridVideoRenderer\":{(.*?)\"navigationEndpoint")
+    upload_chunk_fl_1 = re.compile("simpleText\":\"Streamed")
+    upload_chunk_fl_2 = re.compile("default_live.")
+    upcoming_check = re.compile("\"title\":\"Upcoming live streams\"")
+    upcoming = re.compile("gridVideoRenderer\":{\"videoId\":\"(.*?)\"")
+
+
+class _VideoPatterns:
+    video_id = re.compile('videoId\":\"(.*?)\"')
+    title = re.compile("title\":\"(.*?)\"")
+    duration = re.compile("approxDurationMs\":\"(.*?)\"")
+    upload_date = re.compile("uploadDate\":\"(.*?)\"")
+    author_id = re.compile("channelIds\":\[\"(.*?)\"")
+    description = re.compile("shortDescription\":\"(.*)\",\"isCrawlable")
+    tags = re.compile("<meta name=\"keywords\" content=\"(.*?)\">")
+    is_streamed = re.compile("simpleText\":\"Streamed live")
+    is_premiered = re.compile("dateText\":{\"simpleText\":\"Premiered")
+    views = re.compile("videoViewCountRenderer\":{\"viewCount\":{\"simpleText\":\"(.*?)\"")
+    likes = re.compile("toggledText\":{\"accessibility\":{\"accessibilityData\":{\"label\":\"(.*?) ")
+    thumbnail = re.compile("playerMicroformatRenderer\":{\"thumbnail\":{\"thumbnails\":\[{\"url\":\"(.*?)\"")
+
+
+class _PlaylistPatterns:
+    name = re.compile("{\"title\":\"(.*?)\"")
+    video_count = re.compile("stats\":\[{\"runs\":\[{\"text\":\"(.*?)\"")
+    video_id = re.compile("videoId\":\"(.*?)\"")
+    thumbnail = re.compile("og:image\" content=\"(.*?)\?")
+
+
+class _ExtraPatterns:
+    video_id = re.compile("videoId\":\"(.*?)\"")
+
+
+class _QueryPatterns:
+    channel_id = re.compile("channelId\":\"(.*?)\"")
+    video_id = re.compile("videoId\":\"(.*?)\"")
+    playlist_id = re.compile("playlistId\":\"(.*?)\"")

webscout/Extra/YTToolkit/ytapi/playlist.py

@@ -0,0 +1,59 @@
+import re
+from typing import Dict, Any
+
+from .pool import collect
+from .utils import dup_filter
+from .https import playlist_data
+from .patterns import _PlaylistPatterns as Patterns
+
+
+class Playlist:
+
+    def __init__(self, playlist_id: str):
+        """
+        Represents a YouTube Playlist
+
+        Parameters
+        ----------
+        playlist_id : str
+            The id or url of the playlist
+        """
+        pattern = re.compile('=(.+?)$|^PL(.+?)$')
+        match = pattern.search(playlist_id)
+        if not match:
+            raise ValueError(f'Invalid playlist id: {playlist_id}')
+        if match.group(1):
+            self.id = match.group(1)
+        elif match.group(2):
+            self.id = 'PL' + match.group(2)
+        self._playlist_data = playlist_data(self.id)
+
+    def __repr__(self):
+        return f'<Playlist {self.id}>'
+
+    @property
+    def metadata(self) -> Dict[str, Any]:
+        """
+        Fetches playlist metadata in a dict format
+
+        Returns
+        -------
+        Dict
+            Playlist metadata in a dict format containing keys: id, url, name, video_count, thumbnail,
+        """
+        patterns = [
+            Patterns.name,
+            Patterns.video_count,
+            Patterns.thumbnail,
+            Patterns.video_id,
+        ]
+        ext = collect(lambda x: x.findall(self._playlist_data) or None, patterns)
+        data = [e[0] if e else None for e in ext]
+        return {
+            'id': self.id,
+            'url': 'https://www.youtube.com/playlist?list=' + self.id,
+            'name': data[0] if data else None,
+            'video_count': data[1] if data else None,
+            'thumbnail': data[2] if data else None,
+            'videos': dup_filter(ext[3])
+        }

webscout/Extra/YTToolkit/ytapi/pool.py

@@ -0,0 +1,8 @@
+import concurrent.futures
+from typing import Callable, List, Any
+
+
+def collect(func: Callable, args: List[Any]) -> List[Any]:
+    max_workers = len(args) or 1
+    with concurrent.futures.ThreadPoolExecutor(max_workers) as exe:
+        return list(exe.map(func, args))

webscout/Extra/YTToolkit/ytapi/query.py

@@ -0,0 +1,37 @@
+from .utils import dup_filter
+from .video import Video
+from .channel import Channel
+from .playlist import Playlist
+from .patterns import _QueryPatterns as Patterns
+from typing import Optional, Dict, Any, List
+from .https import find_videos, find_channels, find_playlists
+
+
+class Search:
+
+    @staticmethod
+    def video(keywords: str) -> Optional[Video]:
+        video_ids = Patterns.video_id.findall(find_videos(keywords))
+        return Video(video_ids[0]) if video_ids else None
+
+    @staticmethod
+    def channel(keywords: str) -> Optional[Channel]:
+        channel_ids = Patterns.channel_id.findall(find_channels(keywords))
+        return Channel(channel_ids[0]) if channel_ids else None
+
+    @staticmethod
+    def playlist(keywords: str) -> Optional[Playlist]:
+        playlist_ids = Patterns.playlist_id.findall(find_playlists(keywords))
+        return Playlist(playlist_ids[0]) if playlist_ids else None
+
+    @staticmethod
+    def videos(keywords: str, limit: int = 20) -> Optional[List[str]]:
+        return dup_filter(Patterns.video_id.findall(find_videos(keywords)), limit)
+
+    @staticmethod
+    def channels(keywords: str, limit: int = 20) -> Optional[List[str]]:
+        return dup_filter(Patterns.channel_id.findall(find_channels(keywords)), limit)
+
+    @staticmethod
+    def playlists(keywords: str, limit: int = 20) -> Optional[List[str]]:
+        return dup_filter(Patterns.playlist_id.findall(find_playlists(keywords)), limit)