webscout 7.9__py3-none-any.whl → 8.1__py3-none-any.whl

webscout/Extra/YTToolkit/ytapi/video.py

@@ -1,103 +1,232 @@
-import re
-import json
-from .https import video_data
-
-
-class Video:
-
-    _HEAD = 'https://www.youtube.com/watch?v='
-
-    def __init__(self, video_id: str):
-        """
-        Represents a YouTube video
-
-        Parameters
-        ----------
-        video_id : str
-            The id or url of the video
-        """
-        pattern = re.compile('.be/(.*?)$|=(.*?)$|^(\w{11})$')  # noqa
-        self._matched_id = (
-                pattern.search(video_id).group(1)
-                or pattern.search(video_id).group(2)
-                or pattern.search(video_id).group(3)
-        )
-        if self._matched_id:
-            self._url = self._HEAD + self._matched_id
-            self._video_data = video_data(self._matched_id)
-        else:
-            raise ValueError('invalid video id or url')
-
-    def __repr__(self):
-        return f'<Video {self._url}>'
-
-    @property
-    def metadata(self):
-        """
-        Fetches video metadata in a dict format
-
-        Returns
-        -------
-        Dict
-            Video metadata in a dict format containing keys: title, id, views, duration, author_id,
-            upload_date, url, thumbnails, tags, description
-        """
-        details_pattern = re.compile('videoDetails\":(.*?)\"isLiveContent\":.*?}')
-        upload_date_pattern = re.compile("<meta itemprop=\"uploadDate\" content=\"(.*?)\">")
-        genre_pattern = re.compile("<meta itemprop=\"genre\" content=\"(.*?)\">")
-        like_count_pattern = re.compile("iconType\":\"LIKE\"},\"defaultText\":(.*?)}}")
-        
-        # Add robust error handling
-        raw_details_match = details_pattern.search(self._video_data)
-        if not raw_details_match:
-            # Fallback metadata for search results or incomplete video data
-            return {
-                'title': getattr(self, 'title', None),
-                'id': getattr(self, 'id', None),
-                'views': getattr(self, 'views', None),
-                'streamed': False,
-                'duration': None,
-                'author_id': None,
-                'upload_date': None,
-                'url': f"https://www.youtube.com/watch?v={getattr(self, 'id', '')}" if hasattr(self, 'id') else None,
-                'thumbnails': None,
-                'tags': None,
-                'description': None,
-                'likes': None,
-                'genre': None
-            }
-        
-        raw_details = raw_details_match.group(0)
-        
-        # Add None checking for upload_date
-        upload_date_match = upload_date_pattern.search(self._video_data)
-        upload_date = upload_date_match.group(1) if upload_date_match else None
-        
-        metadata = json.loads(raw_details.replace('videoDetails\":', ''))
-        data = {
-            'title': metadata['title'],
-            'id': metadata['videoId'],
-            'views': metadata.get('viewCount'),
-            'streamed': metadata['isLiveContent'],
-            'duration': metadata['lengthSeconds'],
-            'author_id': metadata['channelId'],
-            'upload_date': upload_date,
-            'url': f"https://www.youtube.com/watch?v={metadata['videoId']}",
-            'thumbnails': metadata.get('thumbnail', {}).get('thumbnails'),
-            'tags': metadata.get('keywords'),
-            'description': metadata.get('shortDescription'),
-        }
-        try:
-            likes_count = like_count_pattern.search(self._video_data).group(1)
-            data['likes'] = json.loads(likes_count + '}}}')[
-                'accessibility'
-            ]['accessibilityData']['label'].split(' ')[0].replace(',', '')
-        except (AttributeError, KeyError, json.decoder.JSONDecodeError):
-            data['likes'] = None
-        try:
-            data['genre'] = genre_pattern.search(self._video_data).group(1)
-        except AttributeError:
-            data['genre'] = None
-        return data
-if __name__ == '__main__':
-    print(Video('https://www.youtube.com/watch?v=9bZkp7q19f0').metadata)
+import re
+import json
+from typing import Dict, Any
+from .https import video_data
+
+
+class Video:
+
+    _HEAD = 'https://www.youtube.com/watch?v='
+
+    def __init__(self, video_id: str):
+        """
+        Represents a YouTube video
+
+        Parameters
+        ----------
+        video_id : str
+            The id or url of the video
+        """
+        pattern = re.compile('.be/(.*?)$|=(.*?)$|^(\w{11})$')  # noqa
+        match = pattern.search(video_id)
+
+        if not match:
+            raise ValueError('Invalid YouTube video ID or URL')
+
+        self._matched_id = (
+                match.group(1)
+                or match.group(2)
+                or match.group(3)
+        )
+
+        if self._matched_id:
+            self._url = self._HEAD + self._matched_id
+            self._video_data = video_data(self._matched_id)
+            # Extract basic info for fallback
+            title_match = re.search('<title>(.*?) - YouTube</title>', self._video_data)
+            self.title = title_match.group(1) if title_match else None
+            self.id = self._matched_id
+        else:
+            raise ValueError('Invalid YouTube video ID or URL')
+
+    def __repr__(self):
+        return f'<Video {self._url}>'
+
+    @property
+    def metadata(self) -> Dict[str, Any]:
+        """
+        Fetches video metadata in a dict format
+
+        Returns
+        -------
+        Dict
+            Video metadata in a dict format containing keys: title, id, views, duration, author_id,
+            upload_date, url, thumbnails, tags, description, likes, genre, etc.
+        """
+        # Multiple patterns to try for video details extraction for robustness
+        details_patterns = [
+            re.compile('videoDetails\":(.*?)\"isLiveContent\":.*?}'),
+            re.compile('videoDetails\":(.*?),\"playerConfig'),
+            re.compile('videoDetails\":(.*?),\"playabilityStatus')
+        ]
+
+        # Other metadata patterns
+        upload_date_pattern = re.compile("<meta itemprop=\"uploadDate\" content=\"(.*?)\">")
+        genre_pattern = re.compile("<meta itemprop=\"genre\" content=\"(.*?)\">")
+        like_count_patterns = [
+            re.compile("iconType\":\"LIKE\"},\"defaultText\":(.*?)}"),
+            re.compile('\"likeCount\":\"(\\d+)\"')
+        ]
+        channel_name_pattern = re.compile('"ownerChannelName":"(.*?)"')
+
+        # Try each pattern for video details
+        raw_details_match = None
+        for pattern in details_patterns:
+            match = pattern.search(self._video_data)
+            if match:
+                raw_details_match = match
+                break
+
+        if not raw_details_match:
+            # Fallback metadata for search results or incomplete video data
+            return {
+                'title': getattr(self, 'title', None),
+                'id': getattr(self, 'id', None),
+                'views': getattr(self, 'views', None),
+                'streamed': False,
+                'duration': None,
+                'author_id': None,
+                'author_name': None,
+                'upload_date': None,
+                'url': f"https://www.youtube.com/watch?v={getattr(self, 'id', '')}" if hasattr(self, 'id') else None,
+                'thumbnails': None,
+                'tags': None,
+                'description': None,
+                'likes': None,
+                'genre': None,
+                'is_age_restricted': 'age-restricted' in self._video_data.lower(),
+                'is_unlisted': 'unlisted' in self._video_data.lower()
+            }
+
+        raw_details = raw_details_match.group(0)
+
+        # Extract upload date
+        upload_date_match = upload_date_pattern.search(self._video_data)
+        upload_date = upload_date_match.group(1) if upload_date_match else None
+
+        # Extract channel name
+        channel_name_match = channel_name_pattern.search(self._video_data)
+        channel_name = channel_name_match.group(1) if channel_name_match else None
+
+        # Parse video details
+        try:
+            # Clean up the JSON string for parsing
+            clean_json = raw_details.replace('videoDetails\":', '')
+            # Handle potential JSON parsing issues
+            if clean_json.endswith(','):
+                clean_json = clean_json[:-1]
+            metadata = json.loads(clean_json)
+
+            data = {
+                'title': metadata.get('title'),
+                'id': metadata.get('videoId', self._matched_id),
+                'views': metadata.get('viewCount'),
+                'streamed': metadata.get('isLiveContent', False),
+                'duration': metadata.get('lengthSeconds'),
+                'author_id': metadata.get('channelId'),
+                'author_name': channel_name or metadata.get('author'),
+                'upload_date': upload_date,
+                'url': f"https://www.youtube.com/watch?v={metadata.get('videoId', self._matched_id)}",
+                'thumbnails': metadata.get('thumbnail', {}).get('thumbnails'),
+                'tags': metadata.get('keywords'),
+                'description': metadata.get('shortDescription'),
+                'is_age_restricted': metadata.get('isAgeRestricted', False) or 'age-restricted' in self._video_data.lower(),
+                'is_unlisted': 'unlisted' in self._video_data.lower(),
+                'is_family_safe': metadata.get('isFamilySafe', True),
+                'is_private': metadata.get('isPrivate', False),
+                'is_live_content': metadata.get('isLiveContent', False),
+                'is_crawlable': metadata.get('isCrawlable', True),
+                'allow_ratings': metadata.get('allowRatings', True)
+            }
+        except (json.JSONDecodeError, KeyError, TypeError) as e:
+            # Fallback to basic metadata if JSON parsing fails
+            return {
+                'title': getattr(self, 'title', None),
+                'id': self._matched_id,
+                'url': self._url,
+                'error': f"Failed to parse video details: {str(e)}"
+            }
+
+        # Try to extract likes count
+        likes = None
+        for pattern in like_count_patterns:
+            try:
+                likes_match = pattern.search(self._video_data)
+                if likes_match:
+                    likes_text = likes_match.group(1)
+                    # Handle different formats of like count
+                    if '{' in likes_text:
+                        likes = json.loads(likes_text + '}}}')['accessibility']['accessibilityData']['label'].split(' ')[0].replace(',', '')
+                    else:
+                        likes = likes_text
+                    break
+            except (AttributeError, KeyError, json.decoder.JSONDecodeError):
+                continue
+
+        data['likes'] = likes
+
+        # Try to extract genre
+        try:
+            genre_match = genre_pattern.search(self._video_data)
+            data['genre'] = genre_match.group(1) if genre_match else None
+        except AttributeError:
+            data['genre'] = None
+
+        return data
+
+
+
+    @property
+    def embed_html(self) -> str:
+        """
+        Get the embed HTML code for this video
+
+        Returns:
+            HTML iframe code for embedding the video
+        """
+        return f'<iframe width="560" height="315" src="https://www.youtube.com/embed/{self._matched_id}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>'
+
+    @property
+    def embed_url(self) -> str:
+        """
+        Get the embed URL for this video
+
+        Returns:
+            URL for embedding the video
+        """
+        return f'https://www.youtube.com/embed/{self._matched_id}'
+
+    @property
+    def thumbnail_url(self) -> str:
+        """
+        Get the thumbnail URL for this video
+
+        Returns:
+            URL of the video thumbnail (high quality)
+        """
+        return f'https://i.ytimg.com/vi/{self._matched_id}/hqdefault.jpg'
+
+    @property
+    def thumbnail_urls(self) -> Dict[str, str]:
+        """
+        Get all thumbnail URLs for this video in different qualities
+
+        Returns:
+            Dictionary of thumbnail URLs with quality labels
+        """
+        return {
+            'default': f'https://i.ytimg.com/vi/{self._matched_id}/default.jpg',
+            'medium': f'https://i.ytimg.com/vi/{self._matched_id}/mqdefault.jpg',
+            'high': f'https://i.ytimg.com/vi/{self._matched_id}/hqdefault.jpg',
+            'standard': f'https://i.ytimg.com/vi/{self._matched_id}/sddefault.jpg',
+            'maxres': f'https://i.ytimg.com/vi/{self._matched_id}/maxresdefault.jpg'
+        }
+
+if __name__ == '__main__':
+    video = Video('https://www.youtube.com/watch?v=9bZkp7q19f0')
+    print(video.metadata)
+
+    # Example of getting comments
+    print("\nFirst 3 comments:")
+    for i, comment in enumerate(video.stream_comments(3), 1):
+        print(f"{i}. {comment['author']}: {comment['text'][:50]}...")

webscout/Provider/AISEARCH/DeepFind.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, Generator, Optional, Union
 
 from webscout.AIbase import AISearch
 from webscout import exceptions
-from webscout import LitAgent
+from webscout.litagent import LitAgent
 
 class Response:
     """A wrapper class for DeepFind API responses.

webscout/Provider/AISEARCH/ISou.py

@@ -2,7 +2,7 @@ import requests
 import json
 import re
 from typing import Dict, Optional, Generator, Any, Union
-from webscout import LitAgent
+from webscout.litagent import LitAgent
 from webscout import exceptions
 from webscout.AIbase import AISearch
 

webscout/Provider/AISEARCH/__init__.py

@@ -1,4 +1,9 @@
 from .felo_search import *
 from .DeepFind import *
 from .ISou import *
-from .genspark_search import *
+from .genspark_search import *
+from .monica_search import *
+from .webpilotai_search import *
+from .hika_search import *
+from .scira_search import *
+from .iask_search import *

webscout/Provider/AISEARCH/felo_search.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, Generator, Optional, Union
 
 from webscout.AIbase import AISearch
 from webscout import exceptions
-from webscout import LitAgent
+from webscout.litagent import LitAgent
 
 
 class Response:

webscout/Provider/AISEARCH/genspark_search.py

@@ -6,7 +6,7 @@ from typing import Dict, Optional, Generator, Union, Any
 
 from webscout.AIbase import AISearch
 from webscout import exceptions
-from webscout import LitAgent
+from webscout.litagent import LitAgent
 
 
 class Response:

webscout/Provider/AISEARCH/hika_search.py

@@ -0,0 +1,194 @@
+import requests
+import hashlib
+import json
+import random
+import time
+import re
+from typing import Dict, Optional, Generator, Union, Any
+
+from webscout.AIbase import AISearch
+from webscout import exceptions
+from webscout.litagent import LitAgent
+
+
+class Response:
+    """A wrapper class for Hika API responses."""
+    def __init__(self, text: str):
+        self.text = text
+    
+    def __str__(self):
+        return self.text
+    
+    def __repr__(self):
+        return self.text
+
+
+class Hika(AISearch):
+    """A class to interact with the Hika AI search API."""
+
+    def __init__(
+        self,
+        timeout: int = 60,
+        proxies: Optional[dict] = None,
+        language: str = "en",
+    ):
+        self.session = requests.Session()
+        self.base_url = "https://api.hika.fyi/api/"
+        self.endpoint = "kbase/web"
+        self.timeout = timeout
+        self.language = language
+        self.last_response = {}
+        
+        self.headers = {
+            "Content-Type": "application/json",
+            "Origin": "https://hika.fyi",
+            "Referer": "https://hika.fyi/",
+            "User-Agent": LitAgent().random()
+        }
+        
+        self.session.headers.update(self.headers)
+        self.proxies = proxies
+
+    def generate_id(self):
+        """Generate a unique ID and hash for the request."""
+        uid = ''.join(random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(10))
+        uid += hex(int(time.time()))[2:]
+        hash_id = hashlib.sha256(f"#{uid}*".encode()).hexdigest()
+        return {"uid": uid, "hashId": hash_id}
+
+    def clean_text(self, text):
+        """Clean all XML tags and control markers from text.
+        
+        Args:
+            text (str): The text to clean
+            
+        Returns:
+            str: Cleaned text
+        """
+        if not text:
+            return ""
+            
+        # Remove XML tags and special markers
+        # First remove <r> tag at the beginning
+        text = text.lstrip("<r>")
+        
+        # Remove any remaining XML tags
+        text = re.sub(r'<[^>]+>', '', text)
+        
+        # Remove [DONE] marker at the end
+        text = re.sub(r'\[DONE\]\s*$', '', text)
+        
+        return text
+
+    def search(
+        self,
+        prompt: str,
+        stream: bool = False,
+        raw: bool = False,
+    ) -> Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
+        """Search using the Hika API and get AI-generated responses."""
+        if not prompt or len(prompt) < 2:
+            raise exceptions.APIConnectionError("Search query must be at least 2 characters long")
+        
+        # Generate ID for this request
+        id_data = self.generate_id()
+        uid, hash_id = id_data["uid"], id_data["hashId"]
+        
+        # Update headers with request-specific values
+        request_headers = {
+            **self.headers,
+            "x-hika": hash_id,
+            "x-uid": uid
+        }
+        
+        # Prepare payload
+        payload = {
+            "keyword": prompt,
+            "language": self.language,
+            "stream": True  # Always request streaming for consistent handling
+        }
+
+        def for_stream():
+            try:
+                with self.session.post(
+                    f"{self.base_url}{self.endpoint}",
+                    json=payload,
+                    headers=request_headers,
+                    stream=True,
+                    timeout=self.timeout,
+                    proxies=self.proxies
+                ) as response:
+                    if not response.ok:
+                        raise exceptions.APIConnectionError(
+                            f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
+                        )
+                    
+                    for line in response.iter_lines(decode_unicode=True):
+                        if line and line.startswith("data: "):
+                            try:
+                                data = json.loads(line[6:])
+                                if "chunk" in data:
+                                    chunk = data["chunk"]
+                                    
+                                    # Skip [DONE] markers completely
+                                    if "[DONE]" in chunk:
+                                        continue
+                                    
+                                    # Clean the chunk
+                                    clean_chunk = self.clean_text(chunk)
+                                    
+                                    if clean_chunk:  # Only yield if there's content after cleaning
+                                        if raw:
+                                            yield {"text": clean_chunk}
+                                        else:
+                                            yield Response(clean_chunk)
+                            except json.JSONDecodeError:
+                                pass
+                                
+            except requests.exceptions.RequestException as e:
+                raise exceptions.APIConnectionError(f"Request failed: {e}")
+
+        def for_non_stream():
+            full_response = ""
+            for chunk in for_stream():
+                if raw:
+                    yield chunk
+                else:
+                    full_response += str(chunk)
+            
+            if not raw:
+                # Clean up the response text one final time
+                cleaned_response = self.format_response(full_response)
+                self.last_response = Response(cleaned_response)
+                return self.last_response
+
+        return for_stream() if stream else for_non_stream()
+    
+    def format_response(self, text: str) -> str:
+        """Format the response text for better readability."""
+        if not text:
+            return ""
+            
+        # First clean any tags or markers
+        cleaned_text = self.clean_text(text)
+        
+        # Remove any empty lines
+        cleaned_text = re.sub(r'\n\s*\n', '\n\n', cleaned_text)
+        
+        # Remove any trailing whitespace
+        cleaned_text = cleaned_text.strip()
+        
+        return cleaned_text
+
+
+if __name__ == "__main__":
+    from rich import print
+    ai = Hika()
+    try:
+        response = ai.search(input(">>> "), stream=True, raw=False)
+        for chunk in response:
+            print(chunk, end="", flush=True)
+    except KeyboardInterrupt:
+        print("\nSearch interrupted by user.")
+    except Exception as e:
+        print(f"\nError: {e}")