webscout 2025.10.15__py3-none-any.whl → 2025.10.16__py3-none-any.whl

webscout/search/engines/yahoo/videos.py

@@ -0,0 +1,302 @@
+"""Yahoo video search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from .base import YahooSearchEngine
+from ...results import VideosResult
+
+
+class YahooVideos(YahooSearchEngine[VideosResult]):
+    """Yahoo video search engine with filters.
+    
+    Features:
+    - Length filters (short, medium, long)
+    - Resolution filters (SD, HD, 4K)
+    - Source filters
+    - Time filters
+    - Pagination support
+    """
+
+    name = "yahoo"
+    category = "videos"
+
+    search_url = "https://video.search.yahoo.com/search/video"
+    search_method = "GET"
+
+    # XPath selectors for video results
+    items_xpath = "//div[@id='results']//div[contains(@class, 'dd') or contains(@class, 'vr')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h3//a/text() | .//a/@title",
+        "url": ".//h3//a/@href | .//a/@href",
+        "thumbnail": ".//img/@src",
+        "duration": ".//span[contains(@class, 'time') or contains(@class, 'duration')]//text()",
+        "views": ".//span[contains(@class, 'views')]//text()",
+        "published": ".//span[contains(@class, 'date') or contains(@class, 'age')]//text()",
+        "source": ".//span[contains(@class, 'source')]//text()",
+    }
+
+    # Filter mappings
+    LENGTH_FILTERS = {
+        "short": "short",    # < 4 minutes
+        "medium": "medium",  # 4-20 minutes
+        "long": "long",      # > 20 minutes
+        "all": "",
+    }
+
+    RESOLUTION_FILTERS = {
+        "sd": "sd",
+        "hd": "hd",
+        "4k": "4k",
+        "all": "",
+    }
+
+    SOURCE_FILTERS = {
+        "youtube": "youtube",
+        "dailymotion": "dailymotion",
+        "vimeo": "vimeo",
+        "metacafe": "metacafe",
+        "all": "",
+    }
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build video search payload.
+        
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter (d, w, m)
+            page: Page number
+            **kwargs: Additional filters:
+                - length: Video length filter
+                - resolution: Video resolution filter
+                - source: Video source filter
+                
+        Returns:
+            Query parameters dictionary
+        """
+        payload = {
+            "p": query,
+            "fr": "sfp",
+            "fr2": "p:s,v:v,m:sb,rgn:top",
+            "ei": "UTF-8",
+        }
+
+        # Pagination
+        if page > 1:
+            # Each page shows ~15-20 videos
+            payload["b"] = f"{(page - 1) * 15 + 1}"
+
+        # Safe search
+        if safesearch == "on":
+            payload["safe"] = "active"
+        elif safesearch == "off":
+            payload["safe"] = "off"
+
+        # Time filter
+        if timelimit:
+            time_map = {
+                "d": "1d",
+                "w": "1w",
+                "m": "1m",
+                "y": "1y",
+            }
+            if timelimit in time_map:
+                payload["age"] = time_map[timelimit]
+
+        # Length filter
+        if "length" in kwargs and kwargs["length"] in self.LENGTH_FILTERS:
+            length_val = self.LENGTH_FILTERS[kwargs["length"]]
+            if length_val:
+                payload["vidlen"] = length_val
+
+        # Resolution filter
+        if "resolution" in kwargs and kwargs["resolution"] in self.RESOLUTION_FILTERS:
+            res_val = self.RESOLUTION_FILTERS[kwargs["resolution"]]
+            if res_val:
+                payload["vidqual"] = res_val
+
+        # Source filter
+        if "source" in kwargs and kwargs["source"] in self.SOURCE_FILTERS:
+            source_val = self.SOURCE_FILTERS[kwargs["source"]]
+            if source_val:
+                payload["site"] = source_val
+
+        return payload
+
+    def extract_video_url(self, href: str) -> str:
+        """Extract actual video URL from Yahoo redirect.
+        
+        Args:
+            href: Yahoo redirect URL
+            
+        Returns:
+            Actual video URL
+        """
+        if not href:
+            return href
+            
+        try:
+            # Parse the URL
+            parsed = urlparse(href)
+            
+            # Check if it's a Yahoo redirect
+            if "r.search.yahoo.com" in parsed.netloc or "/RU=" in href:
+                # Extract the RU parameter
+                if "/RU=" in href:
+                    start = href.find("/RU=") + 4
+                    end = href.find("/RK=", start)
+                    if end == -1:
+                        end = len(href)
+                    from urllib.parse import unquote
+                    return unquote(href[start:end])
+                else:
+                    query_params = parse_qs(parsed.query)
+                    if "url" in query_params:
+                        return query_params["url"][0]
+            
+            return href
+        except Exception:
+            return href
+
+    def post_extract_results(self, results: list[VideosResult]) -> list[VideosResult]:
+        """Post-process video results.
+        
+        Args:
+            results: Raw extracted results
+            
+        Returns:
+            Cleaned results
+        """
+        cleaned_results = []
+        
+        for result in results:
+            # Extract real URL
+            if result.url:
+                result.url = self.extract_video_url(result.url)
+            
+            # Skip invalid results
+            if not result.url or not result.title:
+                continue
+            
+            # Clean thumbnail URL
+            if result.thumbnail and result.thumbnail.startswith("data:"):
+                result.thumbnail = ""
+            
+            cleaned_results.append(result)
+        
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[VideosResult] | None:
+        """Search Yahoo Videos with pagination.
+        
+        Args:
+            query: Video search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter
+            page: Starting page
+            max_results: Maximum results
+            **kwargs: Additional filters (length, resolution, source)
+            
+        Returns:
+            List of VideoResult objects
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 5)
+        
+        while current_page <= max_pages:
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+            
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+            
+            html_text = self.pre_process_html(html_text)
+            page_results = self.extract_results(html_text)
+            
+            if not page_results:
+                break
+            
+            results.extend(page_results)
+            
+            if max_results and len(results) >= max_results:
+                break
+            
+            current_page += 1
+        
+        results = self.post_extract_results(results)
+        
+        if max_results:
+            results = results[:max_results]
+        
+        return results if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        resolution: str | None = None,
+        duration: str | None = None,
+        license_videos: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run video search and return results as dictionaries.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            resolution: Video resolution filter.
+            duration: Video duration filter.
+            license_videos: License filter.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of video result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            resolution=resolution,
+            duration=duration,
+            license_videos=license_videos,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]

webscout/search/engines/yahoo/weather.py

@@ -0,0 +1,220 @@
+"""Yahoo weather search using embedded JSON data."""
+
+from __future__ import annotations
+
+import re
+import json
+from typing import Any
+
+from ...http_client import HttpClient
+
+
+class YahooWeather:
+    """Yahoo weather search using embedded JSON extraction."""
+
+    def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True):
+        """Initialize weather search engine.
+        
+        Args:
+            proxy: Proxy URL.
+            timeout: Request timeout in seconds.
+            verify: Whether to verify SSL certificates.
+        """
+        self.http_client = HttpClient(proxy=proxy, timeout=timeout, verify=verify)
+
+    def request(self, method: str, url: str, **kwargs: Any) -> str | None:
+        """Make a request to the weather service."""
+        try:
+            response = self.http_client.request(method, url, **kwargs)
+            return response.text
+        except Exception:
+            return None
+
+    def run(self, *args, **kwargs) -> list[dict[str, Any]]:
+        """Get weather data from Yahoo.
+        
+        Args:
+            location: Location to get weather for (e.g., "New York", "London", "Bengaluru")
+            
+        Returns:
+            List of weather data dictionaries
+        """
+        location = args[0] if args else kwargs.get("location") or kwargs.get("keywords")
+        
+        if not location:
+            raise ValueError("Location is required for weather search")
+        
+        try:
+            # Use the search endpoint which redirects to the correct weather page
+            search_url = f"https://weather.yahoo.com/search/?q={location.replace(' ', '+')}"
+            
+            # Fetch the page
+            response = self.request("GET", search_url)
+            if not response:
+                return [{
+                    "location": location,
+                    "error": "Failed to fetch weather data from Yahoo"
+                }]
+            
+            # Extract JSON data from the page
+            weather_data = self._extract_json_data(response, location)
+            
+            if weather_data:
+                return [weather_data]
+            
+            # Fallback: try regex parsing
+            return self._parse_weather_html(response, location)
+            
+        except Exception as e:
+            return [{
+                "location": location,
+                "error": f"Failed to fetch weather data: {str(e)}"
+            }]
+    
+    def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None:
+        """Extract weather data from embedded JSON in the page.
+        
+        Yahoo Weather embeds JSON data in script tags that can be parsed.
+        """
+        try:
+            # Look for the main data script tag
+            # Pattern: self.__next_f.push([1,"..JSON data.."])
+            json_pattern = r'self\.__next_f\.push\(\[1,"([^"]+)"\]\)'
+            matches = re.findall(json_pattern, html)
+            
+            weather_info = {}
+            
+            for match in matches:
+                # Unescape the JSON string
+                try:
+                    # The data is escaped, so we need to decode it
+                    decoded = match.encode().decode('unicode_escape')
+                    
+                    # Look for temperature data
+                    temp_match = re.search(r'"temperature":(\d+)', decoded)
+                    if temp_match and not weather_info.get('temperature'):
+                        weather_info['temperature'] = int(temp_match.group(1))
+                    
+                    # Look for condition
+                    condition_match = re.search(r'"iconLabel":"([^"]+)"', decoded)
+                    if condition_match and not weather_info.get('condition'):
+                        weather_info['condition'] = condition_match.group(1)
+                    
+                    # Look for high/low
+                    high_match = re.search(r'"highTemperature":(\d+)', decoded)
+                    if high_match and not weather_info.get('high'):
+                        weather_info['high'] = int(high_match.group(1))
+                    
+                    low_match = re.search(r'"lowTemperature":(\d+)', decoded)
+                    if low_match and not weather_info.get('low'):
+                        weather_info['low'] = int(low_match.group(1))
+                    
+                    # Look for humidity
+                    humidity_match = re.search(r'"value":"(\d+)%"[^}]*"category":"Humidity"', decoded)
+                    if humidity_match and not weather_info.get('humidity'):
+                        weather_info['humidity'] = int(humidity_match.group(1))
+                    
+                    # Look for precipitation probability
+                    precip_match = re.search(r'"probabilityOfPrecipitation":"(\d+)%"', decoded)
+                    if precip_match and not weather_info.get('precipitation_chance'):
+                        weather_info['precipitation_chance'] = int(precip_match.group(1))
+                    
+                    # Look for location name
+                    location_match = re.search(r'"name":"([^"]+)","code":null,"woeid":(\d+)', decoded)
+                    if location_match and not weather_info.get('location_name'):
+                        weather_info['location_name'] = location_match.group(1)
+                        weather_info['woeid'] = int(location_match.group(2))
+                    
+                except Exception:
+                    continue
+            
+            if weather_info and weather_info.get('temperature'):
+                return {
+                    "location": weather_info.get('location_name', location),
+                    "woeid": weather_info.get('woeid'),
+                    "temperature_f": weather_info.get('temperature'),
+                    "condition": weather_info.get('condition'),
+                    "high_f": weather_info.get('high'),
+                    "low_f": weather_info.get('low'),
+                    "humidity_percent": weather_info.get('humidity'),
+                    "precipitation_chance": weather_info.get('precipitation_chance'),
+                    "source": "Yahoo Weather",
+                    "units": "Fahrenheit"
+                }
+            
+            return None
+            
+        except Exception as e:
+            return None
+    
+    def _parse_weather_html(self, html_content: str, location: str) -> list[dict[str, Any]]:
+        """Fallback: Parse weather data from HTML content using regex.
+        
+        Args:
+            html_content: HTML content of weather page
+            location: Location name
+            
+        Returns:
+            List of weather data dictionaries
+        """
+        try:
+            weather_data = {"location": location}
+            
+            # Extract current temperature
+            temp_patterns = [
+                r'<p[^>]*class="[^"]*font-title1[^"]*"[^>]*>(\d+)°</p>',
+                r'>(\d+)°<',
+                r'"temperature":(\d+)',
+            ]
+            
+            for pattern in temp_patterns:
+                match = re.search(pattern, html_content)
+                if match:
+                    weather_data["temperature_f"] = int(match.group(1))
+                    break
+            
+            # Extract condition
+            condition_patterns = [
+                r'"iconLabel":"([^"]+)"',
+                r'aria-label="([^"]*(?:Cloudy|Sunny|Rain|Clear|Thunder|Shower|Fog)[^"]*)"',
+            ]
+            
+            for pattern in condition_patterns:
+                match = re.search(pattern, html_content, re.IGNORECASE)
+                if match:
+                    weather_data["condition"] = match.group(1)
+                    break
+            
+            # Extract high/low
+            high_match = re.search(r'"highTemperature":(\d+)', html_content)
+            if high_match:
+                weather_data["high_f"] = int(high_match.group(1))
+            
+            low_match = re.search(r'"lowTemperature":(\d+)', html_content)
+            if low_match:
+                weather_data["low_f"] = int(low_match.group(1))
+            
+            # Extract humidity
+            humidity_match = re.search(r'Humidity[^>]*>(\d+)%|"value":"(\d+)%"[^}]*"Humidity"', html_content, re.IGNORECASE)
+            if humidity_match:
+                weather_data["humidity_percent"] = int(humidity_match.group(1) or humidity_match.group(2))
+            
+            weather_data["source"] = "Yahoo Weather"
+            weather_data["units"] = "Fahrenheit"
+            
+            # Remove None values
+            weather_data = {k: v for k, v in weather_data.items() if v is not None}
+            
+            if len(weather_data) > 3:  # Has more than just location, source, and units
+                return [weather_data]
+            
+            return [{
+                "location": location,
+                "error": "Could not extract weather data from page"
+            }]
+            
+        except Exception as e:
+            return [{
+                "location": location,
+                "error": f"Failed to parse weather data: {str(e)}"
+            }]

webscout/search/http_client.py

@@ -120,7 +120,7 @@ class HttpClient:
             else:
                 raise WebscoutE(f"{resp.url} returned {resp.status_code}")
                 
-        except curl_cffi.requests.RequestException as ex:
+        except Exception as ex:
             if "time" in str(ex).lower() or "timeout" in str(ex).lower():
                 raise TimeoutE(f"{url} {type(ex).__name__}: {ex}") from ex
             raise WebscoutE(f"{url} {type(ex).__name__}: {ex}") from ex

webscout/search/yahoo_main.py

@@ -0,0 +1,54 @@
+"""Yahoo unified search interface."""
+
+from __future__ import annotations
+from typing import Dict, List, Optional
+from .base import BaseSearch
+from .engines.yahoo.text import YahooText
+from .engines.yahoo.images import YahooImages
+from .engines.yahoo.videos import YahooVideos
+from .engines.yahoo.news import YahooNews
+from .engines.yahoo.suggestions import YahooSuggestions
+from .engines.yahoo.answers import YahooAnswers
+from .engines.yahoo.maps import YahooMaps
+from .engines.yahoo.translate import YahooTranslate
+from .engines.yahoo.weather import YahooWeather
+
+
+class YahooSearch(BaseSearch):
+    """Unified Yahoo search interface."""
+
+    def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooText()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooImages()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def videos(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooVideos()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooNews()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def suggestions(self, keywords: str, region: str = "us") -> List[str]:
+        search = YahooSuggestions()
+        return search.run(keywords, region)
+
+    def answers(self, keywords: str) -> List[Dict[str, str]]:
+        search = YahooAnswers()
+        return search.run(keywords)
+
+    def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = YahooMaps()
+        return search.run(keywords, place, street, city, county, state, country, postalcode, latitude, longitude, radius, max_results)
+
+    def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str = "en") -> List[Dict[str, str]]:
+        search = YahooTranslate()
+        return search.run(keywords, from_lang, to_lang)
+
+    def weather(self, keywords: str) -> List[Dict[str, str]]:
+        search = YahooWeather()
+        return search.run(keywords)

webscout/{auth → server}/__init__.py

@@ -1,16 +1,6 @@
-# webscout/auth/__init__.py
+# webscout/server/__init__.py
 
-from .models import User, APIKey
-from .database import DatabaseManager
-from .api_key_manager import APIKeyManager
-from .rate_limiter import RateLimiter
-from .middleware import AuthMiddleware
 from .schemas import (
-    APIKeyCreateRequest,
-    APIKeyCreateResponse,
-    APIKeyValidationResponse,
-    UserCreateRequest,
-    UserResponse,
     HealthCheckResponse
 )
 # Import server functions lazily to avoid module execution issues
@@ -46,17 +36,6 @@ def initialize_tti_provider_map():
     return _init_tti_provider_map()
 
 __all__ = [
-    "User",
-    "APIKey",
-    "DatabaseManager",
-    "APIKeyManager",
-    "RateLimiter",
-    "AuthMiddleware",
-    "APIKeyCreateRequest",
-    "APIKeyCreateResponse",
-    "APIKeyValidationResponse",
-    "UserCreateRequest",
-    "UserResponse",
     "HealthCheckResponse",
     "create_app",
     "run_api",
@@ -67,4 +46,4 @@ __all__ = [
     "APIError",
     "initialize_provider_map",
     "initialize_tti_provider_map"
-]
+]