webscout 8.2.9__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/engines/yahoo/videos.py

@@ -0,0 +1,302 @@
+"""Yahoo video search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, Optional
+from urllib.parse import parse_qs, urlparse
+
+from ...results import VideosResult
+from .base import YahooSearchEngine
+
+
+class YahooVideos(YahooSearchEngine[VideosResult]):
+    """Yahoo video search engine with filters.
+
+    Features:
+    - Length filters (short, medium, long)
+    - Resolution filters (SD, HD, 4K)
+    - Source filters
+    - Time filters
+    - Pagination support
+    """
+
+    name = "yahoo"
+    category = "videos"
+
+    search_url = "https://video.search.yahoo.com/search/video"
+    search_method = "GET"
+
+    # XPath selectors for video results
+    items_xpath = "//div[@id='results']//div[contains(@class, 'dd') or contains(@class, 'vr')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h3//a/text() | .//a/@title",
+        "url": ".//h3//a/@href | .//a/@href",
+        "thumbnail": ".//img/@src",
+        "duration": ".//span[contains(@class, 'time') or contains(@class, 'duration')]//text()",
+        "views": ".//span[contains(@class, 'views')]//text()",
+        "published": ".//span[contains(@class, 'date') or contains(@class, 'age')]//text()",
+        "source": ".//span[contains(@class, 'source')]//text()",
+    }
+
+    # Filter mappings
+    LENGTH_FILTERS = {
+        "short": "short",    # < 4 minutes
+        "medium": "medium",  # 4-20 minutes
+        "long": "long",      # > 20 minutes
+        "all": "",
+    }
+
+    RESOLUTION_FILTERS = {
+        "sd": "sd",
+        "hd": "hd",
+        "4k": "4k",
+        "all": "",
+    }
+
+    SOURCE_FILTERS = {
+        "youtube": "youtube",
+        "dailymotion": "dailymotion",
+        "vimeo": "vimeo",
+        "metacafe": "metacafe",
+        "all": "",
+    }
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build video search payload.
+
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter (d, w, m)
+            page: Page number
+            **kwargs: Additional filters:
+                - length: Video length filter
+                - resolution: Video resolution filter
+                - source: Video source filter
+
+        Returns:
+            Query parameters dictionary
+        """
+        payload = {
+            "p": query,
+            "fr": "sfp",
+            "fr2": "p:s,v:v,m:sb,rgn:top",
+            "ei": "UTF-8",
+        }
+
+        # Pagination
+        if page > 1:
+            # Each page shows ~15-20 videos
+            payload["b"] = f"{(page - 1) * 15 + 1}"
+
+        # Safe search
+        if safesearch == "on":
+            payload["safe"] = "active"
+        elif safesearch == "off":
+            payload["safe"] = "off"
+
+        # Time filter
+        if timelimit:
+            time_map = {
+                "d": "1d",
+                "w": "1w",
+                "m": "1m",
+                "y": "1y",
+            }
+            if timelimit in time_map:
+                payload["age"] = time_map[timelimit]
+
+        # Length filter
+        if "length" in kwargs and kwargs["length"] in self.LENGTH_FILTERS:
+            length_val = self.LENGTH_FILTERS[kwargs["length"]]
+            if length_val:
+                payload["vidlen"] = length_val
+
+        # Resolution filter
+        if "resolution" in kwargs and kwargs["resolution"] in self.RESOLUTION_FILTERS:
+            res_val = self.RESOLUTION_FILTERS[kwargs["resolution"]]
+            if res_val:
+                payload["vidqual"] = res_val
+
+        # Source filter
+        if "source" in kwargs and kwargs["source"] in self.SOURCE_FILTERS:
+            source_val = self.SOURCE_FILTERS[kwargs["source"]]
+            if source_val:
+                payload["site"] = source_val
+
+        return payload
+
+    def extract_video_url(self, href: str) -> str:
+        """Extract actual video URL from Yahoo redirect.
+
+        Args:
+            href: Yahoo redirect URL
+
+        Returns:
+            Actual video URL
+        """
+        if not href:
+            return href
+
+        try:
+            # Parse the URL
+            parsed = urlparse(href)
+
+            # Check if it's a Yahoo redirect
+            if "r.search.yahoo.com" in parsed.netloc or "/RU=" in href:
+                # Extract the RU parameter
+                if "/RU=" in href:
+                    start = href.find("/RU=") + 4
+                    end = href.find("/RK=", start)
+                    if end == -1:
+                        end = len(href)
+                    from urllib.parse import unquote
+                    return unquote(href[start:end])
+                else:
+                    query_params = parse_qs(parsed.query)
+                    if "url" in query_params:
+                        return query_params["url"][0]
+
+            return href
+        except Exception:
+            return href
+
+    def post_extract_results(self, results: list[VideosResult]) -> list[VideosResult]:
+        """Post-process video results.
+
+        Args:
+            results: Raw extracted results
+
+        Returns:
+            Cleaned results
+        """
+        cleaned_results = []
+
+        for result in results:
+            # Extract real URL
+            if result.url:
+                result.url = self.extract_video_url(result.url)
+
+            # Skip invalid results
+            if not result.url or not result.title:
+                continue
+
+            # Clean thumbnail URL
+            if result.thumbnail and result.thumbnail.startswith("data:"):
+                result.thumbnail = ""
+
+            cleaned_results.append(result)
+
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[VideosResult] | None:
+        """Search Yahoo Videos with pagination.
+
+        Args:
+            query: Video search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter
+            page: Starting page
+            max_results: Maximum results
+            **kwargs: Additional filters (length, resolution, source)
+
+        Returns:
+            List of VideoResult objects
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 5)
+
+        while current_page <= max_pages:
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+
+            html_text = self.pre_process_html(html_text)
+            page_results = self.extract_results(html_text)
+
+            if not page_results:
+                break
+
+            results.extend(page_results)
+
+            if max_results and len(results) >= max_results:
+                break
+
+            current_page += 1
+
+        results = self.post_extract_results(results)
+
+        if max_results:
+            results = results[:max_results]
+
+        return results if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        resolution: str | None = None,
+        duration: str | None = None,
+        license_videos: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run video search and return results as dictionaries.
+
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            resolution: Video resolution filter.
+            duration: Video duration filter.
+            license_videos: License filter.
+            max_results: Maximum number of results.
+
+        Returns:
+            List of video result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            resolution=resolution,
+            duration=duration,
+            license_videos=license_videos,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]

webscout/search/engines/yahoo/weather.py

@@ -0,0 +1,220 @@
+"""Yahoo weather search using embedded JSON data."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Optional
+
+from ...http_client import HttpClient
+
+
+class YahooWeather:
+    """Yahoo weather search using embedded JSON extraction."""
+
+    def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True):
+        """Initialize weather search engine.
+
+        Args:
+            proxy: Proxy URL.
+            timeout: Request timeout in seconds.
+            verify: Whether to verify SSL certificates.
+        """
+        self.http_client = HttpClient(proxy=proxy, timeout=timeout, verify=verify)
+
+    def request(self, method: str, url: str, **kwargs: Any) -> str | None:
+        """Make a request to the weather service."""
+        try:
+            from typing import cast
+            response = self.http_client.request(cast(Any, method), url, **kwargs)
+            return response.text
+        except Exception:
+            return None
+
+    def run(self, *args, **kwargs) -> list[dict[str, Any]]:
+        """Get weather data from Yahoo.
+
+        Args:
+            location: Location to get weather for (e.g., "New York", "London", "Bengaluru")
+
+        Returns:
+            List of weather data dictionaries
+        """
+        location = args[0] if args else kwargs.get("location") or kwargs.get("keywords")
+
+        if not location:
+            raise ValueError("Location is required for weather search")
+
+        try:
+            # Use the search endpoint which redirects to the correct weather page
+            search_url = f"https://weather.yahoo.com/search/?q={location.replace(' ', '+')}"
+
+            # Fetch the page
+            response = self.request("GET", search_url)
+            if not response:
+                return [{
+                    "location": location,
+                    "error": "Failed to fetch weather data from Yahoo"
+                }]
+
+            # Extract JSON data from the page
+            weather_data = self._extract_json_data(response, location)
+
+            if weather_data:
+                return [weather_data]
+
+            # Fallback: try regex parsing
+            return self._parse_weather_html(response, location)
+
+        except Exception as e:
+            return [{
+                "location": location,
+                "error": f"Failed to fetch weather data: {str(e)}"
+            }]
+
+    def _extract_json_data(self, html: str, location: str) -> dict[str, Any] | None:
+        """Extract weather data from embedded JSON in the page.
+
+        Yahoo Weather embeds JSON data in script tags that can be parsed.
+        """
+        try:
+            # Look for the main data script tag
+            # Pattern: self.__next_f.push([1,"..JSON data.."])
+            json_pattern = r'self\.__next_f\.push\(\[1,"([^"]+)"\]\)'
+            matches = re.findall(json_pattern, html)
+
+            weather_info: Dict[str, Any] = {}
+
+            for match in matches:
+                # Unescape the JSON string
+                try:
+                    # The data is escaped, so we need to decode it
+                    decoded = match.encode().decode('unicode_escape')
+
+                    # Look for temperature data
+                    temp_match = re.search(r'"temperature":(\d+)', decoded)
+                    if temp_match and not weather_info.get('temperature'):
+                        weather_info['temperature'] = int(temp_match.group(1))
+
+                    # Look for condition
+                    condition_match = re.search(r'"iconLabel":"([^"]+)"', decoded)
+                    if condition_match and not weather_info.get('condition'):
+                        weather_info['condition'] = condition_match.group(1)
+
+                    # Look for high/low
+                    high_match = re.search(r'"highTemperature":(\d+)', decoded)
+                    if high_match and not weather_info.get('high'):
+                        weather_info['high'] = int(high_match.group(1))
+
+                    low_match = re.search(r'"lowTemperature":(\d+)', decoded)
+                    if low_match and not weather_info.get('low'):
+                        weather_info['low'] = int(low_match.group(1))
+
+                    # Look for humidity
+                    humidity_match = re.search(r'"value":"(\d+)%"[^}]*"category":"Humidity"', decoded)
+                    if humidity_match and not weather_info.get('humidity'):
+                        weather_info['humidity'] = int(humidity_match.group(1))
+
+                    # Look for precipitation probability
+                    precip_match = re.search(r'"probabilityOfPrecipitation":"(\d+)%"', decoded)
+                    if precip_match and not weather_info.get('precipitation_chance'):
+                        weather_info['precipitation_chance'] = int(precip_match.group(1))
+
+                    # Look for location name
+                    location_match = re.search(r'"name":"([^"]+)","code":null,"woeid":(\d+)', decoded)
+                    if location_match and not weather_info.get('location_name'):
+                        weather_info['location_name'] = location_match.group(1)
+                        weather_info['woeid'] = int(location_match.group(2))
+
+                except Exception:
+                    continue
+
+            if weather_info and weather_info.get('temperature'):
+                return {
+                    "location": weather_info.get('location_name', location),
+                    "woeid": weather_info.get('woeid'),
+                    "temperature_f": weather_info.get('temperature'),
+                    "condition": weather_info.get('condition'),
+                    "high_f": weather_info.get('high'),
+                    "low_f": weather_info.get('low'),
+                    "humidity_percent": weather_info.get('humidity'),
+                    "precipitation_chance": weather_info.get('precipitation_chance'),
+                    "source": "Yahoo Weather",
+                    "units": "Fahrenheit"
+                }
+
+            return None
+
+        except Exception:
+            return None
+
+    def _parse_weather_html(self, html_content: str, location: str) -> list[dict[str, Any]]:
+        """Fallback: Parse weather data from HTML content using regex.
+
+        Args:
+            html_content: HTML content of weather page
+            location: Location name
+
+        Returns:
+            List of weather data dictionaries
+        """
+        try:
+            weather_data: dict[str, Any] = {"location": location}
+
+            # Extract current temperature
+            temp_patterns = [
+                r'<p[^>]*class="[^"]*font-title1[^"]*"[^>]*>(\d+)°</p>',
+                r'>(\d+)°<',
+                r'"temperature":(\d+)',
+            ]
+
+            for pattern in temp_patterns:
+                match = re.search(pattern, html_content)
+                if match:
+                    weather_data["temperature_f"] = int(match.group(1))
+                    break
+
+            # Extract condition
+            condition_patterns = [
+                r'"iconLabel":"([^"]+)"',
+                r'aria-label="([^"]*(?:Cloudy|Sunny|Rain|Clear|Thunder|Shower|Fog)[^"]*)"',
+            ]
+
+            for pattern in condition_patterns:
+                match = re.search(pattern, html_content, re.IGNORECASE)
+                if match:
+                    weather_data["condition"] = match.group(1)
+                    break
+
+            # Extract high/low
+            high_match = re.search(r'"highTemperature":(\d+)', html_content)
+            if high_match:
+                weather_data["high_f"] = int(high_match.group(1))
+
+            low_match = re.search(r'"lowTemperature":(\d+)', html_content)
+            if low_match:
+                weather_data["low_f"] = int(low_match.group(1))
+
+            # Extract humidity
+            humidity_match = re.search(r'Humidity[^>]*>(\d+)%|"value":"(\d+)%"[^}]*"Humidity"', html_content, re.IGNORECASE)
+            if humidity_match:
+                weather_data["humidity_percent"] = int(humidity_match.group(1) or humidity_match.group(2))
+
+            weather_data["source"] = "Yahoo Weather"
+            weather_data["units"] = "Fahrenheit"
+
+            # Remove None values
+            weather_data = {k: v for k, v in weather_data.items() if v is not None}
+
+            if len(weather_data) > 3:  # Has more than just location, source, and units
+                return [weather_data]
+
+            return [{
+                "location": location,
+                "error": "Could not extract weather data from page"
+            }]
+
+        except Exception as e:
+            return [{
+                "location": location,
+                "error": f"Failed to parse weather data: {str(e)}"
+            }]

webscout/search/engines/yandex.py

@@ -0,0 +1,67 @@
+"""Yandex search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from random import SystemRandom
+from typing import Any, Optional
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+random = SystemRandom()
+
+
+class Yandex(BaseSearchEngine[TextResult]):
+    """Yandex search engine."""
+
+    name = "yandex"
+    category = "text"
+    provider = "yandex"
+
+    search_url = "https://yandex.com/search/"
+    search_method = "GET"
+
+    items_xpath = "//li[contains(@class, 'serp-item')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//div[contains(@class, 'text-container')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "1", "moderate": "0", "off": "0"}
+        payload = {
+            "text": query,
+            "family": safesearch_base[safesearch.lower()],
+        }
+        if page > 1:
+            payload["p"] = str(page - 1)
+        return payload
+
+    def run(self, *args, **kwargs) -> list[TextResult]:
+        """Run text search on Yandex.
+
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            max_results: Maximum number of results (ignored for now).
+
+        Returns:
+            List of TextResult objects.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        if keywords is None:
+            keywords = ""
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        results = self.search(query=keywords, region=region, safesearch=safesearch)
+        if results and max_results:
+            results = results[:max_results]
+        return results or []

webscout/search/engines/yep/__init__.py

@@ -0,0 +1,13 @@
+"""Yep search engines package."""
+
+from .base import YepBase
+from .images import YepImages
+from .suggestions import YepSuggestions
+from .text import YepSearch as YepTextSearch
+
+__all__ = [
+    "YepBase",
+    "YepTextSearch",
+    "YepImages",
+    "YepSuggestions",
+]

webscout/search/engines/yep/base.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from curl_cffi.requests import Session
+
+from ....litagent import LitAgent
+
+
+class YepBase:
+    """Base class for Yep search engines."""
+
+    def __init__(
+        self,
+        timeout: int = 20,
+        proxies: dict[str, str] | None = None,
+        verify: bool = True,
+        impersonate: str = "chrome110",
+    ):
+        self.base_url = "https://api.yep.com/fs/2/search"
+        self.timeout = timeout
+        from typing import Any, Optional, cast
+        self.session = Session(
+            proxies=cast(Any, proxies),
+            verify=verify,
+            impersonate=cast(Any, impersonate),
+            timeout=timeout,
+        )
+        self.session.headers.update(
+            {
+                **LitAgent().generate_fingerprint(),
+                "Origin": "https://yep.com",
+                "Referer": "https://yep.com/",
+            }
+        )
+