webscout 2025.10.14.1__py3-none-any.whl → 2025.10.16__py3-none-any.whl

webscout/search/engines/yahoo/news.py

@@ -0,0 +1,258 @@
+"""Yahoo news search engine with comprehensive features."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from secrets import token_urlsafe
+from typing import Any
+
+from .base import YahooSearchEngine
+from ...results import NewsResult
+
+
+def extract_image(u: str) -> str:
+    """Sanitize image URL.
+    
+    Args:
+        u: Image URL
+        
+    Returns:
+        Cleaned URL or empty string
+    """
+    if not u:
+        return ""
+    
+    # Skip data URIs
+    if u.startswith("data:image"):
+        return ""
+    
+    return u
+
+
+def extract_source(s: str) -> str:
+    """Remove ' via Yahoo' from source string.
+    
+    Args:
+        s: Source string
+        
+    Returns:
+        Cleaned source name
+    """
+    if not s:
+        return s
+    
+    return s.replace(" via Yahoo", "").replace(" - Yahoo", "").strip()
+
+
+class YahooNews(YahooSearchEngine[NewsResult]):
+    """Yahoo news search engine with advanced filtering.
+    
+    Features:
+    - Time-based filtering
+    - Category filtering
+    - Source filtering
+    - Pagination support
+    - Rich metadata extraction
+    """
+
+    name = "yahoo"
+    category = "news"
+
+    search_url = "https://news.search.yahoo.com/search"
+    search_method = "GET"
+
+    # XPath selectors for news articles
+    items_xpath = "//div[contains(@class, 'NewsArticle') or contains(@class, 'dd') and contains(@class, 'algo')]"
+    elements_xpath: Mapping[str, str] = {
+        "date": ".//span[contains(@class, 'fc-2nd') or contains(@class, 'age') or contains(@class, 's-time')]//text()",
+        "title": ".//h4//a//text() | .//h3//a//text()",
+        "url": ".//h4//a/@href | .//h3//a/@href",
+        "body": ".//p//text() | .//div[contains(@class, 'compText')]//text()",
+        "image": ".//img/@src",
+        "source": ".//span[contains(@class, 's-source') or contains(@class, 'source')]//text()",
+    }
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build news search payload.
+        
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter (d, w, m)
+            page: Page number
+            **kwargs: Additional parameters
+            
+        Returns:
+            Query parameters dictionary
+        """
+        # Generate dynamic URL tokens for tracking
+        self.search_url = (
+            f"https://news.search.yahoo.com/search"
+            f";_ylt={token_urlsafe(24 * 3 // 4)}"
+            f";_ylu={token_urlsafe(47 * 3 // 4)}"
+        )
+        
+        payload = {
+            "p": query,
+            "ei": "UTF-8",
+        }
+        
+        # Pagination - Yahoo news uses 'b' parameter
+        if page > 1:
+            # Each page shows approximately 10 articles
+            payload["b"] = f"{(page - 1) * 10 + 1}"
+        
+        # Time filter
+        if timelimit:
+            time_map = {
+                "d": "1d",   # Past 24 hours
+                "w": "1w",   # Past week
+                "m": "1m",   # Past month
+            }
+            if timelimit in time_map:
+                payload["btf"] = time_map[timelimit]
+        
+        # Additional filters
+        if "category" in kwargs:
+            payload["category"] = kwargs["category"]
+        
+        if "sort" in kwargs:
+            # Sort by relevance or date
+            payload["sort"] = kwargs["sort"]
+        
+        return payload
+
+    def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]:
+        """Post-process news results.
+        
+        Args:
+            results: Raw extracted results
+            
+        Returns:
+            Cleaned news results
+        """
+        cleaned_results = []
+        
+        for result in results:
+            # Clean image URL
+            result.image = extract_image(result.image)
+            
+            # Clean source name
+            result.source = extract_source(result.source)
+            
+            # Extract URL from redirect
+            if result.url and "/RU=" in result.url:
+                from urllib.parse import unquote
+                start = result.url.find("/RU=") + 4
+                end = result.url.find("/RK=", start)
+                if end == -1:
+                    end = len(result.url)
+                result.url = unquote(result.url[start:end])
+            
+            # Filter out results without essential fields
+            if result.title and result.url:
+                cleaned_results.append(result)
+        
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[NewsResult] | None:
+        """Search Yahoo News with pagination.
+        
+        Args:
+            query: News search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter (d, w, m)
+            page: Starting page
+            max_results: Maximum results to return
+            **kwargs: Additional parameters (category, sort)
+            
+        Returns:
+            List of NewsResult objects
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 10)
+        
+        while current_page <= max_pages:
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+            
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+            
+            html_text = self.pre_process_html(html_text)
+            page_results = self.extract_results(html_text)
+            
+            if not page_results:
+                break
+            
+            results.extend(page_results)
+            
+            if max_results and len(results) >= max_results:
+                break
+            
+            current_page += 1
+        
+        results = self.post_extract_results(results)
+        
+        if max_results:
+            results = results[:max_results]
+        
+        return results if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run news search and return results as dictionaries.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of news result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]

webscout/search/engines/yahoo/suggestions.py

@@ -0,0 +1,140 @@
+"""Yahoo search suggestions engine."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .base import YahooSearchEngine
+
+
+class YahooSuggestions(YahooSearchEngine[str]):
+    """Yahoo search suggestions engine.
+    
+    Provides autocomplete suggestions as you type.
+    """
+
+    name = "yahoo"
+    category = "suggestions"
+
+    search_url = "https://search.yahoo.com/sugg/gossip/gossip-us-ura"
+    search_method = "GET"
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build suggestions payload.
+        
+        Args:
+            query: Partial search query
+            region: Region code
+            safesearch: Safe search level (unused)
+            timelimit: Time limit (unused)
+            page: Page number (unused)
+            **kwargs: Additional parameters
+            
+        Returns:
+            Query parameters
+        """
+        payload = {
+            "command": query,
+            "output": "sd1",
+            "nresults": kwargs.get("max_suggestions", 10),
+        }
+        
+        return payload
+
+    def extract_results(self, html_text: str) -> list[str]:
+        """Extract suggestions from JSON response.
+        
+        Args:
+            html_text: JSON response text
+            
+        Returns:
+            List of suggestion strings
+        """
+        try:
+            data = json.loads(html_text)
+            
+            # Yahoo returns suggestions in 'r' key
+            if "r" in data and isinstance(data["r"], list):
+                suggestions = []
+                for item in data["r"]:
+                    if isinstance(item, dict) and "k" in item:
+                        suggestions.append(item["k"])
+                    elif isinstance(item, str):
+                        suggestions.append(item)
+                return suggestions
+            
+            return []
+        except (json.JSONDecodeError, KeyError, TypeError):
+            return []
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[str] | None:
+        """Get search suggestions for a query.
+        
+        Args:
+            query: Partial search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time limit
+            page: Page number
+            max_results: Maximum suggestions
+            **kwargs: Additional parameters
+            
+        Returns:
+            List of suggestion strings
+        """
+        if max_results:
+            kwargs["max_suggestions"] = max_results
+        
+        payload = self.build_payload(
+            query=query,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            page=page,
+            **kwargs
+        )
+        
+        response = self.request(self.search_method, self.search_url, params=payload)
+        if not response:
+            return None
+        
+        suggestions = self.extract_results(response)
+        
+        if max_results:
+            suggestions = suggestions[:max_results]
+        
+        return suggestions if suggestions else None
+
+    def run(self, keywords: str, region: str = "us-en") -> list[str]:
+        """Run suggestions search and return results.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            
+        Returns:
+            List of suggestion strings.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+        )
+        return results if results else []

webscout/search/engines/yahoo/text.py

@@ -0,0 +1,273 @@
+"""Yahoo text search engine with pagination support."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import unquote_plus, urljoin
+
+from .base import YahooSearchEngine
+from ...results import TextResult
+
+
+def extract_url(u: str) -> str:
+    """Extract and sanitize URL from Yahoo redirect.
+    
+    Yahoo uses /RU= redirect URLs that need to be decoded.
+    Example: /url?sa=t&url=https%3A%2F%2Fexample.com
+    """
+    if not u:
+        return u
+    
+    # Handle /RU= redirect format
+    if "/RU=" in u:
+        start = u.find("/RU=") + 4
+        end = u.find("/RK=", start)
+        if end == -1:
+            end = len(u)
+        return unquote_plus(u[start:end])
+    
+    return u
+
+
+class YahooText(YahooSearchEngine[TextResult]):
+    """Yahoo text search engine with full pagination support.
+    
+    Features:
+    - Multi-page navigation like a human
+    - Automatic next page detection
+    - Clean result extraction
+    - Time filter support
+    - Region support
+    """
+
+    name = "yahoo"
+    category = "text"
+
+    search_url = "https://search.yahoo.com/search"
+    search_method = "GET"
+
+    # XPath selectors for result extraction
+    items_xpath = "//div[contains(@class, 'compTitle')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h3//span//text()",
+        "href": ".//a/@href",
+        "body": "./following-sibling::div[contains(@class, 'compText')]//text()",
+    }
+
+    def build_payload(
+        self, 
+        query: str, 
+        region: str, 
+        safesearch: str, 
+        timelimit: str | None, 
+        page: int = 1, 
+        **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build search payload for Yahoo.
+        
+        Args:
+            query: Search query string
+            region: Region code (e.g., 'us-en')
+            safesearch: Safe search level
+            timelimit: Time limit filter (d=day, w=week, m=month)
+            page: Page number (1-indexed)
+            **kwargs: Additional parameters
+            
+        Returns:
+            Dictionary of query parameters
+        """
+        payload = {
+            "p": query,
+            "ei": "UTF-8",
+        }
+        
+        # Pagination: Yahoo uses 'b' parameter for offset
+        # Page 1: no b parameter or b=1
+        # Page 2: b=8 (shows results 8-14)
+        # Page 3: b=15, etc.
+        if page > 1:
+            payload["b"] = f"{(page - 1) * 7 + 1}"
+        
+        # Time filter
+        if timelimit:
+            payload["btf"] = timelimit
+            
+        return payload
+
+    def post_extract_results(self, results: list[TextResult]) -> list[TextResult]:
+        """Post-process and clean extracted results.
+        
+        Args:
+            results: Raw extracted results
+            
+        Returns:
+            Cleaned and filtered results
+        """
+        cleaned_results = []
+        
+        for result in results:
+            # Extract real URL from redirect
+            if result.href:
+                result.href = extract_url(result.href)
+            
+            # Filter out empty results
+            if result.title and result.href:
+                cleaned_results.append(result)
+        
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[TextResult] | None:
+        """Search Yahoo with automatic pagination like a human browser.
+        
+        This method automatically follows pagination links to gather results
+        across multiple pages, similar to how a human would browse search results.
+        
+        Args:
+            query: Search query string
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter (d=day, w=week, m=month, y=year)
+            page: Starting page number
+            max_results: Maximum number of results to return
+            **kwargs: Additional search parameters
+            
+        Returns:
+            List of TextResult objects, or None if search fails
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 10)  # Limit to prevent infinite loops
+        
+        while current_page <= max_pages:
+            # Build payload for current page
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+            
+            # Make request
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+            
+            # Pre-process HTML
+            html_text = self.pre_process_html(html_text)
+            
+            # Extract results from current page
+            page_results = self.extract_results(html_text)
+            if not page_results:
+                break
+            
+            results.extend(page_results)
+            
+            # Check if we have enough results
+            if max_results and len(results) >= max_results:
+                break
+            
+            # Look for next page link
+            tree = self.extract_tree(html_text)
+            next_links = tree.xpath("//a[contains(text(), 'Next') or contains(@class, 'next')]/@href")
+            
+            if not next_links:
+                # Try to find numbered page links
+                page_links = tree.xpath(f"//a[contains(text(), '{current_page + 1}')]/@href")
+                if not page_links:
+                    break
+            
+            current_page += 1
+        
+        # Post-process all results
+        results = self.post_extract_results(results)
+        
+        # Trim to max_results if specified
+        if max_results:
+            results = results[:max_results]
+        
+        return results if results else None
+
+    def search_page(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> list[TextResult] | None:
+        """Search a single page (for compatibility).
+        
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter
+            page: Page number
+            **kwargs: Additional parameters
+            
+        Returns:
+            List of results from the specified page
+        """
+        payload = self.build_payload(
+            query=query,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            page=page,
+            **kwargs
+        )
+        
+        html_text = self.request(self.search_method, self.search_url, params=payload)
+        if not html_text:
+            return None
+        
+        html_text = self.pre_process_html(html_text)
+        results = self.extract_results(html_text)
+        
+        return self.post_extract_results(results) if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        backend: str = "auto",
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run text search and return results as dictionaries.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            backend: Backend type (ignored for Yahoo).
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of search result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]

webscout/search/engines/yahoo/translate.py

@@ -0,0 +1,16 @@
+"""Yahoo translate search."""
+
+from __future__ import annotations
+
+from .base import YahooSearchEngine
+
+
+class YahooTranslate(YahooSearchEngine):
+    """Yahoo translation."""
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Translate text using Yahoo.
+
+        Not supported.
+        """
+        raise NotImplementedError("Yahoo does not support translation")