webscout 2025.10.15__py3-none-any.whl → 2025.10.17__py3-none-any.whl

webscout/search/engines/yahoo/images.py

@@ -0,0 +1,324 @@
+"""Yahoo image search engine with advanced filters."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import urljoin
+
+from .base import YahooSearchEngine
+from ...results import ImagesResult
+
+
+class YahooImages(YahooSearchEngine[ImagesResult]):
+    """Yahoo image search engine with filter support.
+    
+    Features:
+    - Size filters (small, medium, large, wallpaper)
+    - Color filters (color, bw, red, orange, yellow, etc.)
+    - Type filters (photo, clipart, lineart, transparent)
+    - Layout filters (square, wide, tall)
+    - Time filters
+    - Pagination support
+    
+    Note: Yahoo does not support reverse image search (searching by image upload/URL).
+    For reverse image search functionality, use Google Images or Bing Images instead.
+    """
+
+    name = "yahoo"
+    category = "images"
+
+    search_url = "https://images.search.yahoo.com/search/images"
+    search_method = "GET"
+    search_headers = {
+        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1"
+    }
+
+    # XPath selectors
+    items_xpath = "//li[contains(@class, 'ld')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": "@data",
+        "image": "@data", 
+        "thumbnail": "@data",
+        "url": "@data",
+        "source": "@data",
+        "width": "@data",
+        "height": "@data",
+    }
+
+    # Filter mappings
+    SIZE_FILTERS = {
+        "small": "small",
+        "medium": "medium", 
+        "large": "large",
+        "wallpaper": "wallpaper",
+        "all": "",
+    }
+
+    COLOR_FILTERS = {
+        "color": "color",
+        "bw": "bw",
+        "black": "black",
+        "white": "white",
+        "red": "red",
+        "orange": "orange",
+        "yellow": "yellow",
+        "green": "green",
+        "teal": "teal",
+        "blue": "blue",
+        "purple": "purple",
+        "pink": "pink",
+        "brown": "brown",
+        "gray": "gray",
+        "all": "",
+    }
+
+    TYPE_FILTERS = {
+        "photo": "photo",
+        "clipart": "clipart",
+        "lineart": "linedrawing",
+        "transparent": "transparent",
+        "gif": "animatedgif",
+        "all": "",
+    }
+
+    LAYOUT_FILTERS = {
+        "square": "square",
+        "wide": "wide",
+        "tall": "tall",
+        "all": "",
+    }
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build image search payload with filters.
+        
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level (on/moderate/off)
+            timelimit: Time filter (d, w, m)
+            page: Page number
+            **kwargs: Additional filters including:
+                - size: Image size filter
+                - color: Color filter
+                - type: Image type filter
+                - layout: Layout/aspect ratio filter
+                - license: Usage rights filter
+                
+        Returns:
+            Query parameters dictionary
+        """
+        payload = {
+            "p": query,
+        }
+
+        # Pagination - Yahoo images use 'b' parameter
+        if page > 1:
+            # Each page shows approximately 40 images
+            payload["b"] = f"{(page - 1) * 40 + 1}"
+
+        # Safe search
+        if safesearch == "on":
+            payload["safe"] = "active"
+        elif safesearch == "off":
+            payload["safe"] = "off"
+
+        # Time filter
+        if timelimit:
+            time_map = {
+                "d": "1d",  # Past 24 hours
+                "w": "1w",  # Past week
+                "m": "1m",  # Past month
+            }
+            if timelimit in time_map:
+                payload["age"] = time_map[timelimit]
+
+        # Size filter
+        if "size" in kwargs and kwargs["size"] in self.SIZE_FILTERS:
+            size_val = self.SIZE_FILTERS[kwargs["size"]]
+            if size_val:
+                payload["imgsz"] = size_val
+
+        # Color filter
+        if "color" in kwargs and kwargs["color"] in self.COLOR_FILTERS:
+            color_val = self.COLOR_FILTERS[kwargs["color"]]
+            if color_val:
+                payload["imgc"] = color_val
+
+        # Type filter
+        if "type" in kwargs and kwargs["type"] in self.TYPE_FILTERS:
+            type_val = self.TYPE_FILTERS[kwargs["type"]]
+            if type_val:
+                payload["imgt"] = type_val
+
+        # Layout filter
+        if "layout" in kwargs and kwargs["layout"] in self.LAYOUT_FILTERS:
+            layout_val = self.LAYOUT_FILTERS[kwargs["layout"]]
+            if layout_val:
+                payload["imgsp"] = layout_val
+
+        return payload
+
+    def post_extract_results(self, results: list[ImagesResult]) -> list[ImagesResult]:
+        """Post-process image results to parse JSON data.
+        
+        Args:
+            results: Raw extracted results
+            
+        Returns:
+            Cleaned results with proper URLs and metadata
+        """
+        import json
+        from urllib.parse import unquote
+        
+        cleaned_results = []
+        
+        for result in results:
+            # Parse JSON data from the data attribute
+            if result.title and result.title.startswith('{'):
+                try:
+                    data = json.loads(result.title)
+                    
+                    # Extract title
+                    result.title = data.get('desc', '') or data.get('tit', '')
+                    
+                    # Extract URLs
+                    result.url = data.get('rurl', '')
+                    result.thumbnail = data.get('turl', '')
+                    result.image = data.get('turlL', '') or data.get('turl', '')
+                    
+                    # Extract dimensions
+                    result.width = int(data.get('imgW', 0))
+                    result.height = int(data.get('imgH', 0))
+                    
+                except (json.JSONDecodeError, KeyError, ValueError):
+                    # If JSON parsing fails, keep original data
+                    pass
+            
+            # Clean URLs if they exist
+            if result.url:
+                result.url = unquote(result.url)
+            if result.image:
+                result.image = unquote(result.image)
+            if result.thumbnail:
+                result.thumbnail = unquote(result.thumbnail)
+            
+            cleaned_results.append(result)
+        
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[ImagesResult] | None:
+        """Search Yahoo Images with pagination.
+        
+        Args:
+            query: Image search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter
+            page: Starting page
+            max_results: Maximum results to return
+            **kwargs: Additional filters (size, color, type, layout)
+            
+        Returns:
+            List of ImageResult objects
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 5)
+        
+        while current_page <= max_pages:
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+            
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+            
+            html_text = self.pre_process_html(html_text)
+            page_results = self.extract_results(html_text)
+            
+            if not page_results:
+                break
+            
+            results.extend(page_results)
+            
+            if max_results and len(results) >= max_results:
+                break
+            
+            current_page += 1
+        
+        results = self.post_extract_results(results)
+        
+        if max_results:
+            results = results[:max_results]
+        
+        return results if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        size: str | None = None,
+        color: str | None = None,
+        type_image: str | None = None,
+        layout: str | None = None,
+        license_image: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run image search and return results as dictionaries.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            size: Image size filter.
+            color: Color filter.
+            type_image: Image type filter.
+            layout: Layout filter.
+            license_image: License filter.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of image result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            size=size,
+            color=color,
+            type_image=type_image,
+            layout=layout,
+            license_image=license_image,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]

webscout/search/engines/yahoo/maps.py

@@ -0,0 +1,16 @@
+"""Yahoo maps search."""
+
+from __future__ import annotations
+
+from .base import YahooSearchEngine
+
+
+class YahooMaps(YahooSearchEngine):
+    """Yahoo maps search."""
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get maps results from Yahoo.
+
+        Not supported.
+        """
+        raise NotImplementedError("Yahoo does not support maps search")

webscout/search/engines/yahoo/news.py

@@ -0,0 +1,258 @@
+"""Yahoo news search engine with comprehensive features."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from secrets import token_urlsafe
+from typing import Any
+
+from .base import YahooSearchEngine
+from ...results import NewsResult
+
+
+def extract_image(u: str) -> str:
+    """Sanitize image URL.
+    
+    Args:
+        u: Image URL
+        
+    Returns:
+        Cleaned URL or empty string
+    """
+    if not u:
+        return ""
+    
+    # Skip data URIs
+    if u.startswith("data:image"):
+        return ""
+    
+    return u
+
+
+def extract_source(s: str) -> str:
+    """Remove ' via Yahoo' from source string.
+    
+    Args:
+        s: Source string
+        
+    Returns:
+        Cleaned source name
+    """
+    if not s:
+        return s
+    
+    return s.replace(" via Yahoo", "").replace(" - Yahoo", "").strip()
+
+
+class YahooNews(YahooSearchEngine[NewsResult]):
+    """Yahoo news search engine with advanced filtering.
+    
+    Features:
+    - Time-based filtering
+    - Category filtering
+    - Source filtering
+    - Pagination support
+    - Rich metadata extraction
+    """
+
+    name = "yahoo"
+    category = "news"
+
+    search_url = "https://news.search.yahoo.com/search"
+    search_method = "GET"
+
+    # XPath selectors for news articles
+    items_xpath = "//div[contains(@class, 'NewsArticle') or contains(@class, 'dd') and contains(@class, 'algo')]"
+    elements_xpath: Mapping[str, str] = {
+        "date": ".//span[contains(@class, 'fc-2nd') or contains(@class, 'age') or contains(@class, 's-time')]//text()",
+        "title": ".//h4//a//text() | .//h3//a//text()",
+        "url": ".//h4//a/@href | .//h3//a/@href",
+        "body": ".//p//text() | .//div[contains(@class, 'compText')]//text()",
+        "image": ".//img/@src",
+        "source": ".//span[contains(@class, 's-source') or contains(@class, 'source')]//text()",
+    }
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build news search payload.
+        
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter (d, w, m)
+            page: Page number
+            **kwargs: Additional parameters
+            
+        Returns:
+            Query parameters dictionary
+        """
+        # Generate dynamic URL tokens for tracking
+        self.search_url = (
+            f"https://news.search.yahoo.com/search"
+            f";_ylt={token_urlsafe(24 * 3 // 4)}"
+            f";_ylu={token_urlsafe(47 * 3 // 4)}"
+        )
+        
+        payload = {
+            "p": query,
+            "ei": "UTF-8",
+        }
+        
+        # Pagination - Yahoo news uses 'b' parameter
+        if page > 1:
+            # Each page shows approximately 10 articles
+            payload["b"] = f"{(page - 1) * 10 + 1}"
+        
+        # Time filter
+        if timelimit:
+            time_map = {
+                "d": "1d",   # Past 24 hours
+                "w": "1w",   # Past week
+                "m": "1m",   # Past month
+            }
+            if timelimit in time_map:
+                payload["btf"] = time_map[timelimit]
+        
+        # Additional filters
+        if "category" in kwargs:
+            payload["category"] = kwargs["category"]
+        
+        if "sort" in kwargs:
+            # Sort by relevance or date
+            payload["sort"] = kwargs["sort"]
+        
+        return payload
+
+    def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]:
+        """Post-process news results.
+        
+        Args:
+            results: Raw extracted results
+            
+        Returns:
+            Cleaned news results
+        """
+        cleaned_results = []
+        
+        for result in results:
+            # Clean image URL
+            result.image = extract_image(result.image)
+            
+            # Clean source name
+            result.source = extract_source(result.source)
+            
+            # Extract URL from redirect
+            if result.url and "/RU=" in result.url:
+                from urllib.parse import unquote
+                start = result.url.find("/RU=") + 4
+                end = result.url.find("/RK=", start)
+                if end == -1:
+                    end = len(result.url)
+                result.url = unquote(result.url[start:end])
+            
+            # Filter out results without essential fields
+            if result.title and result.url:
+                cleaned_results.append(result)
+        
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[NewsResult] | None:
+        """Search Yahoo News with pagination.
+        
+        Args:
+            query: News search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter (d, w, m)
+            page: Starting page
+            max_results: Maximum results to return
+            **kwargs: Additional parameters (category, sort)
+            
+        Returns:
+            List of NewsResult objects
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 10)
+        
+        while current_page <= max_pages:
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+            
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+            
+            html_text = self.pre_process_html(html_text)
+            page_results = self.extract_results(html_text)
+            
+            if not page_results:
+                break
+            
+            results.extend(page_results)
+            
+            if max_results and len(results) >= max_results:
+                break
+            
+            current_page += 1
+        
+        results = self.post_extract_results(results)
+        
+        if max_results:
+            results = results[:max_results]
+        
+        return results if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run news search and return results as dictionaries.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of news result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]