webscout 2025.10.15__py3-none-any.whl → 2025.10.16__py3-none-any.whl

webscout/search/engines/bing/images.py

@@ -4,10 +4,10 @@ from __future__ import annotations
 
 from typing import Dict, List
 from urllib.parse import urlencode
-from bs4 import BeautifulSoup
 from time import sleep
 
 from .base import BingBase
+from webscout.scout import Scout
 
 
 class BingImagesSearch(BingBase):
@@ -17,6 +17,9 @@ class BingImagesSearch(BingBase):
         safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
         max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
 
+        if max_results is None:
+            max_results = 10
+
         if not keywords:
             raise ValueError("Keywords are mandatory")
 
@@ -59,7 +62,7 @@ class BingImagesSearch(BingBase):
             except Exception as e:
                 raise Exception(f"Failed to fetch images: {str(e)}")
 
-            soup = BeautifulSoup(html, 'html.parser')
+            soup = Scout(html)
             img_tags = soup.select('a.iusc img')
 
             for img in img_tags:

webscout/search/engines/bing/news.py

@@ -4,10 +4,10 @@ from __future__ import annotations
 
 from typing import Dict, List
 from urllib.parse import urlencode
-from bs4 import BeautifulSoup
 from time import sleep
 
 from .base import BingBase
+from webscout.scout import Scout
 
 
 class BingNewsSearch(BingBase):
@@ -17,6 +17,9 @@ class BingNewsSearch(BingBase):
         safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
         max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
 
+        if max_results is None:
+            max_results = 10
+
         if not keywords:
             raise ValueError("Keywords are mandatory")
 
@@ -50,15 +53,14 @@ class BingNewsSearch(BingBase):
             try:
                 response = self.session.get(full_url, timeout=self.timeout)
                 response.raise_for_status()
-                data = response.json()
+                html = response.text
             except Exception as e:
                 raise Exception(f"Failed to fetch news: {str(e)}")
 
-            html = data.get('html', '')
             if not html:
                 break
 
-            soup = BeautifulSoup(html, 'html.parser')
+            soup = Scout(html)
             news_items = soup.select('div.newsitem')
 
             for item in news_items:

webscout/search/engines/bing/text.py

@@ -4,10 +4,10 @@ from __future__ import annotations
 
 from typing import Dict, List
 from urllib.parse import urlencode
-from bs4 import BeautifulSoup
 from time import sleep
 
 from .base import BingBase
+from webscout.scout import Scout
 
 
 class BingTextSearch(BingBase):
@@ -18,6 +18,9 @@ class BingTextSearch(BingBase):
         max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
         unique = kwargs.get("unique", True)
 
+        if max_results is None:
+            max_results = 10
+
         if not keywords:
             raise ValueError("Keywords are mandatory")
 
@@ -46,7 +49,7 @@ class BingTextSearch(BingBase):
         while len(fetched_results) < max_results and urls_to_fetch:
             current_url = urls_to_fetch.pop(0)
             html = fetch_page(current_url)
-            soup = BeautifulSoup(html, 'html.parser')
+            soup = Scout(html)
 
             links = soup.select('ol#b_results > li.b_algo')
             for link in links:

webscout/search/engines/yahoo/__init__.py

@@ -0,0 +1,41 @@
+"""Yahoo search engines package.
+
+This package provides comprehensive Yahoo search functionality including:
+- Text search with multi-page pagination
+- Image search with advanced filters
+- Video search with quality and length filters
+- News search with time filtering
+- Search suggestions/autocomplete
+
+All engines support:
+- Human-like browsing through multiple pages
+- Rich metadata extraction
+- Filter support
+- Clean result formatting
+
+Example:
+    >>> from webscout.search.engines.yahoo import YahooText
+    >>> 
+    >>> # Search with automatic pagination
+    >>> searcher = YahooText()
+    >>> results = searcher.search("python programming", max_results=50)
+    >>> 
+    >>> for result in results:
+    ...     print(f"{result.title}: {result.url}")
+"""
+
+from .base import YahooSearchEngine
+from .images import YahooImages
+from .news import YahooNews
+from .suggestions import YahooSuggestions
+from .text import YahooText
+from .videos import YahooVideos
+
+__all__ = [
+    "YahooSearchEngine",
+    "YahooText",
+    "YahooImages",
+    "YahooVideos",
+    "YahooNews",
+    "YahooSuggestions",
+]

webscout/search/engines/yahoo/answers.py

@@ -0,0 +1,16 @@
+"""Yahoo answers search."""
+
+from __future__ import annotations
+
+from .base import YahooSearchEngine
+
+
+class YahooAnswers(YahooSearchEngine):
+    """Yahoo instant answers."""
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get instant answers from Yahoo.
+
+        Not supported.
+        """
+        raise NotImplementedError("Yahoo does not support instant answers")

webscout/search/engines/yahoo/base.py

@@ -0,0 +1,34 @@
+"""Base class for Yahoo search engines."""
+
+from __future__ import annotations
+
+from secrets import token_urlsafe
+from typing import Any, Generic, TypeVar
+
+from ...base import BaseSearchEngine
+
+T = TypeVar("T")
+
+class YahooSearchEngine(BaseSearchEngine[T], Generic[T]):
+    """Base class for Yahoo search engines.
+    
+    Yahoo search is powered by Bing but has its own interface.
+    All Yahoo searches use dynamic URLs with tokens for tracking.
+    """
+
+    provider = "yahoo"
+    _base_url = "https://search.yahoo.com"
+    
+    def generate_ylt_token(self) -> str:
+        """Generate Yahoo _ylt tracking token."""
+        return token_urlsafe(24 * 3 // 4)
+    
+    def generate_ylu_token(self) -> str:
+        """Generate Yahoo _ylu tracking token."""
+        return token_urlsafe(47 * 3 // 4)
+    
+    def build_search_url(self, base_path: str) -> str:
+        """Build search URL with tracking tokens."""
+        ylt = self.generate_ylt_token()
+        ylu = self.generate_ylu_token()
+        return f"{self._base_url}/{base_path};_ylt={ylt};_ylu={ylu}"

webscout/search/engines/yahoo/images.py

@@ -0,0 +1,324 @@
+"""Yahoo image search engine with advanced filters."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import urljoin
+
+from .base import YahooSearchEngine
+from ...results import ImagesResult
+
+
+class YahooImages(YahooSearchEngine[ImagesResult]):
+    """Yahoo image search engine with filter support.
+    
+    Features:
+    - Size filters (small, medium, large, wallpaper)
+    - Color filters (color, bw, red, orange, yellow, etc.)
+    - Type filters (photo, clipart, lineart, transparent)
+    - Layout filters (square, wide, tall)
+    - Time filters
+    - Pagination support
+    
+    Note: Yahoo does not support reverse image search (searching by image upload/URL).
+    For reverse image search functionality, use Google Images or Bing Images instead.
+    """
+
+    name = "yahoo"
+    category = "images"
+
+    search_url = "https://images.search.yahoo.com/search/images"
+    search_method = "GET"
+    search_headers = {
+        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1"
+    }
+
+    # XPath selectors
+    items_xpath = "//li[contains(@class, 'ld')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": "@data",
+        "image": "@data", 
+        "thumbnail": "@data",
+        "url": "@data",
+        "source": "@data",
+        "width": "@data",
+        "height": "@data",
+    }
+
+    # Filter mappings
+    SIZE_FILTERS = {
+        "small": "small",
+        "medium": "medium", 
+        "large": "large",
+        "wallpaper": "wallpaper",
+        "all": "",
+    }
+
+    COLOR_FILTERS = {
+        "color": "color",
+        "bw": "bw",
+        "black": "black",
+        "white": "white",
+        "red": "red",
+        "orange": "orange",
+        "yellow": "yellow",
+        "green": "green",
+        "teal": "teal",
+        "blue": "blue",
+        "purple": "purple",
+        "pink": "pink",
+        "brown": "brown",
+        "gray": "gray",
+        "all": "",
+    }
+
+    TYPE_FILTERS = {
+        "photo": "photo",
+        "clipart": "clipart",
+        "lineart": "linedrawing",
+        "transparent": "transparent",
+        "gif": "animatedgif",
+        "all": "",
+    }
+
+    LAYOUT_FILTERS = {
+        "square": "square",
+        "wide": "wide",
+        "tall": "tall",
+        "all": "",
+    }
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build image search payload with filters.
+        
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level (on/moderate/off)
+            timelimit: Time filter (d, w, m)
+            page: Page number
+            **kwargs: Additional filters including:
+                - size: Image size filter
+                - color: Color filter
+                - type: Image type filter
+                - layout: Layout/aspect ratio filter
+                - license: Usage rights filter
+                
+        Returns:
+            Query parameters dictionary
+        """
+        payload = {
+            "p": query,
+        }
+
+        # Pagination - Yahoo images use 'b' parameter
+        if page > 1:
+            # Each page shows approximately 40 images
+            payload["b"] = f"{(page - 1) * 40 + 1}"
+
+        # Safe search
+        if safesearch == "on":
+            payload["safe"] = "active"
+        elif safesearch == "off":
+            payload["safe"] = "off"
+
+        # Time filter
+        if timelimit:
+            time_map = {
+                "d": "1d",  # Past 24 hours
+                "w": "1w",  # Past week
+                "m": "1m",  # Past month
+            }
+            if timelimit in time_map:
+                payload["age"] = time_map[timelimit]
+
+        # Size filter
+        if "size" in kwargs and kwargs["size"] in self.SIZE_FILTERS:
+            size_val = self.SIZE_FILTERS[kwargs["size"]]
+            if size_val:
+                payload["imgsz"] = size_val
+
+        # Color filter
+        if "color" in kwargs and kwargs["color"] in self.COLOR_FILTERS:
+            color_val = self.COLOR_FILTERS[kwargs["color"]]
+            if color_val:
+                payload["imgc"] = color_val
+
+        # Type filter
+        if "type" in kwargs and kwargs["type"] in self.TYPE_FILTERS:
+            type_val = self.TYPE_FILTERS[kwargs["type"]]
+            if type_val:
+                payload["imgt"] = type_val
+
+        # Layout filter
+        if "layout" in kwargs and kwargs["layout"] in self.LAYOUT_FILTERS:
+            layout_val = self.LAYOUT_FILTERS[kwargs["layout"]]
+            if layout_val:
+                payload["imgsp"] = layout_val
+
+        return payload
+
+    def post_extract_results(self, results: list[ImagesResult]) -> list[ImagesResult]:
+        """Post-process image results to parse JSON data.
+        
+        Args:
+            results: Raw extracted results
+            
+        Returns:
+            Cleaned results with proper URLs and metadata
+        """
+        import json
+        from urllib.parse import unquote
+        
+        cleaned_results = []
+        
+        for result in results:
+            # Parse JSON data from the data attribute
+            if result.title and result.title.startswith('{'):
+                try:
+                    data = json.loads(result.title)
+                    
+                    # Extract title
+                    result.title = data.get('desc', '') or data.get('tit', '')
+                    
+                    # Extract URLs
+                    result.url = data.get('rurl', '')
+                    result.thumbnail = data.get('turl', '')
+                    result.image = data.get('turlL', '') or data.get('turl', '')
+                    
+                    # Extract dimensions
+                    result.width = int(data.get('imgW', 0))
+                    result.height = int(data.get('imgH', 0))
+                    
+                except (json.JSONDecodeError, KeyError, ValueError):
+                    # If JSON parsing fails, keep original data
+                    pass
+            
+            # Clean URLs if they exist
+            if result.url:
+                result.url = unquote(result.url)
+            if result.image:
+                result.image = unquote(result.image)
+            if result.thumbnail:
+                result.thumbnail = unquote(result.thumbnail)
+            
+            cleaned_results.append(result)
+        
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[ImagesResult] | None:
+        """Search Yahoo Images with pagination.
+        
+        Args:
+            query: Image search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter
+            page: Starting page
+            max_results: Maximum results to return
+            **kwargs: Additional filters (size, color, type, layout)
+            
+        Returns:
+            List of ImageResult objects
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 5)
+        
+        while current_page <= max_pages:
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+            
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+            
+            html_text = self.pre_process_html(html_text)
+            page_results = self.extract_results(html_text)
+            
+            if not page_results:
+                break
+            
+            results.extend(page_results)
+            
+            if max_results and len(results) >= max_results:
+                break
+            
+            current_page += 1
+        
+        results = self.post_extract_results(results)
+        
+        if max_results:
+            results = results[:max_results]
+        
+        return results if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        size: str | None = None,
+        color: str | None = None,
+        type_image: str | None = None,
+        layout: str | None = None,
+        license_image: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run image search and return results as dictionaries.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            size: Image size filter.
+            color: Color filter.
+            type_image: Image type filter.
+            layout: Layout filter.
+            license_image: License filter.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of image result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            size=size,
+            color=color,
+            type_image=type_image,
+            layout=layout,
+            license_image=license_image,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]

webscout/search/engines/yahoo/maps.py

@@ -0,0 +1,16 @@
+"""Yahoo maps search."""
+
+from __future__ import annotations
+
+from .base import YahooSearchEngine
+
+
+class YahooMaps(YahooSearchEngine):
+    """Yahoo maps search."""
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get maps results from Yahoo.
+
+        Not supported.
+        """
+        raise NotImplementedError("Yahoo does not support maps search")