webscout 2025.10.14.1__py3-none-any.whl → 2025.10.16__py3-none-any.whl

webscout/search/engines/bing/text.py

@@ -0,0 +1,106 @@
+"""Bing text search."""
+
+from __future__ import annotations
+
+from typing import Dict, List
+from urllib.parse import urlencode
+from time import sleep
+
+from .base import BingBase
+from webscout.scout import Scout
+
+
+class BingTextSearch(BingBase):
+    def run(self, *args, **kwargs) -> List[Dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+        unique = kwargs.get("unique", True)
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safe_map = {
+            "on": "Strict",
+            "moderate": "Moderate",
+            "off": "Off"
+        }
+        safe = safe_map.get(safesearch.lower(), "Moderate")
+
+        fetched_results = []
+        fetched_links = set()
+
+        def fetch_page(url):
+            try:
+                response = self.session.get(url, timeout=self.timeout)
+                response.raise_for_status()
+                return response.text
+            except Exception as e:
+                raise Exception(f"Failed to fetch page: {str(e)}")
+
+        # Get first page URL
+        url = f'{self.base_url}/search?q={keywords}&search=&form=QBLH'
+        urls_to_fetch = [url]
+
+        while len(fetched_results) < max_results and urls_to_fetch:
+            current_url = urls_to_fetch.pop(0)
+            html = fetch_page(current_url)
+            soup = Scout(html)
+
+            links = soup.select('ol#b_results > li.b_algo')
+            for link in links:
+                if len(fetched_results) >= max_results:
+                    break
+                title_tag = link.select_one('h2')
+                url_tag = link.select_one('h2 a')
+                text_tag = link.select_one('p')
+
+                if title_tag and url_tag and text_tag:
+                    title = title_tag.get_text(strip=True)
+                    href = url_tag.get('href', '')
+                    body = text_tag.get_text(strip=True)
+
+                    # Decode Bing URL if needed
+                    if href.startswith('/ck/a?'):
+                        # Simple unwrap, similar to bing.py
+                        from urllib.parse import parse_qs, urlparse
+                        try:
+                            parsed = urlparse(href)
+                            query_params = parse_qs(parsed.query)
+                            if 'u' in query_params:
+                                encoded_url = query_params['u'][0]
+                                if encoded_url.startswith('a1'):
+                                    encoded_url = encoded_url[2:]
+                                padding = len(encoded_url) % 4
+                                if padding:
+                                    encoded_url += '=' * (4 - padding)
+                                import base64
+                                decoded = base64.urlsafe_b64decode(encoded_url).decode()
+                                href = decoded
+                        except:
+                            pass
+
+                    if unique and href in fetched_links:
+                        continue
+                    fetched_links.add(href)
+
+                    fetched_results.append({
+                        'title': title,
+                        'href': href,
+                        'body': body
+                    })
+
+            # Get next page
+            next_page_tag = soup.select_one('div#b_content nav[role="navigation"] a.sb_pagN')
+            if next_page_tag and next_page_tag.get('href'):
+                next_url = self.base_url + next_page_tag['href']
+                urls_to_fetch.append(next_url)
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return fetched_results[:max_results]

webscout/search/engines/duckduckgo/maps.py

@@ -1,12 +1,25 @@
 from __future__ import annotations
 
 from decimal import Decimal
+from math import sqrt
 
 from ....exceptions import WebscoutE
 from .base import DuckDuckGoBase
 
 
 class DuckDuckGoMaps(DuckDuckGoBase):
+    def _calculate_distance(self, lat_t: Decimal, lon_l: Decimal, lat_b: Decimal, lon_r: Decimal) -> float:
+        """Calculate the Euclidean distance between top-left and bottom-right corners of bounding box."""
+        # Convert to float for math operations
+        lat_t_f = float(lat_t)
+        lon_l_f = float(lon_l)
+        lat_b_f = float(lat_b)
+        lon_r_f = float(lon_r)
+        
+        # Calculate Euclidean distance
+        distance = sqrt((lat_t_f - lat_b_f) ** 2 + (lon_r_f - lon_l_f) ** 2)
+        return distance
+
     def run(self, *args, **kwargs) -> list[dict[str, str]]:
         keywords = args[0] if args else kwargs.get("keywords")
         place = args[1] if len(args) > 1 else kwargs.get("place")

webscout/search/engines/yahoo/__init__.py

@@ -0,0 +1,41 @@
+"""Yahoo search engines package.
+
+This package provides comprehensive Yahoo search functionality including:
+- Text search with multi-page pagination
+- Image search with advanced filters
+- Video search with quality and length filters
+- News search with time filtering
+- Search suggestions/autocomplete
+
+All engines support:
+- Human-like browsing through multiple pages
+- Rich metadata extraction
+- Filter support
+- Clean result formatting
+
+Example:
+    >>> from webscout.search.engines.yahoo import YahooText
+    >>> 
+    >>> # Search with automatic pagination
+    >>> searcher = YahooText()
+    >>> results = searcher.search("python programming", max_results=50)
+    >>> 
+    >>> for result in results:
+    ...     print(f"{result.title}: {result.url}")
+"""
+
+from .base import YahooSearchEngine
+from .images import YahooImages
+from .news import YahooNews
+from .suggestions import YahooSuggestions
+from .text import YahooText
+from .videos import YahooVideos
+
+__all__ = [
+    "YahooSearchEngine",
+    "YahooText",
+    "YahooImages",
+    "YahooVideos",
+    "YahooNews",
+    "YahooSuggestions",
+]

webscout/search/engines/yahoo/answers.py

@@ -0,0 +1,16 @@
+"""Yahoo answers search."""
+
+from __future__ import annotations
+
+from .base import YahooSearchEngine
+
+
+class YahooAnswers(YahooSearchEngine):
+    """Yahoo instant answers."""
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get instant answers from Yahoo.
+
+        Not supported.
+        """
+        raise NotImplementedError("Yahoo does not support instant answers")

webscout/search/engines/yahoo/base.py

@@ -0,0 +1,34 @@
+"""Base class for Yahoo search engines."""
+
+from __future__ import annotations
+
+from secrets import token_urlsafe
+from typing import Any, Generic, TypeVar
+
+from ...base import BaseSearchEngine
+
+T = TypeVar("T")
+
+class YahooSearchEngine(BaseSearchEngine[T], Generic[T]):
+    """Base class for Yahoo search engines.
+    
+    Yahoo search is powered by Bing but has its own interface.
+    All Yahoo searches use dynamic URLs with tokens for tracking.
+    """
+
+    provider = "yahoo"
+    _base_url = "https://search.yahoo.com"
+    
+    def generate_ylt_token(self) -> str:
+        """Generate Yahoo _ylt tracking token."""
+        return token_urlsafe(24 * 3 // 4)
+    
+    def generate_ylu_token(self) -> str:
+        """Generate Yahoo _ylu tracking token."""
+        return token_urlsafe(47 * 3 // 4)
+    
+    def build_search_url(self, base_path: str) -> str:
+        """Build search URL with tracking tokens."""
+        ylt = self.generate_ylt_token()
+        ylu = self.generate_ylu_token()
+        return f"{self._base_url}/{base_path};_ylt={ylt};_ylu={ylu}"

webscout/search/engines/yahoo/images.py

@@ -0,0 +1,324 @@
+"""Yahoo image search engine with advanced filters."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import urljoin
+
+from .base import YahooSearchEngine
+from ...results import ImagesResult
+
+
+class YahooImages(YahooSearchEngine[ImagesResult]):
+    """Yahoo image search engine with filter support.
+    
+    Features:
+    - Size filters (small, medium, large, wallpaper)
+    - Color filters (color, bw, red, orange, yellow, etc.)
+    - Type filters (photo, clipart, lineart, transparent)
+    - Layout filters (square, wide, tall)
+    - Time filters
+    - Pagination support
+    
+    Note: Yahoo does not support reverse image search (searching by image upload/URL).
+    For reverse image search functionality, use Google Images or Bing Images instead.
+    """
+
+    name = "yahoo"
+    category = "images"
+
+    search_url = "https://images.search.yahoo.com/search/images"
+    search_method = "GET"
+    search_headers = {
+        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1"
+    }
+
+    # XPath selectors
+    items_xpath = "//li[contains(@class, 'ld')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": "@data",
+        "image": "@data", 
+        "thumbnail": "@data",
+        "url": "@data",
+        "source": "@data",
+        "width": "@data",
+        "height": "@data",
+    }
+
+    # Filter mappings
+    SIZE_FILTERS = {
+        "small": "small",
+        "medium": "medium", 
+        "large": "large",
+        "wallpaper": "wallpaper",
+        "all": "",
+    }
+
+    COLOR_FILTERS = {
+        "color": "color",
+        "bw": "bw",
+        "black": "black",
+        "white": "white",
+        "red": "red",
+        "orange": "orange",
+        "yellow": "yellow",
+        "green": "green",
+        "teal": "teal",
+        "blue": "blue",
+        "purple": "purple",
+        "pink": "pink",
+        "brown": "brown",
+        "gray": "gray",
+        "all": "",
+    }
+
+    TYPE_FILTERS = {
+        "photo": "photo",
+        "clipart": "clipart",
+        "lineart": "linedrawing",
+        "transparent": "transparent",
+        "gif": "animatedgif",
+        "all": "",
+    }
+
+    LAYOUT_FILTERS = {
+        "square": "square",
+        "wide": "wide",
+        "tall": "tall",
+        "all": "",
+    }
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Build image search payload with filters.
+        
+        Args:
+            query: Search query
+            region: Region code
+            safesearch: Safe search level (on/moderate/off)
+            timelimit: Time filter (d, w, m)
+            page: Page number
+            **kwargs: Additional filters including:
+                - size: Image size filter
+                - color: Color filter
+                - type: Image type filter
+                - layout: Layout/aspect ratio filter
+                - license: Usage rights filter
+                
+        Returns:
+            Query parameters dictionary
+        """
+        payload = {
+            "p": query,
+        }
+
+        # Pagination - Yahoo images use 'b' parameter
+        if page > 1:
+            # Each page shows approximately 40 images
+            payload["b"] = f"{(page - 1) * 40 + 1}"
+
+        # Safe search
+        if safesearch == "on":
+            payload["safe"] = "active"
+        elif safesearch == "off":
+            payload["safe"] = "off"
+
+        # Time filter
+        if timelimit:
+            time_map = {
+                "d": "1d",  # Past 24 hours
+                "w": "1w",  # Past week
+                "m": "1m",  # Past month
+            }
+            if timelimit in time_map:
+                payload["age"] = time_map[timelimit]
+
+        # Size filter
+        if "size" in kwargs and kwargs["size"] in self.SIZE_FILTERS:
+            size_val = self.SIZE_FILTERS[kwargs["size"]]
+            if size_val:
+                payload["imgsz"] = size_val
+
+        # Color filter
+        if "color" in kwargs and kwargs["color"] in self.COLOR_FILTERS:
+            color_val = self.COLOR_FILTERS[kwargs["color"]]
+            if color_val:
+                payload["imgc"] = color_val
+
+        # Type filter
+        if "type" in kwargs and kwargs["type"] in self.TYPE_FILTERS:
+            type_val = self.TYPE_FILTERS[kwargs["type"]]
+            if type_val:
+                payload["imgt"] = type_val
+
+        # Layout filter
+        if "layout" in kwargs and kwargs["layout"] in self.LAYOUT_FILTERS:
+            layout_val = self.LAYOUT_FILTERS[kwargs["layout"]]
+            if layout_val:
+                payload["imgsp"] = layout_val
+
+        return payload
+
+    def post_extract_results(self, results: list[ImagesResult]) -> list[ImagesResult]:
+        """Post-process image results to parse JSON data.
+        
+        Args:
+            results: Raw extracted results
+            
+        Returns:
+            Cleaned results with proper URLs and metadata
+        """
+        import json
+        from urllib.parse import unquote
+        
+        cleaned_results = []
+        
+        for result in results:
+            # Parse JSON data from the data attribute
+            if result.title and result.title.startswith('{'):
+                try:
+                    data = json.loads(result.title)
+                    
+                    # Extract title
+                    result.title = data.get('desc', '') or data.get('tit', '')
+                    
+                    # Extract URLs
+                    result.url = data.get('rurl', '')
+                    result.thumbnail = data.get('turl', '')
+                    result.image = data.get('turlL', '') or data.get('turl', '')
+                    
+                    # Extract dimensions
+                    result.width = int(data.get('imgW', 0))
+                    result.height = int(data.get('imgH', 0))
+                    
+                except (json.JSONDecodeError, KeyError, ValueError):
+                    # If JSON parsing fails, keep original data
+                    pass
+            
+            # Clean URLs if they exist
+            if result.url:
+                result.url = unquote(result.url)
+            if result.image:
+                result.image = unquote(result.image)
+            if result.thumbnail:
+                result.thumbnail = unquote(result.thumbnail)
+            
+            cleaned_results.append(result)
+        
+        return cleaned_results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        max_results: int | None = None,
+        **kwargs: Any,
+    ) -> list[ImagesResult] | None:
+        """Search Yahoo Images with pagination.
+        
+        Args:
+            query: Image search query
+            region: Region code
+            safesearch: Safe search level
+            timelimit: Time filter
+            page: Starting page
+            max_results: Maximum results to return
+            **kwargs: Additional filters (size, color, type, layout)
+            
+        Returns:
+            List of ImageResult objects
+        """
+        results = []
+        current_page = page
+        max_pages = kwargs.get("max_pages", 5)
+        
+        while current_page <= max_pages:
+            payload = self.build_payload(
+                query=query,
+                region=region,
+                safesearch=safesearch,
+                timelimit=timelimit,
+                page=current_page,
+                **kwargs
+            )
+            
+            html_text = self.request(self.search_method, self.search_url, params=payload)
+            if not html_text:
+                break
+            
+            html_text = self.pre_process_html(html_text)
+            page_results = self.extract_results(html_text)
+            
+            if not page_results:
+                break
+            
+            results.extend(page_results)
+            
+            if max_results and len(results) >= max_results:
+                break
+            
+            current_page += 1
+        
+        results = self.post_extract_results(results)
+        
+        if max_results:
+            results = results[:max_results]
+        
+        return results if results else None
+
+    def run(
+        self,
+        keywords: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        size: str | None = None,
+        color: str | None = None,
+        type_image: str | None = None,
+        layout: str | None = None,
+        license_image: str | None = None,
+        max_results: int | None = None,
+    ) -> list[dict[str, str]]:
+        """Run image search and return results as dictionaries.
+        
+        Args:
+            keywords: Search query.
+            region: Region code.
+            safesearch: Safe search level.
+            timelimit: Time filter.
+            size: Image size filter.
+            color: Color filter.
+            type_image: Image type filter.
+            layout: Layout filter.
+            license_image: License filter.
+            max_results: Maximum number of results.
+            
+        Returns:
+            List of image result dictionaries.
+        """
+        results = self.search(
+            query=keywords,
+            region=region,
+            safesearch=safesearch,
+            timelimit=timelimit,
+            size=size,
+            color=color,
+            type_image=type_image,
+            layout=layout,
+            license_image=license_image,
+            max_results=max_results,
+        )
+        if results is None:
+            return []
+        return [result.to_dict() for result in results]

webscout/search/engines/yahoo/maps.py

@@ -0,0 +1,16 @@
+"""Yahoo maps search."""
+
+from __future__ import annotations
+
+from .base import YahooSearchEngine
+
+
+class YahooMaps(YahooSearchEngine):
+    """Yahoo maps search."""
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get maps results from Yahoo.
+
+        Not supported.
+        """
+        raise NotImplementedError("Yahoo does not support maps search")