webscout 8.2.9__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/engines/bing/news.py

@@ -0,0 +1,96 @@
+"""Bing news search."""
+
+from __future__ import annotations
+
+from time import sleep
+from typing import List, Optional
+from urllib.parse import urlencode
+
+from webscout.scout import Scout
+from webscout.search.results import NewsResult
+
+from .base import BingBase
+
+
+class BingNewsSearch(BingBase):
+    name = "bing"
+    category = "news"
+    def run(self, *args, **kwargs) -> List[NewsResult]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safe_map = {
+            "on": "Strict",
+            "moderate": "Moderate",
+            "off": "Off"
+        }
+        safe_map.get(safesearch.lower(), "Moderate")
+
+        # Bing news URL
+        url = f"{self.base_url}/news/infinitescrollajax"
+        params = {
+            'q': keywords,
+            'InfiniteScroll': '1',
+            'first': '1',
+            'SFX': '0',
+            'cc': region.lower(),
+            'setlang': self.lang.split('-')[0]
+        }
+
+        results = []
+        first = 1
+        sfx = 0
+
+        while len(results) < max_results:
+            params['first'] = str(first)
+            params['SFX'] = str(sfx)
+            full_url = f"{url}?{urlencode(params)}"
+
+            try:
+                response = self.session.get(full_url, timeout=self.timeout)
+                response.raise_for_status()
+                html = response.text
+            except Exception as e:
+                raise Exception(f"Failed to fetch news: {str(e)}")
+
+            if not html:
+                break
+
+            soup = Scout(html)
+            news_items = soup.select('div.newsitem')
+
+            for item in news_items:
+                if len(results) >= max_results:
+                    break
+
+                title = item.select_one('a.title')
+                snippet = item.select_one('div.snippet')
+                source = item.select_one('div.source')
+                date = item.select_one('span.date')
+
+                if title:
+                    news_result = NewsResult(
+                        title=title.get_text(strip=True),
+                        url=title.get('href', ''),
+                        body=snippet.get_text(strip=True) if snippet else '',
+                        source=source.get_text(strip=True) if source else '',
+                        date=date.get_text(strip=True) if date else '',
+                        image=""
+                    )
+                    results.append(news_result)
+
+            first += 10
+            sfx += 1
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return results[:max_results]

webscout/search/engines/bing/suggestions.py

@@ -0,0 +1,36 @@
+"""Bing suggestions search."""
+
+from __future__ import annotations
+
+from typing import List, Optional
+from urllib.parse import urlencode
+
+from .base import BingBase
+
+
+class BingSuggestionsSearch(BingBase):
+    name = "bing"
+    category = "suggestions"
+    def run(self, *args, **kwargs) -> List[str]:
+        query = args[0] if args else kwargs.get("query")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "en-US")
+
+        if not query:
+            raise ValueError("Query is mandatory")
+
+        params = {
+            "query": query,
+            "mkt": region
+        }
+        url = f"https://api.bing.com/osjson.aspx?{urlencode(params)}"
+
+        try:
+            response = self.session.get(url, timeout=self.timeout)
+            response.raise_for_status()
+            data = response.json()
+            # Bing suggestions API returns [query, [suggestions]]
+            if len(data) > 1 and isinstance(data[1], list):
+                return data[1]
+            return []
+        except Exception as e:
+            raise Exception(f"Failed to fetch suggestions: {str(e)}")

webscout/search/engines/bing/text.py

@@ -0,0 +1,109 @@
+"""Bing text search."""
+
+from __future__ import annotations
+
+from time import sleep
+from typing import List, Optional
+
+from webscout.scout import Scout
+from webscout.search.results import TextResult
+
+from .base import BingBase
+
+
+class BingTextSearch(BingBase):
+    name = "bing"
+    category = "text"
+    def run(self, *args, **kwargs) -> List[TextResult]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        args[1] if len(args) > 1 else kwargs.get("region", "us")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+        unique = kwargs.get("unique", True)
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safe_map = {
+            "on": "Strict",
+            "moderate": "Moderate",
+            "off": "Off"
+        }
+        safe_map.get(safesearch.lower(), "Moderate")
+
+        fetched_results = []
+        fetched_links = set()
+
+        def fetch_page(url):
+            try:
+                response = self.session.get(url, timeout=self.timeout)
+                response.raise_for_status()
+                return response.text
+            except Exception as e:
+                raise Exception(f"Failed to fetch page: {str(e)}")
+
+        # Get first page URL
+        url = f'{self.base_url}/search?q={keywords}&search=&form=QBLH'
+        urls_to_fetch = [url]
+
+        while len(fetched_results) < max_results and urls_to_fetch:
+            current_url = urls_to_fetch.pop(0)
+            html = fetch_page(current_url)
+            soup = Scout(html)
+
+            links = soup.select('ol#b_results > li.b_algo')
+            for link in links:
+                if len(fetched_results) >= max_results:
+                    break
+                title_tag = link.select_one('h2')
+                url_tag = link.select_one('h2 a')
+                text_tag = link.select_one('p')
+
+                if title_tag and url_tag and text_tag:
+                    title = title_tag.get_text(strip=True)
+                    href = url_tag.get('href', '')
+                    body = text_tag.get_text(strip=True)
+
+                    # Decode Bing URL if needed
+                    if href.startswith('/ck/a?'):
+                        # Simple unwrap, similar to bing.py
+                        from urllib.parse import parse_qs, urlparse
+                        try:
+                            parsed = urlparse(href)
+                            query_params = parse_qs(parsed.query)
+                            if 'u' in query_params:
+                                encoded_url = query_params['u'][0]
+                                if encoded_url.startswith('a1'):
+                                    encoded_url = encoded_url[2:]
+                                padding = len(encoded_url) % 4
+                                if padding:
+                                    encoded_url += '=' * (4 - padding)
+                                import base64
+                                decoded = base64.urlsafe_b64decode(encoded_url).decode()
+                                href = decoded
+                        except Exception:
+                            pass
+
+                    if unique and href in fetched_links:
+                        continue
+                    fetched_links.add(href)
+
+                    fetched_results.append(TextResult(
+                        title=title,
+                        href=href,
+                        body=body
+                    ))
+
+            # Get next page
+            next_page_tag = soup.select_one('div#b_content nav[role="navigation"] a.sb_pagN')
+            if next_page_tag and next_page_tag.get('href'):
+                next_url = self.base_url + next_page_tag['href']
+                urls_to_fetch.append(next_url)
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return fetched_results[:max_results]

webscout/search/engines/brave/__init__.py

@@ -0,0 +1,19 @@
+"""Brave search engine package."""
+
+from .base import BraveBase
+from .images import BraveImages
+from .news import BraveNews
+from .suggestions import BraveSuggestions, SuggestionResult, SuggestionsResponse
+from .text import BraveTextSearch
+from .videos import BraveVideos
+
+__all__ = [
+    "BraveBase",
+    "BraveImages",
+    "BraveNews",
+    "BraveSuggestions",
+    "BraveTextSearch",
+    "BraveVideos",
+    "SuggestionResult",
+    "SuggestionsResponse",
+]

webscout/search/engines/brave/base.py

@@ -0,0 +1,47 @@
+"""Base class for Brave search implementations."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from curl_cffi.requests import Session
+
+from ....litagent import LitAgent
+
+
+class BraveBase:
+    """Base class for Brave search engines."""
+
+    def __init__(
+        self,
+        timeout: int = 10,
+        proxies: dict[str, str] | None = None,
+        verify: bool = True,
+        lang: str = "en-US",
+        sleep_interval: float = 0.0,
+        impersonate: str = "chrome110",
+    ):
+        """Initialize Brave base client.
+
+        Args:
+            timeout: Timeout value for requests.
+            proxies: Dictionary of proxy settings.
+            verify: SSL verification flag.
+            lang: Language setting.
+            sleep_interval: Sleep interval between requests.
+            impersonate: Browser to impersonate.
+        """
+        self.timeout = timeout
+        self.proxies = proxies
+        self.verify = verify
+        self.lang = lang
+        self.sleep_interval = sleep_interval
+        self.base_url = "https://search.brave.com"
+        from typing import cast
+        self.session = Session(
+            proxies=cast(Any, proxies),
+            verify=verify,
+            timeout=timeout,
+            impersonate=cast(Any, impersonate),
+        )
+        self.session.headers.update(LitAgent().generate_fingerprint())

webscout/search/engines/brave/images.py

@@ -0,0 +1,213 @@
+"""Brave images search implementation."""
+
+from __future__ import annotations
+
+import base64
+from typing import Any, Mapping
+
+from webscout.scout import Scout
+
+from ....search.results import ImagesResult
+from .base import BraveBase
+
+
+class BraveImages(BraveBase):
+    """Brave images search engine."""
+
+    name = "brave_images"
+    provider = "brave"
+    category = "images"
+    search_method = "GET"
+    items_xpath = "//button[contains(@class, 'image-result')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//img/@alt",
+        "image": ".//img/@src",
+        "source": "string(.//div[contains(@class, 'metadata')])",
+    }
+    result_type = ImagesResult
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.search_url = f"{self.base_url}/images"
+
+    def build_payload(
+        self,
+        query: str,
+        region: str,
+        safesearch: str,
+        timelimit: str | None,
+        page: int = 1,
+        **_: Any,
+    ) -> dict[str, str]:
+        """Build query parameters for Brave image search."""
+
+        safesearch_map = {"on": "strict", "moderate": "moderate", "off": "off"}
+        payload: dict[str, str] = {
+            "q": query,
+            "source": "web",
+            "safesearch": safesearch_map.get(safesearch.lower(), "moderate"),
+            "page": str(page),
+        }
+
+        # Brave uses "offset" in multiples of 40 (page 1 => 0)
+        if page > 1:
+            payload["offset"] = str((page - 1) * 40)
+
+        if timelimit:
+            payload["tf"] = timelimit
+
+        if region:
+            payload["region"] = region
+
+        return payload
+
+    def run(self, *args: Any, **kwargs: Any) -> list[ImagesResult]:
+        """Run image search on Brave."""
+
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safesearch_value = self.build_payload(keywords, region, safesearch, None)["safesearch"]
+
+        fetched_results: list[ImagesResult] = []
+        fetched_urls: set[str] = set()
+
+        def fetch_page(url: str) -> str:
+            """Fetch page content."""
+            try:
+                response = self.session.get(url, timeout=self.timeout)
+                response.raise_for_status()
+                return response.text
+            except Exception as exc:  # pragma: no cover - network handling
+                raise Exception(f"Failed to fetch page: {str(exc)}") from exc
+
+        page = 1
+        while len(fetched_results) < max_results:
+            params = {
+                "q": keywords,
+                "page": str(page),
+                "safesearch": safesearch_value,
+            }
+            full_url = f"{self.search_url}?{'&'.join(f'{k}={v}' for k, v in params.items())}"
+            html = fetch_page(full_url)
+            soup = Scout(html)
+
+            img_containers = soup.select("button.image-result")
+            if not img_containers:
+                break
+
+            for container in img_containers:
+                if len(fetched_results) >= max_results:
+                    break
+
+                img_elem = container.select_one("img")
+                if not img_elem:
+                    continue
+
+                title = img_elem.get("alt", "")
+                image_url = img_elem.get("src", "")
+
+                source_elem = container.select_one('div[class*="metadata"]')
+                source = source_elem.get_text(strip=True) if source_elem else ""
+
+                width = 0
+                height = 0
+                style = container.get("style", "")
+                if style:
+                    if "--width:" in style:
+                        try:
+                            width = int(style.split("--width:")[1].split(";")[0].strip())
+                        except (ValueError, IndexError):
+                            pass
+                    if "--height:" in style:
+                        try:
+                            height = int(style.split("--height:")[1].split(";")[0].strip())
+                        except (ValueError, IndexError):
+                            pass
+
+                original_url = image_url
+                if image_url and "imgs.search.brave.com" in image_url:
+                    try:
+                        original_url = self._extract_original_url(image_url)
+                    except Exception:
+                        original_url = image_url
+
+                if image_url and image_url not in fetched_urls:
+                    fetched_urls.add(image_url)
+                    fetched_results.append(
+                        ImagesResult(
+                            title=title,
+                            image=image_url,
+                            thumbnail=image_url,
+                            url=original_url,
+                            height=height,
+                            width=width,
+                            source=source,
+                        )
+                    )
+
+            page += 1
+
+            if self.sleep_interval:
+                from time import sleep
+
+                sleep(self.sleep_interval)
+
+        return fetched_results[:max_results]
+
+    def extract_results(self, html_text: str) -> list[ImagesResult]:
+        """Parse Brave image search HTML into results."""
+
+        soup = Scout(html_text)
+        results: list[ImagesResult] = []
+        for container in soup.select("button.image-result"):
+            img_elem = container.select_one("img")
+            if not img_elem:
+                continue
+
+            title = img_elem.get("alt", "")
+            image_url = img_elem.get("src", "")
+            source_elem = container.select_one('div[class*="metadata"]')
+            source = source_elem.get_text(strip=True) if source_elem else ""
+
+            results.append(
+                ImagesResult(
+                    title=title,
+                    image=image_url,
+                    thumbnail=image_url,
+                    url=image_url,
+                    height=0,
+                    width=0,
+                    source=source,
+                )
+            )
+
+        return results
+
+    def _extract_original_url(self, proxy_url: str) -> str:
+        """Extract the original image URL from the Brave proxy URL."""
+
+        try:
+            parts = proxy_url.split("/")
+            base64_part = parts[-1] if parts else ""
+            if base64_part and len(base64_part) > 10:
+                normalized = base64_part.replace("-", "+").replace("_", "/").replace(" ", "")
+                padding = 4 - (len(normalized) % 4)
+                if padding and padding != 4:
+                    normalized += "=" * padding
+
+                decoded = base64.b64decode(normalized).decode("utf-8", errors="ignore")
+                if decoded.startswith("http"):
+                    return decoded
+        except Exception:
+            pass
+
+        return proxy_url