webscout 8.2.9__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/engines/brave/text.py

@@ -0,0 +1,167 @@
+"""Brave text search."""
+
+from __future__ import annotations
+
+from time import sleep
+from typing import List, Optional
+
+from webscout.scout import Scout
+
+from ....search.results import TextResult
+from .base import BraveBase
+
+
+class BraveTextSearch(BraveBase):
+    """Brave text/web search."""
+
+    name = "brave"
+    category = "text"
+
+    def run(self, *args, **kwargs) -> List[TextResult]:
+        """Perform text search on Brave using offset pagination.
+
+        Uses server-rendered HTML and parses result containers with CSS selectors.
+        """
+        from typing import cast
+
+        keywords = args[0] if args else kwargs.get("keywords")
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+        if max_results is None:
+            max_results = 10
+
+        safesearch_map = {"on": "strict", "moderate": "moderate", "off": "off"}
+        safesearch_value = safesearch_map.get(safesearch.lower(), "moderate")
+
+        start_offset = int(kwargs.get("start_offset", 0))
+        offset = start_offset
+
+        fetched_results: List[TextResult] = []
+        fetched_hrefs: set[str] = set()
+
+        def fetch_html(params: dict) -> str:
+            url = f"{self.base_url}/search"
+            # Merge session headers to include fingerprint (User-Agent, etc.)
+            headers = dict(self.session.headers) if getattr(self, "session", None) else {}
+            headers.update({
+                "Referer": "https://search.brave.com/",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+            })
+            attempts = 3
+            backoff = 1.0
+            last_exc: Exception | None = None
+            for attempt in range(attempts):
+                try:
+                    resp = self.session.get(url, params=params, headers=headers, timeout=self.timeout)
+                    resp.raise_for_status()
+                    return resp.text
+                except Exception as exc:  # network or HTTP errors
+                    last_exc = exc
+                    # If it's a 429 / transient server error, back off and retry
+                    try:
+                        code = getattr(exc, "code", None) or getattr(exc, "status_code", None)
+                    except Exception:
+                        code = None
+                    if code in (429, 500, 502, 503, 504):
+                        sleep(backoff)
+                        backoff *= 2
+                        continue
+                    # As a last attempt, try a simple GET without params appended (fallback)
+                    try:
+                        fallback_resp = self.session.get(url + "?" + "&".join(f"{k}={v}" for k, v in params.items()), headers=headers, timeout=self.timeout)
+                        fallback_resp.raise_for_status()
+                        return fallback_resp.text
+                    except Exception:
+                        # Final fallback: try using the requests library (less features but sometimes works)
+                        try:
+                            import requests
+
+                            r = requests.get(url, params=params, headers=headers, timeout=self.timeout, verify=getattr(self, "verify", True))
+                            r.raise_for_status()
+                            return r.text
+                        except Exception:
+                            raise Exception(f"Failed to GET {url} with {params}: {exc}") from exc
+            raise Exception(f"Failed to GET {url} after retries: {last_exc}") from last_exc
+
+        # Pagination: offset param is a 0-based page index
+        while len(fetched_results) < max_results:
+            params = {"q": keywords, "source": "web", "offset": str(offset), "spellcheck": "0"}
+            if safesearch_value:
+                params["safesearch"] = safesearch_value
+
+                html = fetch_html(params)
+            # Parse and extract results using helper
+            page_results = self._parse_results_from_html(html)
+
+            if not page_results:
+                break
+
+            for res in page_results:
+                if len(fetched_results) >= max_results:
+                    break
+                if res.href and res.href not in fetched_hrefs:
+                    fetched_hrefs.add(res.href)
+                    fetched_results.append(res)
+
+            offset += 1
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return fetched_results[:max_results]
+
+    def _parse_results_from_html(self, html: str) -> List[TextResult]:
+        """Parse HTML and extract text search results.
+
+        This method is separated for testability.
+        """
+        soup = Scout(html)
+        containers = soup.select("div.result-content")
+        results: List[TextResult] = []
+
+        for container in containers:
+            a_elem = container.select_one("a[href]")
+            # Title may be in .title.search-snippet-title or nested inside the anchor
+            title_elem = container.select_one(".title.search-snippet-title") or (a_elem.select_one(".title") if a_elem else None)
+
+            # Try multiple snippet locations: inside container, sibling .snippet, parent fallbacks
+            body = ""
+            candidates = []
+            # inside container
+            candidates.append(container.select_one(".generic-snippet"))
+            candidates.append(container.select_one(".snippet .generic-snippet"))
+            candidates.append(container.select_one(".description"))
+            candidates.append(container.select_one(".result-snippet"))
+            candidates.append(container.select_one("p"))
+
+            # sibling .snippet
+            try:
+                fn = getattr(container, "find_next_sibling", None)
+                if callable(fn):
+                    sib = fn("div", class_="snippet")
+                    if sib:
+                        candidates.append(sib.select_one(".generic-snippet"))
+            except Exception:
+                pass
+
+            # parent-level fallbacks
+            if container.parent:
+                candidates.append(container.parent.select_one(".snippet .generic-snippet"))
+                candidates.append(container.parent.select_one(".generic-snippet"))
+
+            for c in candidates:
+                if c:
+                    text = c.get_text(strip=True)
+                    if text:
+                        body = text
+                        break
+
+            if a_elem and title_elem:
+                href = a_elem.get("href", "").strip()
+                title = title_elem.get_text(strip=True)
+                results.append(TextResult(title=title, href=href, body=body))
+
+        return results

webscout/search/engines/brave/videos.py

@@ -0,0 +1,364 @@
+"""Brave videos search implementation.
+
+This module provides a Brave video search engine that parses HTML responses
+from Brave Search to extract video results from YouTube and other video platforms.
+
+Example:
+    >>> from webscout.search.engines.brave.videos import BraveVideos
+    >>> searcher = BraveVideos()
+    >>> results = searcher.run("python tutorial", max_results=10)
+    >>> for video in results:
+    ...     print(f"{video.title} - {video.url}")
+"""
+
+from __future__ import annotations
+
+from time import sleep
+from typing import Any
+
+from webscout.scout import Scout
+
+from ....search.results import VideosResult
+from .base import BraveBase
+
+
+class BraveVideos(BraveBase):
+    """Brave videos search engine.
+
+    Searches Brave Video Search and parses HTML responses to extract
+    video results including title, URL, thumbnail, duration, channel,
+    description, publish date, and view count.
+
+    Attributes:
+        name: Engine identifier name.
+        provider: Provider identifier.
+        category: Search category type.
+        search_url: Base URL for video search.
+    """
+
+    name = "brave_videos"
+    provider = "brave"
+    category = "videos"
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initialize Brave videos search client.
+
+        Args:
+            *args: Positional arguments passed to BraveBase.
+            **kwargs: Keyword arguments passed to BraveBase.
+        """
+        super().__init__(*args, **kwargs)
+        self.search_url = f"{self.base_url}/videos"
+
+    def run(self, *args: Any, **kwargs: Any) -> list[VideosResult]:
+        """Run video search on Brave.
+
+        Args:
+            *args: Positional arguments. First arg is the search query.
+            **kwargs: Keyword arguments including:
+                - keywords: Search query string.
+                - region: Region code (e.g., 'us-en').
+                - safesearch: Safe search level ('on', 'moderate', 'off').
+                - max_results: Maximum number of results to return.
+                - timelimit: Time filter for results.
+
+        Returns:
+            List of VideosResult objects containing video information.
+
+        Raises:
+            ValueError: If no keywords are provided.
+            Exception: If the HTTP request fails.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+        timelimit = kwargs.get("timelimit")
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safesearch_map = {"on": "strict", "moderate": "moderate", "off": "off"}
+        safesearch_str = str(safesearch).lower() if safesearch else "moderate"
+        safesearch_value = safesearch_map.get(safesearch_str, "moderate")
+
+        fetched_results: list[VideosResult] = []
+        fetched_urls: set[str] = set()
+
+        offset = 0
+        while len(fetched_results) < max_results:
+            params: dict[str, str] = {
+                "q": keywords,
+                "source": "web",
+                "safesearch": safesearch_value,
+                "spellcheck": "0",
+            }
+
+            if offset > 0:
+                params["offset"] = str(offset)
+
+            if timelimit:
+                params["tf"] = timelimit
+
+            if region:
+                params["region"] = region
+
+            html = self._fetch_page(params)
+            page_results = self._parse_results_from_html(html)
+
+            if not page_results:
+                break
+
+            for result in page_results:
+                if len(fetched_results) >= max_results:
+                    break
+                if result.url and result.url not in fetched_urls:
+                    fetched_urls.add(result.url)
+                    fetched_results.append(result)
+
+            offset += 1
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return fetched_results[:max_results]
+
+    def _fetch_page(self, params: dict[str, str]) -> str:
+        """Fetch HTML page from Brave videos search.
+
+        Args:
+            params: Query parameters for the request.
+
+        Returns:
+            HTML content of the response.
+
+        Raises:
+            Exception: If the request fails after retries.
+        """
+        headers = dict(self.session.headers) if getattr(self, "session", None) else {}
+        headers.update({
+            "Referer": "https://search.brave.com/",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+        })
+
+        attempts = 3
+        backoff = 1.0
+        last_exc: Exception | None = None
+
+        for attempt in range(attempts):
+            try:
+                resp = self.session.get(
+                    self.search_url,
+                    params=params,
+                    headers=headers,
+                    timeout=self.timeout,
+                )
+                resp.raise_for_status()
+                return resp.text
+            except Exception as exc:
+                last_exc = exc
+                try:
+                    code = getattr(exc, "code", None) or getattr(exc, "status_code", None)
+                except Exception:
+                    code = None
+                if code in (429, 500, 502, 503, 504):
+                    sleep(backoff)
+                    backoff *= 2
+                    continue
+                raise Exception(f"Failed to GET {self.search_url}: {exc}") from exc
+
+        raise Exception(f"Failed to GET {self.search_url} after retries: {last_exc}") from last_exc
+
+    def _parse_results_from_html(self, html: str) -> list[VideosResult]:
+        """Parse HTML and extract video search results.
+
+        Args:
+            html: Raw HTML content from Brave videos search.
+
+        Returns:
+            List of VideosResult objects parsed from the HTML.
+        """
+        soup = Scout(html)
+        results: list[VideosResult] = []
+
+        # Video results are in div.video-snippet containers
+        containers = soup.select("div.video-snippet")
+
+        for container in containers:
+            try:
+                result = self._parse_video_container(container)
+                if result and result.url:
+                    results.append(result)
+            except Exception:
+                # Skip malformed results
+                continue
+
+        return results
+
+    def _parse_video_container(self, container: Any) -> VideosResult | None:
+        """Parse a single video container element.
+
+        Args:
+            container: Scout element representing a video result container.
+
+        Returns:
+            VideosResult object or None if parsing fails.
+        """
+        # Get video URL from main link
+        url = ""
+        link_elem = container.select_one("a[href]")
+        if link_elem:
+            url = link_elem.get("href", "").strip()
+
+        if not url:
+            return None
+
+        # Get thumbnail
+        thumbnail = ""
+        thumb_elem = container.select_one("img.thumb, img.video-thumb")
+        if thumb_elem:
+            thumbnail = thumb_elem.get("src", "").strip()
+
+        # Get duration
+        duration = ""
+        duration_elem = container.select_one(".over-thumbnail-info.duration")
+        if duration_elem:
+            duration = duration_elem.get_text(strip=True)
+
+        # Get title from the result header
+        title = ""
+        title_elem = container.select_one(".snippet-title")
+        if title_elem:
+            title = title_elem.get_text(strip=True)
+
+        # Get channel/uploader
+        uploader = ""
+        channel_elem = container.select_one(".attr.channel")
+        if channel_elem:
+            uploader = channel_elem.get_text(strip=True)
+
+        # Get provider (e.g., YouTube)
+        provider = ""
+        netloc_elem = container.select_one(".netloc.attr")
+        if netloc_elem:
+            provider = netloc_elem.get_text(strip=True)
+
+        # Get description
+        description = ""
+        desc_elem = container.select_one(".snippet-description, p.desc")
+        if desc_elem:
+            description = desc_elem.get_text(strip=True)
+
+        # Get publish date
+        published = ""
+        metrics_elem = container.select_one(".metrics")
+        if metrics_elem:
+            date_elem = metrics_elem.select_one(".attr:first-child")
+            if date_elem:
+                published = date_elem.get_text(strip=True)
+
+        # Get view count from metrics
+        view_count = 0
+        if metrics_elem:
+            view_elems = metrics_elem.select(".attr")
+            for elem in view_elems:
+                text = elem.get_text(strip=True)
+                # Look for view count patterns (e.g., "1.31M", "34.1M", "17K")
+                if any(c.isdigit() for c in text) and not any(
+                    month in text.lower()
+                    for month in [
+                        "jan", "feb", "mar", "apr", "may", "jun",
+                        "jul", "aug", "sep", "oct", "nov", "dec",
+                        "hour", "day", "week", "month", "year", "ago",
+                    ]
+                ):
+                    view_count = self._parse_view_count(text)
+                    break
+
+        return VideosResult(
+            title=title,
+            url=url,
+            thumbnail=thumbnail,
+            duration=duration,
+            uploader=uploader,
+            publisher=uploader,
+            provider=provider,
+            description=description,
+            published=published,
+            statistics={"views": view_count} if view_count else {},
+            content=description,
+            images={"thumbnail": thumbnail} if thumbnail else {},
+        )
+
+    def _parse_view_count(self, text: str) -> int:
+        """Parse view count from text like '1.31M' or '17K'.
+
+        Args:
+            text: View count string with potential suffixes.
+
+        Returns:
+            Integer view count, or 0 if parsing fails.
+        """
+        try:
+            text = text.strip().upper()
+            multiplier = 1
+
+            if text.endswith("K"):
+                multiplier = 1000
+                text = text[:-1]
+            elif text.endswith("M"):
+                multiplier = 1_000_000
+                text = text[:-1]
+            elif text.endswith("B"):
+                multiplier = 1_000_000_000
+                text = text[:-1]
+
+            return int(float(text) * multiplier)
+        except (ValueError, TypeError):
+            return 0
+
+    def extract_results(self, html_text: str) -> list[VideosResult]:
+        """Extract video results from HTML text.
+
+        This is an alias for _parse_results_from_html for API consistency.
+
+        Args:
+            html_text: Raw HTML content from Brave videos search.
+
+        Returns:
+            List of VideosResult objects parsed from the HTML.
+        """
+        return self._parse_results_from_html(html_text)
+
+
+if __name__ == "__main__":
+    # Test the BraveVideos search
+    print("Testing BraveVideos search...")
+
+    searcher = BraveVideos(timeout=15)
+
+    try:
+        # Test basic search
+        results = searcher.run("python programming tutorial", max_results=5)
+
+        print(f"\nFound {len(results)} video results:\n")
+        for i, video in enumerate(results, 1):
+            print(f"{i}. {video.title}")
+            print(f"   URL: {video.url}")
+            print(f"   Duration: {video.duration}")
+            print(f"   Channel: {video.uploader}")
+            print(f"   Provider: {video.provider}")
+            print(f"   Published: {video.published}")
+            if video.statistics:
+                print(f"   Views: {video.statistics.get('views', 'N/A')}")
+            print(f"   Thumbnail: {video.thumbnail[:80]}..." if video.thumbnail else "")
+            print()
+
+    except Exception as e:
+        print(f"Error during search: {e}")
+        import traceback
+        traceback.print_exc()

webscout/search/engines/duckduckgo/__init__.py

@@ -0,0 +1,25 @@
+"""DuckDuckGo search engines package."""
+
+from .answers import DuckDuckGoAnswers
+from .base import DuckDuckGoBase
+from .images import DuckDuckGoImages
+from .maps import DuckDuckGoMaps
+from .news import DuckDuckGoNews
+from .suggestions import DuckDuckGoSuggestions
+from .text import DuckDuckGoTextSearch
+from .translate import DuckDuckGoTranslate
+from .videos import DuckDuckGoVideos
+from .weather import DuckDuckGoWeather
+
+__all__ = [
+    "DuckDuckGoBase",
+    "DuckDuckGoTextSearch",
+    "DuckDuckGoImages",
+    "DuckDuckGoVideos",
+    "DuckDuckGoNews",
+    "DuckDuckGoAnswers",
+    "DuckDuckGoSuggestions",
+    "DuckDuckGoMaps",
+    "DuckDuckGoTranslate",
+    "DuckDuckGoWeather",
+]

webscout/search/engines/duckduckgo/answers.py

@@ -0,0 +1,80 @@
+"""DuckDuckGo answers search."""
+
+from __future__ import annotations
+
+from .base import DuckDuckGoBase
+
+
+class DuckDuckGoAnswers(DuckDuckGoBase):
+    """DuckDuckGo instant answers."""
+
+    name = "duckduckgo"
+    category = "answers"
+
+    def run(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Get instant answers from DuckDuckGo.
+
+        Args:
+            keywords: Search query.
+
+        Returns:
+            List of answer dictionaries.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+
+        assert keywords, "keywords is mandatory"
+
+        payload = {
+            "q": f"what is {keywords}",
+            "format": "json",
+        }
+        resp_content = self._get_url("GET", "https://api.duckduckgo.com/", params=payload).content
+        page_data = self.json_loads(resp_content)
+
+        results = []
+        answer = page_data.get("AbstractText")
+        url = page_data.get("AbstractURL")
+        if answer:
+            results.append(
+                {
+                    "icon": None,
+                    "text": answer,
+                    "topic": None,
+                    "url": url,
+                }
+            )
+
+        # related
+        payload = {
+            "q": f"{keywords}",
+            "format": "json",
+        }
+        resp_content = self._get_url("GET", "https://api.duckduckgo.com/", params=payload).content
+        resp_json = self.json_loads(resp_content)
+        page_data = resp_json.get("RelatedTopics", [])
+
+        for row in page_data:
+            topic = row.get("Name")
+            if not topic:
+                icon = row["Icon"].get("URL")
+                results.append(
+                    {
+                        "icon": f"https://duckduckgo.com{icon}" if icon else "",
+                        "text": row["Text"],
+                        "topic": None,
+                        "url": row["FirstURL"],
+                    }
+                )
+            else:
+                for subrow in row["Topics"]:
+                    icon = subrow["Icon"].get("URL")
+                    results.append(
+                        {
+                            "icon": f"https://duckduckgo.com{icon}" if icon else "",
+                            "text": subrow["Text"],
+                            "topic": topic,
+                            "url": subrow["FirstURL"],
+                        }
+                    )
+
+        return results