webscout 8.2.9__py3-none-any.whl → 2026.1.19__py3-none-any.whl

webscout/search/engines/brave/news.py

@@ -0,0 +1,353 @@
+"""Brave news search implementation.
+
+This module provides a Brave news search engine that parses HTML responses
+from Brave Search to extract news article results.
+
+Example:
+    >>> from webscout.search.engines.brave.news import BraveNews
+    >>> searcher = BraveNews()
+    >>> results = searcher.run("technology news", max_results=10)
+    >>> for article in results:
+    ...     print(f"{article.title} - {article.source}")
+"""
+
+from __future__ import annotations
+
+from time import sleep
+from typing import Any
+
+from webscout.scout import Scout
+
+from ....search.results import NewsResult
+from .base import BraveBase
+
+
+class BraveNews(BraveBase):
+    """Brave news search engine.
+
+    Searches Brave News Search and parses HTML responses to extract
+    news article results including title, URL, source, date, description,
+    and thumbnail image.
+
+    Attributes:
+        name: Engine identifier name.
+        provider: Provider identifier.
+        category: Search category type.
+        search_url: Base URL for news search.
+    """
+
+    name = "brave_news"
+    provider = "brave"
+    category = "news"
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initialize Brave news search client.
+
+        Args:
+            *args: Positional arguments passed to BraveBase.
+            **kwargs: Keyword arguments passed to BraveBase.
+        """
+        super().__init__(*args, **kwargs)
+        self.search_url = f"{self.base_url}/news"
+
+    def run(self, *args: Any, **kwargs: Any) -> list[NewsResult]:
+        """Run news search on Brave.
+
+        Args:
+            *args: Positional arguments. First arg is the search query.
+            **kwargs: Keyword arguments including:
+                - keywords: Search query string.
+                - region: Region code (e.g., 'us-en').
+                - safesearch: Safe search level ('on', 'moderate', 'off').
+                - max_results: Maximum number of results to return.
+                - timelimit: Time filter for results ('d' for day, 'w' for week, etc.).
+
+        Returns:
+            List of NewsResult objects containing news article information.
+
+        Raises:
+            ValueError: If no keywords are provided.
+            Exception: If the HTTP request fails.
+        """
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us-en")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+        timelimit = kwargs.get("timelimit")
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safesearch_map = {"on": "strict", "moderate": "moderate", "off": "off"}
+        safesearch_str = str(safesearch).lower() if safesearch else "moderate"
+        safesearch_value = safesearch_map.get(safesearch_str, "moderate")
+
+        fetched_results: list[NewsResult] = []
+        fetched_urls: set[str] = set()
+
+        offset = 0
+        while len(fetched_results) < max_results:
+            params: dict[str, str] = {
+                "q": keywords,
+                "source": "web",
+                "safesearch": safesearch_value,
+                "spellcheck": "0",
+            }
+
+            if offset > 0:
+                params["offset"] = str(offset)
+
+            if timelimit:
+                params["tf"] = timelimit
+
+            if region:
+                params["region"] = region
+
+            html = self._fetch_page(params)
+            page_results = self._parse_results_from_html(html)
+
+            if not page_results:
+                break
+
+            for result in page_results:
+                if len(fetched_results) >= max_results:
+                    break
+                if result.url and result.url not in fetched_urls:
+                    fetched_urls.add(result.url)
+                    fetched_results.append(result)
+
+            offset += 1
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return fetched_results[:max_results]
+
+    def _fetch_page(self, params: dict[str, str]) -> str:
+        """Fetch HTML page from Brave news search.
+
+        Args:
+            params: Query parameters for the request.
+
+        Returns:
+            HTML content of the response.
+
+        Raises:
+            Exception: If the request fails after retries.
+        """
+        headers = dict(self.session.headers) if getattr(self, "session", None) else {}
+        headers.update({
+            "Referer": "https://search.brave.com/",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+        })
+
+        attempts = 3
+        backoff = 1.0
+        last_exc: Exception | None = None
+
+        for attempt in range(attempts):
+            try:
+                resp = self.session.get(
+                    self.search_url,
+                    params=params,
+                    headers=headers,
+                    timeout=self.timeout,
+                )
+                resp.raise_for_status()
+                return resp.text
+            except Exception as exc:
+                last_exc = exc
+                try:
+                    code = getattr(exc, "code", None) or getattr(exc, "status_code", None)
+                except Exception:
+                    code = None
+                if code in (429, 500, 502, 503, 504):
+                    sleep(backoff)
+                    backoff *= 2
+                    continue
+                raise Exception(f"Failed to GET {self.search_url}: {exc}") from exc
+
+        raise Exception(f"Failed to GET {self.search_url} after retries: {last_exc}") from last_exc
+
+    def _parse_results_from_html(self, html: str) -> list[NewsResult]:
+        """Parse HTML and extract news search results.
+
+        Args:
+            html: Raw HTML content from Brave news search.
+
+        Returns:
+            List of NewsResult objects parsed from the HTML.
+        """
+        soup = Scout(html)
+        results: list[NewsResult] = []
+
+        # News results are in div.snippet containers with data-type="news"
+        containers = soup.select('div.snippet[data-type="news"]')
+
+        # Fallback: try generic snippet containers in main
+        if not containers:
+            main = soup.select_one("main")
+            if main:
+                containers = main.select("div.snippet")
+
+        for container in containers:
+            try:
+                result = self._parse_news_container(container)
+                if result and result.url:
+                    results.append(result)
+            except Exception:
+                # Skip malformed results
+                continue
+
+        return results
+
+    def _parse_news_container(self, container: Any) -> NewsResult | None:
+        """Parse a single news container element.
+
+        Args:
+            container: Scout element representing a news result container.
+
+        Returns:
+            NewsResult object or None if parsing fails.
+        """
+        # Get article URL from result header link
+        url = ""
+        link_elem = container.select_one("a.result-header")
+        if link_elem:
+            url = link_elem.get("href", "").strip()
+
+        if not url:
+            # Try alternate link location
+            link_elem = container.select_one("a[href]")
+            if link_elem:
+                url = link_elem.get("href", "").strip()
+
+        if not url:
+            return None
+
+        # Get title
+        title = ""
+        title_elem = container.select_one(".snippet-title")
+        if title_elem:
+            title = title_elem.get_text(strip=True)
+
+        # Get source (publisher)
+        source = ""
+        source_elem = container.select_one(".netloc")
+        if source_elem:
+            source = source_elem.get_text(strip=True)
+
+        # Get publication date
+        date = ""
+        # Date is typically in a span.attr after the separator
+        cite_elem = container.select_one("cite.snippet-url")
+        if cite_elem:
+            # Look for date patterns (e.g., "5 hours ago", "1 day ago", "January 15, 2026")
+            attr_elems = cite_elem.select(".attr")
+            for elem in attr_elems:
+                text = elem.get_text(strip=True)
+                # Skip separators
+                if text == "•":
+                    continue
+                # Check if this looks like a date
+                if self._is_date_text(text):
+                    date = text
+                    break
+
+        # Get description
+        body = ""
+        desc_elem = container.select_one("p.desc, .snippet-description")
+        if desc_elem:
+            body = desc_elem.get_text(strip=True)
+
+        # Get thumbnail image
+        image = ""
+        img_elem = container.select_one(".image-wrapper img, img")
+        if img_elem:
+            image = img_elem.get("src", "").strip()
+            # Skip favicon images
+            if "favicon" in image.lower() or "size-xs" in (img_elem.get("class") or ""):
+                image = ""
+
+        return NewsResult(
+            title=title,
+            url=url,
+            source=source,
+            date=date,
+            body=body,
+            image=image,
+        )
+
+    def _is_date_text(self, text: str) -> bool:
+        """Check if text appears to be a date.
+
+        Args:
+            text: Text string to check.
+
+        Returns:
+            True if text looks like a date, False otherwise.
+        """
+        text_lower = text.lower()
+
+        # Check for relative time patterns
+        relative_patterns = [
+            "hour", "hours", "minute", "minutes", "second", "seconds",
+            "day", "days", "week", "weeks", "month", "months", "year", "years",
+            "ago", "yesterday", "today",
+        ]
+        if any(pattern in text_lower for pattern in relative_patterns):
+            return True
+
+        # Check for month names
+        months = [
+            "january", "february", "march", "april", "may", "june",
+            "july", "august", "september", "october", "november", "december",
+            "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "oct", "nov", "dec",
+        ]
+        if any(month in text_lower for month in months):
+            return True
+
+        return False
+
+    def extract_results(self, html_text: str) -> list[NewsResult]:
+        """Extract news results from HTML text.
+
+        This is an alias for _parse_results_from_html for API consistency.
+
+        Args:
+            html_text: Raw HTML content from Brave news search.
+
+        Returns:
+            List of NewsResult objects parsed from the HTML.
+        """
+        return self._parse_results_from_html(html_text)
+
+
+if __name__ == "__main__":
+    # Test the BraveNews search
+    print("Testing BraveNews search...")
+
+    searcher = BraveNews(timeout=15)
+
+    try:
+        # Test basic search
+        results = searcher.run("technology", max_results=5)
+
+        print(f"\nFound {len(results)} news results:\n")
+        for i, article in enumerate(results, 1):
+            print(f"{i}. {article.title}")
+            print(f"   URL: {article.url}")
+            print(f"   Source: {article.source}")
+            print(f"   Date: {article.date}")
+            print(f"   Description: {article.body[:100]}..." if article.body else "")
+            print(f"   Image: {article.image[:80]}..." if article.image else "")
+            print()
+
+    except Exception as e:
+        print(f"Error during search: {e}")
+        import traceback
+        traceback.print_exc()

webscout/search/engines/brave/suggestions.py

@@ -0,0 +1,318 @@
+"""Brave search suggestions/autocomplete implementation.
+
+This module provides access to Brave Search's autocomplete/suggestions API
+that returns search suggestions as users type.
+
+Example:
+    >>> from webscout.search.engines.brave.suggestions import BraveSuggestions
+    >>> suggester = BraveSuggestions()
+    >>> suggestions = suggester.run("pyth")
+    >>> for suggestion in suggestions:
+    ...     print(suggestion.query)
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+from .base import BraveBase
+
+
+@dataclass
+class SuggestionResult:
+    """A single search suggestion result.
+
+    Attributes:
+        query: The suggested search query.
+        is_entity: Whether this suggestion represents a known entity.
+        name: Display name for entities.
+        desc: Description for entities.
+        category: Category of the entity (e.g., 'company', 'application').
+        image: URL to entity image/logo if available.
+        is_logo: Whether the image is a logo.
+    """
+
+    query: str = ""
+    is_entity: bool = False
+    name: str = ""
+    desc: str = ""
+    category: str = ""
+    image: str = ""
+    is_logo: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary.
+
+        Returns:
+            Dictionary representation of the suggestion.
+        """
+        return {
+            "query": self.query,
+            "is_entity": self.is_entity,
+            "name": self.name,
+            "desc": self.desc,
+            "category": self.category,
+            "image": self.image,
+            "is_logo": self.is_logo,
+        }
+
+
+@dataclass
+class SuggestionsResponse:
+    """Response containing search suggestions.
+
+    Attributes:
+        query: The original query that was submitted.
+        suggestions: List of suggestion results.
+    """
+
+    query: str = ""
+    suggestions: list[SuggestionResult] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary.
+
+        Returns:
+            Dictionary representation of the response.
+        """
+        return {
+            "query": self.query,
+            "suggestions": [s.to_dict() for s in self.suggestions],
+        }
+
+
+class BraveSuggestions(BraveBase):
+    """Brave search suggestions/autocomplete engine.
+
+    Fetches search suggestions from Brave's autocomplete API endpoint.
+    Supports both simple suggestions and rich entity suggestions with
+    additional metadata like descriptions, categories, and images.
+
+    Attributes:
+        name: Engine identifier name.
+        provider: Provider identifier.
+        category: Search category type.
+        api_url: URL for the suggestions API endpoint.
+    """
+
+    name = "brave_suggestions"
+    provider = "brave"
+    category = "suggestions"
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Initialize Brave suggestions client.
+
+        Args:
+            *args: Positional arguments passed to BraveBase.
+            **kwargs: Keyword arguments passed to BraveBase.
+        """
+        super().__init__(*args, **kwargs)
+        self.api_url = f"{self.base_url}/api/suggest"
+
+    def run(
+        self,
+        query: str,
+        rich: bool = True,
+        country: str | None = None,
+        source: str = "web",
+        max_results: int | None = None,
+    ) -> list[SuggestionResult]:
+        """Get search suggestions for a query.
+
+        Args:
+            query: The partial search query to get suggestions for.
+            rich: Whether to include rich entity information (default True).
+            country: Country code for localized suggestions (e.g., 'us', 'in').
+            source: Search source type (default 'web').
+            max_results: Maximum number of suggestions to return.
+
+        Returns:
+            List of SuggestionResult objects containing suggestions.
+
+        Raises:
+            ValueError: If no query is provided.
+            Exception: If the API request fails.
+        """
+        if not query:
+            raise ValueError("Query is mandatory")
+
+        params: dict[str, str] = {
+            "q": query,
+            "source": source,
+        }
+
+        if rich:
+            params["rich"] = "true"
+
+        if country:
+            params["country"] = country
+
+        response_data = self._fetch_suggestions(params)
+        suggestions = self._parse_response(response_data)
+
+        if max_results is not None:
+            suggestions = suggestions[:max_results]
+
+        return suggestions
+
+    def suggest(
+        self,
+        query: str,
+        rich: bool = True,
+        country: str | None = None,
+    ) -> SuggestionsResponse:
+        """Get search suggestions with full response object.
+
+        This method returns a SuggestionsResponse object containing
+        both the original query and the list of suggestions.
+
+        Args:
+            query: The partial search query to get suggestions for.
+            rich: Whether to include rich entity information.
+            country: Country code for localized suggestions.
+
+        Returns:
+            SuggestionsResponse object with query and suggestions.
+        """
+        suggestions = self.run(query, rich=rich, country=country)
+        return SuggestionsResponse(query=query, suggestions=suggestions)
+
+    def _fetch_suggestions(self, params: dict[str, str]) -> list[Any]:
+        """Fetch suggestions from the Brave API.
+
+        Args:
+            params: Query parameters for the API request.
+
+        Returns:
+            Raw JSON response as a list.
+
+        Raises:
+            Exception: If the request fails.
+        """
+        headers = dict(self.session.headers) if getattr(self, "session", None) else {}
+        headers.update({
+            "Referer": "https://search.brave.com/",
+            "Accept": "*/*",
+            "Accept-Language": "en-US,en;q=0.9",
+        })
+
+        try:
+            resp = self.session.get(
+                self.api_url,
+                params=params,
+                headers=headers,
+                timeout=self.timeout,
+            )
+            resp.raise_for_status()
+            return resp.json()
+        except json.JSONDecodeError as e:
+            raise Exception(f"Failed to parse suggestions response: {e}") from e
+        except Exception as e:
+            raise Exception(f"Failed to fetch suggestions: {e}") from e
+
+    def _parse_response(self, data: list[Any]) -> list[SuggestionResult]:
+        """Parse the API response into SuggestionResult objects.
+
+        The Brave suggestions API returns data in the format:
+        ["query", [suggestion_objects]]
+
+        Each suggestion object can be either:
+        - Simple: {"is_entity": false, "q": "query text"}
+        - Entity: {"is_entity": true, "q": "query", "name": "...", "desc": "...",
+                   "category": "...", "img": "...", "logo": true}
+
+        Args:
+            data: Raw JSON response from the API.
+
+        Returns:
+            List of SuggestionResult objects.
+        """
+        results: list[SuggestionResult] = []
+
+        if not data or len(data) < 2:
+            return results
+
+        # Second element contains the suggestions list
+        suggestions_list = data[1] if len(data) > 1 else []
+
+        for item in suggestions_list:
+            if not isinstance(item, dict):
+                continue
+
+            query_text = item.get("q", "")
+            is_entity = item.get("is_entity", False)
+
+            result = SuggestionResult(
+                query=query_text,
+                is_entity=is_entity,
+                name=item.get("name", ""),
+                desc=item.get("desc", ""),
+                category=item.get("category", ""),
+                image=item.get("img", ""),
+                is_logo=item.get("logo", False),
+            )
+
+            results.append(result)
+
+        return results
+
+    def get_simple_suggestions(self, query: str, max_results: int = 10) -> list[str]:
+        """Get a simple list of suggestion strings.
+
+        This is a convenience method that returns just the query strings
+        without the full SuggestionResult objects.
+
+        Args:
+            query: The partial search query to get suggestions for.
+            max_results: Maximum number of suggestions to return.
+
+        Returns:
+            List of suggestion query strings.
+        """
+        suggestions = self.run(query, rich=False, max_results=max_results)
+        return [s.query for s in suggestions if s.query]
+
+
+if __name__ == "__main__":
+    # Test the BraveSuggestions
+    print("Testing BraveSuggestions...")
+
+    suggester = BraveSuggestions(timeout=10)
+
+    try:
+        # Test basic suggestions
+        print("\n--- Testing with 'pyth' ---")
+        results = suggester.run("pyth")
+
+        print(f"Found {len(results)} suggestions:\n")
+        for i, suggestion in enumerate(results, 1):
+            if suggestion.is_entity:
+                print(f"{i}. [ENTITY] {suggestion.query}")
+                print(f"   Name: {suggestion.name}")
+                print(f"   Description: {suggestion.desc}")
+                print(f"   Category: {suggestion.category}")
+                if suggestion.image:
+                    print(f"   Image: {suggestion.image[:60]}...")
+            else:
+                print(f"{i}. {suggestion.query}")
+            print()
+
+        # Test simple suggestions
+        print("\n--- Testing simple suggestions with 'java' ---")
+        simple = suggester.get_simple_suggestions("java", max_results=5)
+        print("Simple suggestions:")
+        for s in simple:
+            print(f"  - {s}")
+
+        # Test full response object
+        print("\n--- Testing full response with 'react' ---")
+        response = suggester.suggest("react")
+        print(f"Query: {response.query}")
+        print(f"Suggestions count: {len(response.suggestions)}")
+
+    except Exception as e:
+        print(f"Error during test: {e}")
+        import traceback
+        traceback.print_exc()