webscout 8.3.7__py3-none-any.whl → 2025.10.13__py3-none-any.whl

webscout/scout/parsers/lxml_parser.py

@@ -1,178 +1,178 @@
-"""
-Scout LXML Parser - Advanced HTML/XML Parsing with lxml
-"""
-
-import re
-from typing import List, Optional, Dict, Any, Union
-
-from lxml import etree, html as lxml_html
-from ..element import Tag, NavigableString
-
-class LXMLParser:
-    """
-    Advanced HTML/XML parser using lxml library.
-    Provides robust parsing with enhanced error handling and flexibility.
-    """
-    
-    def __init__(self, parser_type: str = 'html'):
-        """
-        Initialize the LXML parser with advanced parsing capabilities.
-        
-        Args:
-            parser_type (str): Type of parser ('html' or 'xml')
-        """
-        self._parser_type = parser_type
-        self._parsing_errors = []
-        
-        # Select appropriate parser based on type
-        if parser_type == 'html':
-            self._parser = lxml_html.HTMLParser(recover=True, encoding='utf-8')
-        elif parser_type == 'xml':
-            self._parser = etree.XMLParser(recover=True, encoding='utf-8')
-        else:
-            raise ValueError(f"Unsupported parser type: {parser_type}")
-    
-    def parse(self, markup: str) -> Tag:
-        """
-        Parse HTML/XML markup and return the root tag.
-        
-        Args:
-            markup (str): HTML/XML content to parse
-        
-        Returns:
-            Tag: Parsed document root
-        """
-        try:
-            # Preprocess markup to handle common issues
-            markup = self._preprocess_markup(markup)
-            
-            # Parse the markup
-            if self._parser_type == 'html':
-                tree = lxml_html.fromstring(markup, parser=self._parser)
-            else:
-                tree = etree.fromstring(markup, parser=self._parser)
-            
-            # Convert lxml element to Scout Tag
-            return self._convert_element(tree)
-        
-        except Exception as e:
-            self._parsing_errors.append(str(e))
-            return Tag('root')
-    
-    def _preprocess_markup(self, markup: str) -> str:
-        """
-        Preprocess markup to handle common parsing issues.
-        
-        Args:
-            markup (str): Raw markup
-        
-        Returns:
-            str: Preprocessed markup
-        """
-        # Remove XML/HTML comments
-        markup = re.sub(r'<!--.*?-->', '', markup, flags=re.DOTALL)
-        
-        # Handle unclosed tags for HTML
-        if self._parser_type == 'html':
-            markup = re.sub(r'<(br|img|input|hr|meta)([^>]*?)(?<!/)>', r'<\1\2 />', markup, flags=re.IGNORECASE)
-        
-        return markup
-    
-    def _convert_element(self, element: Union[etree._Element, lxml_html.HtmlElement]) -> Tag:
-        """
-        Convert lxml element to Scout Tag.
-        
-        Args:
-            element (etree._Element or lxml_html.HtmlElement): lxml element
-        
-        Returns:
-            Tag: Converted Scout Tag
-        """
-        # Create Tag with name and attributes
-        tag = Tag(element.tag, dict(element.attrib))
-        
-        # Add text content
-        if element.text:
-            tag.contents.append(NavigableString(element.text))
-        
-        # Recursively add child elements
-        for child in element:
-            child_tag = self._convert_element(child)
-            child_tag.parent = tag
-            tag.contents.append(child_tag)
-            
-            # Add tail text
-            if child.tail:
-                tail_text = NavigableString(child.tail)
-                tail_text.parent = tag
-                tag.contents.append(tail_text)
-        
-        return tag
-    
-    def get_parsing_errors(self) -> List[str]:
-        """
-        Retrieve parsing errors encountered during processing.
-        
-        Returns:
-            List[str]: List of parsing error messages
-        """
-        return self._parsing_errors
-    
-    def find_all(self, markup: str, tag: Optional[Union[str, List[str]]] = None, 
-                 attrs: Optional[Dict[str, Any]] = None, 
-                 recursive: bool = True, 
-                 text: Optional[str] = None, 
-                 limit: Optional[int] = None) -> List[Tag]:
-        """
-        Find all matching elements in the parsed document.
-        
-        Args:
-            markup (str): HTML/XML content to parse
-            tag (str or List[str], optional): Tag name(s) to search for
-            attrs (dict, optional): Attribute filters
-            recursive (bool): Whether to search recursively
-            text (str, optional): Text content to search for
-            limit (int, optional): Maximum number of results
-        
-        Returns:
-            List[Tag]: List of matching tags
-        """
-        root = self.parse(markup)
-        
-        def matches(element: Tag) -> bool:
-            """Check if an element matches search criteria."""
-            # Tag filter
-            if tag and isinstance(tag, str) and element.name != tag:
-                return False
-            if tag and isinstance(tag, list) and element.name not in tag:
-                return False
-            
-            # Attribute filter
-            if attrs:
-                for key, value in attrs.items():
-                    if key not in element.attrs or element.attrs[key] != value:
-                        return False
-            
-            # Text filter
-            if text:
-                element_text = ' '.join([str(c) for c in element.contents if isinstance(c, NavigableString)])
-                if text not in element_text:
-                    return False
-            
-            return True
-        
-        def collect_matches(element: Tag, results: List[Tag]):
-            """Recursively collect matching elements."""
-            if matches(element):
-                results.append(element)
-                if limit and len(results) >= limit:
-                    return
-            
-            if recursive:
-                for child in element.contents:
-                    if isinstance(child, Tag):
-                        collect_matches(child, results)
-        
-        results = []
-        collect_matches(root, results)
-        return results
+"""
+Scout LXML Parser - Advanced HTML/XML Parsing with lxml
+"""
+
+import re
+from typing import List, Optional, Dict, Any, Union
+
+from lxml import etree, html as lxml_html
+from ..element import Tag, NavigableString
+
+class LXMLParser:
+    """
+    Advanced HTML/XML parser using lxml library.
+    Provides robust parsing with enhanced error handling and flexibility.
+    """
+    
+    def __init__(self, parser_type: str = 'html'):
+        """
+        Initialize the LXML parser with advanced parsing capabilities.
+        
+        Args:
+            parser_type (str): Type of parser ('html' or 'xml')
+        """
+        self._parser_type = parser_type
+        self._parsing_errors = []
+        
+        # Select appropriate parser based on type
+        if parser_type == 'html':
+            self._parser = lxml_html.HTMLParser(recover=True, encoding='utf-8')
+        elif parser_type == 'xml':
+            self._parser = etree.XMLParser(recover=True, encoding='utf-8')
+        else:
+            raise ValueError(f"Unsupported parser type: {parser_type}")
+    
+    def parse(self, markup: str) -> Tag:
+        """
+        Parse HTML/XML markup and return the root tag.
+        
+        Args:
+            markup (str): HTML/XML content to parse
+        
+        Returns:
+            Tag: Parsed document root
+        """
+        try:
+            # Preprocess markup to handle common issues
+            markup = self._preprocess_markup(markup)
+            
+            # Parse the markup
+            if self._parser_type == 'html':
+                tree = lxml_html.fromstring(markup, parser=self._parser)
+            else:
+                tree = etree.fromstring(markup, parser=self._parser)
+            
+            # Convert lxml element to Scout Tag
+            return self._convert_element(tree)
+        
+        except Exception as e:
+            self._parsing_errors.append(str(e))
+            return Tag('root')
+    
+    def _preprocess_markup(self, markup: str) -> str:
+        """
+        Preprocess markup to handle common parsing issues.
+        
+        Args:
+            markup (str): Raw markup
+        
+        Returns:
+            str: Preprocessed markup
+        """
+        # Remove XML/HTML comments
+        markup = re.sub(r'<!--.*?-->', '', markup, flags=re.DOTALL)
+        
+        # Handle unclosed tags for HTML
+        if self._parser_type == 'html':
+            markup = re.sub(r'<(br|img|input|hr|meta)([^>]*?)(?<!/)>', r'<\1\2 />', markup, flags=re.IGNORECASE)
+        
+        return markup
+    
+    def _convert_element(self, element: Union[etree._Element, lxml_html.HtmlElement]) -> Tag:
+        """
+        Convert lxml element to Scout Tag.
+        
+        Args:
+            element (etree._Element or lxml_html.HtmlElement): lxml element
+        
+        Returns:
+            Tag: Converted Scout Tag
+        """
+        # Create Tag with name and attributes
+        tag = Tag(element.tag, dict(element.attrib))
+        
+        # Add text content
+        if element.text:
+            tag.contents.append(NavigableString(element.text))
+        
+        # Recursively add child elements
+        for child in element:
+            child_tag = self._convert_element(child)
+            child_tag.parent = tag
+            tag.contents.append(child_tag)
+            
+            # Add tail text
+            if child.tail:
+                tail_text = NavigableString(child.tail)
+                tail_text.parent = tag
+                tag.contents.append(tail_text)
+        
+        return tag
+    
+    def get_parsing_errors(self) -> List[str]:
+        """
+        Retrieve parsing errors encountered during processing.
+        
+        Returns:
+            List[str]: List of parsing error messages
+        """
+        return self._parsing_errors
+    
+    def find_all(self, markup: str, tag: Optional[Union[str, List[str]]] = None, 
+                 attrs: Optional[Dict[str, Any]] = None, 
+                 recursive: bool = True, 
+                 text: Optional[str] = None, 
+                 limit: Optional[int] = None) -> List[Tag]:
+        """
+        Find all matching elements in the parsed document.
+        
+        Args:
+            markup (str): HTML/XML content to parse
+            tag (str or List[str], optional): Tag name(s) to search for
+            attrs (dict, optional): Attribute filters
+            recursive (bool): Whether to search recursively
+            text (str, optional): Text content to search for
+            limit (int, optional): Maximum number of results
+        
+        Returns:
+            List[Tag]: List of matching tags
+        """
+        root = self.parse(markup)
+        
+        def matches(element: Tag) -> bool:
+            """Check if an element matches search criteria."""
+            # Tag filter
+            if tag and isinstance(tag, str) and element.name != tag:
+                return False
+            if tag and isinstance(tag, list) and element.name not in tag:
+                return False
+            
+            # Attribute filter
+            if attrs:
+                for key, value in attrs.items():
+                    if key not in element.attrs or element.attrs[key] != value:
+                        return False
+            
+            # Text filter
+            if text:
+                element_text = ' '.join([str(c) for c in element.contents if isinstance(c, NavigableString)])
+                if text not in element_text:
+                    return False
+            
+            return True
+        
+        def collect_matches(element: Tag, results: List[Tag]):
+            """Recursively collect matching elements."""
+            if matches(element):
+                results.append(element)
+                if limit and len(results) >= limit:
+                    return
+            
+            if recursive:
+                for child in element.contents:
+                    if isinstance(child, Tag):
+                        collect_matches(child, results)
+        
+        results = []
+        collect_matches(root, results)
+        return results

webscout/scout/utils.py

@@ -1,37 +1,37 @@
-"""
-Utility functions - making life easier! 🛠️
-"""
-
-from typing import Union, Optional
-
-def decode_markup(markup: Union[str, bytes], encoding: Optional[str] = None) -> str:
-    """
-    Decode that markup - no encoding drama! 🎯
-    
-    Args:
-        markup: The raw HTML/XML content
-        encoding: The encoding to use (if known)
-    
-    Returns:
-        Clean decoded string ready for parsing! ✨
-    """
-    if isinstance(markup, str):
-        return markup
-    
-    if encoding:
-        try:
-            return markup.decode(encoding)
-        except UnicodeDecodeError:
-            pass
-    
-    # Try common encodings - we got options! 💪
-    encodings = ['utf-8', 'latin1', 'iso-8859-1', 'ascii']
-    
-    for enc in encodings:
-        try:
-            return markup.decode(enc)
-        except UnicodeDecodeError:
-            continue
-    
-    # Last resort - ignore errors and keep it moving! 🚀
-    return markup.decode('utf-8', errors='ignore')
+"""
+Utility functions - making life easier! 🛠️
+"""
+
+from typing import Union, Optional
+
+def decode_markup(markup: Union[str, bytes], encoding: Optional[str] = None) -> str:
+    """
+    Decode that markup - no encoding drama! 🎯
+    
+    Args:
+        markup: The raw HTML/XML content
+        encoding: The encoding to use (if known)
+    
+    Returns:
+        Clean decoded string ready for parsing! ✨
+    """
+    if isinstance(markup, str):
+        return markup
+    
+    if encoding:
+        try:
+            return markup.decode(encoding)
+        except UnicodeDecodeError:
+            pass
+    
+    # Try common encodings - we got options! 💪
+    encodings = ['utf-8', 'latin1', 'iso-8859-1', 'ascii']
+    
+    for enc in encodings:
+        try:
+            return markup.decode(enc)
+        except UnicodeDecodeError:
+            continue
+    
+    # Last resort - ignore errors and keep it moving! 🚀
+    return markup.decode('utf-8', errors='ignore')

webscout/search/__init__.py

@@ -0,0 +1,51 @@
+"""Webscout search module - unified search interfaces."""
+
+from .base import BaseSearch, BaseSearchEngine
+from .duckduckgo_main import DuckDuckGoSearch
+from .yep_main import YepSearch
+
+# Import new search engines
+from .engines.bing import Bing
+from .engines.brave import Brave
+from .engines.mojeek import Mojeek
+from .engines.yahoo import Yahoo
+from .engines.yandex import Yandex
+from .engines.wikipedia import Wikipedia
+from .engines.bing_news import BingNews
+from .engines.yahoo_news import YahooNews
+
+# Import result models
+from .results import (
+    TextResult,
+    ImagesResult,
+    VideosResult,
+    NewsResult,
+    BooksResult,
+)
+
+__all__ = [
+    # Base classes
+    "BaseSearch",
+    "BaseSearchEngine",
+    
+    # Main search interfaces
+    "DuckDuckGoSearch",
+    "YepSearch",
+    
+    # Individual engines
+    "Bing",
+    "Brave",
+    "Mojeek",
+    "Yahoo",
+    "Yandex",
+    "Wikipedia",
+    "BingNews",
+    "YahooNews",
+    
+    # Result models
+    "TextResult",
+    "ImagesResult",
+    "VideosResult",
+    "NewsResult",
+    "BooksResult",
+]

webscout/search/base.py

@@ -0,0 +1,195 @@
+"""Base class for search engines."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from functools import cached_property
+from typing import Any, Generic, Literal, TypeVar
+
+try:
+    from lxml import html
+    from lxml.etree import HTMLParser as LHTMLParser
+    LXML_AVAILABLE = True
+except ImportError:
+    LXML_AVAILABLE = False
+    html = None  # type: ignore
+    LHTMLParser = None  # type: ignore
+
+from .http_client import HttpClient
+from .results import BooksResult, ImagesResult, NewsResult, TextResult, VideosResult
+
+logger = logging.getLogger(__name__)
+T = TypeVar("T")
+
+
+class BaseSearchEngine(ABC, Generic[T]):
+    """Abstract base class for all search engine backends."""
+
+    name: str  # unique key, e.g. "google"
+    category: Literal["text", "images", "videos", "news", "books"]
+    provider: str  # source of the search results (e.g. "google", "bing", etc.)
+    disabled: bool = False  # if True, the engine is disabled
+    priority: float = 1
+
+    search_url: str
+    search_method: str  # GET or POST
+    search_headers: Mapping[str, str] = {}
+    items_xpath: str = ""
+    elements_xpath: Mapping[str, str] = {}
+    elements_replace: Mapping[str, str] = {}
+
+    def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True):
+        """Initialize search engine.
+        
+        Args:
+            proxy: Proxy URL (supports http/https/socks5).
+            timeout: Request timeout in seconds.
+            verify: Whether to verify SSL certificates.
+        """
+        self.http_client = HttpClient(proxy=proxy, timeout=timeout, verify=verify)
+        self.results: list[T] = []
+
+    @property
+    def result_type(self) -> type[T]:
+        """Get result type based on category."""
+        categories = {
+            "text": TextResult,
+            "images": ImagesResult,
+            "videos": VideosResult,
+            "news": NewsResult,
+            "books": BooksResult,
+        }
+        return categories[self.category]  # type: ignore
+
+    @abstractmethod
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        raise NotImplementedError
+
+    def request(self, method: str, url: str, **kwargs: Any) -> str | None:
+        """Make a request to the search engine."""
+        try:
+            response = self.http_client.request(method, url, **kwargs)  # type: ignore
+            return response.text
+        except Exception as ex:
+            logger.error("Error in %s request: %r", self.name, ex)
+            return None
+
+    @cached_property
+    def parser(self) -> Any:
+        """Get HTML parser."""
+        if not LXML_AVAILABLE:
+            logger.warning("lxml not available, HTML parsing disabled")
+            return None
+        return LHTMLParser(remove_blank_text=True, remove_comments=True, remove_pis=True, collect_ids=False)
+
+    def extract_tree(self, html_text: str) -> Any:
+        """Extract html tree from html text."""
+        if not LXML_AVAILABLE or not self.parser:
+            raise ImportError("lxml is required for HTML parsing")
+        return html.fromstring(html_text, parser=self.parser)
+
+    def pre_process_html(self, html_text: str) -> str:
+        """Pre-process html_text before extracting results."""
+        return html_text
+
+    def extract_results(self, html_text: str) -> list[T]:
+        """Extract search results from html text."""
+        if not LXML_AVAILABLE:
+            raise ImportError("lxml is required for result extraction")
+        
+        html_text = self.pre_process_html(html_text)
+        tree = self.extract_tree(html_text)
+        
+        results = []
+        items = tree.xpath(self.items_xpath) if self.items_xpath else []
+        
+        for item in items:
+            result = self.result_type()
+            for key, xpath in self.elements_xpath.items():
+                try:
+                    data = item.xpath(xpath)
+                    if data:
+                        # Join text nodes or get first attribute
+                        value = "".join(data) if isinstance(data, list) else data
+                        setattr(result, key, value.strip() if isinstance(value, str) else value)
+                except Exception as ex:
+                    logger.debug("Error extracting %s: %r", key, ex)
+            results.append(result)
+        
+        return results
+
+    def post_extract_results(self, results: list[T]) -> list[T]:
+        """Post-process search results."""
+        return results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> list[T] | None:
+        """Search the engine."""
+        payload = self.build_payload(
+            query=query, region=region, safesearch=safesearch, timelimit=timelimit, page=page, **kwargs
+        )
+        if self.search_method == "GET":
+            html_text = self.request(self.search_method, self.search_url, params=payload, headers=self.search_headers)
+        else:
+            html_text = self.request(self.search_method, self.search_url, data=payload, headers=self.search_headers)
+        if not html_text:
+            return None
+        results = self.extract_results(html_text)
+        return self.post_extract_results(results)
+
+
+# Legacy base class for backwards compatibility
+class BaseSearch(ABC):
+    """Base class for synchronous search engines (legacy)."""
+
+    @abstractmethod
+    def text(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Text search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def images(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Images search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def videos(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Videos search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def news(self, *args, **kwargs) -> list[dict[str, str]]:
+        """News search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def answers(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Instant answers."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def suggestions(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Suggestions."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def maps(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Maps search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def translate(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Translate."""
+        raise NotImplementedError