webscout 8.2.7__py3-none-any.whl → 8.2.8__py3-none-any.whl

webscout/scout/parsers/html_parser.py

@@ -0,0 +1,236 @@
+"""
+Scout HTML Parser - Advanced HTML Parsing with Python's Built-in Parser
+"""
+
+import html
+import re
+from html.parser import HTMLParser as StdHTMLParser
+from typing import List
+
+from ..element import Tag, NavigableString
+
+class HTMLParser:
+    """
+    Advanced HTML parser using Python's built-in HTMLParser.
+    Provides robust parsing with enhanced error handling and flexibility.
+    """
+    
+    def __init__(self):
+        """
+        Initialize the HTML parser with advanced parsing capabilities.
+        """
+        self._root = Tag('html')
+        self._current_tag = self._root
+        self._tag_stack = [self._root]
+        self._parsing_errors = []
+    
+    def parse(self, markup: str) -> Tag:
+        """
+        Parse HTML markup and return the root tag.
+        
+        Args:
+            markup (str): HTML content to parse
+        
+        Returns:
+            Tag: Parsed HTML document root
+        """
+        try:
+            # Preprocess markup to handle common issues
+            markup = self._preprocess_markup(markup)
+            
+            # Create a standard HTML parser
+            parser = _ScoutHTMLParser(self)
+            parser.feed(markup)
+            parser.close()
+            
+            return self._root
+        except Exception as e:
+            self._parsing_errors.append(str(e))
+            return self._root
+    
+    def _preprocess_markup(self, markup: str) -> str:
+        """
+        Preprocess HTML markup to handle common parsing issues.
+        
+        Args:
+            markup (str): Raw HTML markup
+        
+        Returns:
+            str: Preprocessed HTML markup
+        """
+        # Decode HTML entities
+        markup = html.unescape(markup)
+        
+        # Handle unclosed tags (basic approach)
+        markup = re.sub(r'<(br|img|input|hr|meta)([^>]*?)(?<!/)>', r'<\1\2 />', markup, flags=re.IGNORECASE)
+        
+        # Remove comments (optional, can be configurable)
+        markup = re.sub(r'<!--.*?-->', '', markup, flags=re.DOTALL)
+        
+        return markup
+    
+    def add_tag(self, tag: Tag):
+        """
+        Add a tag to the current parsing context.
+        
+        Args:
+            tag (Tag): Tag to add
+        """
+        # Set parent-child relationships
+        tag.parent = self._current_tag
+        self._current_tag.contents.append(tag)
+        
+        # Update current tag if it's an opening tag
+        self._current_tag = tag
+        self._tag_stack.append(tag)
+    
+    def add_text(self, text: str):
+        """
+        Add text content to the current tag.
+        
+        Args:
+            text (str): Text content
+        """
+        if text.strip():
+            text_node = NavigableString(text)
+            text_node.parent = self._current_tag
+            self._current_tag.contents.append(text_node)
+    
+    def close_tag(self):
+        """
+        Close the current tag and return to parent context.
+        """
+        if len(self._tag_stack) > 1:
+            self._tag_stack.pop()
+            self._current_tag = self._tag_stack[-1]
+    
+    def get_parsing_errors(self) -> List[str]:
+        """
+        Retrieve parsing errors encountered during HTML processing.
+        
+        Returns:
+            List[str]: List of parsing error messages
+        """
+        return self._parsing_errors
+
+class _ScoutHTMLParser(StdHTMLParser):
+    """
+    Internal HTML parser that integrates with Scout's parsing mechanism.
+    """
+    def __init__(self, scout_parser: HTMLParser):
+        """
+        Initialize the parser with a Scout HTML parser.
+        
+        Args:
+            scout_parser (HTMLParser): Scout's HTML parser instance
+        """
+        super().__init__(convert_charrefs=True)
+        self._scout_parser = scout_parser
+    
+    def handle_starttag(self, tag: str, attrs: List[tuple]):
+        """
+        Handle opening tags during parsing.
+        
+        Args:
+            tag (str): Tag name
+            attrs (List[tuple]): Tag attributes
+        """
+        # Convert attrs to dictionary
+        attrs_dict = dict(attrs)
+        
+        # Create Tag instance
+        new_tag = Tag(tag, attrs_dict)
+        
+        # Add tag to the parsing context
+        self._scout_parser.add_tag(new_tag)
+    
+    def handle_endtag(self, tag: str):
+        """
+        Handle closing tags during parsing.
+        
+        Args:
+            tag (str): Tag name
+        """
+        # Close the current tag
+        self._scout_parser.close_tag()
+    
+    def handle_data(self, data: str):
+        """
+        Handle text data during parsing.
+        
+        Args:
+            data (str): Text content
+        """
+        # Add text to the current tag
+        self._scout_parser.add_text(data)
+    
+    def handle_comment(self, data: str):
+        """
+        Handle HTML comments (optional, can be configured).
+        
+        Args:
+            data (str): Comment content
+        """
+        # Optionally handle comments
+        comment_tag = Tag('comment')
+        comment_tag.attrs['content'] = data
+        self._scout_parser.add_tag(comment_tag)
+    
+    def handle_decl(self, decl: str):
+        """
+        Handle HTML declarations.
+        
+        Args:
+            decl (str): Declaration content
+        """
+        # Create a special tag for declarations
+        decl_tag = Tag('!DOCTYPE')
+        decl_tag.attrs['content'] = decl
+        self._scout_parser.add_tag(decl_tag)
+    
+    def handle_pi(self, data: str):
+        """
+        Handle processing instructions.
+        
+        Args:
+            data (str): Processing instruction content
+        """
+        # Create a special tag for processing instructions
+        pi_tag = Tag('?')
+        pi_tag.attrs['content'] = data
+        self._scout_parser.add_tag(pi_tag)
+    
+    def handle_entityref(self, name: str):
+        """
+        Handle HTML entity references.
+        
+        Args:
+            name (str): Entity reference name
+        """
+        # Convert entity references to their actual characters
+        char = html.entities.html5.get(name, f'&{name};')
+        self._scout_parser.add_text(char)
+    
+    def handle_charref(self, name: str):
+        """
+        Handle character references.
+        
+        Args:
+            name (str): Character reference name
+        """
+        # Convert character references to their actual characters
+        try:
+            if name.startswith('x'):
+                char = chr(int(name[1:], 16))
+            else:
+                char = chr(int(name))
+            self._scout_parser.add_text(char)
+        except ValueError:
+            # Fallback for invalid references
+            self._scout_parser.add_text(f'&#{name};')
+    
+    def close(self):
+        """
+        Finalize parsing and perform cleanup.
+        """
+        super().close()

webscout/scout/parsers/lxml_parser.py

@@ -0,0 +1,178 @@
+"""
+Scout LXML Parser - Advanced HTML/XML Parsing with lxml
+"""
+
+import re
+from typing import List, Optional, Dict, Any, Union
+
+from lxml import etree, html as lxml_html
+from ..element import Tag, NavigableString
+
+class LXMLParser:
+    """
+    Advanced HTML/XML parser using lxml library.
+    Provides robust parsing with enhanced error handling and flexibility.
+    """
+    
+    def __init__(self, parser_type: str = 'html'):
+        """
+        Initialize the LXML parser with advanced parsing capabilities.
+        
+        Args:
+            parser_type (str): Type of parser ('html' or 'xml')
+        """
+        self._parser_type = parser_type
+        self._parsing_errors = []
+        
+        # Select appropriate parser based on type
+        if parser_type == 'html':
+            self._parser = lxml_html.HTMLParser(recover=True, encoding='utf-8')
+        elif parser_type == 'xml':
+            self._parser = etree.XMLParser(recover=True, encoding='utf-8')
+        else:
+            raise ValueError(f"Unsupported parser type: {parser_type}")
+    
+    def parse(self, markup: str) -> Tag:
+        """
+        Parse HTML/XML markup and return the root tag.
+        
+        Args:
+            markup (str): HTML/XML content to parse
+        
+        Returns:
+            Tag: Parsed document root
+        """
+        try:
+            # Preprocess markup to handle common issues
+            markup = self._preprocess_markup(markup)
+            
+            # Parse the markup
+            if self._parser_type == 'html':
+                tree = lxml_html.fromstring(markup, parser=self._parser)
+            else:
+                tree = etree.fromstring(markup, parser=self._parser)
+            
+            # Convert lxml element to Scout Tag
+            return self._convert_element(tree)
+        
+        except Exception as e:
+            self._parsing_errors.append(str(e))
+            return Tag('root')
+    
+    def _preprocess_markup(self, markup: str) -> str:
+        """
+        Preprocess markup to handle common parsing issues.
+        
+        Args:
+            markup (str): Raw markup
+        
+        Returns:
+            str: Preprocessed markup
+        """
+        # Remove XML/HTML comments
+        markup = re.sub(r'<!--.*?-->', '', markup, flags=re.DOTALL)
+        
+        # Handle unclosed tags for HTML
+        if self._parser_type == 'html':
+            markup = re.sub(r'<(br|img|input|hr|meta)([^>]*?)(?<!/)>', r'<\1\2 />', markup, flags=re.IGNORECASE)
+        
+        return markup
+    
+    def _convert_element(self, element: Union[etree._Element, lxml_html.HtmlElement]) -> Tag:
+        """
+        Convert lxml element to Scout Tag.
+        
+        Args:
+            element (etree._Element or lxml_html.HtmlElement): lxml element
+        
+        Returns:
+            Tag: Converted Scout Tag
+        """
+        # Create Tag with name and attributes
+        tag = Tag(element.tag, dict(element.attrib))
+        
+        # Add text content
+        if element.text:
+            tag.contents.append(NavigableString(element.text))
+        
+        # Recursively add child elements
+        for child in element:
+            child_tag = self._convert_element(child)
+            child_tag.parent = tag
+            tag.contents.append(child_tag)
+            
+            # Add tail text
+            if child.tail:
+                tail_text = NavigableString(child.tail)
+                tail_text.parent = tag
+                tag.contents.append(tail_text)
+        
+        return tag
+    
+    def get_parsing_errors(self) -> List[str]:
+        """
+        Retrieve parsing errors encountered during processing.
+        
+        Returns:
+            List[str]: List of parsing error messages
+        """
+        return self._parsing_errors
+    
+    def find_all(self, markup: str, tag: Optional[Union[str, List[str]]] = None, 
+                 attrs: Optional[Dict[str, Any]] = None, 
+                 recursive: bool = True, 
+                 text: Optional[str] = None, 
+                 limit: Optional[int] = None) -> List[Tag]:
+        """
+        Find all matching elements in the parsed document.
+        
+        Args:
+            markup (str): HTML/XML content to parse
+            tag (str or List[str], optional): Tag name(s) to search for
+            attrs (dict, optional): Attribute filters
+            recursive (bool): Whether to search recursively
+            text (str, optional): Text content to search for
+            limit (int, optional): Maximum number of results
+        
+        Returns:
+            List[Tag]: List of matching tags
+        """
+        root = self.parse(markup)
+        
+        def matches(element: Tag) -> bool:
+            """Check if an element matches search criteria."""
+            # Tag filter
+            if tag and isinstance(tag, str) and element.name != tag:
+                return False
+            if tag and isinstance(tag, list) and element.name not in tag:
+                return False
+            
+            # Attribute filter
+            if attrs:
+                for key, value in attrs.items():
+                    if key not in element.attrs or element.attrs[key] != value:
+                        return False
+            
+            # Text filter
+            if text:
+                element_text = ' '.join([str(c) for c in element.contents if isinstance(c, NavigableString)])
+                if text not in element_text:
+                    return False
+            
+            return True
+        
+        def collect_matches(element: Tag, results: List[Tag]):
+            """Recursively collect matching elements."""
+            if matches(element):
+                results.append(element)
+                if limit and len(results) >= limit:
+                    return
+            
+            if recursive:
+                for child in element.contents:
+                    if isinstance(child, Tag):
+                        collect_matches(child, results)
+        
+        results = []
+        collect_matches(root, results)
+        return results

webscout/scout/utils.py

@@ -0,0 +1,37 @@
+"""
+Utility functions - making life easier! 🛠️
+"""
+
+from typing import Union, Optional
+
+def decode_markup(markup: Union[str, bytes], encoding: Optional[str] = None) -> str:
+    """
+    Decode that markup - no encoding drama! 🎯
+    
+    Args:
+        markup: The raw HTML/XML content
+        encoding: The encoding to use (if known)
+    
+    Returns:
+        Clean decoded string ready for parsing! ✨
+    """
+    if isinstance(markup, str):
+        return markup
+    
+    if encoding:
+        try:
+            return markup.decode(encoding)
+        except UnicodeDecodeError:
+            pass
+    
+    # Try common encodings - we got options! 💪
+    encodings = ['utf-8', 'latin1', 'iso-8859-1', 'ascii']
+    
+    for enc in encodings:
+        try:
+            return markup.decode(enc)
+        except UnicodeDecodeError:
+            continue
+    
+    # Last resort - ignore errors and keep it moving! 🚀
+    return markup.decode('utf-8', errors='ignore')