webscout 6.8__py3-none-any.whl → 7.0__py3-none-any.whl

webscout/Extra/markdownlite/__init__.py

@@ -1,862 +0,0 @@
-import sys
-import os
-from webscout.Litlogger import LitLogger, LogFormat, ColorScheme
-from webscout.scout import Scout, Tag
-from textwrap import fill
-import re
-import six
-import html
-import json
-from typing import Union, Dict, Any, Optional, List
-import functools
-
-# Initialize Litlogger
-logger = LitLogger(
-    name="MarkdownLite", 
-    format=LogFormat.DETAILED,
-    color_scheme=ColorScheme.OCEAN
-)
-
-# Decorator for error handling and logging
-def markdown_conversion_error_handler(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            logger.error(f"Markdown conversion error: {e}", exc_info=True)
-            raise
-    return wrapper
-
-# Constants and configuration
-MARKDOWN_CONVERSION_OPTIONS = {
-    'SEMANTIC_CONVERSION': True,
-    'PRESERVE_METADATA': True,
-    'SMART_LISTS': True,
-    'LINK_REWRITING': None,
-    'CUSTOM_ANALYZERS': [],
-    'STRUCTURED_OUTPUT': True,
-    'DEBUG_MODE': False
-}
-
-# Existing utility functions
-def chomp(text):
-    """
-    Strip leading/trailing spaces while preserving prefix/suffix spaces.
-    
-    Args:
-        text (str): Input text to process
-    
-    Returns:
-        tuple: (prefix, suffix, stripped_text)
-    """
-    prefix = ' ' if text and text[0] == ' ' else ''
-    suffix = ' ' if text and text[-1] == ' ' else ''
-    text = text.strip()
-    return (prefix, suffix, text)
-
-def abstract_inline_conversion(markup_fn):
-    """
-    Abstract inline tag conversion with enhanced flexibility.
-    
-    Args:
-        markup_fn (callable): Function to generate markup
-    
-    Returns:
-        callable: Conversion implementation
-    """
-    def implementation(self, el, text, convert_as_inline):
-        markup_prefix = markup_fn(self)
-        if markup_prefix.startswith('<') and markup_prefix.endswith('>'):
-            markup_suffix = '</' + markup_prefix[1:]
-        else:
-            markup_suffix = markup_prefix
-        
-        if el.find_parent(['pre', 'code', 'kbd', 'samp']):
-            return text
-        
-        prefix, suffix, text = chomp(text)
-        if not text:
-            return ''
-        
-        return f'{prefix}{markup_prefix}{text}{markup_suffix}{suffix}'
-    return implementation
-
-class MarkdownConverter(object):
-    class DefaultOptions:
-        autolinks = True
-        bullets = '*+-'  # An iterable of bullet types.
-        code_language = ''
-        code_language_callback = None
-        convert = None
-        default_title = False
-        escape_asterisks = True
-        escape_underscores = True
-        escape_misc = False
-        heading_style = 'underlined'
-        keep_inline_images_in = []
-        newline_style = 'spaces'
-        strip = None
-        strong_em_symbol = '*'
-        sub_symbol = ''
-        sup_symbol = ''
-        wrap = False
-        wrap_width = 80
-        
-        # New options for Scout integration
-        semantic_conversion = False  # Enable semantic-aware conversion
-        preserve_metadata = False  # Keep HTML metadata in markdown
-        smart_lists = True  # Smart list handling
-        link_rewriting = None  # Function for rewriting URLs
-        custom_analyzers = []  # List of custom text analyzers
-        structured_output = False  # Return structured output with metadata
-        
-        # Existing options
-        debug_mode = False
-        handle_unknown_tags = 'ignore'  # 'ignore', 'warn', 'error'
-        preserve_html_comments = False
-        max_depth = 100  # Prevent potential infinite recursion
-        custom_tag_handlers = {}  # Allow custom tag conversion functions
-
-    class Options(DefaultOptions):
-        pass
-
-    # Inline conversion methods
-    convert_b = abstract_inline_conversion(lambda self: 2 * self.options['strong_em_symbol'])
-    convert_del = abstract_inline_conversion(lambda self: '~~')
-    convert_em = abstract_inline_conversion(lambda self: self.options['strong_em_symbol'])
-    convert_i = convert_em
-    convert_s = convert_del
-    convert_strong = convert_b
-    convert_sub = abstract_inline_conversion(lambda self: self.options['sub_symbol'])
-    convert_sup = abstract_inline_conversion(lambda self: self.options['sup_symbol'])
-
-    def __init__(self, **options):
-        # Merge default and user-provided options
-        default_options = {
-            'SEMANTIC_CONVERSION': True,
-            'PRESERVE_METADATA': True,
-            'SMART_LISTS': True,
-            'LINK_REWRITING': None,
-            'CUSTOM_ANALYZERS': [],
-            'STRUCTURED_OUTPUT': True,
-            'DEBUG_MODE': False,
-            # Add max_depth for conversion
-            'max_depth': 10,
-            # Inherit existing default options
-            'strip': None,
-            'convert': None,
-            'heading_style': 'underlined',
-            'newline_style': 'spaces',
-            'strong_em_symbol': '*',
-            'escape_asterisks': True,
-            'escape_underscores': True,
-            'escape_misc': False,
-            'keep_inline_images_in': [],
-            'sub_symbol': '',
-            'sup_symbol': '',
-            'wrap': False
-        }
-        
-        # Update with user options
-        default_options.update(options)
-        self.options = default_options
-        
-        # Setup logging based on debug mode
-        if self.options['DEBUG_MODE']:
-            logger.setLevel(logging.DEBUG)
-        
-        # Initialize metadata and structure
-        self._metadata = {}
-        self._structure = {}
-        self._semantic_info = {}
-
-    @markdown_conversion_error_handler
-    def convert(self, html):
-        """
-        Enhanced conversion with metadata and structure analysis.
-        
-        Args:
-            html (str): HTML content to convert
-        
-        Returns:
-            Union[str, Dict[str, Any]]: Markdown text or structured output
-        """
-        # Handle different Scout result types
-        scout = html if hasattr(html, '_soup') or hasattr(html, 'name') else Scout(html, features='html.parser')
-        
-        # If scout is a search result, get the first result or the original scout
-        if hasattr(scout, '_results') and scout._results:
-            scout = scout._results[0]
-        
-        # Ensure we have a valid Scout object or Tag
-        if not hasattr(scout, '_soup') and not hasattr(scout, 'name'):
-            raise ValueError("Unable to convert input to a valid Scout object")
-        
-        logger.debug(f"Parsing HTML: {str(scout)[:100]}...")
-        
-        # Extract additional information if needed
-        if self.options['PRESERVE_METADATA']:
-            self._metadata = self._extract_metadata(scout)
-            
-        if self.options['SEMANTIC_CONVERSION']:
-            self._structure = self._analyze_structure(scout)
-            self._semantic_info = self._extract_semantic_info(scout)
-        
-        # Convert to markdown
-        markdown = self.convert_soup(scout._soup if hasattr(scout, '_soup') else scout)
-        
-        # Return structured output if requested
-        if self.options['STRUCTURED_OUTPUT']:
-            return {
-                'markdown': markdown,
-                'metadata': self._metadata,
-                'structure': self._structure,
-                'semantic_info': self._semantic_info
-            }
-            
-        return markdown
-
-    def convert_soup(self, soup):
-        """Convert Scout's internal soup object."""
-        return self.process_tag(soup, convert_as_inline=False, children_only=True)
-
-    def process_tag(self, node, convert_as_inline, children_only=False, depth=0):
-        """Enhanced tag processing with semantic awareness."""
-        if depth > self.options['max_depth']:
-            logger.warning(f"Max recursion depth reached at tag: {node.name}")
-            return ''
-
-        # Check for custom tag handlers
-        if hasattr(node, 'name') and node.name in self.options['custom_tag_handlers']:
-            custom_handler = self.options['custom_tag_handlers'][node.name]
-            return custom_handler(node, convert_as_inline)
-
-        text = ''
-
-        # markdown headings or cells can't include
-        # block elements (elements w/newlines)
-        isHeading = re.match(r'h[1-6]', node.name) if hasattr(node, 'name') else False
-        isCell = hasattr(node, 'name') and node.name in ['td', 'th']
-        convert_children_as_inline = convert_as_inline
-
-        if not children_only and (isHeading or isCell):
-            convert_children_as_inline = True
-
-        # Remove whitespace-only textnodes
-        should_remove_inside = should_remove_whitespace_inside(node)
-
-        # Iterate through children
-        for el in node.children:
-            # Skip script, style, and comment-like elements
-            if hasattr(el, 'name') and el.name in ['script', 'style', 'comment']:
-                continue
-
-            # Check if element is a text node that can be stripped
-            if (isinstance(el, str) or 
-                (hasattr(el, 'string') and el.string and str(el.string).strip() == '')):
-                if should_remove_inside and (not el.previous_sibling or not el.next_sibling):
-                    continue
-
-            # Process child elements
-            if isinstance(el, str):
-                text += el
-            elif hasattr(el, 'name'):
-                text_strip = text.rstrip('\n')
-                newlines_left = len(text) - len(text_strip)
-                next_text = self.process_tag(el, convert_children_as_inline)
-                next_text_strip = next_text.lstrip('\n')
-                newlines_right = len(next_text) - len(next_text_strip)
-                newlines = '\n' * max(newlines_left, newlines_right)
-                text = text_strip + newlines + next_text_strip
-
-        if not children_only and hasattr(node, 'name'):
-            convert_fn = getattr(self, 'convert_%s' % node.name, None)
-            if convert_fn and self.should_convert_tag(node.name):
-                text = convert_fn(node, text, convert_as_inline)
-
-        # Apply custom analyzers
-        for analyzer in self.options['custom_analyzers']:
-            text = analyzer(text, node)
-            
-        return text
-
-    def _validate_options(self):
-        """Validate and sanitize converter options."""
-        if self.options['max_depth'] < 1:
-            raise ValueError("max_depth must be a positive integer")
-        
-        if self.options['handle_unknown_tags'] not in ['ignore', 'warn', 'error']:
-            raise ValueError("handle_unknown_tags must be 'ignore', 'warn', or 'error'")
-
-    def process_text(self, el):
-        text = six.text_type(el) or ''
-
-        # normalize whitespace if we're not inside a preformatted element
-        if not el.find_parent('pre'):
-            if self.options['wrap']:
-                text = re.sub(r'[\t ]+', ' ', text)
-            else:
-                text = re.sub(r'[\t \r\n]*[\r\n][\t \r\n]*', '\n', text)
-                text = re.sub(r'[\t ]+', ' ', text)
-
-        # escape special characters if we're not inside a preformatted or code element
-        if not el.find_parent(['pre', 'code', 'kbd', 'samp']):
-            text = self.escape(text)
-
-        # remove leading whitespace at the start or just after a
-        # block-level element; remove traliing whitespace at the end
-        # or just before a block-level element.
-        if (should_remove_whitespace_outside(el.previous_sibling)
-                or (should_remove_whitespace_inside(el.parent)
-                    and not el.previous_sibling)):
-            text = text.lstrip()
-        if (should_remove_whitespace_outside(el.next_sibling)
-                or (should_remove_whitespace_inside(el.parent)
-                    and not el.next_sibling)):
-            text = text.rstrip()
-
-        return text
-
-    def __getattr__(self, attr):
-        # Handle headings
-        m = re.match(r'convert_h(\d+)', attr)
-        if m:
-            n = int(m.group(1))
-
-            def convert_tag(el, text, convert_as_inline):
-                return self._convert_hn(n, el, text, convert_as_inline)
-
-            convert_tag.__name__ = 'convert_h%s' % n
-            setattr(self, convert_tag.__name__, convert_tag)
-            return convert_tag
-
-        raise AttributeError(attr)
-
-    def should_convert_tag(self, tag):
-        tag = tag.lower()
-        strip = self.options['strip']
-        convert = self.options['convert']
-        if strip is not None:
-            return tag not in strip
-        elif convert is not None:
-            return tag in convert
-        else:
-            return True
-
-    def escape(self, text):
-        if not text:
-            return ''
-        if self.options['escape_misc']:
-            text = re.sub(r'([\\&<`[>~=+|])', r'\\\1', text)
-            # A sequence of one or more consecutive '-', preceded and
-            # followed by whitespace or start/end of fragment, might
-            # be confused with an underline of a header, or with a
-            # list marker.
-            text = re.sub(r'(\s|^)(-+(?:\s|$))', r'\1\\\2', text)
-            # A sequence of up to six consecutive '#', preceded and
-            # followed by whitespace or start/end of fragment, might
-            # be confused with an ATX heading.
-            text = re.sub(r'(\s|^)(#{1,6}(?:\s|$))', r'\1\\\2', text)
-            # '.' or ')' preceded by up to nine digits might be
-            # confused with a list item.
-            text = re.sub(r'((?:\s|^)[0-9]{1,9})([.)](?:\s|$))', r'\1\\\2',
-                          text)
-        if self.options['escape_asterisks']:
-            text = text.replace('*', r'\*')
-        if self.options['escape_underscores']:
-            text = text.replace('_', r'\_')
-        return text
-
-    def indent(self, text, columns):
-        return re.sub(r'^', ' ' * columns, text, flags=re.MULTILINE) if text else ''
-
-    def underline(self, text, pad_char):
-        text = (text or '').rstrip()
-        return '\n\n%s\n%s\n\n' % (text, pad_char * len(text)) if text else ''
-
-    def convert_a(self, el, text, convert_as_inline):
-        """Enhanced link conversion with URL rewriting."""
-        if self.options['link_rewriting'] and callable(self.options['link_rewriting']):
-            href = el.get('href')
-            if href:
-                href = self.options['link_rewriting'](href)
-                el['href'] = href
-                
-        prefix, suffix, text = chomp(text)
-        if not text:
-            return ''
-        href = el.get('href')
-        title = el.get('title')
-        # For the replacement see #29: text nodes underscores are escaped
-        if (self.options['autolinks']
-                and text.replace(r'\_', '_') == href
-                and not title
-                and not self.options['default_title']):
-            # Shortcut syntax
-            return '<%s>' % href
-        if self.options['default_title'] and not title:
-            title = href
-        title_part = ' "%s"' % title.replace('"', r'\"') if title else ''
-        return '%s[%s](%s%s)%s' % (prefix, text, href, title_part, suffix) if href else text
-
-    def convert_blockquote(self, el, text, convert_as_inline):
-
-        if convert_as_inline:
-            return ' ' + text.strip() + ' '
-
-        return '\n' + (re.sub(r'^', '> ', text.strip(), flags=re.MULTILINE) + '\n\n') if text else ''
-
-    def convert_br(self, el, text, convert_as_inline):
-        if convert_as_inline:
-            return ""
-
-        if self.options['newline_style'].lower() == 'backslash':
-            return '\\\n'
-        else:
-            return '  \n'
-
-    def convert_code(self, el, text, convert_as_inline):
-        if el.parent.name == 'pre':
-            return text
-        converter = abstract_inline_conversion(lambda self: '`')
-        return converter(self, el, text, convert_as_inline)
-
-    def convert_kbd(self, el, text, convert_as_inline):
-        return self.convert_code(el, text, convert_as_inline)
-
-    def _convert_hn(self, n, el, text, convert_as_inline):
-        """ Method name prefixed with _ to prevent <hn> to call this """
-        if convert_as_inline:
-            return text
-
-        # prevent MemoryErrors in case of very large n
-        n = max(1, min(6, n))
-
-        style = self.options['heading_style'].lower()
-        text = text.strip()
-        if style == 'underlined' and n <= 2:
-            line = '=' if n == 1 else '-'
-            return self.underline(text, line)
-        text = re.sub(r'[\t ]+', ' ', text)
-        hashes = '#' * n
-        if style == 'atx_closed':
-            return '\n%s %s %s\n\n' % (hashes, text, hashes)
-        return '\n%s %s\n\n' % (hashes, text)
-
-    def convert_hr(self, el, text, convert_as_inline):
-        return '\n\n---\n\n'
-
-    def convert_img(self, el, text, convert_as_inline):
-        alt = el.attrs.get('alt', None) or ''
-        src = el.attrs.get('src', None) or ''
-        title = el.attrs.get('title', None) or ''
-        title_part = ' "%s"' % title.replace('"', r'\"') if title else ''
-        if (convert_as_inline
-                and el.parent.name not in self.options['keep_inline_images_in']):
-            return alt
-
-        return '![%s](%s%s)' % (alt, src, title_part)
-
-    def convert_list(self, el, text, convert_as_inline):
-        """Enhanced list conversion with smart handling."""
-        if not self.options['smart_lists']:
-            return super().convert_list(el, text, convert_as_inline)
-            
-        nested = False
-        before_paragraph = False
-        
-        # Smart list processing
-        list_type = el.name
-        is_ordered = list_type == 'ol'
-        start = el.get('start', 1) if is_ordered else None
-        
-        # Process list items
-        items = el.find_all('li', recursive=False)
-        processed_items = []
-        
-        for i, item in enumerate(items):
-            item_text = self.process_tag(item, convert_as_inline)
-            if is_ordered:
-                number = start + i if start else i + 1
-                processed_items.append(f"{number}. {item_text}")
-            else:
-                processed_items.append(f"* {item_text}")
-                
-        return '\n'.join(processed_items)
-
-    def convert_ul(self, el, text, convert_as_inline):
-        return self.convert_list(el, text, convert_as_inline)
-
-    def convert_ol(self, el, text, convert_as_inline):
-        return self.convert_list(el, text, convert_as_inline)
-
-    def convert_li(self, el, text, convert_as_inline):
-        parent = el.parent
-        if parent is not None and parent.name == 'ol':
-            if parent.get("start") and str(parent.get("start")).isnumeric():
-                start = int(parent.get("start"))
-            else:
-                start = 1
-            bullet = '%s.' % (start + parent.index(el))
-        else:
-            depth = -1
-            while el:
-                if el.name == 'ul':
-                    depth += 1
-                el = el.parent
-            bullets = self.options['bullets']
-            bullet = bullets[depth % len(bullets)]
-        bullet = bullet + ' '
-        text = (text or '').strip()
-        text = self.indent(text, len(bullet))
-        if text:
-            text = bullet + text[len(bullet):]
-        return '%s\n' % text
-
-    def convert_p(self, el, text, convert_as_inline):
-        if convert_as_inline:
-            return ' ' + text.strip() + ' '
-        if self.options['wrap']:
-            # Preserve newlines (and preceding whitespace) resulting
-            # from <br> tags.  Newlines in the input have already been
-            # replaced by spaces.
-            lines = text.split('\n')
-            new_lines = []
-            for line in lines:
-                line = line.lstrip()
-                line_no_trailing = line.rstrip()
-                trailing = line[len(line_no_trailing):]
-                line = fill(line,
-                            width=self.options['wrap_width'],
-                            break_long_words=False,
-                            break_on_hyphens=False)
-                new_lines.append(line + trailing)
-            text = '\n'.join(new_lines)
-        return '\n\n%s\n\n' % text if text else ''
-
-    def convert_pre(self, el, text, convert_as_inline):
-        if not text:
-            return ''
-        code_language = self.options['code_language']
-
-        if self.options['code_language_callback']:
-            code_language = self.options['code_language_callback'](el) or code_language
-
-        return '\n```%s\n%s\n```\n' % (code_language, text)
-
-    def convert_script(self, el, text, convert_as_inline):
-        return ''
-
-    def convert_style(self, el, text, convert_as_inline):
-        return ''
-
-    def convert_comment(self, el, text, convert_as_inline):
-        """Handle comment-like elements based on configuration."""
-        if self.options['preserve_html_comments']:
-            return f'<!-- {text} -->'
-        return ''
-
-    def convert_details(self, el, text, convert_as_inline):
-        """Convert HTML5 details and summary tags."""
-        summary = el.find('summary')
-        summary_text = summary.text if summary else 'Details'
-        return f'\n<details>\n<summary>{summary_text}</summary>\n\n{text}\n</details>\n'
-
-    def convert_mark(self, el, text, convert_as_inline):
-        """Convert mark tag with highlighting."""
-        return f'`{text}`'
-
-    def convert_table(self, el, text, convert_as_inline):
-        return '\n\n' + text + '\n'
-
-    def convert_caption(self, el, text, convert_as_inline):
-        return text + '\n'
-
-    def convert_figcaption(self, el, text, convert_as_inline):
-        return '\n\n' + text + '\n\n'
-
-    def convert_td(self, el, text, convert_as_inline):
-        colspan = 1
-        if 'colspan' in el.attrs and el['colspan'].isdigit():
-            colspan = int(el['colspan'])
-        return ' ' + text.strip().replace("\n", " ") + ' |' * colspan
-
-    def convert_th(self, el, text, convert_as_inline):
-        colspan = 1
-        if 'colspan' in el.attrs and el['colspan'].isdigit():
-            colspan = int(el['colspan'])
-        return ' ' + text.strip().replace("\n", " ") + ' |' * colspan
-
-    def convert_tr(self, el, text, convert_as_inline):
-        cells = el.find_all(['td', 'th'])
-        is_headrow = (
-            all([cell.name == 'th' for cell in cells])
-            or (not el.previous_sibling and not el.parent.name == 'tbody')
-            or (not el.previous_sibling and el.parent.name == 'tbody' and len(el.parent.parent.find_all(['thead'])) < 1)
-        )
-        overline = ''
-        underline = ''
-        if is_headrow and not el.previous_sibling:
-            # first row and is headline: print headline underline
-            full_colspan = 0
-            for cell in cells:
-                if 'colspan' in cell.attrs and cell['colspan'].isdigit():
-                    full_colspan += int(cell["colspan"])
-                else:
-                    full_colspan += 1
-            underline += '| ' + ' | '.join(['---'] * full_colspan) + ' |' + '\n'
-        elif (not el.previous_sibling
-              and (el.parent.name == 'table'
-                   or (el.parent.name == 'tbody'
-                       and not el.parent.previous_sibling))):
-            # first row, not headline, and:
-            # - the parent is table or
-            # - the parent is tbody at the beginning of a table.
-            # print empty headline above this row
-            overline += '| ' + ' | '.join([''] * len(cells)) + ' |' + '\n'
-            overline += '| ' + ' | '.join(['---'] * len(cells)) + ' |' + '\n'
-        return overline + '|' + text + '\n' + underline
-
-    def _extract_metadata(self, scout):
-        """
-        Extract metadata from the parsed document.
-        
-        Args:
-            scout (Union[Scout, Tag, ScoutSearchResult]): Parsed object
-        
-        Returns:
-            Dict[str, Any]: Extracted metadata
-        """
-        metadata = {}
-        try:
-            # Handle ScoutSearchResult
-            if hasattr(scout, '_results'):
-                scout = scout._results[0] if scout._results else None
-            
-            if scout is None:
-                return metadata
-
-            # Find head tag
-            head = scout.find('head')
-            if not head and hasattr(scout, 'find_all'):
-                head_list = scout.find_all('head')
-                head = head_list[0] if head_list else None
-
-            if head:
-                # Extract title
-                title_tag = head.find('title') or (head.find_all('title')[0] if head.find_all('title') else None)
-                metadata['title'] = title_tag.get_text() if title_tag else None
-
-                # Extract meta tags
-                metadata['meta'] = {}
-                meta_tags = head.find_all('meta') if hasattr(head, 'find_all') else []
-                for meta in meta_tags:
-                    name = meta.get('name') or meta.get('property')
-                    content = meta.get('content')
-                    if name and content:
-                        metadata['meta'][name] = content
-
-        except Exception as e:
-            logger.warning(f"Metadata extraction failed: {e}")
-        
-        return metadata
-
-    def _extract_semantic_info(self, scout):
-        """
-        Extract semantic information from the document.
-        
-        Args:
-            scout (Union[Scout, Tag, ScoutSearchResult]): Parsed object
-        
-        Returns:
-            Dict[str, Any]: Semantic information
-        """
-        # Handle ScoutSearchResult
-        if hasattr(scout, '_results'):
-            scout = scout._results[0] if scout._results else None
-        
-        if scout is None:
-            return {
-                'language': 'unknown',
-                'text_density': 0,
-                'content_types': {}
-            }
-
-        semantic_info = {
-            'language': 'unknown',
-            'text_density': 0,
-            'content_types': {}
-        }
-
-        try:
-            # Try to find language
-            html_tag = scout.find('html')
-            if not html_tag and hasattr(scout, 'find_all'):
-                html_tags = scout.find_all('html')
-                html_tag = html_tags[0] if html_tags else None
-
-            semantic_info['language'] = html_tag.get('lang', 'unknown') if html_tag else 'unknown'
-            
-            # Calculate text density
-            total_text = scout.get_text() if hasattr(scout, 'get_text') else ''
-            total_html = str(scout)
-            semantic_info['text_density'] = len(total_text) / len(total_html) * 100 if total_html else 0
-
-            # Analyze content types
-            content_types = {}
-            for tag in scout.find_all() if hasattr(scout, 'find_all') else [scout]:
-                tag_type = tag.name
-                content_types[tag_type] = content_types.get(tag_type, 0) + 1
-            
-            semantic_info['content_types'] = content_types
-
-        except Exception as e:
-            logger.warning(f"Semantic info extraction failed: {e}")
-        
-        return semantic_info
-
-    def _analyze_structure(self, scout):
-        """
-        Analyze document structure.
-        
-        Args:
-            scout (Scout): Parsed Scout object
-        
-        Returns:
-            Dict[str, Any]: Document structure information
-        """
-        structure = {
-            'headings': [
-                {'level': h.name, 'text': h.get_text(strip=True)}
-                for h in scout.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
-            ],
-            'sections': [],
-            'links': [
-                {'href': a.get('href'), 'text': a.get_text(strip=True)}
-                for a in scout.find_all('a')
-            ]
-        }
-        return structure
-
-    def _calculate_text_density(self, scout):
-        """
-        Calculate text density of the document.
-        
-        Args:
-            scout (Scout): Parsed Scout object
-        
-        Returns:
-            float: Text density percentage
-        """
-        try:
-            total_text = scout.get_text()
-            total_html = str(scout)
-            return len(total_text) / len(total_html) * 100 if total_html else 0
-        except Exception as e:
-            logger.warning(f"Text density calculation failed: {e}")
-            return 0
-
-    def _analyze_content_types(self, scout):
-        """
-        Analyze content types in the document.
-        
-        Args:
-            scout (Scout): Parsed Scout object
-        
-        Returns:
-            Dict[str, int]: Content type counts
-        """
-        content_types = {}
-        try:
-            for tag in scout.find_all():
-                tag_type = tag.name
-                content_types[tag_type] = content_types.get(tag_type, 0) + 1
-        except Exception as e:
-            logger.warning(f"Content type analysis failed: {e}")
-        return content_types
-
-def markdownify(html: str, **options) -> Union[str, Dict[str, Any]]:
-    """
-    Convert HTML to Markdown with advanced options.
-    
-    Args:
-        html (str): HTML content to convert
-        **options: Conversion options
-    
-    Returns:
-        Union[str, Dict[str, Any]]: Markdown text or structured output
-    """
-    try:
-        # Use Scout's native markdown conversion
-        scout = Scout(html, features='html.parser')
-        
-        # Handle ScoutSearchResult
-        if hasattr(scout, '_results'):
-            scout = scout._results[0] if scout._results else scout
-        
-        # Determine conversion style based on options
-        heading_style = options.get('heading_style', 'ATX')
-        
-        # Custom markdown conversion to preserve formatting
-        def convert_tag(tag):
-            # Handle specific tag types
-            if tag.name == 'h1':
-                return f"# {tag.get_text(strip=True)}\n\n"
-            elif tag.name == 'h2':
-                return f"## {tag.get_text(strip=True)}\n\n"
-            elif tag.name == 'h3':
-                return f"### {tag.get_text(strip=True)}\n\n"
-            elif tag.name == 'p':
-                return f"{tag.get_text(strip=True)}\n\n"
-            elif tag.name == 'strong':
-                return f"**{tag.get_text(strip=True)}**"
-            elif tag.name == 'em':
-                return f"*{tag.get_text(strip=True)}*"
-            elif tag.name == 'ul':
-                return ''.join(f"* {li.get_text(strip=True)}\n" for li in tag.find_all('li'))
-            elif tag.name == 'ol':
-                return ''.join(f"{i+1}. {li.get_text(strip=True)}\n" for i, li in enumerate(tag.find_all('li')))
-            elif tag.name == 'a':
-                return f"[{tag.get_text(strip=True)}]({tag.get('href', '')})"
-            return tag.get_text(strip=True)
-        
-        # Traverse and convert tags
-        markdown_parts = []
-        for tag in scout.find_all():
-            if tag.name in ['h1', 'h2', 'h3', 'p', 'strong', 'em', 'ul', 'ol', 'a']:
-                markdown_parts.append(convert_tag(tag))
-        
-        markdown = '\n'.join(markdown_parts)
-        
-        # If structured output is requested, include additional metadata
-        if options.get('STRUCTURED_OUTPUT', False):
-            # Custom metadata extraction
-            metadata = {}
-            try:
-                head = scout.find('head') or scout.find_all('head')[0] if scout.find_all('head') else None
-                if head:
-                    # Extract title
-                    title_tag = head.find('title') or head.find_all('title')[0] if head.find_all('title') else None
-                    metadata['title'] = title_tag.get_text() if title_tag else None
-                    
-                    # Extract meta tags
-                    metadata['meta'] = {
-                        meta.get('name', meta.get('property')): meta.get('content')
-                        for meta in head.find_all('meta')
-                        if meta.get('name') or meta.get('property')
-                    }
-            except Exception as e:
-                logger.warning(f"Metadata extraction failed: {e}")
-            
-            return {
-                'markdown': markdown,
-                'metadata': metadata,
-                'structure': scout.analyze_page_structure(),
-                'semantic_info': scout.extract_semantic_info()
-            }
-        
-        return markdown
-    except Exception as e:
-        logger.error(f"Markdownify failed: {e}", exc_info=True)
-        return str(e)