h-ai-brain 0.0.15__py3-none-any.whl → 0.0.16__py3-none-any.whl

h_ai/domain/web_docs/gitbook/text_page.py

@@ -1,46 +0,0 @@
-import hashlib
-from dataclasses import dataclass, field
-from typing import Dict, List
-
-from ....domain.web_docs.gitbook.text_chapter import TextChapter
-
-
-@dataclass
-class TextPage:
-    """Represents text on a page from a web document"""
-    url: str = ""
-    title: str = ""
-    content: str = ""
-    last_updated: str = ""
-
-    index: int = 0
-    toc_level: int = 0
-    parent_id: str = ""
-
-    chapters: List[TextChapter] = field(default_factory=list)
-    links: Dict[str, str] = field(default_factory=dict)  # Text -> URL
-
-    id: str = field(init=False, default="")
-    content_hash: str = field(init=False, default="")
-
-    def __post_init__(self):
-        self.id = hashlib.md5(self.url.encode()).hexdigest()
-        # Generate a content hash for deduplication
-        self.content_hash = hashlib.md5(self.content.encode()).hexdigest() if self.content else ""
-
-    def to_dict(self):
-        """Convert this TextPage instance to a serializable dictionary"""
-        result = {
-            'url': self.url,
-            'title': self.title,
-            #'content': self.content,
-            'last_updated': self.last_updated,
-            'index': self.index,
-            'toc_level': self.toc_level,
-            'parent_id': self.parent_id,
-            'id': self.id,
-            'content_hash': self.content_hash,
-            'links': self.links,
-            'chapters': [chapter.to_dict() for chapter in self.chapters]
-        }
-        return result

h_ai/domain/web_docs/gitbook_web_fetcher_service.py

@@ -1,171 +0,0 @@
-import asyncio
-from typing import Optional, Set, Dict, List
-from urllib.parse import urlparse
-
-import aiohttp
-
-from ...domain.web_docs.gitbook.text_page import TextPage
-from ...domain.webpages.web_text_fetcher_repository import WebTextFetcherRepository
-from ...infrastructure.beautifulsoup.soup_processor import SoupProcessor
-
-
-class GitbookWebFetcherService(WebTextFetcherRepository):
-
-    def __init__(self, url: str):
-        self.headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5'
-        }
-        self.base_url = url.rstrip('/')
-        self.base_domain = urlparse(self.base_url).netloc
-
-        self.delay = 1.0  # Delay between requests in seconds
-        self.max_retries = 3
-        self.retry_delay = 2  # Initial retry delay in seconds
-        self.concurrent_requests = 3  # Number of concurrent requests
-
-        self.visited_urls: Set[str] = set()
-        self.content_hashes: Dict[str, str] = {}  # Hash -> URL
-        self.pages: Dict[str, TextPage] = {}
-
-    async def fetch(self) -> Optional[List[TextPage]]:
-        timeout = aiohttp.ClientTimeout(total=180)  # 3 minutes total timeout
-        async with aiohttp.ClientSession(headers=self.headers, timeout=timeout) as session:
-            # Start with main page
-
-            await self._process_url(session, self.base_url)
-
-            # Wait for all tasks to complete
-            await asyncio.sleep(0)
-
-            # Sort pages by index
-            sorted_pages = sorted(
-                self.pages.values(),
-                key=lambda p: p.index
-            )
-            return sorted_pages
-
-    async def _process_url(self, session: aiohttp.ClientSession, url: str) -> None:
-        if url in self.visited_urls:
-            return
-        print(f"Processing {url}")
-        self.visited_urls.add(url)
-
-        # Fetch page content
-        html_content = await self._fetch_page(session, url)
-        if not html_content:
-            return
-
-        # Extract page content
-        page = await GitbookWebFetcherService.extract_page_content(url, html_content)
-        if not page:
-            return
-
-        # Check for duplicate content
-        if page.content_hash in self.content_hashes:
-            return
-
-        # Set page index
-        page.index = len(self.pages)
-
-        # Add page to collection
-        self.pages[url] = page
-        self.content_hashes[page.content_hash] = url
-
-        # Extract navigation links from this page
-        nav_links = await GitbookWebFetcherService.gitbook_extract_navigation(self.base_url, html_content)
-
-        #Process the discovered links
-        for link in nav_links:
-            if link not in self.visited_urls:
-                # Add delay between requests
-                await asyncio.sleep(self.delay)
-                # Process the URL
-                await self._process_url(session, link)
-
-    @staticmethod
-    async def extract_page_content(url: str, html_content: str) -> Optional[TextPage]:
-        try:
-            soup_processor = SoupProcessor(html_content)
-
-            title = soup_processor.extract_title()
-            if not title:
-                title = urlparse(url).path.split('/')[-1] or "Index"
-                title = title.replace('-', ' ').replace('_', ' ').title()
-
-            last_updated = soup_processor.extract_last_updated_refs_from_soup()
-
-            body_tag = soup_processor.find_body_content()
-            if body_tag is None:
-                return None
-            soup_processor.clean_template_usage(body_tag)
-            chapters = soup_processor.extract_chapters(content=body_tag)
-
-            return TextPage(
-                url=url,
-                title=title,
-                content=html_content,
-                last_updated=last_updated,
-                chapters=chapters,
-            )
-        except Exception as e:
-            return None
-
-    @staticmethod
-    async def gitbook_extract_navigation(base_url: str, html_content: str) -> List[str]:
-        """Extract navigation links from a page"""
-        try:
-
-            soup_processor = SoupProcessor(html_content)
-
-            nav_links = []
-            processed_urls = set()
-
-            # Extract links from modern layout
-            nav_links.extend(soup_processor.gitbook_extract_modern_nav(base_url, processed_urls))
-
-            # Extract links from traditional layout
-            nav_links.extend(soup_processor.gitbook_extract_traditional_nav(base_url, processed_urls))
-
-            # Extract links from pagination elements
-            nav_links.extend(soup_processor.gitbook_extract_pagination_links(base_url, processed_urls))
-
-            # Extract links from search for specific class patterns
-            nav_links.extend(soup_processor.gitbook_extract_class_based_nav(base_url, processed_urls))
-
-            # Remove duplicates while preserving order
-            return list(dict.fromkeys(nav_links))
-
-        except Exception as e:
-            return []
-
-    async def _fetch_page(self, session: aiohttp.ClientSession, url: str) -> Optional[str]:
-        """Fetch a page with retry logic"""
-        retry_count = 0
-        current_delay = self.retry_delay
-
-        while retry_count < self.max_retries:
-            try:
-                async with session.get(url) as response:
-                    if response.status == 429:  # Rate limit
-                        retry_after = response.headers.get('Retry-After', '60')
-                        wait_time = int(retry_after)
-
-                        await asyncio.sleep(wait_time)
-                        retry_count += 1
-                        continue
-
-                    if response.status == 200:
-                        return await response.text()
-                    else:
-                        return None
-
-            except (aiohttp.ClientError, asyncio.TimeoutError) as e:
-                if retry_count < self.max_retries - 1:
-                    await asyncio.sleep(current_delay)
-                    current_delay *= 2  # Exponential backoff
-                    retry_count += 1
-                else:
-                    return None
-        return None

h_ai/domain/web_docs/web_docs_link_detector.py

@@ -1,28 +0,0 @@
-from typing import List
-
-from ...domain.webpages.web_fetcher_repository import WebFetcherRepository
-from ...domain.web_docs.doc_link_scorer_service import DocLinkScorerService
-from ...domain.web_docs.web_link import WebLink
-from ...infrastructure.beautifulsoup.soup_processor import SoupProcessor
-
-
-class WebDocsLinkDetector:
-    def __init__(self, doc_link_scorer: DocLinkScorerService, web_fetcher: WebFetcherRepository, confidence_threshold: float = 0.5):
-        self.doc_link_scorer = doc_link_scorer
-        self.web_fetcher = web_fetcher
-        self.confidence_threshold = confidence_threshold
-
-    async def find_docs_links(self, website_url: str) -> List[WebLink]:
-        doc_links = []
-
-        web_content = await self.web_fetcher.fetch(website_url)
-        if not web_content:
-            return doc_links
-
-        soup_processor = SoupProcessor(web_content)
-        web_links = soup_processor.extract_links(website_url)
-        for web_link in web_links:
-            score = self.doc_link_scorer.score(web_link.url, web_link.title)
-            if score >= self.confidence_threshold:
-                doc_links.append(web_link)
-        return doc_links

h_ai/domain/web_docs/web_link.py

@@ -1,11 +0,0 @@
-class WebLink:
-    def __init__(self, url: str, title: str):
-        self.url = url
-        self.title = title
-
-    def to_dict(self):
-        """Convert this WebLink instance to a serializable dictionary"""
-        return {
-            'url': self.url,
-            'title': self.title
-        }

h_ai/domain/webpages/web_fetcher_repository.py

@@ -1,10 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Optional
-
-
-class WebFetcherRepository(ABC):
-
-    @abstractmethod
-    async def fetch(self, url: str) -> Optional[str]:
-        """Fetch the content of the given URL."""
-        pass

h_ai/domain/webpages/web_text_fetcher_repository.py

@@ -1,12 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List, Optional
-
-from ...domain.web_docs.gitbook.text_page import TextPage
-
-
-class WebTextFetcherRepository(ABC):
-
-    @abstractmethod
-    async def fetch(self) -> Optional[List[TextPage]]:
-        """Fetch all content"""
-        pass

h_ai/infrastructure/beautifulsoup/soup_processor.py

@@ -1,240 +0,0 @@
-import logging
-import re
-from typing import List, Optional
-from urllib.parse import urljoin
-
-from bs4 import BeautifulSoup, Tag
-
-from ...domain.web_docs.gitbook.text_chapter import TextChapter
-from ...domain.web_docs.web_link import WebLink
-
-logger = logging.getLogger(__name__)
-
-class SoupProcessor:
-    def __init__(self, html_content:str):
-        self.soup = BeautifulSoup(html_content, 'html.parser')
-
-    def extract_links(self, base_url: str) -> List[WebLink]:
-        """Extract links from a page"""
-        web_links = []
-        links = self.soup.find_all('a', href=True)
-        for link in links:
-            href = link.get('href', '').strip()
-            if not href or href.startswith('#') or href.startswith('javascript:'):
-                continue
-
-            full_url = urljoin(base_url, href)
-            link_text = link.get_text().strip()
-            web_link = WebLink(url=full_url, title=link_text)
-            web_links.append(web_link)
-        return web_links
-
-    def normalize_url(self, href, base_url) -> Optional[str]:
-        """Normalize URL to absolute form and filter out non-content URLs"""
-        # Skip fragment-only URLs
-        if href.startswith('#'):
-            return None
-
-        # Skip external links
-        if href.startswith(('http://', 'https://')) and not href.startswith(base_url):
-            return None
-
-        # Skip resource URLs
-        if href.endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg', '.pdf', '.zip', '.js', '.css')):
-            return None
-
-        # Convert to absolute URL if needed
-        full_url = href
-        if not href.startswith(('http://', 'https://')):
-            full_url = urljoin(base_url, href)
-
-        # Make sure URL belongs to the same domain
-        if not full_url.startswith(base_url):
-            return None
-
-        return full_url
-
-    def extract_last_updated_refs_from_soup(self) -> str:
-        datetime_value = ""
-
-        # Find and remove elements containing "Last updated" text
-        for element in self.soup.find_all(string=lambda text: text and "Last updated" in text):
-            # Get the parent element and remove it
-            parent = element.parent
-            if parent:
-                parent.decompose()
-
-        return datetime_value
-
-    def extract_title(self) -> Optional[str]:
-        """Extract the title of the page using multiple strategies"""
-        # Strategy 1: Look for h1
-        h1 = self.soup.find('h1')
-        if h1:
-            return h1.get_text(strip=True)
-
-        # Strategy 2: Look for title tag
-        title_tag = self.soup.find('title')
-        if title_tag:
-            title_text = title_tag.get_text(strip=True)
-            title_parts = re.split(r'[|\-–]', title_text)
-            return title_parts[0].strip()
-
-        # Strategy 3: Try to find GitBook-specific title elements
-        gitbook_title = self.soup.find('span', {'data-testid': 'page.title'})
-        if gitbook_title:
-            return gitbook_title.get_text(strip=True)
-
-        return None
-
-    def find_body_content(self) -> Optional[Tag]:
-        body_content = self.soup.find('body')
-        if body_content:
-            return body_content
-        return None
-
-    def gitbook_extract_modern_nav(self, base_url, processed_urls):
-        """Extract navigation from modern GitBook layout"""
-        nav_links = []
-
-        # Look for navigation sidebar
-        sidebar = self.soup.select_one('div[data-testid="page.desktopTableOfContents"]')
-        if sidebar:
-            for link in sidebar.find_all('a', href=True):
-                full_url = self.normalize_url(link['href'], base_url)
-                if full_url and full_url not in processed_urls:
-                    nav_links.append(full_url)
-                    processed_urls.add(full_url)
-
-        return nav_links
-
-    def gitbook_extract_traditional_nav(self, base_url, processed_urls):
-        """Extract navigation from traditional GitBook layout"""
-        nav_links = []
-
-        # Find GitBook navigation elements
-        nav_elements = self.soup.find_all(['nav', 'aside'])
-        for nav in nav_elements:
-            # Look for lists that typically contain the navigation
-            nav_lists = nav.find_all(['ol', 'ul'])
-            for nav_list in nav_lists:
-                for li in nav_list.find_all('li'):
-                    link = li.find('a', href=True)
-                    if link:
-                        full_url = self.normalize_url(link['href'], base_url)
-                        if full_url and full_url not in processed_urls:
-                            nav_links.append(full_url)
-                            processed_urls.add(full_url)
-
-        # Try summary element which is common in GitBook
-        summary = self.soup.find('ul', {'class': 'summary'})
-        if summary:
-            for link in summary.find_all('a', href=True):
-                full_url = self.normalize_url(link['href'], base_url)
-                if full_url and full_url not in processed_urls:
-                    nav_links.append(full_url)
-                    processed_urls.add(full_url)
-
-        return nav_links
-
-    def gitbook_extract_pagination_links(self, base_url, processed_urls):
-        """Extract navigation from pagination elements"""
-        nav_links = []
-
-        # Find pagination links (next/prev)
-        selectors = [
-            'a[aria-label="Next"]',
-            'a[aria-label="Previous"]',
-            'a.navigation-next',
-            'a.navigation-prev',
-            'a:has(svg[data-icon="arrow-right"])',
-            'a:has(svg[data-icon="arrow-left"])'
-        ]
-
-        for selector in selectors:
-            try:
-                for link in self.soup.select(selector):
-                    if link.has_attr('href'):
-                        full_url = self.normalize_url(link['href'], base_url)
-                        if full_url and full_url not in processed_urls:
-                            nav_links.append(full_url)
-                            processed_urls.add(full_url)
-            except Exception:
-                continue
-
-        return nav_links
-
-    def gitbook_extract_class_based_nav(self, base_url, processed_urls):
-        """Extract navigation based on common GitBook class patterns"""
-        nav_links = []
-
-        # Common class patterns for navigation in GitBook
-        class_patterns = [
-            'nav-', 'menu-', 'sidebar-', 'toc-', '-nav', '-menu', '-sidebar', '-toc'
-        ]
-
-        # Look for elements with these class patterns
-        for pattern in class_patterns:
-            elements = self.soup.find_all(class_=lambda c: c and pattern in c)
-            for element in elements:
-                for link in element.find_all('a', href=True):
-                    full_url = self.normalize_url(link['href'], base_url)
-                    if full_url and full_url not in processed_urls:
-                        nav_links.append(full_url)
-                        processed_urls.add(full_url)
-
-        return nav_links
-
-    @staticmethod
-    def clean_template_usage(content: Tag):
-        if not content or not isinstance(content, Tag):
-            return None
-        # Step 1: Build a mapping of template IDs to hidden div content
-        template_map = {}
-        # Find all hidden divs with IDs like S:*
-        for hidden_div in content.find_all('div', {'hidden': True}, id=re.compile(r'S:\d+')):
-            div_id = hidden_div.get('id')
-            # Store the first child (e.g., <a> tag) or the entire content
-            if hidden_div.contents:
-                template_map[div_id] = hidden_div.contents[0] if len(hidden_div.contents) == 1 else hidden_div
-
-        # Step 2: Replace <template> tags with content from hidden divs based on $RS logic
-        for template in content.find_all('template', id=re.compile(r'P:\d+')):
-            template_id = template.get('id')  # e.g., P:2
-            # Convert P:* to S:* to match the hidden div (assuming $RS("S:2", "P:2") pattern)
-            source_id = f"S:{template_id.split(':')[1]}"  # e.g., S:2
-            if source_id in template_map:
-                # Replace the template with the content from the hidden div
-                replacement = template_map[source_id]
-                # If it's a Tag, use it directly; if it's a div, extract its contents
-                if isinstance(replacement, Tag):
-                    template.replace_with(replacement)
-                else:
-                    template.replace_with(replacement.contents[0])
-
-    @staticmethod
-    def extract_chapters(content: Tag) -> List[TextChapter]:
-        chapters = []
-
-        # Create a default chapter for content before any heading
-        default_chapter = TextChapter(heading="Introduction", level=0)
-        current_chapter = default_chapter
-        chapters.append(default_chapter)
-
-        for element in content.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p']):
-            if element.name.startswith('h'):
-                # Extract heading level (h1=1, h2=2, etc.)
-                level = int(element.name[1])
-                heading_text = element.get_text(strip=True)
-
-                # Create a new chapter
-                current_chapter = TextChapter(heading=heading_text, level=level)
-                chapters.append(current_chapter)
-
-            elif element.name == 'p' and current_chapter is not None:
-                paragraph_text = element.get_text(strip=True)
-                if paragraph_text:
-                    current_chapter.paragraphs.append(paragraph_text)
-
-        # Remove any chapters without content if they're not top-level
-        return [ch for ch in chapters if ch.paragraphs or ch.level <= 2]

h_ai/infrastructure/playwright/playwright_web_content_fetcher.py

@@ -1,64 +0,0 @@
-import logging
-from typing import Optional
-
-from playwright.async_api import async_playwright
-
-from ...domain.webpages.web_fetcher_repository import WebFetcherRepository
-
-logger = logging.getLogger(__name__)
-
-class PlayWrightWebContentFetcher(WebFetcherRepository):
-    def __init__(self):
-        self.headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5'
-        }
-        self.page_load_timeout = 60
-        self.wait_for_idle = True
-
-    async def fetch(self, url: str) -> Optional[str]:
-        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True, args=[
-            '--disable-dev-shm-usage',  # Required for Docker
-            '--no-sandbox',  # Required for Docker non-root user
-            '--disable-setuid-sandbox',  # Required for Docker security
-            '--disable-gpu',  # Reduces resource usage
-            ])
-
-            logger.debug(
-                f"Launching headless browser with user agent: {self.headers.get('User-Agent')}"
-            )
-            try:
-                context = await browser.new_context(
-                    user_agent=self.headers.get('User-Agent')
-                )
-                page = await context.new_page()
-
-                # Set timeout
-                page.set_default_timeout(self.page_load_timeout * 1000)  # Convert to ms
-
-                page.on("console", lambda msg: logger.debug(f"Browser console {url}: {msg.text}"))
-
-                # Navigate to the URL
-                await page.goto(url, timeout=self.page_load_timeout * 1000, wait_until='domcontentloaded')
-
-                # Wait for network to be idle if requested
-                if self.wait_for_idle:
-                    logger.debug(f"Waiting for network idle on {url}")
-
-                    await page.wait_for_load_state("networkidle", timeout=self.page_load_timeout * 1000)
-                    logger.debug(
-                        f"Network idle on {url} after {self.page_load_timeout} seconds"
-                    )
-
-                logger.debug(f"Successfully fetched {url} with headless browser")
-
-                # Get the rendered HTML
-                return await page.content()
-
-            except Exception as e:
-                logger.error(f"Error fetching {url} with headless browser: {str(e)}")
-                return None
-            finally:
-                await browser.close()