h-ai-brain 0.0.1__py3-none-any.whl

h_ai/domain/webpages/web_text_fetcher_repository.py

@@ -0,0 +1,12 @@
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from ...domain.web_docs.gitbook.text_page import TextPage
+
+
+class WebTextFetcherRepository(ABC):
+
+    @abstractmethod
+    async def fetch(self) -> Optional[List[TextPage]]:
+        """Fetch all content"""
+        pass

h_ai/infrastructure/beautifulsoup/soup_processor.py

@@ -0,0 +1,240 @@
+import logging
+import re
+from typing import List, Optional
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup, Tag
+
+from ...domain.web_docs.gitbook.text_chapter import TextChapter
+from ...domain.web_docs.web_link import WebLink
+
+logger = logging.getLogger(__name__)
+
+class SoupProcessor:
+    def __init__(self, html_content:str):
+        self.soup = BeautifulSoup(html_content, 'html.parser')
+
+    def extract_links(self, base_url: str) -> List[WebLink]:
+        """Extract links from a page"""
+        web_links = []
+        links = self.soup.find_all('a', href=True)
+        for link in links:
+            href = link.get('href', '').strip()
+            if not href or href.startswith('#') or href.startswith('javascript:'):
+                continue
+
+            full_url = urljoin(base_url, href)
+            link_text = link.get_text().strip()
+            web_link = WebLink(url=full_url, title=link_text)
+            web_links.append(web_link)
+        return web_links
+
+    def normalize_url(self, href, base_url) -> Optional[str]:
+        """Normalize URL to absolute form and filter out non-content URLs"""
+        # Skip fragment-only URLs
+        if href.startswith('#'):
+            return None
+
+        # Skip external links
+        if href.startswith(('http://', 'https://')) and not href.startswith(base_url):
+            return None
+
+        # Skip resource URLs
+        if href.endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg', '.pdf', '.zip', '.js', '.css')):
+            return None
+
+        # Convert to absolute URL if needed
+        full_url = href
+        if not href.startswith(('http://', 'https://')):
+            full_url = urljoin(base_url, href)
+
+        # Make sure URL belongs to the same domain
+        if not full_url.startswith(base_url):
+            return None
+
+        return full_url
+
+    def extract_last_updated_refs_from_soup(self) -> str:
+        datetime_value = ""
+
+        # Find and remove elements containing "Last updated" text
+        for element in self.soup.find_all(string=lambda text: text and "Last updated" in text):
+            # Get the parent element and remove it
+            parent = element.parent
+            if parent:
+                parent.decompose()
+
+        return datetime_value
+
+    def extract_title(self) -> Optional[str]:
+        """Extract the title of the page using multiple strategies"""
+        # Strategy 1: Look for h1
+        h1 = self.soup.find('h1')
+        if h1:
+            return h1.get_text(strip=True)
+
+        # Strategy 2: Look for title tag
+        title_tag = self.soup.find('title')
+        if title_tag:
+            title_text = title_tag.get_text(strip=True)
+            title_parts = re.split(r'[|\-–]', title_text)
+            return title_parts[0].strip()
+
+        # Strategy 3: Try to find GitBook-specific title elements
+        gitbook_title = self.soup.find('span', {'data-testid': 'page.title'})
+        if gitbook_title:
+            return gitbook_title.get_text(strip=True)
+
+        return None
+
+    def find_body_content(self) -> Optional[Tag]:
+        body_content = self.soup.find('body')
+        if body_content:
+            return body_content
+        return None
+
+    def gitbook_extract_modern_nav(self, base_url, processed_urls):
+        """Extract navigation from modern GitBook layout"""
+        nav_links = []
+
+        # Look for navigation sidebar
+        sidebar = self.soup.select_one('div[data-testid="page.desktopTableOfContents"]')
+        if sidebar:
+            for link in sidebar.find_all('a', href=True):
+                full_url = self.normalize_url(link['href'], base_url)
+                if full_url and full_url not in processed_urls:
+                    nav_links.append(full_url)
+                    processed_urls.add(full_url)
+
+        return nav_links
+
+    def gitbook_extract_traditional_nav(self, base_url, processed_urls):
+        """Extract navigation from traditional GitBook layout"""
+        nav_links = []
+
+        # Find GitBook navigation elements
+        nav_elements = self.soup.find_all(['nav', 'aside'])
+        for nav in nav_elements:
+            # Look for lists that typically contain the navigation
+            nav_lists = nav.find_all(['ol', 'ul'])
+            for nav_list in nav_lists:
+                for li in nav_list.find_all('li'):
+                    link = li.find('a', href=True)
+                    if link:
+                        full_url = self.normalize_url(link['href'], base_url)
+                        if full_url and full_url not in processed_urls:
+                            nav_links.append(full_url)
+                            processed_urls.add(full_url)
+
+        # Try summary element which is common in GitBook
+        summary = self.soup.find('ul', {'class': 'summary'})
+        if summary:
+            for link in summary.find_all('a', href=True):
+                full_url = self.normalize_url(link['href'], base_url)
+                if full_url and full_url not in processed_urls:
+                    nav_links.append(full_url)
+                    processed_urls.add(full_url)
+
+        return nav_links
+
+    def gitbook_extract_pagination_links(self, base_url, processed_urls):
+        """Extract navigation from pagination elements"""
+        nav_links = []
+
+        # Find pagination links (next/prev)
+        selectors = [
+            'a[aria-label="Next"]',
+            'a[aria-label="Previous"]',
+            'a.navigation-next',
+            'a.navigation-prev',
+            'a:has(svg[data-icon="arrow-right"])',
+            'a:has(svg[data-icon="arrow-left"])'
+        ]
+
+        for selector in selectors:
+            try:
+                for link in self.soup.select(selector):
+                    if link.has_attr('href'):
+                        full_url = self.normalize_url(link['href'], base_url)
+                        if full_url and full_url not in processed_urls:
+                            nav_links.append(full_url)
+                            processed_urls.add(full_url)
+            except Exception:
+                continue
+
+        return nav_links
+
+    def gitbook_extract_class_based_nav(self, base_url, processed_urls):
+        """Extract navigation based on common GitBook class patterns"""
+        nav_links = []
+
+        # Common class patterns for navigation in GitBook
+        class_patterns = [
+            'nav-', 'menu-', 'sidebar-', 'toc-', '-nav', '-menu', '-sidebar', '-toc'
+        ]
+
+        # Look for elements with these class patterns
+        for pattern in class_patterns:
+            elements = self.soup.find_all(class_=lambda c: c and pattern in c)
+            for element in elements:
+                for link in element.find_all('a', href=True):
+                    full_url = self.normalize_url(link['href'], base_url)
+                    if full_url and full_url not in processed_urls:
+                        nav_links.append(full_url)
+                        processed_urls.add(full_url)
+
+        return nav_links
+
+    @staticmethod
+    def clean_template_usage(content: Tag):
+        if not content or not isinstance(content, Tag):
+            return None
+        # Step 1: Build a mapping of template IDs to hidden div content
+        template_map = {}
+        # Find all hidden divs with IDs like S:*
+        for hidden_div in content.find_all('div', {'hidden': True}, id=re.compile(r'S:\d+')):
+            div_id = hidden_div.get('id')
+            # Store the first child (e.g., <a> tag) or the entire content
+            if hidden_div.contents:
+                template_map[div_id] = hidden_div.contents[0] if len(hidden_div.contents) == 1 else hidden_div
+
+        # Step 2: Replace <template> tags with content from hidden divs based on $RS logic
+        for template in content.find_all('template', id=re.compile(r'P:\d+')):
+            template_id = template.get('id')  # e.g., P:2
+            # Convert P:* to S:* to match the hidden div (assuming $RS("S:2", "P:2") pattern)
+            source_id = f"S:{template_id.split(':')[1]}"  # e.g., S:2
+            if source_id in template_map:
+                # Replace the template with the content from the hidden div
+                replacement = template_map[source_id]
+                # If it's a Tag, use it directly; if it's a div, extract its contents
+                if isinstance(replacement, Tag):
+                    template.replace_with(replacement)
+                else:
+                    template.replace_with(replacement.contents[0])
+
+    @staticmethod
+    def extract_chapters(content: Tag) -> List[TextChapter]:
+        chapters = []
+
+        # Create a default chapter for content before any heading
+        default_chapter = TextChapter(heading="Introduction", level=0)
+        current_chapter = default_chapter
+        chapters.append(default_chapter)
+
+        for element in content.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p']):
+            if element.name.startswith('h'):
+                # Extract heading level (h1=1, h2=2, etc.)
+                level = int(element.name[1])
+                heading_text = element.get_text(strip=True)
+
+                # Create a new chapter
+                current_chapter = TextChapter(heading=heading_text, level=level)
+                chapters.append(current_chapter)
+
+            elif element.name == 'p' and current_chapter is not None:
+                paragraph_text = element.get_text(strip=True)
+                if paragraph_text:
+                    current_chapter.paragraphs.append(paragraph_text)
+
+        # Remove any chapters without content if they're not top-level
+        return [ch for ch in chapters if ch.paragraphs or ch.level <= 2]

h_ai/infrastructure/llm/data_handler.py

@@ -0,0 +1,30 @@
+import json
+
+
+def parse_json_data(json_string: str) -> dict |None:
+    try:
+        # Check for empty string
+        if not json_string or not json_string.strip():
+            print("Empty JSON string provided")
+            return {}
+
+        # First try to extract JSON from Markdown code blocks
+        import re
+        match = re.search(r'```(?:json)?\s*\n(.*?)\n```', json_string, re.DOTALL)
+        if match:
+            json_string = match.group(1).strip()
+        else:
+            # If no code block found, try the existing logic
+            if json_string.strip().startswith('json'):
+                json_string = json_string.strip()[4:].strip()
+
+        # Parse the JSON data
+        data = json.loads(json_string)
+
+        return data
+    except json.JSONDecodeError as e:
+        print(f"Error parsing JSON data: {e}")
+        return {}
+    except Exception as e:
+        print(f"Unexpected error while parsing JSON data: {e}")
+        return {}

h_ai/infrastructure/llm/llm_response_cleaner.py

@@ -0,0 +1,21 @@
+
+
+def clean_llm_response(response_text: str) -> str:
+    clean_deepseek = _strip_think_tag_from_deepseek_response(_replace_role_tags_from_response(response_text))
+    clean_tags = _replace_role_tags_from_response(clean_deepseek)
+    return clean_tags
+
+def _strip_think_tag_from_deepseek_response(response_text: str) -> str:
+    """
+    Removes any content enclosed within the "<think>" tags from the response text,
+    """
+    think_start = response_text.find("<think>")
+    think_end = response_text.find("</think>")
+    if think_start != -1 and think_end != -1:
+        return response_text[think_end + 8:].strip()
+    return response_text
+
+def _replace_role_tags_from_response(response_text: str) -> str:
+    return (response_text.replace("<|assistant|>", "")
+            .replace("<|user|>", "")
+            .replace("<|system|>", ""))

h_ai/infrastructure/llm/ollama/models/ollama_chat_message.py

@@ -0,0 +1,13 @@
+from typing import Dict
+
+
+class OllamaChatMessage:
+    def __init__(self, role: str, content: str):
+        self.role = role
+        self.content = content
+
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "role": self.role,
+            "content": self.content
+        }

h_ai/infrastructure/llm/ollama/models/ollama_chat_session.py

@@ -0,0 +1,12 @@
+from typing import List
+
+from .....infrastructure.llm.ollama.models.ollama_chat_message import OllamaChatMessage
+
+
+class OllamaChatSession:
+    def __init__(self, session_id: str, messages: List[OllamaChatMessage]):
+        self.session_id = session_id
+        self.messages = messages
+
+    def add_message(self, message: OllamaChatMessage):
+        self.messages.append(message)

h_ai/infrastructure/llm/ollama/ollama_chat_repository.py

@@ -0,0 +1,56 @@
+from typing import Optional, List
+
+import requests
+
+from ....domain.reasoning.llm_chat_repository import LlmChatRepository
+from ....infrastructure.llm.llm_response_cleaner import clean_llm_response
+from ....infrastructure.llm.ollama.models.ollama_chat_message import OllamaChatMessage
+from ....infrastructure.llm.ollama.models.ollama_chat_session import OllamaChatSession
+
+
+class OllamaChatRepository(LlmChatRepository):
+
+    def __init__(self, api_url: str, model_name: str, system_prompts: list[str] = None, temperature: float = None, seed: int = None):
+        self.api_url = api_url
+        self.model_name = model_name
+        self.temperature = temperature
+        self.seed = seed
+        self.system_prompts = system_prompts
+
+    def chat(self, user_message: str, session_id: str) -> Optional[str]:
+
+        messages = [OllamaChatMessage("user", user_message)]
+        for system_prompt in self.system_prompts:
+            messages.append(OllamaChatMessage("system", system_prompt))
+        session = OllamaChatSession(session_id, messages)
+
+        return self._call_ollama_api(session.messages)
+
+    def _call_ollama_api(self, messages: List[OllamaChatMessage]) -> Optional[str]:
+        url = f"{self.api_url}/chat"
+        formatted_messages = [message.to_dict() for message in messages]
+        payload = {
+            "model": self.model_name,
+            "messages": formatted_messages,
+            "stream": False,
+            "temperature": "0.6"
+        }
+        if self.temperature:
+            payload["temperature"] = self.temperature
+        if self.seed:
+            payload["seed"] = self.seed
+
+        try:
+            print(payload)
+            response = requests.post(url, json=payload)
+            response.raise_for_status()
+            full_response = response.json()["message"]["content"]
+            print(full_response)
+            return clean_llm_response(full_response)
+
+        except requests.exceptions.RequestException as e:
+            print(f"Error occurred during API call: {e}")
+            return None
+
+
+

h_ai/infrastructure/llm/ollama/ollama_generate_repository.py

@@ -0,0 +1,53 @@
+import uuid
+
+import requests
+
+from ....domain.reasoning.llm_generate_respository import LlmGenerateRepository
+
+
+class OllamaGenerateRepository(LlmGenerateRepository):
+
+    def __init__(self, api_url: str, model_name: str, system_prompt: str = None, temperature: float = None, seed: int = None):
+        self.model_name = model_name
+        self.system_prompt = system_prompt
+        self.api_url = api_url
+        self.temperature = temperature
+        self.seed = seed
+
+
+    def generate(self, user_prompt: str, system_prompt: str = None, session_id: str = None) -> str|None:
+        url = f"{self.api_url}/generate"
+        random_guid = uuid.uuid4()
+        guid_str = str(random_guid)
+        system_prompt = system_prompt or self.system_prompt
+        payload = {
+            "model": self.model_name,
+            "prompt": user_prompt,
+            "system": system_prompt,
+            "stream": False,
+            "session": guid_str,
+            "num_ctx": "5000",
+            "temperature": "0.6"
+        }
+
+        if session_id:
+            payload["session"] = session_id
+        if self.seed:
+            payload["seed"] = self.seed
+        if self.temperature:
+            payload["temperature"] = self.temperature
+
+        try:
+            print(payload)
+            response = requests.post(url, json=payload)
+            response.raise_for_status()
+
+            print(response.json())
+
+            response_content = response.json()["response"]
+            return clean_llm_response(response_content)
+
+        except requests.exceptions.RequestException as e:
+            print(f"Error occurred during API call: {e}")
+            return None
+

h_ai/infrastructure/llm/ollama/ollama_tool_repository.py

@@ -0,0 +1,138 @@
+import json
+from typing import Dict, Any
+
+import requests
+
+from ..prompt_helper import replace_placeholders
+from ....domain.reasoning.llm_tool_repository import LlmToolRepository
+from ....domain.reasoning.tool_message import ToolMessage
+
+
+class OllamaToolRepository(LlmToolRepository):
+
+    def __init__(self, api_url: str, model_name: str, tools: list, tool_prompt: str, tool_response_prompt: str, temperature: float = None, seed: int = None):
+        self.model_name = model_name
+        self.api_url = api_url
+        self.tools = tools
+        self.tool_prompt = tool_prompt
+        self.tool_response_prompt = tool_response_prompt
+        self.temperature = temperature
+        self.seed = seed
+
+    def find_tools_in_message(self, message: str) -> list[ToolMessage] | None:
+        url = f"{self.api_url}/generate"
+
+        placeholders = {
+            "$available_tools_placeholder": {json.dumps(self.tools, indent=2)},
+        }
+        system_message_tools = replace_placeholders(self.tool_prompt, placeholders)
+
+        payload = {
+            "model": self.model_name,
+            "prompt": message,
+            "stream": False,
+            "system": system_message_tools,
+            "num_ctx": "2500",
+            "temperature": "0.6"
+        }
+
+        if self.seed:
+            payload["seed"] = self.seed
+        if self.temperature:
+            payload["temperature"] = self.temperature
+
+        try:
+            print(payload)
+            response = requests.post(url, json=payload)
+            response.raise_for_status()
+
+            print(response.json())
+
+            found_tool_definitions = self._process_tool_response(response.json())
+            if found_tool_definitions:
+                tool_messages = []
+                for tool_definition in found_tool_definitions:
+                    tool_message = ToolMessage()
+                    tool_message.method_name = tool_definition.get("tool")
+                    tool_message.method_params = tool_definition.get("parameters",{})
+                    tool_messages.append(tool_message)
+
+                return tool_messages
+            else:
+                return None
+
+        except requests.exceptions.RequestException as e:
+            print(f"Error occurred during API call: {e}")
+            return None
+
+    def build_tool_response_prompt(self, question:str, tool_results: list[str]):
+        placeholders = {
+            "$tools_question_placeholder": question,
+            "$tools_response_context_placeholder": tool_results,
+        }
+        return replace_placeholders(self.tool_response_prompt, placeholders)
+
+    def _process_tool_response(self, response: Dict[str, Any]) -> list[dict] | None:
+        try:
+            text = response.get('response', '')
+
+            tool_calls = self._parse_json_string(text)
+
+            if isinstance(tool_calls, dict):
+                tool_calls = [tool_calls]
+
+            if isinstance(tool_calls, list):
+                valid_calls = [
+                    call for call in tool_calls
+                    if isinstance(call, dict) and 'tool' in call
+                ]
+                return valid_calls if valid_calls else None
+
+        except json.JSONDecodeError:
+            return None
+
+        return None
+
+    def _parse_json_string(self, input_str: str) -> list[dict] | None:
+        if not input_str or not input_str.strip():
+            return None
+
+        try:
+            return json.loads(input_str)
+        except json.JSONDecodeError:
+            cleaned = self._clean_json_string(input_str)
+            if not cleaned:
+                return None
+
+            json_objects = []
+            current_pos = 0
+
+            while True:
+                try:
+                    start = cleaned.find('{', current_pos)
+                    if start == -1:
+                        break
+
+                    decoder = json.JSONDecoder()
+                    obj, end = decoder.raw_decode(cleaned[start:])
+                    json_objects.append(obj)
+                    current_pos = start + end
+                except json.JSONDecodeError:
+                    current_pos += 1
+                    continue
+                except Exception:
+                    break
+
+            if not json_objects:
+                return None
+            return json_objects
+
+    @staticmethod
+    def _clean_json_string(input_str: str) -> str:
+        cleaned = input_str.replace('```json', '').replace('```', '')
+        cleaned = cleaned.strip("'").strip('"')
+        cleaned = cleaned.strip()
+        return cleaned
+
+
+

h_ai/infrastructure/llm/prompt_helper.py

@@ -0,0 +1,7 @@
+def replace_placeholders(prompt_string, placeholder_dict):
+    result = prompt_string
+
+    for placeholder, replacement in placeholder_dict.items():
+        result = result.replace(placeholder, str(replacement))
+
+    return result

h_ai/infrastructure/llm/prompt_loader.py

@@ -0,0 +1,18 @@
+import json
+import os
+
+
+class PromptLoader:
+    def __init__(self, file_path):
+        # Resolve relative path to absolute path
+        absolute_file_path = os.path.abspath(file_path)
+
+        with open(absolute_file_path, "r") as file:
+            self.config = json.load(file)
+
+    def get_config_value(self, key):
+        return self.config.get(key)
+
+    def get_entire_config(self):
+        return json.dumps(self.config, indent=2)
+

h_ai/infrastructure/playwright/playwright_web_content_fetcher.py

@@ -0,0 +1,48 @@
+import logging
+from typing import Optional
+
+from playwright.async_api import async_playwright
+
+from ...domain.webpages.web_fetcher_repository import WebFetcherRepository
+
+logger = logging.getLogger(__name__)
+
+class PlayWrightWebContentFetcher(WebFetcherRepository):
+    def __init__(self):
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5'
+        }
+        self.page_load_timeout = 30
+        self.wait_for_idle = True
+
+    async def fetch(self, url: str) -> Optional[str]:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch()
+            try:
+                context = await browser.new_context(
+                    user_agent=self.headers.get('User-Agent')
+                )
+                page = await context.new_page()
+
+                # Set timeout
+                page.set_default_timeout(self.page_load_timeout * 1000)  # Convert to ms
+
+                # Navigate to the URL
+                await page.goto(url)
+
+                # Wait for network to be idle if requested
+                if self.wait_for_idle:
+                    await page.wait_for_load_state("networkidle")
+
+                logger.debug(f"Successfully fetched {url} with headless browser")
+
+                # Get the rendered HTML
+                return await page.content()
+
+            except Exception as e:
+                logger.error(f"Error fetching {url} with headless browser: {str(e)}")
+                return None
+            finally:
+                await browser.close()