h-ai-brain 0.0.1__py3-none-any.whl

h_ai/__init__.py

@@ -0,0 +1,5 @@
+__all__ = ['HaiService', 'WebDocsService']
+
+from .application.web_docs_service import WebDocsService
+
+from .application.hai_service import HaiService

h_ai/application/hai_service.py

@@ -0,0 +1,48 @@
+import datetime
+
+from h_message_bus import NatsPublisherAdapter
+from h_message_bus.domain.twitter_get_user_request_message import TwitterGetUserRequestMessage
+from h_message_bus.domain.twitter_get_user_response_message import TwitterGetUserResponseMessage
+from h_message_bus.domain.vector_save_request_message import VectorSaveRequestMessage
+
+from ..application.web_docs_service import WebDocsService
+
+
+class HaiService:
+    def __init__(self, nats_publisher_adapter: NatsPublisherAdapter):
+        self.nats_publisher_adapter = nats_publisher_adapter
+        self.web_docs_service = WebDocsService()
+
+    async def detect_and_store_documentation(self, twitter_screen_name: str):
+        req_message = TwitterGetUserRequestMessage.create_message(twitter_screen_name)
+        response = await self.nats_publisher_adapter.request(req_message)
+        twitter_user = TwitterGetUserResponseMessage.from_hai_message(response)
+
+        if twitter_user.url is not None:
+            print(f"Documentation found for {twitter_user.screen_name}: {twitter_user.url}")
+            docs = await self.web_docs_service.discover_documentation(twitter_user.url)
+
+            for doc in docs:
+                collection_name = f"{twitter_user.screen_name}_docs"
+                chapters = doc.chapters
+                for chapter in chapters:
+                    i = 0
+                    for text in chapter.paragraphs:
+                        document_id = f"{doc.title}_{chapter.heading}_{i}"
+
+                        req_metadata = {
+                            "source": doc.url,
+                            "updated": datetime.datetime.utcnow()
+                        }
+                        i = i + 1
+
+                        request = VectorSaveRequestMessage.create_message(
+                            collection_name=collection_name,
+                            document_id=document_id,
+                            content=text,
+                            metadata=req_metadata)
+
+                        await self.nats_publisher_adapter.publish(request)
+
+        else:
+            print(f"No documentation found for {twitter_user.screen_name}")

h_ai/application/web_docs_service.py

@@ -0,0 +1,35 @@
+from typing import List, Optional
+
+from ..domain.web_docs.doc_link_scorer_service import DocLinkScorerService
+from ..domain.web_docs.documentation_pattern_repository import DocumentationPatternRepository
+from ..domain.web_docs.gitbook.text_page import TextPage
+from ..domain.web_docs.gitbook_web_fetcher_service import GitbookWebFetcherService
+from ..domain.web_docs.web_docs_link_detector import WebDocsLinkDetector
+from ..domain.web_docs.web_link import WebLink
+from ..infrastructure.playwright.playwright_web_content_fetcher import PlayWrightWebContentFetcher
+
+
+class WebDocsService:
+
+    def __init__(self):
+        self.pattern_repo = DocumentationPatternRepository()
+        self.scorer = DocLinkScorerService(self.pattern_repo)
+        self.headless_browser = PlayWrightWebContentFetcher()
+        self.web_link_detector = WebDocsLinkDetector(
+            self.scorer,
+            self.headless_browser)
+
+
+    async def discover_documentation(self, website_url: str) -> Optional[List[TextPage]]:
+        detected_links = await self.detect_documentation_links(website_url)
+        for link in detected_links:
+            gitbook_fetcher = GitbookWebFetcherService(link.url)
+            gitbook_pages = await gitbook_fetcher.fetch()
+            return gitbook_pages
+
+    async def detect_documentation_links(self, website_url: str) -> List[WebLink]:
+        """
+        Function to detect documentation links from a website
+        Returns a list of potential documentation root URLs
+        """
+        return await self.web_link_detector.find_docs_links(website_url)

h_ai/domain/reasoning/llm_chat_repository.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class LlmChatRepository(ABC):
+
+    @abstractmethod
+    def chat(self, user_message: str, session_id: str) -> Optional[str]:
+        ...

h_ai/domain/reasoning/llm_generate_respository.py

@@ -0,0 +1,6 @@
+from typing import Protocol
+
+
+class LlmGenerateRepository(Protocol):
+    def generate(self, user_prompt: str, system_prompt: str, session_id: str = None) -> str:
+        ...

h_ai/domain/reasoning/llm_tool_repository.py

@@ -0,0 +1,14 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from ...domain.reasoning.tool_message import ToolMessage
+
+
+class LlmToolRepository(ABC):
+    @abstractmethod
+    def find_tools_in_message(self, message: str) -> List[ToolMessage] | None:
+        ...
+
+    @abstractmethod
+    def build_tool_response_prompt(self, question: str, tool_results: list[str])-> str|None:
+        ...

h_ai/domain/reasoning/text_analysis.py

@@ -0,0 +1,149 @@
+import uuid
+from typing import List, Dict, Union, Optional
+from dataclasses import dataclass
+
+
+@dataclass
+class Sentiment:
+    """Domain object representing the sentiment of analyzed text"""
+    score: float
+    label: str  # 'negative', 'neutral', or 'positive'
+
+
+@dataclass
+class DetectedEntity:
+    """Domain object representing an entity detected in analyzed text"""
+    name: str
+    type: str
+    mentions: List[str] = None
+
+    def __post_init__(self):
+        if self.mentions is None:
+            self.mentions = []
+
+
+@dataclass
+class DetectedRelationship:
+    """Domain object representing a relationship between entities detected in text"""
+    source_entity_name: str
+    relation_type: str
+    target_entity_name: str
+    confidence: float = 0.0  # Confidence score of this relationship detection
+
+
+@dataclass
+class ExtractedFact:
+    """Domain object representing a factual statement extracted from text"""
+    statement: str
+    referenced_entities: List[str] = None  # Names of entities referenced in this fact
+
+    def __post_init__(self):
+        if self.referenced_entities is None:
+            self.referenced_entities = []
+
+
+@dataclass
+class TextAnalysisResult:
+    """Root aggregate for the result of analyzing a piece of text"""
+    text_id: str
+    category: str
+    topic: str
+    sentiment: Sentiment
+    entities: List[DetectedEntity]
+    relationships: List[DetectedRelationship]
+    facts: List[ExtractedFact]
+
+    def __init__(self, text_id: str, category: str, topic: str,
+                 sentiment_score: float = 0.0,
+                 sentiment_label: str = "neutral",
+                 entities: Optional[List[Dict[str, Union[str, List[str]]]]] = None,
+                 relationships: Optional[List[Dict[str, Union[str, float]]]] = None,
+                 facts: Optional[List[Dict[str, Union[str, List[str]]]]] = None):
+
+        self.text_id = text_id
+        self.category = category
+        self.topic = topic
+        self.sentiment = Sentiment(sentiment_score, sentiment_label)
+
+        # Process entities
+        self.entities = []
+        if entities:
+            for entity_data in entities:
+                entity = DetectedEntity(
+                    name=entity_data["name"],
+                    type=entity_data["type"],
+                    mentions=entity_data.get("mentions", [])
+                )
+                self.entities.append(entity)
+
+        # Process relationships
+        self.relationships = []
+        if relationships:
+            for rel_data in relationships:
+                rel = DetectedRelationship(
+                    source_entity_name=rel_data["source_entity_name"],
+                    relation_type=rel_data["relation_type"],
+                    target_entity_name=rel_data["target_entity_name"],
+                    confidence=rel_data.get("confidence", 0.0)
+                )
+                self.relationships.append(rel)
+
+        # Process facts
+        self.facts = []
+        if facts:
+            for fact_data in facts:
+                fact = ExtractedFact(
+                    statement=fact_data["statement"],
+                    referenced_entities=fact_data.get("referenced_entities", [])
+                )
+                self.facts.append(fact)
+
+    def to_dict(self) -> Dict[str, Union[str, Dict, List]]:
+        """Convert the TextAnalysisResult to a dictionary"""
+        return {
+            "text_id": self.text_id,
+            "category": self.category,
+            "topic": self.topic,
+            "sentiment": {
+                "score": self.sentiment.score,
+                "label": self.sentiment.label
+            },
+            "entities": [
+                {
+                    "name": entity.name,
+                    "type": entity.type,
+                    "mentions": entity.mentions
+                } for entity in self.entities
+            ],
+            "relationships": [
+                {
+                    "source_entity_name": rel.source_entity_name,
+                    "relation_type": rel.relation_type,
+                    "target_entity_name": rel.target_entity_name,
+                    "confidence": rel.confidence
+                } for rel in self.relationships
+            ],
+            "facts": [
+                {
+                    "statement": fact.statement,
+                    "referenced_entities": fact.referenced_entities
+                } for fact in self.facts
+            ]
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict) -> 'TextAnalysisResult':
+        """Create a TextAnalysisResult from a dictionary"""
+        random_guid = uuid.uuid4()
+        random_guid_str = str(uuid.uuid4())
+
+        return cls(
+            text_id=random_guid_str,
+            category=data["category"],
+            topic=data["topic"],
+            sentiment_score=data["sentiment"]["score"],
+            sentiment_label=data["sentiment"]["label"],
+            entities=data.get("entities", []),
+            relationships=data.get("relationships", []),
+            facts=data.get("facts", [])
+        )

h_ai/domain/reasoning/tool_message.py

@@ -0,0 +1,4 @@
+
+class ToolMessage:
+    method_name: str
+    method_params: dict

h_ai/domain/web_docs/doc_link_scorer_service.py

@@ -0,0 +1,45 @@
+import re
+from urllib.parse import urlparse
+
+from ...domain.web_docs.documentation_pattern_repository import DocumentationPatternRepository
+
+
+class DocLinkScorerService:
+    """Service for scoring potential documentation links"""
+
+    def __init__(self, pattern_repo: DocumentationPatternRepository):
+        self.pattern_repo = pattern_repo
+
+    def score(self, full_url: str, link_text: str) -> float:
+        """
+        Score a link based on how likely it is to be documentation
+        Returns a value between 0.0 and 1.0
+        """
+        score = 0.0
+        max_score = 3.0  # Maximum possible score
+
+        # Parse the URL
+        parsed_url = urlparse(full_url)
+        domain = parsed_url.netloc
+        path = parsed_url.path
+
+        # Check domain patterns
+        for doc_domain in self.pattern_repo.doc_domains:
+            if doc_domain in domain:
+                score += 1.0
+                break
+
+        # Check path patterns
+        for path_pattern in self.pattern_repo.doc_path_patterns:
+            if re.search(path_pattern, path):
+                score += 1.0
+                break
+
+        # Check link text patterns
+        for text_pattern in self.pattern_repo.doc_text_patterns:
+            if re.search(text_pattern, link_text):
+                score += 1.0
+                break
+
+        # Normalize score to 0.0-1.0 range
+        return min(score / max_score, 1.0)

h_ai/domain/web_docs/documentation_pattern_repository.py

@@ -0,0 +1,44 @@
+class DocumentationPatternRepository:
+    """Repository of patterns that indicate documentation links"""
+
+    def __init__(self):
+        # Domain patterns that commonly host documentation
+        self.doc_domains = [
+            "gitbook.io",
+            "readthedocs.io",
+            "docs.github.com",
+            "developer.mozilla.org",
+            "confluence.",
+            "zendesk.com",
+            "help.",
+            "support.",
+            "wiki.",
+        ]
+
+        # URL path patterns that commonly indicate documentation
+        self.doc_path_patterns = [
+            r"/docs/",
+            r"/documentation/",
+            r"/guide/",
+            r"/manual/",
+            r"/help/",
+            r"/knowledge/",
+            r"/support/",
+            r"/api/",
+            r"/reference/",
+            r"/wiki/",
+        ]
+
+        # Link text patterns that suggest documentation
+        self.doc_text_patterns = [
+            r"(?i)documentation",
+            r"(?i)docs",
+            r"(?i)developer guide",
+            r"(?i)user guide",
+            r"(?i)knowledge base",
+            r"(?i)help center",
+            r"(?i)manual",
+            r"(?i)api reference",
+            r"(?i)getting started",
+            r"(?i)learn more",
+        ]

h_ai/domain/web_docs/gitbook/text_chapter.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass, field
+from typing import List
+
+
+@dataclass
+class TextChapter:
+    """Represents a chapter/section in a page defined by a heading."""
+    heading: str
+    level: int  # h1=1, h2=2, etc.
+    paragraphs: List[str] = field(default_factory=list)
+
+    def to_dict(self):
+        """Convert this TextChapter instance to a serializable dictionary"""
+        return {
+            'heading': self.heading,
+            'level': self.level,
+            'paragraphs': self.paragraphs
+        }

h_ai/domain/web_docs/gitbook/text_page.py

@@ -0,0 +1,46 @@
+import hashlib
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+from src.h_ai.domain.web_docs.gitbook.text_chapter import TextChapter
+
+
+@dataclass
+class TextPage:
+    """Represents text on a page from a web document"""
+    url: str = ""
+    title: str = ""
+    content: str = ""
+    last_updated: str = ""
+
+    index: int = 0
+    toc_level: int = 0
+    parent_id: str = ""
+
+    chapters: List[TextChapter] = field(default_factory=list)
+    links: Dict[str, str] = field(default_factory=dict)  # Text -> URL
+
+    id: str = field(init=False, default="")
+    content_hash: str = field(init=False, default="")
+
+    def __post_init__(self):
+        self.id = hashlib.md5(self.url.encode()).hexdigest()
+        # Generate a content hash for deduplication
+        self.content_hash = hashlib.md5(self.content.encode()).hexdigest() if self.content else ""
+
+    def to_dict(self):
+        """Convert this TextPage instance to a serializable dictionary"""
+        result = {
+            'url': self.url,
+            'title': self.title,
+            #'content': self.content,
+            'last_updated': self.last_updated,
+            'index': self.index,
+            'toc_level': self.toc_level,
+            'parent_id': self.parent_id,
+            'id': self.id,
+            'content_hash': self.content_hash,
+            'links': self.links,
+            'chapters': [chapter.to_dict() for chapter in self.chapters]
+        }
+        return result

h_ai/domain/web_docs/gitbook_web_fetcher_service.py

@@ -0,0 +1,172 @@
+import asyncio
+import json
+from typing import Optional, Set, Dict, List
+from urllib.parse import urlparse
+
+import aiohttp
+
+from ...domain.web_docs.gitbook.text_page import TextPage
+from ...domain.webpages.web_text_fetcher_repository import WebTextFetcherRepository
+from ...infrastructure.beautifulsoup.soup_processor import SoupProcessor
+
+
+class GitbookWebFetcherService(WebTextFetcherRepository):
+
+    def __init__(self, url: str):
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5'
+        }
+        self.base_url = url.rstrip('/')
+        self.base_domain = urlparse(self.base_url).netloc
+
+        self.delay = 1.0  # Delay between requests in seconds
+        self.max_retries = 3
+        self.retry_delay = 2  # Initial retry delay in seconds
+        self.concurrent_requests = 3  # Number of concurrent requests
+
+        self.visited_urls: Set[str] = set()
+        self.content_hashes: Dict[str, str] = {}  # Hash -> URL
+        self.pages: Dict[str, TextPage] = {}
+
+    async def fetch(self) -> Optional[List[TextPage]]:
+        timeout = aiohttp.ClientTimeout(total=180)  # 3 minutes total timeout
+        async with aiohttp.ClientSession(headers=self.headers, timeout=timeout) as session:
+            # Start with main page
+
+            await self._process_url(session, self.base_url)
+
+            # Wait for all tasks to complete
+            await asyncio.sleep(0)
+
+            # Sort pages by index
+            sorted_pages = sorted(
+                self.pages.values(),
+                key=lambda p: p.index
+            )
+            return sorted_pages
+
+    async def _process_url(self, session: aiohttp.ClientSession, url: str) -> None:
+        if url in self.visited_urls:
+            return
+        print(f"Processing {url}")
+        self.visited_urls.add(url)
+
+        # Fetch page content
+        html_content = await self._fetch_page(session, url)
+        if not html_content:
+            return
+
+        # Extract page content
+        page = await GitbookWebFetcherService.extract_page_content(url, html_content)
+        if not page:
+            return
+
+        # Check for duplicate content
+        if page.content_hash in self.content_hashes:
+            return
+
+        # Set page index
+        page.index = len(self.pages)
+
+        # Add page to collection
+        self.pages[url] = page
+        self.content_hashes[page.content_hash] = url
+
+        # Extract navigation links from this page
+        nav_links = await GitbookWebFetcherService.gitbook_extract_navigation(self.base_url, html_content)
+
+        #Process the discovered links
+        for link in nav_links:
+            if link not in self.visited_urls:
+                # Add delay between requests
+                await asyncio.sleep(self.delay)
+                # Process the URL
+                await self._process_url(session, link)
+
+    @staticmethod
+    async def extract_page_content(url: str, html_content: str) -> Optional[TextPage]:
+        try:
+            soup_processor = SoupProcessor(html_content)
+
+            title = soup_processor.extract_title()
+            if not title:
+                title = urlparse(url).path.split('/')[-1] or "Index"
+                title = title.replace('-', ' ').replace('_', ' ').title()
+
+            last_updated = soup_processor.extract_last_updated_refs_from_soup()
+
+            body_tag = soup_processor.find_body_content()
+            if body_tag is None:
+                return None
+            soup_processor.clean_template_usage(body_tag)
+            chapters = soup_processor.extract_chapters(content=body_tag)
+
+            return TextPage(
+                url=url,
+                title=title,
+                content=html_content,
+                last_updated=last_updated,
+                chapters=chapters,
+            )
+        except Exception as e:
+            return None
+
+    @staticmethod
+    async def gitbook_extract_navigation(base_url: str, html_content: str) -> List[str]:
+        """Extract navigation links from a page"""
+        try:
+
+            soup_processor = SoupProcessor(html_content)
+
+            nav_links = []
+            processed_urls = set()
+
+            # Extract links from modern layout
+            nav_links.extend(soup_processor.gitbook_extract_modern_nav(base_url, processed_urls))
+
+            # Extract links from traditional layout
+            nav_links.extend(soup_processor.gitbook_extract_traditional_nav(base_url, processed_urls))
+
+            # Extract links from pagination elements
+            nav_links.extend(soup_processor.gitbook_extract_pagination_links(base_url, processed_urls))
+
+            # Extract links from search for specific class patterns
+            nav_links.extend(soup_processor.gitbook_extract_class_based_nav(base_url, processed_urls))
+
+            # Remove duplicates while preserving order
+            return list(dict.fromkeys(nav_links))
+
+        except Exception as e:
+            return []
+
+    async def _fetch_page(self, session: aiohttp.ClientSession, url: str) -> Optional[str]:
+        """Fetch a page with retry logic"""
+        retry_count = 0
+        current_delay = self.retry_delay
+
+        while retry_count < self.max_retries:
+            try:
+                async with session.get(url) as response:
+                    if response.status == 429:  # Rate limit
+                        retry_after = response.headers.get('Retry-After', '60')
+                        wait_time = int(retry_after)
+
+                        await asyncio.sleep(wait_time)
+                        retry_count += 1
+                        continue
+
+                    if response.status == 200:
+                        return await response.text()
+                    else:
+                        return None
+
+            except (aiohttp.ClientError, asyncio.TimeoutError) as e:
+                if retry_count < self.max_retries - 1:
+                    await asyncio.sleep(current_delay)
+                    current_delay *= 2  # Exponential backoff
+                    retry_count += 1
+                else:
+                    return None
+        return None

h_ai/domain/web_docs/web_docs_link_detector.py

@@ -0,0 +1,26 @@
+from typing import List
+
+from ...domain.webpages.web_fetcher_repository import WebFetcherRepository
+from ...domain.web_docs.doc_link_scorer_service import DocLinkScorerService
+from ...domain.web_docs.web_link import WebLink
+from ...infrastructure.beautifulsoup.soup_processor import SoupProcessor
+
+
+class WebDocsLinkDetector:
+    def __init__(self, doc_link_scorer: DocLinkScorerService, web_fetcher: WebFetcherRepository, confidence_threshold: float = 0.5):
+        self.doc_link_scorer = doc_link_scorer
+        self.web_fetcher = web_fetcher
+        self.confidence_threshold = confidence_threshold
+
+    async def find_docs_links(self, website_url: str) -> List[WebLink]:
+        doc_links = []
+
+        web_content = await self.web_fetcher.fetch(website_url)
+
+        soup_processor = SoupProcessor(web_content)
+        web_links = soup_processor.extract_links(website_url)
+        for web_link in web_links:
+            score = self.doc_link_scorer.score(web_link.url, web_link.title)
+            if score >= self.confidence_threshold:
+                doc_links.append(web_link)
+        return doc_links

h_ai/domain/web_docs/web_link.py

@@ -0,0 +1,11 @@
+class WebLink:
+    def __init__(self, url: str, title: str):
+        self.url = url
+        self.title = title
+
+    def to_dict(self):
+        """Convert this WebLink instance to a serializable dictionary"""
+        return {
+            'url': self.url,
+            'title': self.title
+        }

h_ai/domain/webpages/web_fetcher_repository.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class WebFetcherRepository(ABC):
+
+    @abstractmethod
+    async def fetch(self, url: str) -> Optional[str]:
+        """Fetch the content of the given URL."""
+        pass