h-ai-brain 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

h_ai/application/web_docs_service.py

@@ -23,6 +23,7 @@ class WebDocsService:
     async def discover_documentation(self, website_url: str) -> Optional[List[TextPage]]:
         detected_links = await self.detect_documentation_links(website_url)
         for link in detected_links:
+            print(f"Fetching {link.url}")
             gitbook_fetcher = GitbookWebFetcherService(link.url)
             gitbook_pages = await gitbook_fetcher.fetch()
             return gitbook_pages

h_ai/domain/web_docs/web_docs_link_detector.py

@@ -16,6 +16,8 @@ class WebDocsLinkDetector:
         doc_links = []
 
         web_content = await self.web_fetcher.fetch(website_url)
+        if not web_content:
+            return doc_links
 
         soup_processor = SoupProcessor(web_content)
         web_links = soup_processor.extract_links(website_url)

h_ai/infrastructure/playwright/playwright_web_content_fetcher.py

@@ -14,12 +14,15 @@ class PlayWrightWebContentFetcher(WebFetcherRepository):
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
             'Accept-Language': 'en-US,en;q=0.5'
         }
-        self.page_load_timeout = 30
+        self.page_load_timeout = 60
         self.wait_for_idle = True
 
     async def fetch(self, url: str) -> Optional[str]:
         async with async_playwright() as p:
-            browser = await p.chromium.launch()
+            browser = await p.chromium.launch(headless=True)
+            logger.debug(
+                f"Launching headless browser with user agent: {self.headers.get('User-Agent')}"
+            )
             try:
                 context = await browser.new_context(
                     user_agent=self.headers.get('User-Agent')
@@ -29,12 +32,19 @@ class PlayWrightWebContentFetcher(WebFetcherRepository):
                 # Set timeout
                 page.set_default_timeout(self.page_load_timeout * 1000)  # Convert to ms
 
+                page.on("console", lambda msg: logger.debug(f"Browser console {url}: {msg.text}"))
+
                 # Navigate to the URL
-                await page.goto(url)
+                await page.goto(url, timeout=self.page_load_timeout * 1000, wait_until='domcontentloaded')
 
                 # Wait for network to be idle if requested
                 if self.wait_for_idle:
-                    await page.wait_for_load_state("networkidle")
+                    logger.debug(f"Waiting for network idle on {url}")
+
+                    await page.wait_for_load_state("networkidle", timeout=self.page_load_timeout * 1000)
+                    logger.debug(
+                        f"Network idle on {url} after {self.page_load_timeout} seconds"
+                    )
 
                 logger.debug(f"Successfully fetched {url} with headless browser")
 

{h_ai_brain-0.0.3.dist-info → h_ai_brain-0.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h_ai_brain
-Version: 0.0.3
+Version: 0.0.5
 Summary: AI Research agent API
 Author-email: shoebill <shoebill.hai@gmail.com>
 Classifier: Programming Language :: Python :: 3

{h_ai_brain-0.0.3.dist-info → h_ai_brain-0.0.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 h_ai/__init__.py,sha256=bmHMDoui52Q73UvXdHslQ3w_LubmhRuKRlrjOYyCP8c,153
 h_ai/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 h_ai/application/hai_service.py,sha256=DpSx1_QYS_3N5OwY5U4My0WwH-e4tvRHNeJxSduGoZM,2200
-h_ai/application/web_docs_service.py,sha256=45Pxhm279pGm3xWx00qOrS5pFjLHrbZTEhHTVXqyxJQ,1609
+h_ai/application/web_docs_service.py,sha256=YiPBfPyjlloDq6CIOP0u7F1jNBK-elYRU8xl4qJ1oVc,1652
 h_ai/application/system_prompts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 h_ai/application/system_prompts/roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 h_ai/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,7 +15,7 @@ h_ai/domain/web_docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 h_ai/domain/web_docs/doc_link_scorer_service.py,sha256=EmLSOaX7BCUQcKHZquaUt-Ps_DssZrRqpch5MgbUhAc,1444
 h_ai/domain/web_docs/documentation_pattern_repository.py,sha256=VhNzP3PUqgg9MaWhBVefj13XNxRBh6ZPUt-KH70ww2w,1302
 h_ai/domain/web_docs/gitbook_web_fetcher_service.py,sha256=Ye-TcuwgW1fhIY8x6v9_-pmPN9pVFWzlOpwRt-4teaA,6490
-h_ai/domain/web_docs/web_docs_link_detector.py,sha256=ePiIvMfT0fLiQNKEeZeU_MaATL568DtiajYUETwbn10,1133
+h_ai/domain/web_docs/web_docs_link_detector.py,sha256=NyMKFNs-41bqrxx6u-9GqIufy7pkDF_-_f1h8HECBK8,1192
 h_ai/domain/web_docs/web_link.py,sha256=J4KC3MmjkvWlAPDdEdjcqAZCvuSnJMahudCohiBk3wk,307
 h_ai/domain/web_docs/gitbook/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 h_ai/domain/web_docs/gitbook/text_chapter.py,sha256=XiZZlt128uA6oq-4bQfOpFf-djoHiQEsWpFDzRCNXdk,534
@@ -39,10 +39,10 @@ h_ai/infrastructure/llm/ollama/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 h_ai/infrastructure/llm/ollama/models/ollama_chat_message.py,sha256=ZIz4PQ3869vI3xAYYufPrxXpacajRDtOI8RDl5Dm9RQ,305
 h_ai/infrastructure/llm/ollama/models/ollama_chat_session.py,sha256=GZ_ddpbWa8iy6NZq50vokUFVZBiX0WNa81z9-r9RzTY,392
 h_ai/infrastructure/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-h_ai/infrastructure/playwright/playwright_web_content_fetcher.py,sha256=DHXkx91SP55PUT_o3ED1Z6uB0ijN-DeTXCnxdnXkDFw,1817
-h_ai_brain-0.0.3.dist-info/licenses/LICENSE,sha256=SbvpEU5JIU3yzMMkyzrI0dGqHDoJR_lMKGdl6GZHsy4,11558
-h_ai_brain-0.0.3.dist-info/licenses/NOTICE.txt,sha256=vxeIKUiGqAePLvDW4AVm3Xh-3BcsvMtCMn1tbsr9zsE,668
-h_ai_brain-0.0.3.dist-info/METADATA,sha256=Xdn_wNKqOOU1fpqiyST5Yu83gWqBSj1R8cZ6VJuIufA,735
-h_ai_brain-0.0.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-h_ai_brain-0.0.3.dist-info/top_level.txt,sha256=3MChDBWvDJV4cEHuZhzeODxQ4ewtw-arOuyaDOc6sIo,5
-h_ai_brain-0.0.3.dist-info/RECORD,,
+h_ai/infrastructure/playwright/playwright_web_content_fetcher.py,sha256=fcf8ZxIR2bLU1rV378ASQUaI26PvaX7sYqQb1SDlkt4,2401
+h_ai_brain-0.0.5.dist-info/licenses/LICENSE,sha256=SbvpEU5JIU3yzMMkyzrI0dGqHDoJR_lMKGdl6GZHsy4,11558
+h_ai_brain-0.0.5.dist-info/licenses/NOTICE.txt,sha256=vxeIKUiGqAePLvDW4AVm3Xh-3BcsvMtCMn1tbsr9zsE,668
+h_ai_brain-0.0.5.dist-info/METADATA,sha256=d6J1f3gTeJaS90RwHDfPLuwPila1TzYFxbRInOqHqHM,735
+h_ai_brain-0.0.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+h_ai_brain-0.0.5.dist-info/top_level.txt,sha256=3MChDBWvDJV4cEHuZhzeODxQ4ewtw-arOuyaDOc6sIo,5
+h_ai_brain-0.0.5.dist-info/RECORD,,