h-ai-brain 0.0.4__tar.gz → 0.0.5__tar.gz

{h_ai_brain-0.0.4/src/h_ai_brain.egg-info → h_ai_brain-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h_ai_brain
-Version: 0.0.4
+Version: 0.0.5
 Summary: AI Research agent API
 Author-email: shoebill <shoebill.hai@gmail.com>
 Classifier: Programming Language :: Python :: 3

{h_ai_brain-0.0.4 → h_ai_brain-0.0.5}/pyproject.toml

@@ -4,7 +4,7 @@
 
    [project]
    name = "h_ai_brain"
-   version = "0.0.4"
+   version = "0.0.5"
    authors = [
        {name = "shoebill", email = "shoebill.hai@gmail.com"},
    ]

{h_ai_brain-0.0.4 → h_ai_brain-0.0.5}/src/h_ai/application/web_docs_service.py

@@ -23,6 +23,7 @@ class WebDocsService:
     async def discover_documentation(self, website_url: str) -> Optional[List[TextPage]]:
         detected_links = await self.detect_documentation_links(website_url)
         for link in detected_links:
+            print(f"Fetching {link.url}")
             gitbook_fetcher = GitbookWebFetcherService(link.url)
             gitbook_pages = await gitbook_fetcher.fetch()
             return gitbook_pages

{h_ai_brain-0.0.4 → h_ai_brain-0.0.5}/src/h_ai/domain/web_docs/web_docs_link_detector.py

@@ -16,6 +16,8 @@ class WebDocsLinkDetector:
         doc_links = []
 
         web_content = await self.web_fetcher.fetch(website_url)
+        if not web_content:
+            return doc_links
 
         soup_processor = SoupProcessor(web_content)
         web_links = soup_processor.extract_links(website_url)

{h_ai_brain-0.0.4 → h_ai_brain-0.0.5}/src/h_ai/infrastructure/playwright/playwright_web_content_fetcher.py

@@ -19,7 +19,10 @@ class PlayWrightWebContentFetcher(WebFetcherRepository):
 
     async def fetch(self, url: str) -> Optional[str]:
         async with async_playwright() as p:
-            browser = await p.chromium.launch()
+            browser = await p.chromium.launch(headless=True)
+            logger.debug(
+                f"Launching headless browser with user agent: {self.headers.get('User-Agent')}"
+            )
             try:
                 context = await browser.new_context(
                     user_agent=self.headers.get('User-Agent')
@@ -29,12 +32,19 @@ class PlayWrightWebContentFetcher(WebFetcherRepository):
                 # Set timeout
                 page.set_default_timeout(self.page_load_timeout * 1000)  # Convert to ms
 
+                page.on("console", lambda msg: logger.debug(f"Browser console {url}: {msg.text}"))
+
                 # Navigate to the URL
-                await page.goto(url)
+                await page.goto(url, timeout=self.page_load_timeout * 1000, wait_until='domcontentloaded')
 
                 # Wait for network to be idle if requested
                 if self.wait_for_idle:
-                    await page.wait_for_load_state("networkidle")
+                    logger.debug(f"Waiting for network idle on {url}")
+
+                    await page.wait_for_load_state("networkidle", timeout=self.page_load_timeout * 1000)
+                    logger.debug(
+                        f"Network idle on {url} after {self.page_load_timeout} seconds"
+                    )
 
                 logger.debug(f"Successfully fetched {url} with headless browser")
 

{h_ai_brain-0.0.4 → h_ai_brain-0.0.5/src/h_ai_brain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: h_ai_brain
-Version: 0.0.4
+Version: 0.0.5
 Summary: AI Research agent API
 Author-email: shoebill <shoebill.hai@gmail.com>
 Classifier: Programming Language :: Python :: 3