zrb 1.13.1__py3-none-any.whl → 1.21.33__py3-none-any.whl

zrb/builtin/llm/tool/web.py

@@ -1,179 +1,116 @@
-import json
 from collections.abc import Callable
+from typing import Any
+from urllib.parse import urljoin
 
+from zrb.builtin.llm.tool.search import brave, searxng, serpapi
+from zrb.config.config import CFG
+from zrb.config.llm_config import llm_config
 
-async def open_web_page(url: str) -> str:
-    """
-    Fetches and parses the textual content of a given web page URL.
-
-    Use this tool to "read" a web page. It strips away HTML tags, scripts, and other non-textual elements to provide the clean text content. It also extracts any hyperlinks found on the page. This is useful when you need to understand the content of a specific URL that you have discovered through a search or from another source.
+_DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"  # noqa
 
-    Args:
-        url (str): The full URL of the web page to open (e.g., "https://example.com/article").
 
-    Returns:
-        str: A JSON object containing the cleaned text `content` of the page and a list of `links_on_page`.
+async def open_web_page(url: str) -> dict[str, Any]:
     """
+    Fetches, parses, and converts a web page to readable Markdown.
+    Preserves semantic structure, removes non-essentials, and extracts all absolute links.
 
-    async def get_page_content(page_url: str):
-        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"  # noqa
-        try:
-            from playwright.async_api import async_playwright
-
-            async with async_playwright() as p:
-                browser = await p.chromium.launch(headless=True)
-                page = await browser.new_page()
-                await page.set_extra_http_headers({"User-Agent": user_agent})
-                try:
-                    # Navigate to the URL with a timeout of 30 seconds
-                    await page.goto(page_url, wait_until="networkidle", timeout=30000)
-                    # Wait for the content to load
-                    await page.wait_for_load_state("domcontentloaded")
-                    # Get the page content
-                    content = await page.content()
-                    # Extract all links from the page
-                    links = await page.eval_on_selector_all(
-                        "a[href]",
-                        """
-                        (elements) => elements.map(el => {
-                            const href = el.getAttribute('href');
-                            if (href && !href.startsWith('#') && !href.startsWith('/')) {
-                                return href;
-                            }
-                            return null;
-                        }).filter(href => href !== null)
-                    """,
-                    )
-                    return {"content": content, "links_on_page": links}
-                finally:
-                    await browser.close()
-        except BaseException:
-            import requests
-
-            response = requests.get(url, headers={"User-Agent": user_agent})
-            if response.status_code != 200:
-                msg = f"Unable to retrieve search results. Status code: {response.status_code}"
-                raise Exception(msg)
-            return {"content": response.text, "links_on_page": []}
-
-    result = await get_page_content(url)
-    # Parse the HTML content
-    return json.dumps(parse_html_text(result["content"]))
-
-
-def create_search_internet_tool(serp_api_key: str) -> Callable[[str, int], str]:
-    """
-    Creates a tool that searches the internet using the SerpAPI Google Search API.
+    **EFFICIENCY TIP:**
+    Use this tool to read the full content of a specific search result or article.
+    It returns clean Markdown and a list of links, which is perfect for deep-diving
+    into a topic without navigating a browser UI.
 
-    This factory returns a function that can be used to find information on the web. The generated tool is the primary way to answer general knowledge questions or to find information on topics you are unfamiliar with.
+    Example:
+    open_web_page(url='https://www.example.com/article')
 
     Args:
-        serp_api_key (str): The API key for SerpAPI.
+        url (str): The full URL of the web page.
 
     Returns:
-        Callable: A function that takes a search query and returns a list of search results.
+        dict: Markdown content and a list of absolute links.
     """
-
-    def search_internet(query: str, num_results: int = 10) -> str:
-        """
-        Performs an internet search using Google and returns a summary of the results.
-
-        Use this tool to find information on the web, answer general knowledge questions, or research topics.
-
-        Args:
-            query (str): The search query.
-            num_results (int, optional): The desired number of search results. Defaults to 10.
-
-        Returns:
-            str: A JSON object containing the parsed text content from the search results page.
-        """
+    html_content, links = await _fetch_page_content(url)
+    markdown_content = _convert_html_to_markdown(html_content)
+    return {"content": markdown_content, "links_on_page": links}
+
+
+def create_search_internet_tool() -> Callable:
+    if llm_config.default_search_internet_tool is not None:
+        return llm_config.default_search_internet_tool
+    if (
+        CFG.SEARCH_INTERNET_METHOD.strip().lower() == "serpapi"
+        and CFG.SERPAPI_KEY != ""
+    ):
+        return serpapi.search_internet
+    if (
+        CFG.SEARCH_INTERNET_METHOD.strip().lower() == "brave"
+        and CFG.BRAVE_API_KEY != ""
+    ):
+        return brave.search_internet
+    return searxng.search_internet
+
+
+async def _fetch_page_content(url: str) -> tuple[str, list[str]]:
+    """Fetches the HTML content and all absolute links from a URL."""
+    try:
+        from playwright.async_api import async_playwright
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            page = await browser.new_page()
+            await page.set_extra_http_headers({"User-Agent": _DEFAULT_USER_AGENT})
+            try:
+                await page.goto(url, wait_until="networkidle", timeout=30000)
+                await page.wait_for_load_state("domcontentloaded")
+                content = await page.content()
+                links = await page.eval_on_selector_all(
+                    "a[href]",
+                    """
+                    (elements, baseUrl) => elements.map(el => {
+                        const href = el.getAttribute('href');
+                        if (!href || href.startsWith('#')) return null;
+                        try {
+                            return new URL(href, baseUrl).href;
+                        } catch (e) {
+                            return null;
+                        }
+                    }).filter(href => href !== null)
+                    """,
+                    url,
+                )
+                return content, links
+                # return json.dumps({"content": content, "links": links})
+            finally:
+                await browser.close()
+    except Exception:
         import requests
+        from bs4 import BeautifulSoup
 
-        response = requests.get(
-            "https://serpapi.com/search",
-            headers={
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"  # noqa
-            },
-            params={
-                "q": query,
-                "num": num_results,
-                "hl": "en",
-                "safe": "off",
-                "api_key": serp_api_key,
-            },
-        )
+        response = requests.get(url, headers={"User-Agent": _DEFAULT_USER_AGENT})
         if response.status_code != 200:
             raise Exception(
-                f"Error: Unable to retrieve search results (status code: {response.status_code})"  # noqa
+                f"Unable to retrieve page content. Status code: {response.status_code}"
             )
-        return json.dumps(parse_html_text(response.text))
-
-    return search_internet
-
-
-def search_wikipedia(query: str) -> str:
-    """
-    Searches for articles on Wikipedia.
-
-    This is a specialized search tool for querying Wikipedia. It's best for when the user is asking for definitions, historical information, or biographical details that are likely to be found on an encyclopedia.
-
-    Args:
-        query (str): The search term or question.
-
-    Returns:
-        str: The raw JSON response from the Wikipedia API, containing a list of search results.
-    """
-    import requests
-
-    params = {"action": "query", "list": "search", "srsearch": query, "format": "json"}
-    response = requests.get("https://en.wikipedia.org/w/api.php", params=params)
-    return response.json()
-
-
-def search_arxiv(query: str, num_results: int = 10) -> str:
-    """
-    Searches for academic papers and preprints on ArXiv.
-
-    Use this tool when the user's query is scientific or technical in nature and they are likely looking for research papers, articles, or academic publications.
-
-    Args:
-        query (str): The search query, which can include keywords, author names, or titles.
-        num_results (int, optional): The maximum number of results to return. Defaults to 10.
-
-    Returns:
-        str: The raw XML response from the ArXiv API, containing a list of matching papers.
-    """
-    import requests
-
-    params = {"search_query": f"all:{query}", "start": 0, "max_results": num_results}
-    response = requests.get("http://export.arxiv.org/api/query", params=params)
-    return response.content
-
-
-def parse_html_text(html_text: str) -> dict[str, str]:
+        content = response.text
+        soup = BeautifulSoup(content, "html.parser")
+        links = [
+            urljoin(url, a["href"])
+            for a in soup.find_all("a", href=True)
+            if not a["href"].startswith("#")
+        ]
+        return content, links
+        # return json.dumps({"content": content, "links": links})
+
+
+def _convert_html_to_markdown(html_text: str) -> str:
+    """Converts HTML content to a clean Markdown representation."""
     from bs4 import BeautifulSoup
+    from markdownify import markdownify as md
 
-    ignored_tags = [
-        "script",
-        "link",
-        "meta",
-        "style",
-        "code",
-        "footer",
-        "nav",
-        "header",
-        "aside",
-    ]
     soup = BeautifulSoup(html_text, "html.parser")
-    links = []
-    for anchor in soup.find_all("a"):
-        if not anchor or "href" not in anchor.attrs:
-            continue
-        link: str = anchor["href"]
-        if link.startswith("#") or link.startswith("/"):
-            continue
-        links.append(link)
-    for tag in soup(ignored_tags):
+    # Remove non-content tags
+    for tag in soup(
+        ["script", "link", "meta", "style", "header", "footer", "nav", "aside"]
+    ):
         tag.decompose()
-    html_text = soup.get_text(separator=" ", strip=True)
-    return {"content": html_text, "links_on_page": links}
+    # Convert the cleaned HTML to Markdown
+    return md(str(soup))

zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/entity/add_entity_util.py

@@ -204,7 +204,7 @@ def update_migration_metadata_file(ctx: AnyContext, migration_metadata_file_path
     app_name = os.path.basename(APP_DIR)
     existing_migration_metadata_code = read_file(migration_metadata_file_path)
     write_file(
-        file_path=migration_metadata_file_path,
+        abs_file_path=migration_metadata_file_path,
         content=[
             _get_migration_import_schema_code(
                 existing_migration_metadata_code, app_name, ctx.input.entity
@@ -251,7 +251,7 @@ def update_client_file(ctx: AnyContext, client_file_path: str):
     snake_plural_entity_name = to_snake_case(ctx.input.plural)
     pascal_entity_name = to_pascal_case(ctx.input.entity)
     write_file(
-        file_path=client_file_path,
+        abs_file_path=client_file_path,
         content=[
             _get_import_schema_for_client_code(
                 existing_code=existing_client_code, entity_name=ctx.input.entity
@@ -305,7 +305,7 @@ def update_api_client_file(ctx: AnyContext, api_client_file_path: str):
     snake_module_name = to_snake_case(ctx.input.module)
     pascal_module_name = to_pascal_case(ctx.input.module)
     write_file(
-        file_path=api_client_file_path,
+        abs_file_path=api_client_file_path,
         content=[
             f"from {app_name}.module.{snake_module_name}.service.{snake_entity_name}.{snake_entity_name}_service_factory import {snake_entity_name}_service",  # noqa
             prepend_code_to_module(
@@ -327,7 +327,7 @@ def update_direct_client_file(ctx: AnyContext, direct_client_file_path: str):
     snake_module_name = to_snake_case(ctx.input.module)
     pascal_module_name = to_pascal_case(ctx.input.module)
     write_file(
-        file_path=direct_client_file_path,
+        abs_file_path=direct_client_file_path,
         content=[
             f"from {app_name}.module.{snake_module_name}.service.{snake_entity_name}.{snake_entity_name}_service_factory import {snake_entity_name}_service",  # noqa
             prepend_code_to_module(
@@ -348,7 +348,7 @@ def update_route_file(ctx: AnyContext, route_file_path: str):
     app_name = os.path.basename(APP_DIR)
     module_name = to_snake_case(ctx.input.module)
     write_file(
-        file_path=route_file_path,
+        abs_file_path=route_file_path,
         content=[
             f"from {app_name}.module.{module_name}.service.{entity_name}.{entity_name}_service_factory import {entity_name}_service",  # noqa
             append_code_to_function(
@@ -370,7 +370,7 @@ def update_gateway_subroute_file(ctx: AnyContext, module_gateway_subroute_path:
     pascal_entity_name = to_pascal_case(ctx.input.entity)
     existing_gateway_subroute_code = read_file(module_gateway_subroute_path)
     write_file(
-        file_path=module_gateway_subroute_path,
+        abs_file_path=module_gateway_subroute_path,
         content=[
             _get_import_client_for_gateway_subroute_code(
                 existing_gateway_subroute_code, module_name=ctx.input.module
@@ -456,7 +456,7 @@ def update_gateway_navigation_config_file(
         },
     ).strip()
     write_file(
-        file_path=gateway_navigation_config_file_path,
+        abs_file_path=gateway_navigation_config_file_path,
         content=[
             existing_gateway_navigation_config_code,
             new_navigation_config_code,

zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/module/add_module_util.py

@@ -73,7 +73,7 @@ def update_app_zrb_config_file(ctx: AnyContext, zrb_config_file_path: str):
         ]
     )
     write_file(
-        file_path=zrb_config_file_path,
+        abs_file_path=zrb_config_file_path,
         content=append_key_to_dict(
             original_code=existing_zrb_config_code,
             dictionary_name="MICROSERVICES_ENV_VARS",
@@ -86,7 +86,7 @@ def update_app_zrb_config_file(ctx: AnyContext, zrb_config_file_path: str):
 def update_app_zrb_task_file(ctx: AnyContext, zrb_task_file_path: str):
     existing_zrb_task_code = read_file(zrb_task_file_path)
     write_file(
-        file_path=zrb_task_file_path,
+        abs_file_path=zrb_task_file_path,
         content=[
             existing_zrb_task_code.strip(),
             "",
@@ -126,7 +126,7 @@ def _get_task_definition_code(existing_code: str, module_name: str) -> str | Non
 def update_app_main_file(ctx: AnyContext, app_main_file_path: str):
     existing_app_main_code = read_file(app_main_file_path)
     write_file(
-        file_path=app_main_file_path,
+        abs_file_path=app_main_file_path,
         content=[
             _get_import_module_route_code(existing_app_main_code, ctx.input.module),
             existing_app_main_code,
@@ -187,7 +187,7 @@ def update_gateway_route_file(ctx: AnyContext, gateway_route_file_path: str):
     existing_gateway_route_code = read_file(gateway_route_file_path)
     snake_module_name = to_snake_case(ctx.input.module)
     write_file(
-        file_path=gateway_route_file_path,
+        abs_file_path=gateway_route_file_path,
         content=[
             _get_module_subroute_import(existing_gateway_route_code, ctx.input.module),
             append_code_to_function(
@@ -233,7 +233,7 @@ def update_gateway_navigation_config_file(
         },
     ).strip()
     write_file(
-        file_path=gateway_navigation_config_file_path,
+        abs_file_path=gateway_navigation_config_file_path,
         content=[
             existing_gateway_navigation_config_code,
             new_navigation_config_code,

zrb/builtin/project/add/fastapp/fastapp_util.py

@@ -22,7 +22,7 @@ def is_project_zrb_init_file(ctx: AnyContext, file_path: str) -> bool:
 def update_project_zrb_init_file(ctx: AnyContext, zrb_init_path: str):
     existing_zrb_init_code = read_file(zrb_init_path)
     write_file(
-        file_path=zrb_init_path,
+        abs_file_path=zrb_init_path,
         content=[
             _get_import_load_file_code(existing_zrb_init_code),
             existing_zrb_init_code.strip(),