not-again-ai 0.19.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

not_again_ai/data/__init__.py

@@ -1,7 +0,0 @@
-import importlib.util
-
-if importlib.util.find_spec("playwright") is None:
-    raise ImportError(
-        "not_again_ai.data requires the 'data' extra to be installed. "
-        "You can install it using 'pip install not_again_ai[data]'."
-    )

not_again_ai/data/brave_search_api.py

@@ -0,0 +1,203 @@
+import os
+
+import httpx
+from loguru import logger
+from pydantic import BaseModel
+
+
+class SearchWebResult(BaseModel):
+    title: str
+    url: str
+    description: str
+    netloc: str | None = None
+
+
+class SearchWebResults(BaseModel):
+    results: list[SearchWebResult]
+
+
+async def search(
+    query: str,
+    count: int = 20,
+    offset: int = 0,
+    country: str = "US",
+    search_lang: str = "en",
+    ui_lang: str = "en-US",
+    freshness: str | None = None,
+    timezone: str = "America/New_York",
+    state: str = "MA",
+    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.",
+) -> SearchWebResults:
+    """
+    Search using Brave Search API.
+
+    Args:
+        query: The search query string
+        count: Number of search results to return (1-20, default 10)
+        offset: Number of search results to skip (default 0)
+        country: Country code for search results (default "US")
+        search_lang: Language for search (default "en")
+        ui_lang: User interface language (default "en-US")
+        freshness: Freshness of results ("pd", "pw", "pm", "py" or YYYY-MM-DDtoYYYY-MM-DD or None)
+        timezone: Timezone for search results (default "America/New_York")
+        state: State for search results (default "MA")
+        user_agent: User agent string for the request (default is a common browser UA)
+
+    Returns:
+        SearchWebResults: A model containing the search results
+
+    Raises:
+        httpx.HTTPError: If the request fails
+        ValueError: If BRAVE_SEARCH_API_KEY is not set
+    """
+    api_key = os.getenv("BRAVE_SEARCH_API_KEY")
+    if not api_key:
+        raise ValueError("BRAVE_SEARCH_API_KEY environment variable is not set")
+
+    url = "https://api.search.brave.com/res/v1/web/search"
+
+    headers = {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip",
+        "X-Subscription-Token": api_key,
+        "X-Loc-Country": country,
+        "X-Loc-Timezone": timezone,
+        "X-Loc-State": state,
+        "User-Agent": user_agent,
+    }
+
+    params: dict[str, str | int | bool] = {
+        "q": query,
+        "count": count,
+        "offset": offset,
+        "country": country,
+        "search_lang": search_lang,
+        "ui_lang": ui_lang,
+        "text_decorations": False,
+        "spellcheck": False,
+        "units": "imperial",
+        "extra_snippets": False,
+        "safesearch": "off",
+    }
+
+    # Add optional parameters if provided
+    if freshness:
+        params["freshness"] = freshness
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url, headers=headers, params=params)
+            response.raise_for_status()
+            data = response.json()
+            results_list: list[SearchWebResult] = []
+            for item in data.get("web", {}).get("results", []):
+                result = SearchWebResult(
+                    title=item.get("title", ""),
+                    url=item.get("url", ""),
+                    description=item.get("snippet", ""),
+                    netloc=item.get("meta_url", {}).get("netloc", None),
+                )
+                results_list.append(result)
+            return SearchWebResults(results=results_list)
+
+    except httpx.HTTPError as e:
+        logger.error(f"HTTP error during Brave search: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error during Brave search: {e}")
+        raise
+
+
+class SearchNewsResult(BaseModel):
+    title: str
+    url: str
+    description: str
+    age: str
+    netloc: str | None = None
+
+
+class SearchNewsResults(BaseModel):
+    results: list[SearchNewsResult]
+
+
+async def search_news(
+    query: str,
+    count: int = 20,
+    offset: int = 0,
+    country: str = "US",
+    search_lang: str = "en",
+    ui_lang: str = "en-US",
+    freshness: str | None = None,
+    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.",
+) -> SearchNewsResults:
+    """
+    Search news using Brave News Search API.
+
+    Args:
+        query: The search query string
+        count: Number of news results to return (1-20, default 20)
+        offset: Number of search results to skip (default 0)
+        country: Country code for search results (default "US")
+        search_lang: Language for search (default "en")
+        ui_lang: User interface language (default "en-US")
+        freshness: Freshness of results ("pd", "pw", "pm", "py" or YYYY-MM-DDtoYYYY-MM-DD or None)
+        user_agent: User agent string for the request (default is a common browser UA)
+
+    Returns:
+        SearchNewsResults: A model containing the news search results
+
+    Raises:
+        httpx.HTTPError: If the request fails
+        ValueError: If BRAVE_SEARCH_API_KEY is not set
+    """
+    api_key = os.getenv("BRAVE_SEARCH_API_KEY")
+    if not api_key:
+        raise ValueError("BRAVE_SEARCH_API_KEY environment variable is not set")
+
+    url = "https://api.search.brave.com/res/v1/news/search"
+
+    headers = {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip",
+        "X-Subscription-Token": api_key,
+        "User-Agent": user_agent,
+    }
+
+    params: dict[str, str | int | bool] = {
+        "q": query,
+        "count": count,
+        "offset": offset,
+        "country": country,
+        "search_lang": search_lang,
+        "ui_lang": ui_lang,
+        "spellcheck": False,
+        "safesearch": "off",
+    }
+
+    # Add optional parameters if provided
+    if freshness:
+        params["freshness"] = freshness
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url, headers=headers, params=params)
+            response.raise_for_status()
+            data = response.json()
+            results_list: list[SearchNewsResult] = []
+            for item in data.get("results", []):
+                result = SearchNewsResult(
+                    title=item.get("title", ""),
+                    url=item.get("url", ""),
+                    description=item.get("description", ""),
+                    age=item.get("age"),
+                    netloc=item.get("meta_url", {}).get("netloc", None),
+                )
+                results_list.append(result)
+            return SearchNewsResults(results=results_list)
+
+    except httpx.HTTPError as e:
+        logger.error(f"HTTP error during Brave news search: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error during Brave news search: {e}")
+        raise

not_again_ai/data/web.py

@@ -1,56 +1,160 @@
-from loguru import logger
-from playwright.sync_api import Browser, Playwright, sync_playwright
+import asyncio
+import io
+import mimetypes
+from pathlib import Path
+import re
+from urllib.parse import urlparse
 
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+import httpx
+from markitdown import MarkItDown, StreamInfo
+from pydantic import BaseModel
 
-def create_browser(headless: bool = True) -> tuple[Playwright, Browser]:
-    """Creates and returns a new Playwright instance and browser.
 
-    Args:
-        headless (bool, optional): Whether to run the browser in headless mode. Defaults to True.
+class Link(BaseModel):
+    url: str
+    text: str
 
-    Returns:
-        tuple[Playwright, Browser]: A tuple containing the Playwright instance and browser.
+
+class URLResult(BaseModel):
+    url: str
+    markdown: str
+    links: list[Link] = []
+
+
+async def _markitdown_bytes_to_str(file_bytes: bytes, filename_extension: str) -> str:
     """
-    pwright = sync_playwright().start()
-    browser = pwright.chromium.launch(
-        headless=headless,
-        chromium_sandbox=False,
-        timeout=15000,
-    )
-    return pwright, browser
+    Convert a file using MarkItDown defaults.
+    """
+    with io.BytesIO(file_bytes) as temp:
+        result = await asyncio.to_thread(
+            MarkItDown(enable_plugins=False).convert,
+            source=temp,
+            stream_info=StreamInfo(extension=filename_extension),
+        )
+        text = result.text_content
+    return text
 
 
-def get_raw_web_content(url: str, browser: Browser | None = None, headless: bool = True) -> str:
-    """Fetches raw web content from a given URL using Playwright.
+def _detect_pdf_extension(url: str) -> bool:
+    """
+    Detect if the URL is a PDF based on its extension.
+    """
+    parsed_url = urlparse(url)
+    filename = Path(parsed_url.path).name
+    return mimetypes.guess_type(filename)[0] == "application/pdf"
 
-    Args:
-        url (str): The URL to fetch content from.
-        browser (Browser | None, optional): An existing browser instance to use. Defaults to None.
-        headless (bool, optional): Whether to run the browser in headless mode. Defaults to True.
 
-    Returns:
-        str: The raw web content.
+def _detect_google_sheets(url: str) -> bool:
+    """
+    Detect if the URL is a Google Sheets document.
     """
-    p = None
-    try:
-        if browser is None:
-            p, browser = create_browser(headless)
+    is_google_sheets = url.startswith("https://docs.google.com/spreadsheets/")
+    return is_google_sheets
+
+
+async def _handle_pdf_content(url: str) -> URLResult:
+    md = MarkItDown(enable_plugins=False)
+    result = md.convert(url)
+    url_result = URLResult(
+        url=url,
+        markdown=result.markdown or "",
+        links=[],
+    )
+    return url_result
+
+
+async def _handle_google_sheets_content(url: str) -> URLResult:
+    """
+    Handle Google Sheets by using the export URL to get the raw content.
+    """
+    edit_pattern = r"https://docs\.google\.com/spreadsheets/d/([a-zA-Z0-9-_]+)/edit"
+    export_pattern = r"https://docs\.google\.com/spreadsheets/d/([a-zA-Z0-9-_]+)/export\?format=csv"
+
+    # Check if it's already an export URL
+    export_match = re.search(export_pattern, url)
+    if export_match:
+        export_url = url
+    else:
+        # Check if it's an edit URL and extract document ID
+        edit_match = re.search(edit_pattern, url)
+        if edit_match:
+            doc_id = edit_match.group(1)
+            export_url = f"https://docs.google.com/spreadsheets/d/{doc_id}/export?format=csv&gid=0"
+        else:
+            return await _handle_web_content(url)
 
-        page = browser.new_page(
-            accept_downloads=False,
-            java_script_enabled=True,
-            viewport={"width": 1366, "height": 768},
-            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.3",
+    async with httpx.AsyncClient(follow_redirects=True) as client:
+        response = await client.get(export_url)
+        response.raise_for_status()
+        csv_bytes = response.content
+
+    # Convert CSV to markdown using MarkItDown
+    markdown_content = await _markitdown_bytes_to_str(csv_bytes, ".csv")
+
+    url_result = URLResult(
+        url=url,
+        markdown=markdown_content,
+        links=[],
+    )
+    return url_result
+
+
+async def _handle_web_content(url: str) -> URLResult:
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        verbose=False,
+        user_agent_mode="random",
+        java_script_enabled=True,
+    )
+    run_config = CrawlerRunConfig(
+        scan_full_page=True,
+        user_agent_mode="random",
+        cache_mode=CacheMode.DISABLED,
+        markdown_generator=DefaultMarkdownGenerator(),
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=url,
+            config=run_config,
         )
-        page.goto(url)
-        content = page.content()
-        page.close()
-        return content
-    except Exception as e:
-        logger.error(f"Failed to get web content: {e}")
-        return ""
-    finally:
-        if browser:
-            browser.close()
-        if p:
-            p.stop()
+
+        if result.response_headers.get("content-type") == "application/pdf":
+            return await _handle_pdf_content(url)
+
+    links: list[Link] = []
+    seen_urls: set[str] = set()
+    combined_link_data = result.links.get("internal", []) + result.links.get("external", [])
+    for link_data in combined_link_data:
+        href = link_data.get("href", "")
+        if href and href not in seen_urls:
+            seen_urls.add(href)
+            link = Link(
+                url=href,
+                text=link_data.get("title", "") or link_data.get("text", ""),
+            )
+            links.append(link)
+
+    url_result = URLResult(
+        url=url,
+        markdown=result.markdown or "",
+        links=links,
+    )
+    return url_result
+
+
+async def process_url(url: str) -> URLResult:
+    """
+    Process a URL to extract content and convert it to Markdown and links
+    """
+    if _detect_pdf_extension(url):
+        url_result = await _handle_pdf_content(url)
+    elif _detect_google_sheets(url):
+        url_result = await _handle_google_sheets_content(url)
+    else:
+        url_result = await _handle_web_content(url)
+    return url_result

{not_again_ai-0.19.0.dist-info → not_again_ai-0.20.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: not-again-ai
-Version: 0.19.0
+Version: 0.20.0
 Summary: Designed to once and for all collect all the little things that come up over and over again in AI projects and put them in one place.
 Project-URL: Homepage, https://github.com/DaveCoDev/not-again-ai
 Project-URL: Documentation, https://davecodev.github.io/not-again-ai/
@@ -22,8 +22,9 @@ Requires-Python: >=3.11
 Requires-Dist: loguru<1.0,>=0.7
 Requires-Dist: pydantic<3.0,>=2.11
 Provides-Extra: data
-Requires-Dist: playwright<2.0,>=1.51; extra == 'data'
-Requires-Dist: pytest-playwright<1.0,>=0.7; extra == 'data'
+Requires-Dist: crawl4ai<1.0,>=0.6; extra == 'data'
+Requires-Dist: httpx<1.0,>=0.28; extra == 'data'
+Requires-Dist: markitdown[pdf]==0.1.2; extra == 'data'
 Provides-Extra: llm
 Requires-Dist: anthropic<1.0,>=0.50; extra == 'llm'
 Requires-Dist: azure-identity<2.0,>=1.21; extra == 'llm'
@@ -83,7 +84,9 @@ The package is split into subpackages, so you can install only the parts you nee
 
 ### Data
 1. `pip install not_again_ai[data]`
-1. `playwright install` to download the browser binaries.
+1. `crawl4ai-setup` to run crawl4ai post-installation setup.
+1. Set the `BRAVE_SEARCH_API_KEY` environment variable to use the Brave Search API for web data extraction.
+   1. Get the API key from https://api-dashboard.search.brave.com/app/keys. You must have at least the Free "Data for Search" subscription.
 
 
 ### LLM
@@ -312,3 +315,5 @@ Default settings are configured in [`.vscode/settings.json`](./.vscode/settings.
 
 # Attributions
 [python-blueprint](https://github.com/johnthagen/python-blueprint) for the Python package skeleton.
+
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.

{not_again_ai-0.19.0.dist-info → not_again_ai-0.20.0.dist-info}/RECORD

@@ -3,8 +3,9 @@ not_again_ai/py.typed,sha256=UaCuPFa3H8UAakbt-5G8SPacldTOGvJv18pPjUJ5gDY,93
 not_again_ai/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 not_again_ai/base/file_system.py,sha256=KNQmacO4Q__CQuq2oPzWrg3rQO48n3evglc9bNiP7KM,949
 not_again_ai/base/parallel.py,sha256=fcYhKBYBWvob84iKp3O93wvFFdXeidljZsShgBLTNGA,3448
-not_again_ai/data/__init__.py,sha256=1jF6mwvtB2PT7IEc3xpbRtZm3g3Lyf8zUqH4AEE4qlQ,244
-not_again_ai/data/web.py,sha256=wjx9cc33jcoJBGonYCIpwygPBFOwz7F-dx_ominmbnI,1838
+not_again_ai/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+not_again_ai/data/brave_search_api.py,sha256=cTH7smRf8ITPpcQEcoIEvJo6VAST9_mg7FSL88xdGMc,6603
+not_again_ai/data/web.py,sha256=LLWM5SkgI3-ILhtXYYuDm_eVnij3uS4tL059RyMq0lU,4737
 not_again_ai/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 not_again_ai/llm/chat_completion/__init__.py,sha256=HozawvdRkTFgq8XR16GJUHN1ukEa4Ya68wVPVrl-afs,250
 not_again_ai/llm/chat_completion/interface.py,sha256=OU6ghG7RlveahkHZWdRHFg0uzbSrSh2Dz7u5-4rrypA,2700
@@ -39,7 +40,7 @@ not_again_ai/viz/distributions.py,sha256=OyWwJaNI6lMRm_iSrhq-CORLNvXfeuLSgDtVo3u
 not_again_ai/viz/scatterplot.py,sha256=5CUOWeknbBOaZPeX9oPin5sBkRKEwk8qeFH45R-9LlY,2292
 not_again_ai/viz/time_series.py,sha256=pOGZqXp_2nd6nKo-PUQNCtmMh__69jxQ6bQibTGLwZA,5212
 not_again_ai/viz/utils.py,sha256=hN7gwxtBt3U6jQni2K8j5m5pCXpaJDoNzGhBBikEU28,238
-not_again_ai-0.19.0.dist-info/METADATA,sha256=LeCIas912YMtvKEJcChoPqYrM3ay_EurUZehvlQ9t8o,12004
-not_again_ai-0.19.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-not_again_ai-0.19.0.dist-info/licenses/LICENSE,sha256=btjOgNGpp-ux5xOo1Gx1MddxeWtT9sof3s3Nui29QfA,1071
-not_again_ai-0.19.0.dist-info/RECORD,,
+not_again_ai-0.20.0.dist-info/METADATA,sha256=sYV6N2dmYarZnbjIqLoCrjfrsa8m1G-1rYxX3XV89tI,12389
+not_again_ai-0.20.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+not_again_ai-0.20.0.dist-info/licenses/LICENSE,sha256=btjOgNGpp-ux5xOo1Gx1MddxeWtT9sof3s3Nui29QfA,1071
+not_again_ai-0.20.0.dist-info/RECORD,,