not-again-ai 0.19.0__tar.gz → 0.20.0__tar.gz

{not_again_ai-0.19.0 → not_again_ai-0.20.0}/.github/copilot-instructions.md

@@ -7,4 +7,5 @@
 - If the user is using Pydantic, it is version >=2.10
 - Always prefer pathlib for dealing with files. Use `Path.open` instead of `open`.
 - Prefer to use pendulum instead of datetime
-- Prefer to use loguru instead of logging
+- Prefer to use loguru instead of logging
+- Prefer httpx for HTTP requests instead of requests

{not_again_ai-0.19.0 → not_again_ai-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: not-again-ai
-Version: 0.19.0
+Version: 0.20.0
 Summary: Designed to once and for all collect all the little things that come up over and over again in AI projects and put them in one place.
 Project-URL: Homepage, https://github.com/DaveCoDev/not-again-ai
 Project-URL: Documentation, https://davecodev.github.io/not-again-ai/
@@ -22,8 +22,9 @@ Requires-Python: >=3.11
 Requires-Dist: loguru<1.0,>=0.7
 Requires-Dist: pydantic<3.0,>=2.11
 Provides-Extra: data
-Requires-Dist: playwright<2.0,>=1.51; extra == 'data'
-Requires-Dist: pytest-playwright<1.0,>=0.7; extra == 'data'
+Requires-Dist: crawl4ai<1.0,>=0.6; extra == 'data'
+Requires-Dist: httpx<1.0,>=0.28; extra == 'data'
+Requires-Dist: markitdown[pdf]==0.1.2; extra == 'data'
 Provides-Extra: llm
 Requires-Dist: anthropic<1.0,>=0.50; extra == 'llm'
 Requires-Dist: azure-identity<2.0,>=1.21; extra == 'llm'
@@ -83,7 +84,9 @@ The package is split into subpackages, so you can install only the parts you nee
 
 ### Data
 1. `pip install not_again_ai[data]`
-1. `playwright install` to download the browser binaries.
+1. `crawl4ai-setup` to run crawl4ai post-installation setup.
+1. Set the `BRAVE_SEARCH_API_KEY` environment variable to use the Brave Search API for web data extraction.
+   1. Get the API key from https://api-dashboard.search.brave.com/app/keys. You must have at least the Free "Data for Search" subscription.
 
 
 ### LLM
@@ -312,3 +315,5 @@ Default settings are configured in [`.vscode/settings.json`](./.vscode/settings.
 
 # Attributions
 [python-blueprint](https://github.com/johnthagen/python-blueprint) for the Python package skeleton.
+
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.

{not_again_ai-0.19.0 → not_again_ai-0.20.0}/README.md

@@ -39,7 +39,9 @@ The package is split into subpackages, so you can install only the parts you nee
 
 ### Data
 1. `pip install not_again_ai[data]`
-1. `playwright install` to download the browser binaries.
+1. `crawl4ai-setup` to run crawl4ai post-installation setup.
+1. Set the `BRAVE_SEARCH_API_KEY` environment variable to use the Brave Search API for web data extraction.
+   1. Get the API key from https://api-dashboard.search.brave.com/app/keys. You must have at least the Free "Data for Search" subscription.
 
 
 ### LLM
@@ -268,3 +270,5 @@ Default settings are configured in [`.vscode/settings.json`](./.vscode/settings.
 
 # Attributions
 [python-blueprint](https://github.com/johnthagen/python-blueprint) for the Python package skeleton.
+
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.

{not_again_ai-0.19.0 → not_again_ai-0.20.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "not-again-ai"
-version = "0.19.0"
+version = "0.20.0"
 description = "Designed to once and for all collect all the little things that come up over and over again in AI projects and put them in one place."
 authors = [
     { name = "DaveCoDev", email = "dave.co.dev@gmail.com" }
@@ -34,8 +34,9 @@ Repository = "https://github.com/DaveCoDev/not-again-ai"
 
 [project.optional-dependencies]
 data = [
-    "playwright>=1.51,<2.0",
-    "pytest-playwright>=0.7,<1.0",
+    "Crawl4AI>=0.6,<1.0",
+    "httpx>=0.28,<1.0",
+    "markitdown[pdf]==0.1.2"
 ]
 llm = [
     "anthropic>=0.50,<1.0",
@@ -140,7 +141,7 @@ filterwarnings = [
     "error",
     # Add additional warning suppressions as needed here. For example, if a third-party library
     # is throwing a deprecation warning that needs to be fixed upstream:
-    # "ignore::DeprecationWarning:typer",
+    "ignore::DeprecationWarning",
     "ignore::pytest.PytestUnraisableExceptionWarning"
 ]
 asyncio_mode = "auto"

not_again_ai-0.20.0/src/not_again_ai/data/brave_search_api.py

@@ -0,0 +1,203 @@
+import os
+
+import httpx
+from loguru import logger
+from pydantic import BaseModel
+
+
+class SearchWebResult(BaseModel):
+    title: str
+    url: str
+    description: str
+    netloc: str | None = None
+
+
+class SearchWebResults(BaseModel):
+    results: list[SearchWebResult]
+
+
+async def search(
+    query: str,
+    count: int = 20,
+    offset: int = 0,
+    country: str = "US",
+    search_lang: str = "en",
+    ui_lang: str = "en-US",
+    freshness: str | None = None,
+    timezone: str = "America/New_York",
+    state: str = "MA",
+    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.",
+) -> SearchWebResults:
+    """
+    Search using Brave Search API.
+
+    Args:
+        query: The search query string
+        count: Number of search results to return (1-20, default 10)
+        offset: Number of search results to skip (default 0)
+        country: Country code for search results (default "US")
+        search_lang: Language for search (default "en")
+        ui_lang: User interface language (default "en-US")
+        freshness: Freshness of results ("pd", "pw", "pm", "py" or YYYY-MM-DDtoYYYY-MM-DD or None)
+        timezone: Timezone for search results (default "America/New_York")
+        state: State for search results (default "MA")
+        user_agent: User agent string for the request (default is a common browser UA)
+
+    Returns:
+        SearchWebResults: A model containing the search results
+
+    Raises:
+        httpx.HTTPError: If the request fails
+        ValueError: If BRAVE_SEARCH_API_KEY is not set
+    """
+    api_key = os.getenv("BRAVE_SEARCH_API_KEY")
+    if not api_key:
+        raise ValueError("BRAVE_SEARCH_API_KEY environment variable is not set")
+
+    url = "https://api.search.brave.com/res/v1/web/search"
+
+    headers = {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip",
+        "X-Subscription-Token": api_key,
+        "X-Loc-Country": country,
+        "X-Loc-Timezone": timezone,
+        "X-Loc-State": state,
+        "User-Agent": user_agent,
+    }
+
+    params: dict[str, str | int | bool] = {
+        "q": query,
+        "count": count,
+        "offset": offset,
+        "country": country,
+        "search_lang": search_lang,
+        "ui_lang": ui_lang,
+        "text_decorations": False,
+        "spellcheck": False,
+        "units": "imperial",
+        "extra_snippets": False,
+        "safesearch": "off",
+    }
+
+    # Add optional parameters if provided
+    if freshness:
+        params["freshness"] = freshness
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url, headers=headers, params=params)
+            response.raise_for_status()
+            data = response.json()
+            results_list: list[SearchWebResult] = []
+            for item in data.get("web", {}).get("results", []):
+                result = SearchWebResult(
+                    title=item.get("title", ""),
+                    url=item.get("url", ""),
+                    description=item.get("snippet", ""),
+                    netloc=item.get("meta_url", {}).get("netloc", None),
+                )
+                results_list.append(result)
+            return SearchWebResults(results=results_list)
+
+    except httpx.HTTPError as e:
+        logger.error(f"HTTP error during Brave search: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error during Brave search: {e}")
+        raise
+
+
+class SearchNewsResult(BaseModel):
+    title: str
+    url: str
+    description: str
+    age: str
+    netloc: str | None = None
+
+
+class SearchNewsResults(BaseModel):
+    results: list[SearchNewsResult]
+
+
+async def search_news(
+    query: str,
+    count: int = 20,
+    offset: int = 0,
+    country: str = "US",
+    search_lang: str = "en",
+    ui_lang: str = "en-US",
+    freshness: str | None = None,
+    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.",
+) -> SearchNewsResults:
+    """
+    Search news using Brave News Search API.
+
+    Args:
+        query: The search query string
+        count: Number of news results to return (1-20, default 20)
+        offset: Number of search results to skip (default 0)
+        country: Country code for search results (default "US")
+        search_lang: Language for search (default "en")
+        ui_lang: User interface language (default "en-US")
+        freshness: Freshness of results ("pd", "pw", "pm", "py" or YYYY-MM-DDtoYYYY-MM-DD or None)
+        user_agent: User agent string for the request (default is a common browser UA)
+
+    Returns:
+        SearchNewsResults: A model containing the news search results
+
+    Raises:
+        httpx.HTTPError: If the request fails
+        ValueError: If BRAVE_SEARCH_API_KEY is not set
+    """
+    api_key = os.getenv("BRAVE_SEARCH_API_KEY")
+    if not api_key:
+        raise ValueError("BRAVE_SEARCH_API_KEY environment variable is not set")
+
+    url = "https://api.search.brave.com/res/v1/news/search"
+
+    headers = {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip",
+        "X-Subscription-Token": api_key,
+        "User-Agent": user_agent,
+    }
+
+    params: dict[str, str | int | bool] = {
+        "q": query,
+        "count": count,
+        "offset": offset,
+        "country": country,
+        "search_lang": search_lang,
+        "ui_lang": ui_lang,
+        "spellcheck": False,
+        "safesearch": "off",
+    }
+
+    # Add optional parameters if provided
+    if freshness:
+        params["freshness"] = freshness
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url, headers=headers, params=params)
+            response.raise_for_status()
+            data = response.json()
+            results_list: list[SearchNewsResult] = []
+            for item in data.get("results", []):
+                result = SearchNewsResult(
+                    title=item.get("title", ""),
+                    url=item.get("url", ""),
+                    description=item.get("description", ""),
+                    age=item.get("age"),
+                    netloc=item.get("meta_url", {}).get("netloc", None),
+                )
+                results_list.append(result)
+            return SearchNewsResults(results=results_list)
+
+    except httpx.HTTPError as e:
+        logger.error(f"HTTP error during Brave news search: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error during Brave news search: {e}")
+        raise

not_again_ai-0.20.0/src/not_again_ai/data/web.py

@@ -0,0 +1,160 @@
+import asyncio
+import io
+import mimetypes
+from pathlib import Path
+import re
+from urllib.parse import urlparse
+
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+import httpx
+from markitdown import MarkItDown, StreamInfo
+from pydantic import BaseModel
+
+
+class Link(BaseModel):
+    url: str
+    text: str
+
+
+class URLResult(BaseModel):
+    url: str
+    markdown: str
+    links: list[Link] = []
+
+
+async def _markitdown_bytes_to_str(file_bytes: bytes, filename_extension: str) -> str:
+    """
+    Convert a file using MarkItDown defaults.
+    """
+    with io.BytesIO(file_bytes) as temp:
+        result = await asyncio.to_thread(
+            MarkItDown(enable_plugins=False).convert,
+            source=temp,
+            stream_info=StreamInfo(extension=filename_extension),
+        )
+        text = result.text_content
+    return text
+
+
+def _detect_pdf_extension(url: str) -> bool:
+    """
+    Detect if the URL is a PDF based on its extension.
+    """
+    parsed_url = urlparse(url)
+    filename = Path(parsed_url.path).name
+    return mimetypes.guess_type(filename)[0] == "application/pdf"
+
+
+def _detect_google_sheets(url: str) -> bool:
+    """
+    Detect if the URL is a Google Sheets document.
+    """
+    is_google_sheets = url.startswith("https://docs.google.com/spreadsheets/")
+    return is_google_sheets
+
+
+async def _handle_pdf_content(url: str) -> URLResult:
+    md = MarkItDown(enable_plugins=False)
+    result = md.convert(url)
+    url_result = URLResult(
+        url=url,
+        markdown=result.markdown or "",
+        links=[],
+    )
+    return url_result
+
+
+async def _handle_google_sheets_content(url: str) -> URLResult:
+    """
+    Handle Google Sheets by using the export URL to get the raw content.
+    """
+    edit_pattern = r"https://docs\.google\.com/spreadsheets/d/([a-zA-Z0-9-_]+)/edit"
+    export_pattern = r"https://docs\.google\.com/spreadsheets/d/([a-zA-Z0-9-_]+)/export\?format=csv"
+
+    # Check if it's already an export URL
+    export_match = re.search(export_pattern, url)
+    if export_match:
+        export_url = url
+    else:
+        # Check if it's an edit URL and extract document ID
+        edit_match = re.search(edit_pattern, url)
+        if edit_match:
+            doc_id = edit_match.group(1)
+            export_url = f"https://docs.google.com/spreadsheets/d/{doc_id}/export?format=csv&gid=0"
+        else:
+            return await _handle_web_content(url)
+
+    async with httpx.AsyncClient(follow_redirects=True) as client:
+        response = await client.get(export_url)
+        response.raise_for_status()
+        csv_bytes = response.content
+
+    # Convert CSV to markdown using MarkItDown
+    markdown_content = await _markitdown_bytes_to_str(csv_bytes, ".csv")
+
+    url_result = URLResult(
+        url=url,
+        markdown=markdown_content,
+        links=[],
+    )
+    return url_result
+
+
+async def _handle_web_content(url: str) -> URLResult:
+    browser_config = BrowserConfig(
+        browser_type="chromium",
+        headless=True,
+        verbose=False,
+        user_agent_mode="random",
+        java_script_enabled=True,
+    )
+    run_config = CrawlerRunConfig(
+        scan_full_page=True,
+        user_agent_mode="random",
+        cache_mode=CacheMode.DISABLED,
+        markdown_generator=DefaultMarkdownGenerator(),
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=url,
+            config=run_config,
+        )
+
+        if result.response_headers.get("content-type") == "application/pdf":
+            return await _handle_pdf_content(url)
+
+    links: list[Link] = []
+    seen_urls: set[str] = set()
+    combined_link_data = result.links.get("internal", []) + result.links.get("external", [])
+    for link_data in combined_link_data:
+        href = link_data.get("href", "")
+        if href and href not in seen_urls:
+            seen_urls.add(href)
+            link = Link(
+                url=href,
+                text=link_data.get("title", "") or link_data.get("text", ""),
+            )
+            links.append(link)
+
+    url_result = URLResult(
+        url=url,
+        markdown=result.markdown or "",
+        links=links,
+    )
+    return url_result
+
+
+async def process_url(url: str) -> URLResult:
+    """
+    Process a URL to extract content and convert it to Markdown and links
+    """
+    if _detect_pdf_extension(url):
+        url_result = await _handle_pdf_content(url)
+    elif _detect_google_sheets(url):
+        url_result = await _handle_google_sheets_content(url)
+    else:
+        url_result = await _handle_web_content(url)
+    return url_result

not_again_ai-0.20.0/tests/data/test_brave_search_api.py

@@ -0,0 +1,34 @@
+from typing import Any
+
+import pytest
+
+from not_again_ai.data.brave_search_api import search, search_news
+
+
+@pytest.mark.parametrize(
+    ("query", "search_params"),
+    [
+        ("brave search", {}),
+        ("python programming", {"count": 2, "country": "US"}),
+        ("machine learning", {"count": 4, "search_lang": "en", "freshness": "pw"}),
+        ("AI news", {"count": 1, "offset": 5, "country": "GB", "ui_lang": "en-GB"}),
+    ],
+)
+async def test_brave_search_api(query: str, search_params: dict[str, Any]) -> None:
+    """Test the Brave Search API with a sample query and optional parameters."""
+    content = await search(query=query, **search_params)
+    assert content.results, f"No results returned for query: {query}"
+
+
+@pytest.mark.skip("API Cost")
+@pytest.mark.parametrize(
+    ("query", "search_params"),
+    [
+        ("latest tech news", {}),
+        ("AI breakthrough", {"count": 3, "country": "US"}),
+    ],
+)
+async def test_brave_search_news_api(query: str, search_params: dict[str, Any]) -> None:
+    """Test the Brave News Search API with a sample query and optional parameters."""
+    content = await search_news(query=query, **search_params)
+    assert content.results, f"No news results returned for query: {query}"

not_again_ai-0.20.0/tests/data/test_web.py

@@ -0,0 +1,20 @@
+import pytest
+
+from not_again_ai.data.web import process_url
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://example.com",
+        "https://github.com/unclecode/crawl4ai",
+        "https://arxiv.org/pdf/1710.02298",
+        "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+        "https://www.nascar.com/news/nascar-craftsman-truck-series/",
+        "https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit?gid=0#gid=0",
+        "https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/export?format=csv&gid=0",
+    ],
+)
+async def test_process_url(url: str) -> None:
+    content = await process_url(url)
+    assert content, f"Content should not be empty for URL: {url}"