llmsbrieftxt 1.6.0__py3-none-any.whl

llmsbrieftxt/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.1"

llmsbrieftxt/cli.py

@@ -0,0 +1,276 @@
+"""Command-line interface for llmsbrieftxt."""
+
+import argparse
+import asyncio
+import os
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+from llmsbrieftxt.constants import (
+    DEFAULT_CACHE_DIR,
+    DEFAULT_CONCURRENT_SUMMARIES,
+    DEFAULT_CRAWL_DEPTH,
+    DEFAULT_OPENAI_MODEL,
+    DOCS_DIR,
+    ESTIMATED_TOKENS_PER_PAGE_INPUT,
+    ESTIMATED_TOKENS_PER_PAGE_OUTPUT,
+    OPENAI_PRICING,
+)
+from llmsbrieftxt.main import generate_llms_txt
+
+
+def parse_args(test_args: list[str] | None = None) -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Generate llms-brief.txt files from documentation websites",
+        epilog="Example: llmsbrieftxt https://docs.python.org/3/",
+    )
+
+    # Positional argument for URL
+    parser.add_argument("url", help="URL of the documentation site to process")
+
+    parser.add_argument(
+        "--model",
+        default=DEFAULT_OPENAI_MODEL,
+        help=f"OpenAI model to use (default: {DEFAULT_OPENAI_MODEL})",
+    )
+
+    parser.add_argument(
+        "--max-concurrent-summaries",
+        type=int,
+        default=DEFAULT_CONCURRENT_SUMMARIES,
+        help=f"Maximum number of concurrent LLM requests (default: {DEFAULT_CONCURRENT_SUMMARIES})",
+    )
+
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help=f"Output file path (default: {DOCS_DIR}/<domain>.txt)",
+    )
+
+    parser.add_argument(
+        "--show-urls",
+        action="store_true",
+        help="Preview discovered URLs with cost estimate (no processing or API calls)",
+    )
+
+    parser.add_argument(
+        "--max-urls", type=int, help="Maximum number of URLs to discover and process"
+    )
+
+    parser.add_argument(
+        "--depth",
+        type=int,
+        default=DEFAULT_CRAWL_DEPTH,
+        help=f"Maximum crawl depth (default: {DEFAULT_CRAWL_DEPTH})",
+    )
+
+    parser.add_argument(
+        "--cache-dir",
+        type=str,
+        default=DEFAULT_CACHE_DIR,
+        help=f"Cache directory path (default: {DEFAULT_CACHE_DIR})",
+    )
+
+    parser.add_argument(
+        "--use-cache-only",
+        action="store_true",
+        help="Use only cached summaries, skip API calls for new pages",
+    )
+
+    parser.add_argument(
+        "--force-refresh",
+        action="store_true",
+        help="Ignore cache and regenerate all summaries",
+    )
+
+    parser.add_argument(
+        "--yes",
+        "-y",
+        action="store_true",
+        help="Skip confirmation prompts (useful for automation)",
+    )
+
+    return parser.parse_args(test_args)
+
+
+def validate_url(url: str) -> bool:
+    """Validate that the URL is well-formed and uses http/https scheme."""
+    try:
+        parsed = urlparse(url)
+        return bool(parsed.scheme in ("http", "https") and parsed.netloc)
+    except Exception:
+        return False
+
+
+def check_openai_api_key() -> bool:
+    """Check if OPENAI_API_KEY is set in environment."""
+    return bool(os.environ.get("OPENAI_API_KEY"))
+
+
+def estimate_cost(num_pages: int, model: str) -> str:
+    """
+    Estimate the API cost for processing a given number of pages.
+
+    Args:
+        num_pages: Number of pages to process
+        model: OpenAI model name
+
+    Returns:
+        Formatted cost estimate string
+    """
+    if model not in OPENAI_PRICING:
+        return "Cost estimation not available for this model"
+
+    input_price, output_price = OPENAI_PRICING[model]
+
+    # Calculate total tokens
+    total_input_tokens = num_pages * ESTIMATED_TOKENS_PER_PAGE_INPUT
+    total_output_tokens = num_pages * ESTIMATED_TOKENS_PER_PAGE_OUTPUT
+
+    # Calculate cost (prices are per 1M tokens)
+    input_cost = (total_input_tokens / 1_000_000) * input_price
+    output_cost = (total_output_tokens / 1_000_000) * output_price
+    total_cost = input_cost + output_cost
+
+    if total_cost < 0.01:
+        return f"~${total_cost:.4f}"
+    elif total_cost < 1.00:
+        return f"~${total_cost:.3f}"
+    else:
+        return f"~${total_cost:.2f}"
+
+
+def get_output_path(url: str, custom_output: str | None = None) -> Path:
+    """
+    Get the output file path for a given URL.
+
+    Args:
+        url: The URL being processed
+        custom_output: Optional custom output path
+
+    Returns:
+        Path object for the output file
+    """
+    if custom_output:
+        # Expand environment variables and user home directory
+        expanded = os.path.expandvars(custom_output)
+        return Path(expanded).expanduser()
+
+    # Extract domain from URL
+    parsed = urlparse(url)
+    domain = parsed.netloc or parsed.path.split("/")[0]
+
+    # Remove www. prefix if present
+    if domain.startswith("www."):
+        domain = domain[4:]
+
+    # Ensure ~/.claude/docs/ exists
+    docs_dir = Path(DOCS_DIR).expanduser()
+    docs_dir.mkdir(parents=True, exist_ok=True)
+
+    return docs_dir / f"{domain}.txt"
+
+
+def main() -> None:
+    """Main entry point for the CLI."""
+    args = parse_args()
+
+    try:
+        # Validate URL
+        if not validate_url(args.url):
+            print("Error: Invalid URL", file=sys.stderr)
+            print(
+                f"Please provide a valid HTTP or HTTPS URL. Got: {args.url}",
+                file=sys.stderr,
+            )
+            print("Example: https://docs.python.org/3/", file=sys.stderr)
+            sys.exit(1)
+
+        # Validate depth parameter
+        if args.depth < 1:
+            print("Error: --depth must be at least 1", file=sys.stderr)
+            sys.exit(1)
+
+        # Check for conflicting cache flags
+        if args.use_cache_only and args.force_refresh:
+            print(
+                "Error: Cannot use --use-cache-only and --force-refresh together",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        # Check for API key (unless just showing URLs or using cache only)
+        if (
+            not args.show_urls
+            and not args.use_cache_only
+            and not check_openai_api_key()
+        ):
+            print("Error: OPENAI_API_KEY not found", file=sys.stderr)
+            print("Please set your OpenAI API key:", file=sys.stderr)
+            print("  export OPENAI_API_KEY='sk-your-api-key-here'", file=sys.stderr)
+            print("", file=sys.stderr)
+            print(
+                "Get your API key from: https://platform.openai.com/api-keys",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        # Determine output path
+        output_path = get_output_path(args.url, args.output)
+
+        # Expand cache directory path
+        cache_dir = Path(os.path.expandvars(args.cache_dir)).expanduser()
+
+        # Print configuration
+        print(f"Processing URL: {args.url}")
+        if not args.show_urls:
+            print(f"Using model: {args.model}")
+        print(f"Crawl depth: {args.depth}")
+        print(f"Output: {output_path}")
+        if args.max_urls:
+            print(f"Max URLs: {args.max_urls}")
+        if args.use_cache_only:
+            print("Mode: Cache-only (no API calls)")
+        elif args.force_refresh:
+            print("Mode: Force refresh (ignoring cache)")
+
+        # Run generation
+        result = asyncio.run(
+            generate_llms_txt(
+                url=args.url,
+                llm_name=args.model,
+                max_concurrent_summaries=args.max_concurrent_summaries,
+                output_path=str(output_path),
+                show_urls=args.show_urls,
+                max_urls=args.max_urls,
+                max_depth=args.depth,
+                cache_dir=str(cache_dir),
+                use_cache_only=args.use_cache_only,
+                force_refresh=args.force_refresh,
+                skip_confirmation=args.yes,
+            )
+        )
+
+        # Show cost estimate and failed URLs if available
+        if args.show_urls and result:
+            num_urls_value = result.get("num_urls", 0)
+            # Type guard to ensure we have an int
+            if isinstance(num_urls_value, int):
+                print(
+                    f"\nEstimated cost for {num_urls_value} pages: {estimate_cost(num_urls_value, args.model)}"
+                )
+            print("Note: Actual cost may vary based on page content size and caching")
+
+    except KeyboardInterrupt:
+        print("\nOperation cancelled by user.", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error: {str(e)}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

llmsbrieftxt/constants.py

@@ -0,0 +1,62 @@
+"""Configuration constants for llmsbrieftxt package."""
+
+# Concurrency
+DEFAULT_CONCURRENT_SUMMARIES = 10
+
+# Default Models
+DEFAULT_OPENAI_MODEL = "gpt-5-mini"
+
+# Docs Directory
+DOCS_DIR = "~/.claude/docs"  # Will be expanded to full path at runtime
+
+# Default Cache Directory
+DEFAULT_CACHE_DIR = ".llmsbrieftxt_cache"
+
+# Default Crawl Depth
+DEFAULT_CRAWL_DEPTH = 3
+
+# OpenAI Pricing (per 1M tokens) - prices subject to change
+# Format: {model: (input_price, output_price)}
+# Note: Verify current pricing at https://openai.com/api/pricing/
+OPENAI_PRICING = {
+    "gpt-5-mini": (0.15, 0.60),  # $0.15 input, $0.60 output per 1M tokens
+    "gpt-4o-mini": (0.15, 0.60),
+    "gpt-4o": (2.50, 10.00),
+    "gpt-4-turbo": (10.00, 30.00),
+    "gpt-4": (30.00, 60.00),
+}
+
+# Estimated tokens per page for cost calculation
+# These estimates are based on typical documentation page sizes:
+# - Input: ~2000-4000 words per doc page → ~3000 tokens (conservative estimate)
+# - Output: ~300 tokens for structured PageSummary with all fields
+# Accuracy: Estimates typically within ±30% of actual cost
+# Pages with code examples or very long content may exceed these estimates
+ESTIMATED_TOKENS_PER_PAGE_INPUT = 3000
+ESTIMATED_TOKENS_PER_PAGE_OUTPUT = 400
+
+
+# Prompt Templates
+DEFAULT_SUMMARY_PROMPT = """You are a specialized content analyzer creating structured summaries for llms-brief.txt files. Your role is to help LLMs understand web content by providing comprehensive yet concise summaries.
+
+Focus on:
+- What information and resources are available
+- When and why an LLM should reference this content
+- Key insights and practical applications
+
+Guidelines:
+1. Be specific and actionable - avoid vague descriptions
+2. Focus on practical utility - what can someone DO with this information?
+3. Identify unique value - what makes this page worth referencing?
+4. Target 500-800 tokens total across all fields (roughly 2-4 sentences per field)
+5. Write from the perspective of helping an LLM know when to use this resource
+
+Provide structured summaries with:
+- Core information and resources available (2-3 detailed sentences)
+- Specific scenarios when this page should be referenced (3-5 concrete use cases)
+- The most valuable insights or capabilities offered (2-3 key points)
+- Related domains and topics for context (brief but comprehensive list)
+- Searchable keywords for discovery (5-10 specific terms)
+- A single-sentence executive summary (15-25 words)
+
+Aim for depth over brevity - each field should contain substantive, actionable information while remaining concise."""

llmsbrieftxt/crawler.py

@@ -0,0 +1,358 @@
+"""Simple web crawler with sitemap support and BFS fallback."""
+
+import asyncio
+import contextlib
+import io
+import logging
+import sys
+from collections.abc import Generator
+from urllib.parse import urljoin, urlparse
+
+import httpx
+from bs4 import BeautifulSoup, Tag
+from tenacity import retry, stop_after_attempt, wait_exponential
+from usp.tree import sitemap_tree_for_homepage  # type: ignore[import-untyped]
+
+from llmsbrieftxt.url_filters import URLFilter
+from llmsbrieftxt.url_utils import URLCanonicalizer
+
+logger = logging.getLogger(__name__)
+
+
+@contextlib.contextmanager
+def suppress_output() -> Generator[None, None, None]:
+    """Suppress stdout and stderr temporarily.
+
+    USP library prints noisy errors to console.
+    Suppress them when sitemap parsing fails (expected for SPA sites).
+    """
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    try:
+        sys.stdout = io.StringIO()
+        sys.stderr = io.StringIO()
+        yield
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+
+
+class RobustDocCrawler:
+    """Production-ready documentation crawler with sitemap and BFS fallback."""
+
+    def __init__(
+        self,
+        max_urls: int | None = None,
+        max_depth: int = 3,
+        max_concurrent: int = 10,
+    ):
+        """Initialize the crawler.
+
+        Args:
+            max_urls: Maximum number of URLs to crawl (None = unlimited)
+            max_depth: Maximum crawl depth
+            max_concurrent: Maximum concurrent requests
+        """
+        self.max_urls = max_urls or 500
+        self.max_depth = max_depth
+        self.max_concurrent = max_concurrent
+        self.discovered_urls: set[str] = set()
+        self.user_agent = (
+            "llmsbrieftxt-bot/1.0 (+https://github.com/stevennevins/llmsbrief)"
+        )
+        self.url_canonicalizer = URLCanonicalizer(keep_fragments=False)
+        self.url_filter = URLFilter()
+        self.semaphore = asyncio.Semaphore(max_concurrent)
+
+    async def discover_urls(self, base_url: str) -> set[str]:
+        """Discover URLs using sitemap or BFS crawling.
+
+        Priority:
+        1. Sitemap (fastest, most complete)
+        2. BFS crawling (fallback)
+
+        Args:
+            base_url: The starting URL
+
+        Returns:
+            Set of discovered URLs
+        """
+        self.discovered_urls = set()
+
+        # Strategy 1: Try sitemap first
+        logger.info("Strategy 1: Checking for sitemap...")
+        sitemap_urls = await self._discover_from_sitemap(base_url)
+        if sitemap_urls:
+            logger.info(f"Sitemap discovery successful: {len(sitemap_urls)} URLs")
+            self.discovered_urls = sitemap_urls
+            return self._apply_limits(sitemap_urls)
+
+        # Strategy 2: Fall back to BFS crawling
+        logger.info("Strategy 2: Using BFS crawler...")
+        crawled_urls = await self._crawl_bfs(base_url)
+        logger.info(f"BFS discovery complete: {len(crawled_urls)} URLs")
+        self.discovered_urls = crawled_urls
+        return self._apply_limits(crawled_urls)
+
+    async def _discover_from_sitemap(self, base_url: str) -> set[str]:
+        """Discover URLs from sitemap.xml files.
+
+        Args:
+            base_url: The base URL to discover sitemap for
+
+        Returns:
+            Set of URLs found in sitemaps
+        """
+        urls: set[str] = set()
+
+        try:
+            # Try standard sitemap location
+            parsed = urlparse(base_url)
+            base_domain = f"{parsed.scheme}://{parsed.netloc}"
+            sitemap_url = f"{base_domain}/sitemap.xml"
+            logger.info("Trying standard sitemap location")
+
+            # Parse sitemap (with timeout)
+            # Suppress noisy errors from USP library (expected for SPA sites)
+            try:
+                logger.info(f"Parsing sitemap: {sitemap_url}")
+                with suppress_output():
+                    tree = await asyncio.wait_for(
+                        asyncio.to_thread(sitemap_tree_for_homepage, sitemap_url),  # type: ignore[reportUnknownArgumentType]
+                        timeout=30.0,
+                    )
+                sitemap_pages = [page.url for page in tree.all_pages()]
+
+                if sitemap_pages:
+                    logger.info(
+                        f"Found {len(sitemap_pages)} URLs in sitemap {sitemap_url}"
+                    )
+
+                    # Filter URLs to only those under base path
+                    base_path = self._get_base_path(base_url)
+                    for url in sitemap_pages:
+                        if self._is_under_base_path(url, base_path):
+                            urls.add(url)
+            except asyncio.TimeoutError:
+                logger.debug(f"Timeout parsing sitemap {sitemap_url}")
+
+        except Exception as e:
+            logger.debug(f"Sitemap discovery failed: {e}")
+
+        return urls
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        reraise=True,
+    )
+    async def _fetch_with_retry(self, url: str, client: httpx.AsyncClient) -> str:
+        """Fetch URL with retry logic and concurrency limiting."""
+        async with self.semaphore:
+            response = await client.get(url, timeout=30.0, follow_redirects=True)
+            response.raise_for_status()
+            return response.text
+
+    async def _crawl_bfs(self, base_url: str) -> set[str]:
+        """Crawl using breadth-first search.
+
+        Args:
+            base_url: The starting URL
+
+        Returns:
+            Set of discovered URLs
+        """
+        discovered: set[str] = set()
+        to_visit: set[str] = {base_url}
+        visited: set[str] = set()
+        base_path = self._get_base_path(base_url)
+
+        # Print initial discovery status
+        print("Discovering URLs: 0 found", end="", flush=True)
+
+        async with httpx.AsyncClient(
+            follow_redirects=True, timeout=httpx.Timeout(30.0)
+        ) as client:
+            for depth in range(self.max_depth):
+                if not to_visit or len(discovered) >= self.max_urls:
+                    break
+
+                logger.info(f"Depth {depth}: {len(to_visit)} URLs to visit")
+                current_level = list(to_visit)
+                to_visit: set[str] = set()
+
+                # Process in batches
+                for i in range(0, len(current_level), self.max_concurrent):
+                    batch = current_level[i : i + self.max_concurrent]
+                    tasks = [
+                        self._extract_links(url, client, base_path) for url in batch
+                    ]
+                    results = await asyncio.gather(*tasks, return_exceptions=True)
+
+                    for url, result in zip(batch, results, strict=False):
+                        visited.add(url)
+                        discovered.add(url)
+
+                        # Update live counter
+                        print(
+                            f"\rDiscovering URLs: {len(discovered)} found",
+                            end="",
+                            flush=True,
+                        )
+
+                        if not isinstance(result, Exception) and isinstance(
+                            result, set
+                        ):
+                            # Add new URLs to visit
+                            for link in result:
+                                if (
+                                    link not in visited
+                                    and link not in discovered
+                                    and link not in to_visit
+                                    and len(discovered) < self.max_urls
+                                ):
+                                    to_visit.add(link)
+
+                logger.info(
+                    f"Depth {depth} complete: {len(discovered)} URLs discovered"
+                )
+
+        # Final newline after live counter
+        print()  # Move to next line
+        return discovered
+
+    async def _extract_links(
+        self, url: str, client: httpx.AsyncClient, base_path: str
+    ) -> set[str]:
+        """Extract links from a page."""
+        links: set[str] = set()
+
+        try:
+            html = await self._fetch_with_retry(url, client)
+            soup = BeautifulSoup(html, "html.parser")
+
+            for anchor in soup.find_all("a", href=True):
+                if not isinstance(anchor, Tag):
+                    continue
+                href_value = anchor.get("href")
+                if (
+                    href_value
+                    and isinstance(href_value, str)
+                    and self._is_valid_doc_link(href_value)
+                ):
+                    href = href_value
+                    absolute_url = urljoin(url, href)
+                    if self._is_under_base_path(absolute_url, base_path):
+                        links.add(absolute_url)
+
+        except Exception as e:
+            logger.debug(f"Failed to extract links from {url}: {e}")
+
+        return links
+
+    def _get_base_path(self, url: str) -> str:
+        """Extract the base path from a URL for scope filtering.
+
+        Args:
+            url: The URL to extract base path from
+
+        Returns:
+            Base path string
+        """
+        parsed = urlparse(url)
+
+        # For GitHub repos, extract user/repo
+        if "github.com" in parsed.netloc:
+            parts = parsed.path.strip("/").split("/")
+            if len(parts) >= 2:
+                # Handle special GitHub paths
+                if parts[2:3] == ["wiki"]:
+                    return f"/{parts[0]}/{parts[1]}/wiki/"
+                elif parts[2:4] == ["tree"]:
+                    # Include branch in path
+                    if len(parts) >= 4:
+                        return f"/{parts[0]}/{parts[1]}/tree/{parts[3]}/"
+                    return f"/{parts[0]}/{parts[1]}/"
+                elif parts[2:3] == ["blob"]:
+                    # Single file path
+                    return f"/{parts[0]}/{parts[1]}/"
+                else:
+                    # Standard repo root
+                    return f"/{parts[0]}/{parts[1]}/"
+
+        # For regular docs, use the full path up to the last segment
+        path = parsed.path
+        if not path or path == "/":
+            return "/"
+
+        # If path looks like a file, remove the filename
+        if "." in path.split("/")[-1]:
+            path = "/".join(path.split("/")[:-1]) + "/"
+        elif not path.endswith("/"):
+            path = path + "/"
+
+        return path
+
+    def _is_under_base_path(self, url: str, base_path: str) -> bool:
+        """Check if URL is under the base path.
+
+        Args:
+            url: The URL to check
+            base_path: The base path to compare against
+
+        Returns:
+            True if URL is under base path
+        """
+        parsed = urlparse(url)
+        url_path = parsed.path if parsed.path else "/"
+
+        # Ensure consistent trailing slashes
+        if not url_path.endswith("/") and "." not in url_path.split("/")[-1]:
+            url_path = url_path + "/"
+
+        return url_path.startswith(base_path)
+
+    def _is_valid_doc_link(self, link: str) -> bool:
+        """Check if a link is likely to be documentation using filtering.
+
+        Args:
+            link: The link to check
+
+        Returns:
+            True if link appears to be documentation
+        """
+        # Skip invalid links
+        if not link or link.startswith("#") or link.startswith("javascript:"):
+            return False
+
+        # Use URLFilter for extension-based filtering
+        return self.url_filter.should_include(link)
+
+    def _apply_limits(self, urls: set[str]) -> set[str]:
+        """Apply canonicalization, deduplication, and max_urls limit.
+
+        Args:
+            urls: Set of URLs to process
+
+        Returns:
+            Canonicalized, deduplicated, and limited set of URLs
+        """
+        # Convert set to list for deduplication (preserves order)
+        url_list = list(urls)
+
+        # Deduplicate using URL canonicalizer
+        unique_urls = self.url_canonicalizer.deduplicate(url_list)
+
+        duplicates_removed = len(url_list) - len(unique_urls)
+        if duplicates_removed > 0:
+            logger.info(
+                f"Removed {duplicates_removed} duplicate URLs "
+                f"({len(url_list)} -> {len(unique_urls)})"
+            )
+
+        # Apply max_urls limit
+        if self.max_urls and len(unique_urls) > self.max_urls:
+            logger.info(f"Limiting {len(unique_urls)} URLs to {self.max_urls}")
+            unique_urls = unique_urls[: self.max_urls]
+
+        return set(unique_urls)