llmsbrieftxt 1.6.0__py3-none-any.whl

llmsbrieftxt/summarizer.py

@@ -0,0 +1,303 @@
+import asyncio
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+import openai
+from openai import AsyncOpenAI
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+from tqdm import tqdm
+
+from .constants import (
+    DEFAULT_CONCURRENT_SUMMARIES,
+    DEFAULT_OPENAI_MODEL,
+    DEFAULT_SUMMARY_PROMPT,
+)
+from .schema import Document, PageSummary
+
+logger = logging.getLogger(__name__)
+
+
+# Fallback summary used when LLM summarization fails
+FALLBACK_SUMMARY = PageSummary(
+    content_analysis="This page contains web content relevant to the topic.",
+    primary_use_cases="When accessing general web content",
+    key_takeaways="Contains general information",
+    related_topics="Web content",
+    keywords="web, content, information",
+    concise_summary="This page contains web content relevant to the topic.",
+)
+
+
+class Summarizer:
+    def __init__(
+        self,
+        llm_name: str = DEFAULT_OPENAI_MODEL,
+        summary_prompt: str | None = None,
+        max_concurrent: int = DEFAULT_CONCURRENT_SUMMARIES,
+    ) -> None:
+        self.llm_name = llm_name
+        self.max_concurrent = max_concurrent
+        self.summary_prompt = summary_prompt or DEFAULT_SUMMARY_PROMPT
+        self.client = self._init_client()
+        self.semaphore = asyncio.Semaphore(max_concurrent)
+        # Cache JSON schema to avoid regenerating on every request
+        self._schema_cache = PageSummary.model_json_schema()
+        self._schema_cache["additionalProperties"] = False
+        # Track token usage for cost reporting (protected by lock for thread safety)
+        self.total_input_tokens = 0
+        self.total_output_tokens = 0
+        self._usage_lock = asyncio.Lock()
+
+    def _init_client(self) -> AsyncOpenAI:
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "OPENAI_API_KEY environment variable is required. Please set your OpenAI API key in your environment variables."
+            )
+        base_url = os.getenv("OPENAI_BASE_URL")
+        if base_url:
+            return AsyncOpenAI(api_key=api_key, base_url=base_url)
+        return AsyncOpenAI(api_key=api_key)
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=2, min=4, max=60),
+        retry=retry_if_exception_type(
+            (
+                openai.RateLimitError,
+                openai.APITimeoutError,
+                openai.APIConnectionError,
+            )
+        ),
+        reraise=True,
+    )
+    async def _summarize_with_retry(
+        self,
+        doc: Any,
+        loop: Any,
+        messages: list[dict[str, str]],
+        schema: dict[str, Any],
+    ) -> Any:
+        """Make LLM API call with retry logic for transient failures."""
+        return await self.client.chat.completions.create(  # type: ignore[call-overload]
+            model=self.llm_name,
+            messages=messages,  # type: ignore[arg-type]
+            response_format={  # type: ignore[typeddict-item]
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "page_summary",
+                    "schema": schema,
+                    "strict": True,
+                },
+            },
+        )
+
+    async def _summarize(self, doc: Any, loop: Any) -> PageSummary:
+        """Summarize document using OpenAI API."""
+        url = doc.metadata.get("source", "unknown")
+        try:
+            # Truncate content if it's too long (keep first 10000 chars for now)
+            content = doc.page_content[:10000]
+
+            # Build messages with combined system prompt
+            messages = [
+                {"role": "system", "content": self.summary_prompt},
+                {
+                    "role": "user",
+                    "content": f"Analyze and summarize the following webpage content:\n\n{content}",
+                },
+            ]
+
+            # Use cached schema
+            schema = self._schema_cache
+
+            response = await self._summarize_with_retry(doc, loop, messages, schema)
+
+            # Track token usage (protected by lock to prevent race conditions)
+            if response and hasattr(response, "usage") and response.usage:
+                async with self._usage_lock:
+                    self.total_input_tokens += response.usage.prompt_tokens
+                    self.total_output_tokens += response.usage.completion_tokens
+
+            # Validate response
+            if not response:
+                raise ValueError("No response object from API")
+
+            if not response.choices:
+                raise ValueError(f"No choices in response: {response}")
+
+            if not response.choices[0].message:
+                raise ValueError(
+                    f"No message in response choice: {response.choices[0]}"
+                )
+
+            if not response.choices[0].message.content:
+                # Check if there's a finish reason that explains why
+                finish_reason = (
+                    response.choices[0].finish_reason
+                    if hasattr(response.choices[0], "finish_reason")
+                    else "unknown"
+                )
+                raise ValueError(
+                    f"Empty content in response. Finish reason: {finish_reason}"
+                )
+
+            content = response.choices[0].message.content.strip()
+            if not content:
+                raise ValueError("Empty response content after stripping")
+
+            # Parse JSON response
+            try:
+                parsed_response = PageSummary(**json.loads(content))
+            except json.JSONDecodeError as je:
+                raise ValueError(
+                    f"Invalid JSON response: {je}. Content: {content[:200]}..."
+                ) from je
+
+            # Return structured response for formatting
+            return parsed_response
+
+        except Exception as e:
+            # Log with full traceback for debugging
+            logger.exception(
+                f"Failed to summarize {url}: {str(e)}",
+                exc_info=e,
+                extra={
+                    "url": url,
+                    "model": self.llm_name,
+                },
+            )
+            # Return cached fallback PageSummary object
+            return FALLBACK_SUMMARY
+
+    async def summarize_document(
+        self, doc: Any, cache_file: Path | None = None
+    ) -> str | None:
+        async with self.semaphore:
+            url = doc.metadata.get("source", "")
+            try:
+                loop = asyncio.get_event_loop()
+                page_summary = await self._summarize(doc, loop)
+
+                # Format the summary with new structure
+                title = doc.metadata.get("title", url.split("/")[-1])
+                formatted_summary = f"Title: [{title}]({url})\nKeywords: {page_summary.keywords}\nSummary: {page_summary.concise_summary}\n\n"
+
+                # Update cache if provided
+                if cache_file:
+                    self._update_cache(cache_file, url, formatted_summary)
+
+                return formatted_summary
+            except Exception as e:
+                logger.exception(
+                    f"Error summarizing {url}: {str(e)}",
+                    exc_info=e,
+                    extra={"url": url},
+                )
+                return None
+
+    def _update_cache(self, cache_file: Path, url: str, summary: str) -> None:
+        """Update the cache file with a new summary (simple version for single-user CLI)."""
+        try:
+            # Read existing cache
+            cache_data = {}
+            if cache_file.exists():
+                cache_data = json.loads(cache_file.read_text())
+
+            # Update and write
+            cache_data[url] = summary
+            cache_file.write_text(json.dumps(cache_data, indent=2))
+        except Exception as e:
+            logger.exception(
+                f"Could not update cache: {str(e)}",
+                exc_info=e,
+                extra={"cache_file": str(cache_file), "url": url},
+            )
+
+    async def summarize_all(
+        self,
+        docs: list[Document],
+        existing_summaries: dict[str, str] | None = None,
+        cache_file: Path | None = None,
+    ) -> tuple[list[str], dict[str, int]]:
+        # Reset token counters at start of each run
+        self.total_input_tokens = 0
+        self.total_output_tokens = 0
+
+        existing_summaries = existing_summaries or {}
+        summaries: list[str] = []
+        docs_to_process: list[Document] = []
+
+        # Separate cached from new documents
+        for doc in docs:
+            url = doc.metadata.get("source", "")
+            if url in existing_summaries:
+                summaries.append(existing_summaries[url])
+            else:
+                docs_to_process.append(doc)
+
+        if len(existing_summaries) > 0:
+            print(f"Using {len(existing_summaries)} cached summaries")
+
+        if not docs_to_process:
+            usage_stats = {
+                "input_tokens": self.total_input_tokens,
+                "output_tokens": self.total_output_tokens,
+            }
+            return summaries, usage_stats
+
+        # Process new documents with progress bar
+        print(
+            f"Summarizing {len(docs_to_process)} documents (max {self.max_concurrent} concurrent)..."
+        )
+
+        tasks = [self.summarize_document(doc, cache_file) for doc in docs_to_process]
+
+        # Use tqdm to track completion
+        failed_count = 0
+        with tqdm(
+            total=len(docs_to_process), desc="Generating summaries", unit="doc"
+        ) as pbar:
+            results: list[str | None | Exception] = []
+            for coro in asyncio.as_completed(tasks):
+                result = await coro
+                results.append(result)
+                if result is None or isinstance(result, Exception):
+                    failed_count += 1
+                    pbar.set_postfix({"failed": failed_count})  # type: ignore[reportUnknownMemberType]
+                pbar.update(1)
+
+        # Collect successful summaries
+        for result in results:
+            if isinstance(result, str):
+                summaries.append(result)
+
+        # Log any failures with full context
+        for result, doc in zip(results, docs_to_process, strict=False):
+            if isinstance(result, Exception):
+                url = doc.metadata.get("source", "unknown")
+                logger.exception(
+                    f"Failed to summarize document {url}: {str(result)}",
+                    exc_info=result,
+                    extra={"url": url},
+                )
+
+        success_count = len(results) - failed_count
+        print(
+            f"Summarization complete: {success_count} successful, {failed_count} failed"
+        )
+
+        # Return summaries and usage statistics
+        usage_stats = {
+            "input_tokens": self.total_input_tokens,
+            "output_tokens": self.total_output_tokens,
+        }
+        return summaries, usage_stats

llmsbrieftxt/url_filters.py

@@ -0,0 +1,75 @@
+"""Simple URL filtering for documentation crawling."""
+
+import logging
+import re
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+
+class URLFilter:
+    """Simple URL filtering based on file extensions."""
+
+    # File extensions to skip (assets, downloads, media)
+    FILE_EXTENSION_PATTERNS: list[str] = [
+        r"\.(pdf|zip|tar|gz|exe|dmg|iso)$",  # Downloads
+        r"\.(css|js|map)$",  # Web assets
+        r"\.(woff2?|ttf|eot)$",  # Fonts
+        r"\.(png|jpe?g|gif|svg|webp|ico|bmp)$",  # Images
+        r"\.(mp4|webm|avi|mov|mp3|wav|ogg)$",  # Media
+    ]
+
+    def __init__(self) -> None:
+        """Initialize URL filter with compiled patterns."""
+        self.file_extension_patterns = [
+            re.compile(p, re.IGNORECASE) for p in self.FILE_EXTENSION_PATTERNS
+        ]
+        logger.debug(
+            f"URLFilter initialized with {len(self.file_extension_patterns)} patterns"
+        )
+
+    def should_include(self, url: str) -> bool:
+        """
+        Determine if URL should be included in crawl.
+
+        Logic:
+        - Skip URLs with file extensions (downloads, assets, media)
+        - Include everything else (documentation pages)
+
+        Args:
+            url: URL to check
+
+        Returns:
+            True if URL should be crawled, False otherwise
+        """
+        parsed = urlparse(url)
+        path = parsed.path.lower()
+
+        # Check file extensions
+        for pattern in self.file_extension_patterns:
+            if pattern.search(path):
+                logger.debug(f"URL skipped by file extension: {url}")
+                return False
+
+        # Include by default
+        return True
+
+    def filter_urls(self, urls: list[str]) -> list[str]:
+        """
+        Filter a list of URLs.
+
+        Args:
+            urls: List of URLs to filter
+
+        Returns:
+            Filtered list of URLs
+        """
+        filtered = [url for url in urls if self.should_include(url)]
+        skipped_count = len(urls) - len(filtered)
+
+        if skipped_count > 0:
+            logger.info(
+                f"Filtered {skipped_count} URLs ({len(filtered)}/{len(urls)} remaining)"
+            )
+
+        return filtered

llmsbrieftxt/url_utils.py

@@ -0,0 +1,73 @@
+"""Simple URL deduplication utilities."""
+
+from urllib.parse import urlparse, urlunparse
+
+
+class URLCanonicalizer:
+    """Simple URL canonicalization for documentation sites."""
+
+    def __init__(self, keep_fragments: bool = False):
+        """
+        Initialize URL canonicalizer.
+
+        Args:
+            keep_fragments: Keep URL fragments (for sites using hash routing).
+                           Default is False to treat #section1 and #section2 as same URL.
+        """
+        self.keep_fragments = keep_fragments
+
+    def canonicalize(self, url: str) -> str:
+        """
+        Normalize URL for deduplication.
+
+        Simple normalization:
+        - Lowercase scheme and domain
+        - Remove fragments (unless keep_fragments=True)
+        - Normalize trailing slashes for directory paths
+
+        Args:
+            url: URL to canonicalize
+
+        Returns:
+            Canonicalized URL string
+        """
+        parsed = urlparse(url)
+
+        # Normalize scheme and domain to lowercase
+        scheme = parsed.scheme.lower()
+        netloc = parsed.netloc.lower()
+        path = parsed.path
+
+        # Normalize trailing slashes: add to directory-like paths
+        if path and not path.endswith("/"):
+            # If no file extension, treat as directory
+            last_segment = path.split("/")[-1]
+            if "." not in last_segment:
+                path = path + "/"
+
+        # Remove fragment unless keeping them
+        fragment = parsed.fragment if self.keep_fragments else ""
+
+        # Reconstruct URL
+        return urlunparse((scheme, netloc, path, parsed.params, parsed.query, fragment))
+
+    def deduplicate(self, urls: list[str]) -> list[str]:
+        """
+        Remove duplicate URLs from list while preserving order.
+
+        Args:
+            urls: List of URLs (may contain duplicates)
+
+        Returns:
+            List of unique URLs (first occurrence preserved)
+        """
+        seen: set[str] = set()
+        unique: list[str] = []
+
+        for url in urls:
+            canonical = self.canonicalize(url)
+            if canonical not in seen:
+                seen.add(canonical)
+                unique.append(url)  # Keep original URL
+
+        return unique