agno 2.2.13__py3-none-any.whl

agno/knowledge/reader/text_reader.py

@@ -0,0 +1,115 @@
+import asyncio
+import uuid
+from pathlib import Path
+from typing import IO, Any, List, Optional, Union
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_info, logger
+
+
+class TextReader(Reader):
+    """Reader for Text files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Text readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TXT]
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_info(f"Reading: {file}")
+                file_name = name or file.stem
+                file_contents = file.read_text(self.encoding or "utf-8")
+            else:
+                file_name = name or file.name.split(".")[0]
+                log_info(f"Reading uploaded file: {file_name}")
+                file.seek(0)
+                file_contents = file.read().decode(self.encoding or "utf-8")
+
+            documents = [
+                Document(
+                    name=file_name,
+                    id=str(uuid.uuid4()),
+                    content=file_contents,
+                )
+            ]
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+        except Exception as e:
+            logger.error(f"Error reading: {file}: {e}")
+            return []
+
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+
+                log_info(f"Reading asynchronously: {file}")
+                file_name = name or file.stem
+
+                try:
+                    import aiofiles
+
+                    async with aiofiles.open(file, "r", encoding=self.encoding or "utf-8") as f:
+                        file_contents = await f.read()
+                except ImportError:
+                    logger.warning("aiofiles not installed, using synchronous file I/O")
+                    file_contents = file.read_text(self.encoding or "utf-8")
+            else:
+                log_info(f"Reading uploaded file asynchronously: {file.name}")
+                file_name = name or file.name.split(".")[0]
+                file.seek(0)
+                file_contents = file.read().decode(self.encoding or "utf-8")
+
+            document = Document(
+                name=file_name,
+                id=str(uuid.uuid4()),
+                content=file_contents,
+            )
+
+            if self.chunk:
+                return await self._async_chunk_document(document)
+            return [document]
+        except Exception as e:
+            logger.error(f"Error reading asynchronously: {file}: {e}")
+            return []
+
+    async def _async_chunk_document(self, document: Document) -> List[Document]:
+        if not self.chunk or not document:
+            return [document]
+
+        async def process_chunk(chunk_doc: Document) -> Document:
+            return chunk_doc
+
+        chunked_documents = self.chunk_document(document)
+
+        if not chunked_documents:
+            return [document]
+
+        tasks = [process_chunk(chunk_doc) for chunk_doc in chunked_documents]
+        return await asyncio.gather(*tasks)

agno/knowledge/reader/web_search_reader.py

@@ -0,0 +1,372 @@
+import asyncio
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Optional, Set
+from urllib.parse import urlparse
+
+import httpx
+
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, logger
+
+try:
+    from bs4 import BeautifulSoup, Tag  # noqa: F401
+except ImportError:
+    raise ImportError("The `bs4` package is not installed. Please install it via `pip install beautifulsoup4`.")
+
+try:
+    from ddgs import DDGS
+except ImportError:
+    raise ImportError("The `ddgs` package is not installed. Please install it via `pip install ddgs`.")
+
+
+@dataclass
+class WebSearchReader(Reader):
+    """Reader that uses web search to find content for a given query"""
+
+    search_timeout: int = 10
+
+    request_timeout: int = 30
+    delay_between_requests: float = 2.0  # Increased default delay
+    max_retries: int = 3
+    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+
+    # Search engine configuration
+    search_engine: Literal["duckduckgo", "google"] = "duckduckgo"
+    search_delay: float = 3.0  # Delay between search requests
+    max_search_retries: int = 2  # Retries for search operations
+
+    # Rate limiting
+    rate_limit_delay: float = 5.0  # Delay when rate limited
+    exponential_backoff: bool = True
+
+    # Internal state
+    _visited_urls: Set[str] = field(default_factory=set)
+    _last_search_time: float = field(default=0.0, init=False)
+
+    # Override default chunking strategy
+    chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking()
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Web Search readers."""
+        return [
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TOPIC]
+
+    def _respect_rate_limits(self):
+        """Ensure we don't exceed rate limits"""
+        current_time = time.time()
+        time_since_last_search = current_time - self._last_search_time
+
+        if time_since_last_search < self.search_delay:
+            sleep_time = self.search_delay - time_since_last_search
+            log_debug(f"Rate limiting: sleeping for {sleep_time:.2f} seconds")
+            time.sleep(sleep_time)
+
+        self._last_search_time = time.time()
+
+    def _perform_duckduckgo_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using DuckDuckGo with rate limiting"""
+        log_debug(f"Performing DuckDuckGo search for: {query}")
+
+        for attempt in range(self.max_search_retries):
+            try:
+                self._respect_rate_limits()
+
+                ddgs = DDGS(timeout=self.search_timeout)
+                search_results = ddgs.text(query=query, max_results=self.max_results)
+
+                # Convert to list and extract relevant fields
+                results = []
+                for result in search_results:
+                    results.append(
+                        {
+                            "title": result.get("title", ""),
+                            "url": result.get("href", ""),
+                            "description": result.get("body", ""),
+                        }
+                    )
+
+                log_debug(f"Found {len(results)} search results")
+                return results
+
+            except Exception as e:
+                logger.warning(f"DuckDuckGo search attempt {attempt + 1} failed: {e}")
+                if "rate limit" in str(e).lower() or "429" in str(e):
+                    # Rate limited - wait longer
+                    wait_time = (
+                        self.rate_limit_delay * (2**attempt) if self.exponential_backoff else self.rate_limit_delay
+                    )
+                    logger.info(f"Rate limited, waiting {wait_time} seconds before retry")
+                    time.sleep(wait_time)
+                elif attempt < self.max_search_retries - 1:
+                    # Other error - shorter wait
+                    time.sleep(self.search_delay)
+                else:
+                    logger.error(f"All DuckDuckGo search attempts failed: {e}")
+                    return []
+        return []
+
+    def _perform_google_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using Google (requires googlesearch-python)"""
+        log_debug(f"Performing Google search for: {query}")
+
+        try:
+            from googlesearch import search
+        except ImportError:
+            logger.error("Google search requires 'googlesearch-python'. Install with: pip install googlesearch-python")
+            return []
+
+        for attempt in range(self.max_search_retries):
+            try:
+                self._respect_rate_limits()
+
+                results = []
+                # Use the basic search function without unsupported parameters
+                # The googlesearch-python library's search function only accepts basic parameters
+                search_results = search(query)
+
+                # Convert iterator to list and limit results
+                result_list = list(search_results)[: self.max_results]
+
+                for result in result_list:
+                    # The search function returns URLs as strings
+                    results.append(
+                        {
+                            "title": "",  # Google search doesn't provide titles directly
+                            "url": result,
+                            "description": "",  # Google search doesn't provide descriptions directly
+                        }
+                    )
+
+                log_debug(f"Found {len(results)} Google search results")
+                return results
+
+            except Exception as e:
+                logger.warning(f"Google search attempt {attempt + 1} failed: {e}")
+                if attempt < self.max_search_retries - 1:
+                    time.sleep(self.search_delay)
+                else:
+                    logger.error(f"All Google search attempts failed: {e}")
+                    return []
+
+        return []
+
+    def _perform_web_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using the configured search engine"""
+        if self.search_engine == "duckduckgo":
+            return self._perform_duckduckgo_search(query)
+        elif self.search_engine == "google":
+            return self._perform_google_search(query)
+        else:
+            logger.error(f"Unsupported search engine: {self.search_engine}")
+            return []
+
+    def _is_valid_url(self, url: str) -> bool:
+        """Check if URL is valid and not already visited"""
+        try:
+            parsed = urlparse(url)
+            return bool(parsed.scheme in ["http", "https"] and parsed.netloc and url not in self._visited_urls)
+        except Exception:
+            return False
+
+    def _extract_text_from_html(self, html_content: str, url: str) -> str:
+        """Extract clean text content from HTML"""
+        try:
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            # Remove script and style elements
+            for script in soup(["script", "style"]):
+                script.decompose()
+
+            # Get text content
+            text = soup.get_text()
+
+            # Clean up whitespace
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            text = " ".join(chunk for chunk in chunks if chunk)
+
+            return text
+
+        except Exception as e:
+            logger.warning(f"Error extracting text from {url}: {e}")
+            return html_content
+
+    def _fetch_url_content(self, url: str) -> Optional[str]:
+        """Fetch content from a URL with retry logic"""
+        headers = {"User-Agent": self.user_agent}
+
+        for attempt in range(self.max_retries):
+            try:
+                response = httpx.get(url, headers=headers, timeout=self.request_timeout, follow_redirects=True)
+                response.raise_for_status()
+
+                # Check if it's HTML content
+                content_type = response.headers.get("content-type", "").lower()
+                if "text/html" in content_type:
+                    return self._extract_text_from_html(response.text, url)
+                else:
+                    # For non-HTML content, return as-is
+                    return response.text
+
+            except Exception as e:
+                logger.warning(f"Attempt {attempt + 1} failed for {url}: {e}")
+                if attempt < self.max_retries - 1:
+                    time.sleep(random.uniform(1, 3))  # Random delay between retries
+                continue
+
+        logger.error(f"Failed to fetch content from {url} after {self.max_retries} attempts")
+        return None
+
+    def _create_document_from_url(self, url: str, content: str, search_result: Dict[str, str]) -> Document:
+        """Create a Document object from URL content and search result metadata"""
+        # Use the URL as the document ID
+        doc_id = url
+
+        # Use the search result title as the document name, fallback to URL
+        doc_name = search_result.get("title", urlparse(url).netloc)
+
+        # Create metadata with search information
+        meta_data = {
+            "url": url,
+            "search_title": search_result.get("title", ""),
+            "search_description": search_result.get("description", ""),
+            "source": "web_search",
+            "search_engine": self.search_engine,
+        }
+
+        return Document(id=doc_id, name=doc_name, content=content, meta_data=meta_data)
+
+    def read(self, query: str) -> List[Document]:
+        """Read content for a given query by performing web search and fetching content"""
+        if not query:
+            raise ValueError("Query cannot be empty")
+
+        log_debug(f"Starting web search reader for query: {query}")
+
+        # Perform web search
+        search_results = self._perform_web_search(query)
+        if not search_results:
+            logger.warning(f"No search results found for query: {query}")
+            return []
+
+        documents: List[Document] = []
+
+        for result in search_results:
+            url = result.get("url", "")
+
+            # Skip if URL is invalid or already visited
+            if not self._is_valid_url(url):
+                continue
+
+            # Mark URL as visited
+            self._visited_urls.add(url)
+
+            # Add delay between requests to be respectful
+            if len(documents) > 0:
+                time.sleep(self.delay_between_requests)
+
+            # Fetch content from URL
+            content = self._fetch_url_content(url)
+            if content is None:
+                continue
+
+            # Create document
+            document = self._create_document_from_url(url, content, result)
+
+            # Apply chunking if enabled
+            if self.chunk:
+                chunked_docs = self.chunk_document(document)
+                documents.extend(chunked_docs)
+            else:
+                documents.append(document)
+
+            # Stop if we've reached max_results
+            if len(documents) >= self.max_results:
+                break
+
+        log_debug(f"Created {len(documents)} documents from web search")
+        return documents
+
+    async def async_read(self, query: str) -> List[Document]:
+        """Asynchronously read content for a given query"""
+        if not query:
+            raise ValueError("Query cannot be empty")
+
+        log_debug(f"Starting async web search reader for query: {query}")
+
+        # Perform web search (synchronous operation)
+        search_results = self._perform_web_search(query)
+        if not search_results:
+            logger.warning(f"No search results found for query: {query}")
+            return []
+
+        # Create tasks for fetching content from each URL
+        async def fetch_url_async(result: Dict[str, str]) -> Optional[Document]:
+            url = result.get("url", "")
+
+            # Skip if URL is invalid or already visited
+            if not self._is_valid_url(url):
+                return None
+
+            # Mark URL as visited
+            self._visited_urls.add(url)
+
+            try:
+                headers = {"User-Agent": self.user_agent}
+                async with httpx.AsyncClient(timeout=self.request_timeout) as client:
+                    response = await client.get(url, headers=headers, follow_redirects=True)
+                    response.raise_for_status()
+
+                    content_type = response.headers.get("content-type", "").lower()
+                    if "text/html" in content_type:
+                        content = self._extract_text_from_html(response.text, url)
+                    else:
+                        content = response.text
+
+                    document = self._create_document_from_url(url, content, result)
+                    return document
+
+            except Exception as e:
+                logger.warning(f"Error fetching {url}: {e}")
+                return None
+
+        # Create tasks for all URLs
+        tasks = [fetch_url_async(result) for result in search_results]
+
+        # Execute all tasks concurrently with delays
+        documents = []
+        for i, task in enumerate(tasks):
+            if i > 0:  # Add delay between requests (except for the first one)
+                await asyncio.sleep(self.delay_between_requests)
+
+            doc = await task
+            if doc is not None:
+                # Apply chunking if enabled
+                if self.chunk:
+                    chunked_docs = await self.chunk_documents_async([doc])
+                    documents.extend(chunked_docs)
+                else:
+                    documents.append(doc)
+
+                # Stop if we've reached max_results
+                if len(documents) >= self.max_results:
+                    break
+
+        log_debug(f"Created {len(documents)} documents from async web search")
+        return documents