agno 1.8.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

agno/knowledge/reader/s3_reader.py

@@ -0,0 +1,101 @@
+import asyncio
+from io import BytesIO
+from pathlib import Path
+from typing import List, Optional
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.pdf_reader import PDFReader
+from agno.knowledge.reader.text_reader import TextReader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_info, logger
+
+try:
+    from agno.aws.resource.s3.object import S3Object  # type: ignore
+except (ModuleNotFoundError, ImportError):
+    raise ImportError("`agno-aws` not installed. Please install using `pip install agno-aws`")
+
+try:
+    import textract  # noqa: F401
+except ImportError:
+    raise ImportError("`textract` not installed. Please install it via `pip install textract`.")
+
+try:
+    from pypdf import PdfReader as DocumentReader  # noqa: F401
+except ImportError:
+    raise ImportError("`pypdf` not installed. Please install it via `pip install pypdf`.")
+
+
+class S3Reader(Reader):
+    """Reader for S3 files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for S3 readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.FILE, ContentType.URL, ContentType.TEXT]
+
+    def read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        try:
+            log_info(f"Reading S3 file: {s3_object.uri}")
+
+            # Read PDF files
+            if s3_object.uri.endswith(".pdf"):
+                object_resource = s3_object.get_resource()
+                object_body = object_resource.get()["Body"]
+                doc_name = (
+                    s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+                    if name is None
+                    else name
+                )
+                return PDFReader().read(pdf=BytesIO(object_body.read()), name=doc_name)
+
+            # Read text files
+            else:
+                doc_name = (
+                    s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+                    if name is None
+                    else name
+                )
+                obj_name = s3_object.name.split("/")[-1]
+                temporary_file = Path("storage").joinpath(obj_name)
+                s3_object.download(temporary_file)
+
+                # TODO: Before we were using textract here. Needed?
+                # s3_object.download(temporary_file)
+                # doc_content = textract.process(temporary_file)
+                # documents = [
+                #     Document(
+                #         name=doc_name,
+                #         id=doc_name,
+                #         content=doc_content.decode("utf-8"),
+                #     )
+                # ]
+
+                documents = TextReader().read(file=temporary_file, name=doc_name)
+
+                temporary_file.unlink()
+                return documents
+
+        except Exception as e:
+            logger.error(f"Error reading: {s3_object.uri}: {e}")
+
+        return []
+
+    async def async_read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        """Asynchronously read S3 files by running the synchronous read operation in a thread."""
+        return await asyncio.to_thread(self.read, name, s3_object)

agno/{document → knowledge}/reader/text_reader.py

@@ -1,27 +1,48 @@
 import asyncio
 import uuid
 from pathlib import Path
-from typing import IO, Any, List, Union
+from typing import IO, Any, List, Optional, Union
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_info, logger
 
 
 class TextReader(Reader):
     """Reader for Text files"""
 
-    def read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Text readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TXT]
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
                 log_info(f"Reading: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
                 file_contents = file.read_text("utf-8")
             else:
-                log_info(f"Reading uploaded file: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
+                log_info(f"Reading uploaded file: {file_name}")
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 
@@ -42,14 +63,14 @@ class TextReader(Reader):
             logger.error(f"Error reading: {file}: {e}")
             return []
 
-    async def async_read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
 
                 log_info(f"Reading asynchronously: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
 
                 try:
                     import aiofiles
@@ -61,7 +82,7 @@ class TextReader(Reader):
                     file_contents = file.read_text("utf-8")
             else:
                 log_info(f"Reading uploaded file asynchronously: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 

agno/knowledge/reader/url_reader.py

@@ -0,0 +1,128 @@
+from io import BytesIO
+from os.path import basename
+from pathlib import Path
+from typing import List, Optional
+from urllib.parse import urlparse
+
+import httpx
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.csv_reader import CSVReader
+from agno.knowledge.reader.pdf_reader import PDFReader
+from agno.knowledge.types import ContentType
+from agno.utils.http import async_fetch_with_retry, fetch_with_retry
+from agno.utils.log import log_debug
+
+
+class URLReader(Reader):
+    """Reader for general URL content"""
+
+    def __init__(
+        self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), proxy: Optional[str] = None, **kwargs
+    ):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+        self.proxy = proxy
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for URL readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
+    def read(
+        self, url: str, id: Optional[str] = None, name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
+        if not url:
+            raise ValueError("No url provided")
+
+        log_debug(f"Reading: {url}")
+
+        # Retry the request up to 3 times with exponential backoff
+        response = fetch_with_retry(url, proxy=self.proxy)
+
+        documents = self._create_documents(
+            url=url, text=response.text, content=response.content, id=id, name=name, password=password
+        )
+
+        if not self.chunk:
+            return documents
+
+        chunked_documents = []
+        for document in documents:
+            chunked_documents.append(self.chunk_document(document))
+        return [doc for sublist in chunked_documents for doc in sublist]
+
+    async def async_read(
+        self, url: str, id: Optional[str] = None, name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
+        """Async version of read method"""
+        if not url:
+            raise ValueError("No url provided")
+
+        log_debug(f"Reading async: {url}")
+        client_args = {"proxy": self.proxy} if self.proxy else {}
+        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
+            response = await async_fetch_with_retry(url, client=client)
+
+        documents = self._create_documents(
+            url=url, text=response.text, content=response.content, id=id, name=name, password=password
+        )
+
+        if not self.chunk:
+            return documents
+
+        return await self.chunk_documents_async(documents)
+
+    def _create_documents(
+        self,
+        url: str,
+        text: str,
+        content: bytes,
+        id: Optional[str] = None,
+        name: Optional[str] = None,
+        password: Optional[str] = None,
+    ) -> List[Document]:
+        """Helper method to create a document from URL content"""
+
+        # Determine file extension from URL
+        parsed_url = urlparse(url)
+        url_path = Path(parsed_url.path)  # type: ignore
+        file_extension = url_path.suffix.lower()
+
+        # Read the document using the appropriate reader
+        if file_extension == ".csv":
+            filename = basename(parsed_url.path) or "data.csv"
+            return CSVReader().read(file=BytesIO(content), name=filename)
+        elif file_extension == ".pdf":
+            if password:
+                return PDFReader().read(pdf=BytesIO(content), name=name, password=password)
+            else:
+                return PDFReader().read(pdf=BytesIO(content), name=name)
+        else:
+            doc_name = name or parsed_url.path.strip("/").replace("/", "_").replace(" ", "_")
+            if not doc_name:
+                doc_name = parsed_url.netloc
+            if not doc_name:
+                doc_name = url
+
+        return [
+            Document(
+                name=doc_name,
+                id=id or doc_name,
+                meta_data={"url": url},
+                content=text,
+                size=len(text),
+            )
+        ]

agno/knowledge/reader/web_search_reader.py

@@ -0,0 +1,366 @@
+import asyncio
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Optional, Set
+from urllib.parse import urlparse
+
+import httpx
+
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, logger
+
+try:
+    from bs4 import BeautifulSoup, Tag  # noqa: F401
+except ImportError:
+    raise ImportError("The `bs4` package is not installed. Please install it via `pip install beautifulsoup4`.")
+
+try:
+    from ddgs import DDGS
+except ImportError:
+    raise ImportError("The `ddgs` package is not installed. Please install it via `pip install ddgs`.")
+
+
+@dataclass
+class WebSearchReader(Reader):
+    """Reader that uses web search to find content for a given query"""
+
+    search_timeout: int = 10
+
+    request_timeout: int = 30
+    delay_between_requests: float = 2.0  # Increased default delay
+    max_retries: int = 3
+    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+
+    # Search engine configuration
+    search_engine: Literal["duckduckgo", "google"] = "duckduckgo"
+    search_delay: float = 3.0  # Delay between search requests
+    max_search_retries: int = 2  # Retries for search operations
+
+    # Rate limiting
+    rate_limit_delay: float = 5.0  # Delay when rate limited
+    exponential_backoff: bool = True
+
+    # Internal state
+    _visited_urls: Set[str] = field(default_factory=set)
+    _last_search_time: float = field(default=0.0, init=False)
+
+    # Override default chunking strategy
+    chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking()
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Web Search readers."""
+        return [
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TOPIC]
+
+    def _respect_rate_limits(self):
+        """Ensure we don't exceed rate limits"""
+        current_time = time.time()
+        time_since_last_search = current_time - self._last_search_time
+
+        if time_since_last_search < self.search_delay:
+            sleep_time = self.search_delay - time_since_last_search
+            log_debug(f"Rate limiting: sleeping for {sleep_time:.2f} seconds")
+            time.sleep(sleep_time)
+
+        self._last_search_time = time.time()
+
+    def _perform_duckduckgo_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using DuckDuckGo with rate limiting"""
+        log_debug(f"Performing DuckDuckGo search for: {query}")
+
+        for attempt in range(self.max_search_retries):
+            try:
+                self._respect_rate_limits()
+
+                ddgs = DDGS(timeout=self.search_timeout)
+                search_results = ddgs.text(query=query, max_results=self.max_results)
+
+                # Convert to list and extract relevant fields
+                results = []
+                for result in search_results:
+                    results.append(
+                        {
+                            "title": result.get("title", ""),
+                            "url": result.get("link", ""),
+                            "description": result.get("body", ""),
+                        }
+                    )
+
+                log_debug(f"Found {len(results)} search results")
+                return results
+
+            except Exception as e:
+                logger.warning(f"DuckDuckGo search attempt {attempt + 1} failed: {e}")
+                if "rate limit" in str(e).lower() or "429" in str(e):
+                    # Rate limited - wait longer
+                    wait_time = (
+                        self.rate_limit_delay * (2**attempt) if self.exponential_backoff else self.rate_limit_delay
+                    )
+                    logger.info(f"Rate limited, waiting {wait_time} seconds before retry")
+                    time.sleep(wait_time)
+                elif attempt < self.max_search_retries - 1:
+                    # Other error - shorter wait
+                    time.sleep(self.search_delay)
+                else:
+                    logger.error(f"All DuckDuckGo search attempts failed: {e}")
+                    return []
+        return []
+
+    def _perform_google_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using Google (requires googlesearch-python)"""
+        log_debug(f"Performing Google search for: {query}")
+
+        try:
+            from googlesearch import search
+        except ImportError:
+            logger.error("Google search requires 'googlesearch-python'. Install with: pip install googlesearch-python")
+            return []
+
+        for attempt in range(self.max_search_retries):
+            try:
+                self._respect_rate_limits()
+
+                results = []
+                search_results = search(query, num_results=self.max_results, stop=self.max_results)
+
+                for result in search_results:
+                    results.append(
+                        {
+                            "title": getattr(result, "title", ""),
+                            "url": getattr(result, "url", ""),
+                            "description": getattr(result, "description", ""),
+                        }
+                    )
+
+                log_debug(f"Found {len(results)} Google search results")
+                return results
+
+            except Exception as e:
+                logger.warning(f"Google search attempt {attempt + 1} failed: {e}")
+                if attempt < self.max_search_retries - 1:
+                    time.sleep(self.search_delay)
+                else:
+                    logger.error(f"All Google search attempts failed: {e}")
+                    return []
+
+        return []
+
+    def _perform_web_search(self, query: str) -> List[Dict[str, str]]:
+        """Perform web search using the configured search engine"""
+        if self.search_engine == "duckduckgo":
+            return self._perform_duckduckgo_search(query)
+        elif self.search_engine == "google":
+            return self._perform_google_search(query)
+        else:
+            logger.error(f"Unsupported search engine: {self.search_engine}")
+            return []
+
+    def _is_valid_url(self, url: str) -> bool:
+        """Check if URL is valid and not already visited"""
+        try:
+            parsed = urlparse(url)
+            return bool(parsed.scheme in ["http", "https"] and parsed.netloc and url not in self._visited_urls)
+        except Exception:
+            return False
+
+    def _extract_text_from_html(self, html_content: str, url: str) -> str:
+        """Extract clean text content from HTML"""
+        try:
+            soup = BeautifulSoup(html_content, "html.parser")
+
+            # Remove script and style elements
+            for script in soup(["script", "style"]):
+                script.decompose()
+
+            # Get text content
+            text = soup.get_text()
+
+            # Clean up whitespace
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            text = " ".join(chunk for chunk in chunks if chunk)
+
+            return text
+
+        except Exception as e:
+            logger.warning(f"Error extracting text from {url}: {e}")
+            return html_content
+
+    def _fetch_url_content(self, url: str) -> Optional[str]:
+        """Fetch content from a URL with retry logic"""
+        headers = {"User-Agent": self.user_agent}
+
+        for attempt in range(self.max_retries):
+            try:
+                response = httpx.get(url, headers=headers, timeout=self.request_timeout, follow_redirects=True)
+                response.raise_for_status()
+
+                # Check if it's HTML content
+                content_type = response.headers.get("content-type", "").lower()
+                if "text/html" in content_type:
+                    return self._extract_text_from_html(response.text, url)
+                else:
+                    # For non-HTML content, return as-is
+                    return response.text
+
+            except Exception as e:
+                logger.warning(f"Attempt {attempt + 1} failed for {url}: {e}")
+                if attempt < self.max_retries - 1:
+                    time.sleep(random.uniform(1, 3))  # Random delay between retries
+                continue
+
+        logger.error(f"Failed to fetch content from {url} after {self.max_retries} attempts")
+        return None
+
+    def _create_document_from_url(self, url: str, content: str, search_result: Dict[str, str]) -> Document:
+        """Create a Document object from URL content and search result metadata"""
+        # Use the URL as the document ID
+        doc_id = url
+
+        # Use the search result title as the document name, fallback to URL
+        doc_name = search_result.get("title", urlparse(url).netloc)
+
+        # Create metadata with search information
+        meta_data = {
+            "url": url,
+            "search_title": search_result.get("title", ""),
+            "search_description": search_result.get("description", ""),
+            "source": "web_search",
+            "search_engine": self.search_engine,
+        }
+
+        return Document(id=doc_id, name=doc_name, content=content, meta_data=meta_data)
+
+    def read(self, query: str) -> List[Document]:
+        """Read content for a given query by performing web search and fetching content"""
+        if not query:
+            raise ValueError("Query cannot be empty")
+
+        log_debug(f"Starting web search reader for query: {query}")
+
+        # Perform web search
+        search_results = self._perform_web_search(query)
+        if not search_results:
+            logger.warning(f"No search results found for query: {query}")
+            return []
+
+        documents: List[Document] = []
+
+        for result in search_results:
+            url = result.get("url", "")
+
+            # Skip if URL is invalid or already visited
+            if not self._is_valid_url(url):
+                continue
+
+            # Mark URL as visited
+            self._visited_urls.add(url)
+
+            # Add delay between requests to be respectful
+            if len(documents) > 0:
+                time.sleep(self.delay_between_requests)
+
+            # Fetch content from URL
+            content = self._fetch_url_content(url)
+            if content is None:
+                continue
+
+            # Create document
+            document = self._create_document_from_url(url, content, result)
+
+            # Apply chunking if enabled
+            if self.chunk:
+                chunked_docs = self.chunk_document(document)
+                documents.extend(chunked_docs)
+            else:
+                documents.append(document)
+
+            # Stop if we've reached max_results
+            if len(documents) >= self.max_results:
+                break
+
+        log_debug(f"Created {len(documents)} documents from web search")
+        return documents
+
+    async def async_read(self, query: str) -> List[Document]:
+        """Asynchronously read content for a given query"""
+        if not query:
+            raise ValueError("Query cannot be empty")
+
+        log_debug(f"Starting async web search reader for query: {query}")
+
+        # Perform web search (synchronous operation)
+        search_results = self._perform_web_search(query)
+        if not search_results:
+            logger.warning(f"No search results found for query: {query}")
+            return []
+
+        # Create tasks for fetching content from each URL
+        async def fetch_url_async(result: Dict[str, str]) -> Optional[Document]:
+            url = result.get("url", "")
+
+            # Skip if URL is invalid or already visited
+            if not self._is_valid_url(url):
+                return None
+
+            # Mark URL as visited
+            self._visited_urls.add(url)
+
+            try:
+                headers = {"User-Agent": self.user_agent}
+                async with httpx.AsyncClient(timeout=self.request_timeout) as client:
+                    response = await client.get(url, headers=headers, follow_redirects=True)
+                    response.raise_for_status()
+
+                    content_type = response.headers.get("content-type", "").lower()
+                    if "text/html" in content_type:
+                        content = self._extract_text_from_html(response.text, url)
+                    else:
+                        content = response.text
+
+                    document = self._create_document_from_url(url, content, result)
+                    return document
+
+            except Exception as e:
+                logger.warning(f"Error fetching {url}: {e}")
+                return None
+
+        # Create tasks for all URLs
+        tasks = [fetch_url_async(result) for result in search_results]
+
+        # Execute all tasks concurrently with delays
+        documents = []
+        for i, task in enumerate(tasks):
+            if i > 0:  # Add delay between requests (except for the first one)
+                await asyncio.sleep(self.delay_between_requests)
+
+            doc = await task
+            if doc is not None:
+                # Apply chunking if enabled
+                if self.chunk:
+                    chunked_docs = await self.chunk_documents_async([doc])
+                    documents.extend(chunked_docs)
+                else:
+                    documents.append(doc)
+
+                # Stop if we've reached max_results
+                if len(documents) >= self.max_results:
+                    break
+
+        log_debug(f"Created {len(documents)} documents from async web search")
+        return documents