agno 1.8.0__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/knowledge/reader/s3_reader.py

@@ -0,0 +1,171 @@
+import asyncio
+from io import BytesIO
+from pathlib import Path
+from typing import List, Optional
+from uuid import uuid4
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_info, logger
+
+try:
+    from agno.aws.resource.s3.object import S3Object  # type: ignore
+except (ModuleNotFoundError, ImportError):
+    raise ImportError("`agno-aws` not installed. Please install using `pip install agno-aws`")
+
+try:
+    import textract  # noqa: F401
+except ImportError:
+    raise ImportError("`textract` not installed. Please install it via `pip install textract`.")
+
+try:
+    from pypdf import PdfReader as DocumentReader  # noqa: F401
+except ImportError:
+    raise ImportError("`pypdf` not installed. Please install it via `pip install pypdf`.")
+
+
+class S3Reader(Reader):
+    """Reader for S3 files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for S3 readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.FILE, ContentType.URL, ContentType.TEXT]
+
+    def read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        try:
+            log_info(f"Reading S3 file: {s3_object.uri}")
+
+            if s3_object.uri.endswith(".pdf"):
+                return S3PDFReader().read(name, s3_object)
+            else:
+                return S3TextReader().read(name, s3_object)
+
+        except Exception as e:
+            logger.error(f"Error reading: {s3_object.uri}: {e}")
+        return []
+
+    async def async_read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        """Asynchronously read S3 files by running the synchronous read operation in a thread."""
+        return await asyncio.to_thread(self.read, name, s3_object)
+
+
+class S3TextReader(Reader):
+    """Reader for text files on S3"""
+
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for S3 text readers."""
+        return [
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+        ]
+
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TEXT]
+
+    def read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        try:
+            log_info(f"Reading text file: {s3_object.uri}")
+
+            doc_name = s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+            if name is not None:
+                doc_name = name
+            obj_name = s3_object.name.split("/")[-1]
+            temporary_file = Path("storage").joinpath(obj_name)
+            s3_object.download(temporary_file)
+
+            log_info(f"Parsing: {temporary_file}")
+            doc_content = textract.process(temporary_file)
+            documents = [
+                Document(
+                    name=doc_name,
+                    id=doc_name,
+                    content=doc_content.decode("utf-8"),
+                )
+            ]
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+
+            log_debug(f"Deleting: {temporary_file}")
+            temporary_file.unlink()
+            return documents
+        except Exception as e:
+            logger.error(f"Error reading: {s3_object.uri}: {e}")
+        return []
+
+    async def async_read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        """Asynchronously read text files from S3 by running the synchronous read operation in a thread.
+
+        Args:
+            s3_object (S3Object): The S3 object to read
+
+        Returns:
+            List[Document]: List of documents from the text file
+        """
+        return await asyncio.to_thread(self.read, name, s3_object)
+
+
+class S3PDFReader(Reader):
+    """Reader for PDF files on S3"""
+
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.FILE]
+
+    def read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        try:
+            log_info(f"Reading PDF file: {s3_object.uri}")
+
+            object_resource = s3_object.get_resource()
+            object_body = object_resource.get()["Body"]
+            doc_name = s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+            if name is not None:
+                doc_name = name
+            doc_reader = DocumentReader(BytesIO(object_body.read()))
+            documents = [
+                Document(
+                    name=doc_name,
+                    id=str(uuid4()),
+                    meta_data={"page": page_number},
+                    content=page.extract_text(),
+                )
+                for page_number, page in enumerate(doc_reader.pages, start=1)
+            ]
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+        except Exception:
+            raise
+
+    async def async_read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        """Asynchronously read PDF files from S3 by running the synchronous read operation in a thread.
+
+        Args:
+            s3_object (S3Object): The S3 object to read
+
+        Returns:
+            List[Document]: List of documents from the PDF file
+        """
+        return await asyncio.to_thread(self.read, name, s3_object)

agno/{document → knowledge}/reader/text_reader.py

@@ -1,27 +1,48 @@
 import asyncio
 import uuid
 from pathlib import Path
-from typing import IO, Any, List, Union
+from typing import IO, Any, List, Optional, Union
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_info, logger
 
 
 class TextReader(Reader):
     """Reader for Text files"""
 
-    def read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Text readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TXT]
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
                 log_info(f"Reading: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
                 file_contents = file.read_text("utf-8")
             else:
-                log_info(f"Reading uploaded file: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
+                log_info(f"Reading uploaded file: {file_name}")
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 
@@ -42,14 +63,14 @@ class TextReader(Reader):
             logger.error(f"Error reading: {file}: {e}")
             return []
 
-    async def async_read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
 
                 log_info(f"Reading asynchronously: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
 
                 try:
                     import aiofiles
@@ -61,7 +82,7 @@ class TextReader(Reader):
                     file_contents = file.read_text("utf-8")
             else:
                 log_info(f"Reading uploaded file asynchronously: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 

agno/knowledge/reader/url_reader.py

@@ -0,0 +1,84 @@
+from typing import List, Optional
+from urllib.parse import urlparse
+
+import httpx
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.http import async_fetch_with_retry, fetch_with_retry
+from agno.utils.log import log_debug
+
+
+class URLReader(Reader):
+    """Reader for general URL content"""
+
+    def __init__(
+        self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), proxy: Optional[str] = None, **kwargs
+    ):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+        self.proxy = proxy
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for URL readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
+    def read(self, url: str, id: Optional[str] = None, name: Optional[str] = None) -> List[Document]:
+        if not url:
+            raise ValueError("No url provided")
+
+        log_debug(f"Reading: {url}")
+        # Retry the request up to 3 times with exponential backoff
+        response = fetch_with_retry(url, proxy=self.proxy)
+
+        document = self._create_document(url, response.text, id, name)
+        if self.chunk:
+            return self.chunk_document(document)
+        return [document]
+
+    async def async_read(self, url: str, id: Optional[str] = None, name: Optional[str] = None) -> List[Document]:
+        """Async version of read method"""
+        if not url:
+            raise ValueError("No url provided")
+
+        log_debug(f"Reading async: {url}")
+        client_args = {"proxy": self.proxy} if self.proxy else {}
+        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
+            response = await async_fetch_with_retry(url, client=client)
+
+        document = self._create_document(url, response.text, id, name)
+        if self.chunk:
+            return await self.chunk_documents_async([document])
+        return [document]
+
+    def _create_document(
+        self, url: str, content: str, id: Optional[str] = None, name: Optional[str] = None
+    ) -> Document:
+        """Helper method to create a document from URL content"""
+        parsed_url = urlparse(url)
+        doc_name = name or parsed_url.path.strip("/").replace("/", "_").replace(" ", "_")
+        if not doc_name:
+            doc_name = parsed_url.netloc
+        if not doc_name:
+            doc_name = url
+
+        return Document(
+            name=doc_name,
+            id=id or doc_name,
+            meta_data={"url": url},
+            content=content,
+            size=len(content),
+        )