agno 2.0.1__py3-none-any.whl → 2.3.0__py3-none-any.whl

agno/knowledge/reader/pdf_reader.py

@@ -4,11 +4,12 @@ from pathlib import Path
 from typing import IO, Any, List, Optional, Tuple, Union
 from uuid import uuid4
 
-from agno.knowledge.chunking.strategy import ChunkingStrategyType
+from agno.knowledge.chunking.document import DocumentChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
 from agno.knowledge.document.base import Document
 from agno.knowledge.reader.base import Reader
 from agno.knowledge.types import ContentType
-from agno.utils.log import log_error, log_info, logger
+from agno.utils.log import log_debug, log_error
 
 try:
     from pypdf import PdfReader as DocumentReader  # noqa: F401
@@ -117,6 +118,10 @@ def _clean_page_numbers(
     page_numbers = [find_page_number(content) for content in page_content_list]
     if all(x is None or x > 5 for x in page_numbers):
         # This approach won't work reliably for higher page numbers.
+        page_content_list = [
+            f"\n{page_content_list[i]}\n{extra_content[i]}" if extra_content else page_content_list[i]
+            for i in range(len(page_content_list))
+        ]
         return page_content_list, None
 
     # Possible range shifts to detect page numbering
@@ -179,6 +184,7 @@ class BasePDFReader(Reader):
         page_start_numbering_format: Optional[str] = None,
         page_end_numbering_format: Optional[str] = None,
         password: Optional[str] = None,
+        chunking_strategy: Optional[ChunkingStrategy] = DocumentChunking(chunk_size=5000),
         **kwargs,
     ):
         if page_start_numbering_format is None:
@@ -191,11 +197,7 @@ class BasePDFReader(Reader):
         self.page_end_numbering_format = page_end_numbering_format
         self.password = password
 
-        if self.chunking_strategy is None:
-            from agno.knowledge.chunking.document import DocumentChunking
-
-            self.chunking_strategy = DocumentChunking(chunk_size=5000)
-        super().__init__(**kwargs)
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
 
     @classmethod
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
@@ -214,6 +216,19 @@ class BasePDFReader(Reader):
             chunked_documents.extend(self.chunk_document(document))
         return chunked_documents
 
+    def _get_doc_name(self, pdf_source: Union[str, Path, IO[Any]], name: Optional[str] = None) -> str:
+        """Determines the document name from the source or a provided name."""
+        try:
+            if name:
+                return name
+            if isinstance(pdf_source, str):
+                return pdf_source.split("/")[-1].split(".")[0].replace(" ", "_")
+            # Assumes a file-like object with a .name attribute
+            return pdf_source.name.split(".")[0]
+        except Exception:
+            # The original code had a bug here, it should check `name` first.
+            return name or "pdf"
+
     def _decrypt_pdf(self, doc_reader: DocumentReader, doc_name: str, password: Optional[str] = None) -> bool:
         if not doc_reader.is_encrypted:
             return True
@@ -221,13 +236,13 @@ class BasePDFReader(Reader):
         # Use provided password or fall back to instance password
         pdf_password = password or self.password
         if not pdf_password:
-            logger.error(f'PDF file "{doc_name}" is password protected but no password provided')
+            log_error(f'PDF file "{doc_name}" is password protected but no password provided')
             return False
 
         try:
             decrypted_pdf = doc_reader.decrypt(pdf_password)
             if decrypted_pdf:
-                log_info(f'Successfully decrypted PDF file "{doc_name}" with user password')
+                log_debug(f'Successfully decrypted PDF file "{doc_name}" with user password')
                 return True
             else:
                 log_error(f'Failed to decrypt PDF file "{doc_name}": incorrect password')
@@ -261,7 +276,6 @@ class BasePDFReader(Reader):
 
         if self.chunk:
             return self._build_chunked_documents(documents)
-
         return documents
 
     def _pdf_reader_to_documents(
@@ -329,40 +343,14 @@ class PDFReader(BasePDFReader):
     def read(
         self, pdf: Union[str, Path, IO[Any]], name: Optional[str] = None, password: Optional[str] = None
     ) -> List[Document]:
-        try:
-            if name:
-                doc_name = name
-            elif isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
-            else:
-                doc_name = pdf.name.split(".")[0]
-        except Exception:
-            doc_name = "pdf"
-
-        log_info(f"Reading: {doc_name}")
-
-        try:
-            DocumentReader(pdf)
-        except PdfStreamError as e:
-            logger.error(f"Error reading PDF: {e}")
-            return []
-
-        try:
-            if isinstance(pdf, str):
-                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
-            else:
-                doc_name = name or pdf.name.split(".")[0]
-        except Exception:
-            doc_name = name or "pdf"
-
-        log_info(f"Reading: {doc_name}")
+        doc_name = self._get_doc_name(pdf, name)
+        log_debug(f"Reading: {doc_name}")
 
         try:
             pdf_reader = DocumentReader(pdf)
         except PdfStreamError as e:
-            logger.error(f"Error reading PDF: {e}")
+            log_error(f"Error reading PDF: {e}")
             return []
-
         # Handle PDF decryption
         if not self._decrypt_pdf(pdf_reader, doc_name, password):
             return []
@@ -379,21 +367,13 @@ class PDFReader(BasePDFReader):
         if pdf is None:
             log_error("No pdf provided")
             return []
-
-        try:
-            if isinstance(pdf, str):
-                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
-            else:
-                doc_name = pdf.name.split(".")[0]
-        except Exception:
-            doc_name = name or "pdf"
-
-        log_info(f"Reading: {doc_name}")
+        doc_name = self._get_doc_name(pdf, name)
+        log_debug(f"Reading: {doc_name}")
 
         try:
             pdf_reader = DocumentReader(pdf)
         except PdfStreamError as e:
-            logger.error(f"Error reading PDF: {e}")
+            log_error(f"Error reading PDF: {e}")
             return []
 
         # Handle PDF decryption
@@ -413,16 +393,13 @@ class PDFImageReader(BasePDFReader):
         if not pdf:
             raise ValueError("No pdf provided")
 
+        doc_name = self._get_doc_name(pdf, name)
+        log_debug(f"Reading: {doc_name}")
         try:
-            if isinstance(pdf, str):
-                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
-            else:
-                doc_name = pdf.name.split(".")[0]
-        except Exception:
-            doc_name = "pdf"
-
-        log_info(f"Reading: {doc_name}")
-        pdf_reader = DocumentReader(pdf)
+            pdf_reader = DocumentReader(pdf)
+        except PdfStreamError as e:
+            log_error(f"Error reading PDF: {e}")
+            return []
 
         # Handle PDF decryption
         if not self._decrypt_pdf(pdf_reader, doc_name, password):
@@ -437,16 +414,14 @@ class PDFImageReader(BasePDFReader):
         if not pdf:
             raise ValueError("No pdf provided")
 
-        try:
-            if isinstance(pdf, str):
-                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
-            else:
-                doc_name = pdf.name.split(".")[0]
-        except Exception:
-            doc_name = "pdf"
+        doc_name = self._get_doc_name(pdf, name)
+        log_debug(f"Reading: {doc_name}")
 
-        log_info(f"Reading: {doc_name}")
-        pdf_reader = DocumentReader(pdf)
+        try:
+            pdf_reader = DocumentReader(pdf)
+        except PdfStreamError as e:
+            log_error(f"Error reading PDF: {e}")
+            return []
 
         # Handle PDF decryption
         if not self._decrypt_pdf(pdf_reader, doc_name, password):

agno/knowledge/reader/pptx_reader.py

@@ -0,0 +1,101 @@
+import asyncio
+from pathlib import Path
+from typing import IO, Any, List, Optional, Union
+from uuid import uuid4
+
+from agno.knowledge.chunking.document import DocumentChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error
+
+try:
+    from pptx import Presentation  # type: ignore
+except ImportError:
+    raise ImportError("The `python-pptx` package is not installed. Please install it via `pip install python-pptx`.")
+
+
+class PPTXReader(Reader):
+    """Reader for PPTX files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = DocumentChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for PPTX readers."""
+        return [
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.PPTX]
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        """Read a pptx file and return a list of documents"""
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_debug(f"Reading: {file}")
+                presentation = Presentation(str(file))
+                doc_name = name or file.stem
+            else:
+                log_debug(f"Reading uploaded file: {getattr(file, 'name', 'pptx_file')}")
+                presentation = Presentation(file)
+                doc_name = name or (
+                    getattr(file, "name", "pptx_file").split(".")[0] if hasattr(file, "name") else "pptx_file"
+                )
+
+            # Extract text from all slides
+            slide_texts = []
+            for slide_number, slide in enumerate(presentation.slides, 1):
+                slide_text = f"Slide {slide_number}:\n"
+
+                # Extract text from shapes that contain text
+                text_content = []
+                for shape in slide.shapes:
+                    if hasattr(shape, "text") and shape.text.strip():
+                        text_content.append(shape.text.strip())
+
+                if text_content:
+                    slide_text += "\n".join(text_content)
+                else:
+                    slide_text += "(No text content)"
+
+                slide_texts.append(slide_text)
+
+            doc_content = "\n\n".join(slide_texts)
+
+            documents = [
+                Document(
+                    name=doc_name,
+                    id=str(uuid4()),
+                    content=doc_content,
+                )
+            ]
+
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+
+        except Exception as e:
+            log_error(f"Error reading file: {e}")
+            return []
+
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        """Asynchronously read a pptx file and return a list of documents"""
+        try:
+            return await asyncio.to_thread(self.read, file, name)
+        except Exception as e:
+            log_error(f"Error reading file asynchronously: {e}")
+            return []

agno/knowledge/reader/reader_factory.py

@@ -16,8 +16,7 @@ class ReaderFactory:
         from agno.knowledge.reader.pdf_reader import PDFReader
 
         config: Dict[str, Any] = {
-            "chunk": True,
-            "chunk_size": 100,
+            "name": "PDF Reader",
             "description": "Processes PDF documents with OCR support for images and text extraction",
         }
         config.update(kwargs)
@@ -35,6 +34,18 @@ class ReaderFactory:
         config.update(kwargs)
         return CSVReader(**config)
 
+    @classmethod
+    def _get_field_labeled_csv_reader(cls, **kwargs) -> Reader:
+        """Get Field Labeled CSV reader instance."""
+        from agno.knowledge.reader.field_labeled_csv_reader import FieldLabeledCSVReader
+
+        config: Dict[str, Any] = {
+            "name": "Field Labeled CSV Reader",
+            "description": "Converts CSV rows to field-labeled text format for enhanced readability and context",
+        }
+        config.update(kwargs)
+        return FieldLabeledCSVReader(**config)
+
     @classmethod
     def _get_docx_reader(cls, **kwargs) -> Reader:
         """Get Docx reader instance."""
@@ -47,6 +58,18 @@ class ReaderFactory:
         config.update(kwargs)
         return DocxReader(**config)
 
+    @classmethod
+    def _get_pptx_reader(cls, **kwargs) -> Reader:
+        """Get PPTX reader instance."""
+        from agno.knowledge.reader.pptx_reader import PPTXReader
+
+        config: Dict[str, Any] = {
+            "name": "PPTX Reader",
+            "description": "Extracts text content from Microsoft PowerPoint presentations (.pptx format)",
+        }
+        config.update(kwargs)
+        return PPTXReader(**config)
+
     @classmethod
     def _get_json_reader(cls, **kwargs) -> Reader:
         """Get JSON reader instance."""
@@ -109,6 +132,21 @@ class ReaderFactory:
         config.update(kwargs)
         return FirecrawlReader(**config)
 
+    @classmethod
+    def _get_tavily_reader(cls, **kwargs) -> Reader:
+        """Get Tavily reader instance."""
+        from agno.knowledge.reader.tavily_reader import TavilyReader
+
+        config: Dict[str, Any] = {
+            "api_key": kwargs.get("api_key") or os.getenv("TAVILY_API_KEY"),
+            "extract_format": "markdown",
+            "extract_depth": "basic",
+            "name": "Tavily Reader",
+            "description": "Extracts content from URLs using Tavily's Extract API with markdown or text output",
+        }
+        config.update(kwargs)
+        return TavilyReader(**config)
+
     @classmethod
     def _get_youtube_reader(cls, **kwargs) -> Reader:
         """Get YouTube reader instance."""
@@ -189,8 +227,10 @@ class ReaderFactory:
             return cls.create_reader("pdf")
         elif extension in [".csv", "text/csv"]:
             return cls.create_reader("csv")
-        elif extension in [".docx", ".doc"]:
+        elif extension in [".docx", ".doc", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"]:
             return cls.create_reader("docx")
+        elif extension == ".pptx":
+            return cls.create_reader("pptx")
         elif extension == ".json":
             return cls.create_reader("json")
         elif extension in [".md", ".markdown"]:
@@ -210,8 +250,8 @@ class ReaderFactory:
         if any(domain in url_lower for domain in ["youtube.com", "youtu.be"]):
             return cls.create_reader("youtube")
 
-        # Default to URL reader
-        return cls.create_reader("url")
+        # Default to website reader
+        return cls.create_reader("website")
 
     @classmethod
     def get_all_reader_keys(cls) -> List[str]:
@@ -228,7 +268,12 @@ class ReaderFactory:
                 reader_keys.append(reader_key)
 
         # Define priority order for URL readers
-        url_reader_priority = ["url", "website", "firecrawl", "pdf_url", "csv_url", "youtube", "web_search"]
+        url_reader_priority = [
+            "website",
+            "firecrawl",
+            "tavily",
+            "youtube",
+        ]
 
         # Sort with URL readers in priority order, others alphabetically
         def sort_key(reader_key):

agno/knowledge/reader/s3_reader.py

@@ -10,7 +10,7 @@ from agno.knowledge.reader.base import Reader
 from agno.knowledge.reader.pdf_reader import PDFReader
 from agno.knowledge.reader.text_reader import TextReader
 from agno.knowledge.types import ContentType
-from agno.utils.log import log_info, logger
+from agno.utils.log import log_debug, log_error
 
 try:
     from agno.aws.resource.s3.object import S3Object  # type: ignore
@@ -51,7 +51,7 @@ class S3Reader(Reader):
 
     def read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
         try:
-            log_info(f"Reading S3 file: {s3_object.uri}")
+            log_debug(f"Reading S3 file: {s3_object.uri}")
 
             # Read PDF files
             if s3_object.uri.endswith(".pdf"):
@@ -74,25 +74,13 @@ class S3Reader(Reader):
                 obj_name = s3_object.name.split("/")[-1]
                 temporary_file = Path("storage").joinpath(obj_name)
                 s3_object.download(temporary_file)
-
-                # TODO: Before we were using textract here. Needed?
-                # s3_object.download(temporary_file)
-                # doc_content = textract.process(temporary_file)
-                # documents = [
-                #     Document(
-                #         name=doc_name,
-                #         id=doc_name,
-                #         content=doc_content.decode("utf-8"),
-                #     )
-                # ]
-
                 documents = TextReader().read(file=temporary_file, name=doc_name)
 
                 temporary_file.unlink()
                 return documents
 
         except Exception as e:
-            logger.error(f"Error reading: {s3_object.uri}: {e}")
+            log_error(f"Error reading: {s3_object.uri}: {e}")
 
         return []
 

agno/knowledge/reader/tavily_reader.py

@@ -0,0 +1,194 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Dict, List, Literal, Optional
+
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, logger
+
+try:
+    from tavily import TavilyClient  # type: ignore[attr-defined]
+except ImportError:
+    raise ImportError(
+        "The `tavily-python` package is not installed. Please install it via `pip install tavily-python`."
+    )
+
+
+@dataclass
+class TavilyReader(Reader):
+    api_key: Optional[str] = None
+    params: Optional[Dict] = None
+    extract_format: Literal["markdown", "text"] = "markdown"
+    extract_depth: Literal["basic", "advanced"] = "basic"
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        params: Optional[Dict] = None,
+        extract_format: Literal["markdown", "text"] = "markdown",
+        extract_depth: Literal["basic", "advanced"] = "basic",
+        chunk: bool = True,
+        chunk_size: int = 5000,
+        chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking(),
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> None:
+        """
+        Initialize TavilyReader for extracting content from URLs using Tavily's Extract API.
+
+        Args:
+            api_key: Tavily API key (or use TAVILY_API_KEY env var)
+            params: Additional parameters to pass to the extract API
+            extract_format: Output format - "markdown" or "text"
+            extract_depth: Extraction depth - "basic" (1 credit/5 URLs) or "advanced" (2 credits/5 URLs)
+            chunk: Whether to chunk the extracted content
+            chunk_size: Size of chunks when chunking is enabled
+            chunking_strategy: Strategy to use for chunking
+            name: Name of the reader
+            description: Description of the reader
+        """
+        # Initialize base Reader (handles chunk_size / strategy)
+        super().__init__(
+            chunk=chunk, chunk_size=chunk_size, chunking_strategy=chunking_strategy, name=name, description=description
+        )
+
+        # Tavily-specific attributes
+        self.api_key = api_key
+        self.params = params or {}
+        self.extract_format = extract_format
+        self.extract_depth = extract_depth
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Tavily readers."""
+        return [
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
+    def _extract(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Internal method to extract content from a URL using Tavily's Extract API.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document (defaults to URL)
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        log_debug(f"Extracting content from: {url}")
+
+        client = TavilyClient(api_key=self.api_key)
+
+        # Prepare extract parameters
+        extract_params = {
+            "urls": [url],
+            "depth": self.extract_depth,
+        }
+
+        # Add optional params if provided
+        if self.params:
+            extract_params.update(self.params)
+
+        try:
+            # Call Tavily Extract API
+            response = client.extract(**extract_params)
+
+            # Extract content from response
+            if not response or "results" not in response:
+                logger.warning(f"No results received for URL: {url}")
+                return [Document(name=name or url, id=url, content="")]
+
+            results = response.get("results", [])
+            if not results:
+                logger.warning(f"Empty results for URL: {url}")
+                return [Document(name=name or url, id=url, content="")]
+
+            # Get the first result (since we're extracting a single URL)
+            result = results[0]
+
+            # Check if extraction failed
+            if "failed_reason" in result:
+                logger.warning(f"Extraction failed for {url}: {result['failed_reason']}")
+                return [Document(name=name or url, id=url, content="")]
+
+            # Get raw content
+            content = result.get("raw_content", "")
+
+            if content is None:
+                content = ""
+                logger.warning(f"No content received for URL: {url}")
+
+            # Debug logging
+            log_debug(f"Received content type: {type(content)}")
+            log_debug(f"Content length: {len(content) if content else 0}")
+
+            # Create documents
+            documents = []
+            if self.chunk and content:
+                documents.extend(self.chunk_document(Document(name=name or url, id=url, content=content)))
+            else:
+                documents.append(Document(name=name or url, id=url, content=content))
+
+            return documents
+
+        except Exception as e:
+            logger.error(f"Error extracting content from {url}: {e}")
+            return [Document(name=name or url, id=url, content="")]
+
+    async def _async_extract(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Internal async method to extract content from a URL.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        log_debug(f"Async extracting content from: {url}")
+
+        # Use asyncio.to_thread to run the synchronous extract in a thread
+        return await asyncio.to_thread(self._extract, url, name)
+
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Reads content from a URL using Tavily Extract API.
+
+        This is the public API method that users should call.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        return self._extract(url, name)
+
+    async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Asynchronously reads content from a URL using Tavily Extract API.
+
+        This is the public API method that users should call for async operations.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        return await self._async_extract(url, name)