agno 1.8.1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/{document → knowledge}/reader/markdown_reader.py

@@ -1,27 +1,54 @@
 import asyncio
 import uuid
 from pathlib import Path
-from typing import IO, Any, List, Union
+from typing import IO, Any, List, Optional, Union
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.markdown import MarkdownChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_info, logger
 
 
 class MarkdownReader(Reader):
     """Reader for Markdown files"""
 
-    def read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Markdown readers."""
+        return [
+            ChunkingStrategyType.MARKDOWN_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.MARKDOWN]
+
+    def __init__(
+        self,
+        chunking_strategy: Optional[ChunkingStrategy] = MarkdownChunking(),
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> None:
+        super().__init__(chunking_strategy=chunking_strategy, name=name, description=description)
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
                 log_info(f"Reading: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
                 file_contents = file.read_text("utf-8")
             else:
                 log_info(f"Reading uploaded file: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 
@@ -36,14 +63,14 @@ class MarkdownReader(Reader):
             logger.error(f"Error reading: {file}: {e}")
             return []
 
-    async def async_read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
 
                 log_info(f"Reading asynchronously: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
 
                 try:
                     import aiofiles
@@ -55,7 +82,7 @@ class MarkdownReader(Reader):
                     file_contents = file.read_text("utf-8")
             else:
                 log_info(f"Reading uploaded file asynchronously: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 

agno/{document → knowledge}/reader/pdf_reader.py

@@ -4,8 +4,10 @@ from pathlib import Path
 from typing import IO, Any, List, Optional, Tuple, Union
 from uuid import uuid4
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.strategy import ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.http import async_fetch_with_retry, fetch_with_retry
 from agno.utils.log import log_error, log_info, logger
 
@@ -95,9 +97,9 @@ def _clean_page_numbers(
     Notes:
         - The function scans for page numbers using a regular expression that matches digits at the start or end of a string.
         - It evaluates several potential starting points for numbering (-2, -1, 0, 1, 2 shifts) to determine the most consistent sequence.
-        - If at least a specified ratio of pages (defined by `PAGE_NUMBERING_CORRECTNESS_RATIO_FOR_REMOVAL`) has correct sequential numbering, 
+        - If at least a specified ratio of pages (defined by `PAGE_NUMBERING_CORRECTNESS_RATIO_FOR_REMOVAL`) has correct sequential numbering,
           the page numbers are processed.
-        - If page numbers are found, the function will add formatted page numbers to each page's content if `page_start_numbering_format` or 
+        - If page numbers are found, the function will add formatted page numbers to each page's content if `page_start_numbering_format` or
           `page_end_numbering_format` is provided.
     """
     assert len(extra_content) == 0 or len(extra_content) == len(page_content_list), (
@@ -190,8 +192,23 @@ class BasePDFReader(Reader):
         self.page_end_numbering_format = page_end_numbering_format
         self.password = password
 
+        if self.chunking_strategy is None:
+            from agno.knowledge.chunking.document import DocumentChunking
+
+            self.chunking_strategy = DocumentChunking(chunk_size=5000)
         super().__init__(**kwargs)
 
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for PDF readers."""
+        return [
+            ChunkingStrategyType.DOCUMENT_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKING,
+            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKING,
+        ]
+
     def _build_chunked_documents(self, documents: List[Document]) -> List[Document]:
         chunked_documents: List[Document] = []
         for document in documents:
@@ -306,9 +323,17 @@ class BasePDFReader(Reader):
 class PDFReader(BasePDFReader):
     """Reader for PDF files"""
 
-    def read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.PDF]
+
+    def read(
+        self, pdf: Union[str, Path, IO[Any]], name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         try:
-            if isinstance(pdf, str):
+            if name:
+                doc_name = name
+            elif isinstance(pdf, str):
                 doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
                 doc_name = pdf.name.split(".")[0]
@@ -317,6 +342,22 @@ class PDFReader(BasePDFReader):
 
         log_info(f"Reading: {doc_name}")
 
+        try:
+            DocumentReader(pdf)
+        except PdfStreamError as e:
+            logger.error(f"Error reading PDF: {e}")
+            return []
+
+        try:
+            if isinstance(pdf, str):
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+            else:
+                doc_name = name or pdf.name.split(".")[0]
+        except Exception:
+            doc_name = name or "pdf"
+
+        log_info(f"Reading: {doc_name}")
+
         try:
             pdf_reader = DocumentReader(pdf)
         except PdfStreamError as e:
@@ -330,14 +371,23 @@ class PDFReader(BasePDFReader):
         # Read and chunk.
         return self._pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=True)
 
-    async def async_read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
+    async def async_read(
+        self,
+        pdf: Optional[Union[str, Path, IO[Any]]] = None,
+        name: Optional[str] = None,
+        password: Optional[str] = None,
+    ) -> List[Document]:
+        if pdf is None:
+            log_error("No pdf provided")
+            return []
+
         try:
             if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
                 doc_name = pdf.name.split(".")[0]
         except Exception:
-            doc_name = "pdf"
+            doc_name = name or "pdf"
 
         log_info(f"Reading: {doc_name}")
 
@@ -362,7 +412,11 @@ class PDFUrlReader(BasePDFReader):
         super().__init__(password=password, **kwargs)
         self.proxy = proxy
 
-    def read(self, url: str, password: Optional[str] = None) -> List[Document]:
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
+    def read(self, url: str, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
         if not url:
             raise ValueError("No url provided")
 
@@ -373,7 +427,7 @@ class PDFUrlReader(BasePDFReader):
         # Retry the request up to 3 times with exponential backoff
         response = fetch_with_retry(url, proxy=self.proxy)
 
-        doc_name = url.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+        doc_name = name or url.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
         pdf_reader = DocumentReader(BytesIO(response.content))
 
         # Handle PDF decryption
@@ -383,7 +437,7 @@ class PDFUrlReader(BasePDFReader):
         # Read and chunk.
         return self._pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=False)
 
-    async def async_read(self, url: str, password: Optional[str] = None) -> List[Document]:
+    async def async_read(self, url: str, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
         if not url:
             raise ValueError("No url provided")
 
@@ -397,7 +451,7 @@ class PDFUrlReader(BasePDFReader):
         async with httpx.AsyncClient(**client_args) as client:  # type: ignore
             response = await async_fetch_with_retry(url, client=client)
 
-        doc_name = url.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+        doc_name = name or url.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
         pdf_reader = DocumentReader(BytesIO(response.content))
 
         # Handle PDF decryption
@@ -411,13 +465,15 @@ class PDFUrlReader(BasePDFReader):
 class PDFImageReader(BasePDFReader):
     """Reader for PDF files with text and images extraction"""
 
-    def read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
+    def read(
+        self, pdf: Union[str, Path, IO[Any]], name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         if not pdf:
             raise ValueError("No pdf provided")
 
         try:
             if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
                 doc_name = pdf.name.split(".")[0]
         except Exception:
@@ -433,13 +489,15 @@ class PDFImageReader(BasePDFReader):
         # Read and chunk.
         return self._pdf_reader_to_documents(pdf_reader, doc_name, read_images=True, use_uuid_for_id=False)
 
-    async def async_read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
+    async def async_read(
+        self, pdf: Union[str, Path, IO[Any]], name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         if not pdf:
             raise ValueError("No pdf provided")
 
         try:
             if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
                 doc_name = pdf.name.split(".")[0]
         except Exception:
@@ -463,7 +521,7 @@ class PDFUrlImageReader(BasePDFReader):
         super().__init__(password=password, **kwargs)
         self.proxy = proxy
 
-    def read(self, url: str, password: Optional[str] = None) -> List[Document]:
+    def read(self, url: str, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
         if not url:
             raise ValueError("No url provided")
 
@@ -475,7 +533,7 @@ class PDFUrlImageReader(BasePDFReader):
         log_info(f"Reading: {url}")
         response = httpx.get(url, proxy=self.proxy) if self.proxy else httpx.get(url)
 
-        doc_name = url.split("/")[-1].split(".")[0].replace(" ", "_")
+        doc_name = name or url.split("/")[-1].split(".")[0].replace(" ", "_")
         pdf_reader = DocumentReader(BytesIO(response.content))
 
         # Handle PDF decryption
@@ -485,7 +543,7 @@ class PDFUrlImageReader(BasePDFReader):
         # Read and chunk.
         return self._pdf_reader_to_documents(pdf_reader, doc_name, read_images=True, use_uuid_for_id=False)
 
-    async def async_read(self, url: str, password: Optional[str] = None) -> List[Document]:
+    async def async_read(self, url: str, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
         if not url:
             raise ValueError("No url provided")
 
@@ -500,7 +558,7 @@ class PDFUrlImageReader(BasePDFReader):
             response = await client.get(url)
             response.raise_for_status()
 
-        doc_name = url.split("/")[-1].split(".")[0].replace(" ", "_")
+        doc_name = name or url.split("/")[-1].split(".")[0].replace(" ", "_")
         pdf_reader = DocumentReader(BytesIO(response.content))
 
         # Handle PDF decryption

agno/knowledge/reader/reader_factory.py

@@ -0,0 +1,275 @@
+import os
+from typing import Any, Callable, Dict, List, Optional
+
+from agno.knowledge.reader.base import Reader
+
+
+class ReaderFactory:
+    """Factory for creating and managing document readers with lazy loading."""
+
+    # Cache for instantiated readers
+    _reader_cache: Dict[str, Reader] = {}
+
+    @classmethod
+    def _get_pdf_reader(cls, **kwargs) -> Reader:
+        """Get PDF reader instance."""
+        from agno.knowledge.reader.pdf_reader import PDFReader
+
+        config: Dict[str, Any] = {"chunk": True, "chunk_size": 100}
+        config.update(kwargs)
+        return PDFReader(**config)
+
+    @classmethod
+    def _get_csv_reader(cls, **kwargs) -> Reader:
+        """Get CSV reader instance."""
+        from agno.knowledge.reader.csv_reader import CSVReader
+
+        config: Dict[str, Any] = {"name": "CSV Reader", "description": "Reads CSV files"}
+        config.update(kwargs)
+        return CSVReader(**config)
+
+    @classmethod
+    def _get_docx_reader(cls, **kwargs) -> Reader:
+        """Get Docx reader instance."""
+        from agno.knowledge.reader.docx_reader import DocxReader
+
+        config: Dict[str, Any] = {"name": "Docx Reader", "description": "Reads Docx files"}
+        config.update(kwargs)
+        return DocxReader(**config)
+
+    @classmethod
+    def _get_json_reader(cls, **kwargs) -> Reader:
+        """Get JSON reader instance."""
+        from agno.knowledge.reader.json_reader import JSONReader
+
+        config: Dict[str, Any] = {"name": "JSON Reader", "description": "Reads JSON files"}
+        config.update(kwargs)
+        return JSONReader(**config)
+
+    @classmethod
+    def _get_markdown_reader(cls, **kwargs) -> Reader:
+        """Get Markdown reader instance."""
+        from agno.knowledge.reader.markdown_reader import MarkdownReader
+
+        config: Dict[str, Any] = {"name": "Markdown Reader", "description": "Reads Markdown files"}
+        config.update(kwargs)
+        return MarkdownReader(**config)
+
+    @classmethod
+    def _get_text_reader(cls, **kwargs) -> Reader:
+        """Get Text reader instance."""
+        from agno.knowledge.reader.text_reader import TextReader
+
+        config: Dict[str, Any] = {"name": "Text Reader", "description": "Reads Text files"}
+        config.update(kwargs)
+        return TextReader(**config)
+
+    @classmethod
+    def _get_url_reader(cls, **kwargs) -> Reader:
+        """Get URL reader instance."""
+        from agno.knowledge.reader.url_reader import URLReader
+
+        config: Dict[str, Any] = {"name": "URL Reader", "description": "Reads URLs"}
+        config.update(kwargs)
+        return URLReader(**config)
+
+    @classmethod
+    def _get_website_reader(cls, **kwargs) -> Reader:
+        """Get Website reader instance."""
+        from agno.knowledge.reader.website_reader import WebsiteReader
+
+        config: Dict[str, Any] = {"name": "Website Reader", "description": "Reads Website files"}
+        config.update(kwargs)
+        return WebsiteReader(**config)
+
+    @classmethod
+    def _get_firecrawl_reader(cls, **kwargs) -> Reader:
+        """Get Firecrawl reader instance."""
+        from agno.knowledge.reader.firecrawl_reader import FirecrawlReader
+
+        config: Dict[str, Any] = {
+            "api_key": kwargs.get("api_key") or os.getenv("FIRECRAWL_API_KEY"),
+            "mode": "crawl",
+            "name": "Firecrawl Reader",
+            "description": "Crawls websites",
+        }
+        config.update(kwargs)
+        return FirecrawlReader(**config)
+
+    @classmethod
+    def _get_youtube_reader(cls, **kwargs) -> Reader:
+        """Get YouTube reader instance."""
+        from agno.knowledge.reader.youtube_reader import YouTubeReader
+
+        config: Dict[str, Any] = {"name": "YouTube Reader", "description": "Reads YouTube videos"}
+        config.update(kwargs)
+        return YouTubeReader(**config)
+
+    @classmethod
+    def _get_pdf_url_reader(cls, **kwargs) -> Reader:
+        """Get PDF URL reader instance."""
+        from agno.knowledge.reader.pdf_reader import PDFUrlReader
+
+        config: Dict[str, Any] = {"name": "PDF URL Reader", "description": "Reads PDF URLs"}
+        config.update(kwargs)
+        return PDFUrlReader(**config)
+
+    @classmethod
+    def _get_csv_url_reader(cls, **kwargs) -> Reader:
+        """Get CSV URL reader instance."""
+        from agno.knowledge.reader.csv_reader import CSVUrlReader
+
+        config: Dict[str, Any] = {"name": "CSV URL Reader", "description": "Reads CSV URLs"}
+        config.update(kwargs)
+        return CSVUrlReader(**config)
+
+    @classmethod
+    def _get_s3_reader(cls, **kwargs) -> Reader:
+        """Get S3 reader instance."""
+        from agno.knowledge.reader.s3_reader import S3Reader
+
+        config: Dict[str, Any] = {"name": "S3 Reader", "description": "Reads S3 files"}
+        config.update(kwargs)
+        return S3Reader(**config)
+
+    @classmethod
+    def _get_gcs_reader(cls, **kwargs) -> Reader:
+        """Get GCS reader instance."""
+        from agno.knowledge.reader.gcs_reader import GCSReader
+
+        config: Dict[str, Any] = {"name": "GCS Reader", "description": "Reads GCS files"}
+        config.update(kwargs)
+        return GCSReader(**config)
+
+    @classmethod
+    def _get_arxiv_reader(cls, **kwargs) -> Reader:
+        """Get Arxiv reader instance."""
+        from agno.knowledge.reader.arxiv_reader import ArxivReader
+
+        config: Dict[str, Any] = {"name": "Arxiv Reader", "description": "Reads Arxiv papers"}
+        config.update(kwargs)
+        return ArxivReader(**config)
+
+    @classmethod
+    def _get_wikipedia_reader(cls, **kwargs) -> Reader:
+        """Get Wikipedia reader instance."""
+        from agno.knowledge.reader.wikipedia_reader import WikipediaReader
+
+        config: Dict[str, Any] = {"name": "Wikipedia Reader", "description": "Reads Wikipedia articles"}
+        config.update(kwargs)
+        return WikipediaReader(**config)
+
+    @classmethod
+    def _get_web_search_reader(cls, **kwargs) -> Reader:
+        """Get Web Search reader instance."""
+        from agno.knowledge.reader.web_search_reader import WebSearchReader
+
+        config: Dict[str, Any] = {"name": "Web Search Reader", "description": "Performs web searches"}
+        config.update(kwargs)
+        return WebSearchReader(**config)
+
+    @classmethod
+    def _get_reader_method(cls, reader_key: str) -> Callable[[], Reader]:
+        """Get the appropriate reader method for the given key."""
+        method_name = f"_get_{reader_key}_reader"
+        if not hasattr(cls, method_name):
+            raise ValueError(f"Unknown reader: {reader_key}")
+        return getattr(cls, method_name)
+
+    @classmethod
+    def create_reader(cls, reader_key: str, **kwargs) -> Reader:
+        """Create a reader instance with the given key and optional overrides."""
+        if reader_key in cls._reader_cache:
+            return cls._reader_cache[reader_key]
+
+        # Get the reader method and create the instance
+        reader_method = cls._get_reader_method(reader_key)
+        reader = reader_method(**kwargs)
+
+        # Cache the reader
+        cls._reader_cache[reader_key] = reader
+
+        return reader
+
+    @classmethod
+    def get_reader_for_extension(cls, extension: str) -> Reader:
+        """Get the appropriate reader for a file extension."""
+        extension = extension.lower()
+
+        if extension in [".pdf", "application/pdf"]:
+            return cls.create_reader("pdf")
+        elif extension in [".csv", "text/csv"]:
+            return cls.create_reader("csv")
+        elif extension in [".docx", ".doc"]:
+            return cls.create_reader("docx")
+        elif extension == ".json":
+            return cls.create_reader("json")
+        elif extension in [".md", ".markdown"]:
+            return cls.create_reader("markdown")
+        elif extension in [".txt", ".text"]:
+            return cls.create_reader("text")
+        else:
+            # Default to text reader for unknown extensions
+            return cls.create_reader("text")
+
+    @classmethod
+    def get_reader_for_url(cls, url: str) -> Reader:
+        """Get the appropriate reader for a URL."""
+        url_lower = url.lower()
+
+        # Check for YouTube URLs
+        if any(domain in url_lower for domain in ["youtube.com", "youtu.be"]):
+            return cls.create_reader("youtube")
+
+        # Default to URL reader
+        return cls.create_reader("url")
+
+    @classmethod
+    def get_reader_for_url_file(cls, extension: str) -> Reader:
+        """Get the appropriate reader for a URL file extension."""
+        extension = extension.lower()
+
+        if extension == ".pdf":
+            return cls.create_reader("pdf_url")
+        elif extension == ".csv":
+            return cls.create_reader("csv_url")
+        else:
+            return cls.create_reader("url")
+
+    @classmethod
+    def get_all_reader_keys(cls) -> List[str]:
+        """Get all available reader keys."""
+        # Extract reader keys from method names
+        reader_keys = []
+        for attr_name in dir(cls):
+            if attr_name.startswith("_get_") and attr_name.endswith("_reader"):
+                reader_key = attr_name[5:-7]  # Remove "_get_" prefix and "_reader" suffix
+                reader_keys.append(reader_key)
+        return reader_keys
+
+    @classmethod
+    def create_all_readers(cls) -> Dict[str, Reader]:
+        """Create all readers and return them as a dictionary."""
+        readers = {}
+        for reader_key in cls.get_all_reader_keys():
+            readers[reader_key] = cls.create_reader(reader_key)
+        return readers
+
+    @classmethod
+    def clear_cache(cls):
+        """Clear the reader cache."""
+        cls._reader_cache.clear()
+
+    @classmethod
+    def register_reader(
+        cls,
+        key: str,
+        reader_method,
+        name: str,
+        description: str,
+        extensions: Optional[List[str]] = None,
+    ):
+        """Register a new reader type."""
+        # Add the reader method to the class
+        method_name = f"_get_{key}_reader"
+        setattr(cls, method_name, classmethod(reader_method))