agno 1.8.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

agno/{document → knowledge}/reader/markdown_reader.py

@@ -1,27 +1,76 @@
 import asyncio
 import uuid
 from pathlib import Path
-from typing import IO, Any, List, Union
+from typing import IO, Any, List, Optional, Union
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_info, logger
 
+DEFAULT_CHUNKER_STRATEGY: ChunkingStrategy
+
+# Try to import MarkdownChunking, fallback to FixedSizeChunking if not available
+try:
+    from agno.knowledge.chunking.markdown import MarkdownChunking
+
+    DEFAULT_CHUNKER_STRATEGY = MarkdownChunking()
+    MARKDOWN_CHUNKER_AVAILABLE = True
+except ImportError:
+    from agno.knowledge.chunking.fixed import FixedSizeChunking
+
+    DEFAULT_CHUNKER_STRATEGY = FixedSizeChunking()
+    MARKDOWN_CHUNKER_AVAILABLE = False
+
 
 class MarkdownReader(Reader):
     """Reader for Markdown files"""
 
-    def read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Markdown readers."""
+        strategies = [
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+        ]
+
+        # Only include MarkdownChunking if it's available
+        if MARKDOWN_CHUNKER_AVAILABLE:
+            strategies.insert(0, ChunkingStrategyType.MARKDOWN_CHUNKER)
+
+        return strategies
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.MARKDOWN]
+
+    def __init__(
+        self,
+        chunking_strategy: Optional[ChunkingStrategy] = None,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> None:
+        # Use the default chunking strategy if none provided
+        if chunking_strategy is None:
+            chunking_strategy = DEFAULT_CHUNKER_STRATEGY
+
+        super().__init__(chunking_strategy=chunking_strategy, name=name, description=description)
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
                 log_info(f"Reading: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
                 file_contents = file.read_text("utf-8")
             else:
                 log_info(f"Reading uploaded file: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 
@@ -36,14 +85,14 @@ class MarkdownReader(Reader):
             logger.error(f"Error reading: {file}: {e}")
             return []
 
-    async def async_read(self, file: Union[Path, IO[Any]]) -> List[Document]:
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
                     raise FileNotFoundError(f"Could not find file: {file}")
 
                 log_info(f"Reading asynchronously: {file}")
-                file_name = file.stem
+                file_name = name or file.stem
 
                 try:
                     import aiofiles
@@ -55,7 +104,7 @@ class MarkdownReader(Reader):
                     file_contents = file.read_text("utf-8")
             else:
                 log_info(f"Reading uploaded file asynchronously: {file.name}")
-                file_name = file.name.split(".")[0]
+                file_name = name or file.name.split(".")[0]
                 file.seek(0)
                 file_contents = file.read().decode("utf-8")
 

agno/{document → knowledge}/reader/pdf_reader.py

@@ -4,9 +4,10 @@ from pathlib import Path
 from typing import IO, Any, List, Optional, Tuple, Union
 from uuid import uuid4
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.http import async_fetch_with_retry, fetch_with_retry
+from agno.knowledge.chunking.strategy import ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_error, log_info, logger
 
 try:
@@ -95,9 +96,9 @@ def _clean_page_numbers(
     Notes:
         - The function scans for page numbers using a regular expression that matches digits at the start or end of a string.
         - It evaluates several potential starting points for numbering (-2, -1, 0, 1, 2 shifts) to determine the most consistent sequence.
-        - If at least a specified ratio of pages (defined by `PAGE_NUMBERING_CORRECTNESS_RATIO_FOR_REMOVAL`) has correct sequential numbering, 
+        - If at least a specified ratio of pages (defined by `PAGE_NUMBERING_CORRECTNESS_RATIO_FOR_REMOVAL`) has correct sequential numbering,
           the page numbers are processed.
-        - If page numbers are found, the function will add formatted page numbers to each page's content if `page_start_numbering_format` or 
+        - If page numbers are found, the function will add formatted page numbers to each page's content if `page_start_numbering_format` or
           `page_end_numbering_format` is provided.
     """
     assert len(extra_content) == 0 or len(extra_content) == len(page_content_list), (
@@ -190,8 +191,23 @@ class BasePDFReader(Reader):
         self.page_end_numbering_format = page_end_numbering_format
         self.password = password
 
+        if self.chunking_strategy is None:
+            from agno.knowledge.chunking.document import DocumentChunking
+
+            self.chunking_strategy = DocumentChunking(chunk_size=5000)
         super().__init__(**kwargs)
 
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for PDF readers."""
+        return [
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
     def _build_chunked_documents(self, documents: List[Document]) -> List[Document]:
         chunked_documents: List[Document] = []
         for document in documents:
@@ -205,19 +221,19 @@ class BasePDFReader(Reader):
         # Use provided password or fall back to instance password
         pdf_password = password or self.password
         if not pdf_password:
-            logger.error(f"PDF {doc_name} is password protected but no password provided")
+            logger.error(f'PDF file "{doc_name}" is password protected but no password provided')
             return False
 
         try:
             decrypted_pdf = doc_reader.decrypt(pdf_password)
             if decrypted_pdf:
-                log_info(f"Successfully decrypted PDF {doc_name} with user password")
+                log_info(f'Successfully decrypted PDF file "{doc_name}" with user password')
                 return True
             else:
-                log_error(f"Failed to decrypt PDF {doc_name}: incorrect password")
+                log_error(f'Failed to decrypt PDF file "{doc_name}": incorrect password')
                 return False
         except Exception as e:
-            log_error(f"Error decrypting PDF {doc_name}: {e}")
+            log_error(f'Error decrypting PDF file "{doc_name}": {e}')
             return False
 
     def _create_documents(self, pdf_content: List[str], doc_name: str, use_uuid_for_id: bool, page_number_shift):
@@ -306,9 +322,17 @@ class BasePDFReader(Reader):
 class PDFReader(BasePDFReader):
     """Reader for PDF files"""
 
-    def read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.PDF]
+
+    def read(
+        self, pdf: Union[str, Path, IO[Any]], name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         try:
-            if isinstance(pdf, str):
+            if name:
+                doc_name = name
+            elif isinstance(pdf, str):
                 doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
                 doc_name = pdf.name.split(".")[0]
@@ -318,26 +342,18 @@ class PDFReader(BasePDFReader):
         log_info(f"Reading: {doc_name}")
 
         try:
-            pdf_reader = DocumentReader(pdf)
+            DocumentReader(pdf)
         except PdfStreamError as e:
             logger.error(f"Error reading PDF: {e}")
             return []
 
-        # Handle PDF decryption
-        if not self._decrypt_pdf(pdf_reader, doc_name, password):
-            return []
-
-        # Read and chunk.
-        return self._pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=True)
-
-    async def async_read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
-                doc_name = pdf.name.split(".")[0]
+                doc_name = name or pdf.name.split(".")[0]
         except Exception:
-            doc_name = "pdf"
+            doc_name = name or "pdf"
 
         log_info(f"Reading: {doc_name}")
 
@@ -351,73 +367,55 @@ class PDFReader(BasePDFReader):
         if not self._decrypt_pdf(pdf_reader, doc_name, password):
             return []
 
-        # Read and chunk.
-        return await self._async_pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=True)
-
-
-class PDFUrlReader(BasePDFReader):
-    """Reader for PDF files from URL"""
-
-    def __init__(self, proxy: Optional[str] = None, password: Optional[str] = None, **kwargs):
-        super().__init__(password=password, **kwargs)
-        self.proxy = proxy
-
-    def read(self, url: str, password: Optional[str] = None) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        from io import BytesIO
-
-        log_info(f"Reading: {url}")
-
-        # Retry the request up to 3 times with exponential backoff
-        response = fetch_with_retry(url, proxy=self.proxy)
-
-        doc_name = url.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-        pdf_reader = DocumentReader(BytesIO(response.content))
+        # Read and chunk
+        return self._pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=True)
 
-        # Handle PDF decryption
-        if not self._decrypt_pdf(pdf_reader, doc_name, password):
+    async def async_read(
+        self,
+        pdf: Optional[Union[str, Path, IO[Any]]] = None,
+        name: Optional[str] = None,
+        password: Optional[str] = None,
+    ) -> List[Document]:
+        if pdf is None:
+            log_error("No pdf provided")
             return []
 
-        # Read and chunk.
-        return self._pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=False)
-
-    async def async_read(self, url: str, password: Optional[str] = None) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        from io import BytesIO
-
-        import httpx
-
-        log_info(f"Reading: {url}")
+        try:
+            if isinstance(pdf, str):
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+            else:
+                doc_name = pdf.name.split(".")[0]
+        except Exception:
+            doc_name = name or "pdf"
 
-        client_args = {"proxy": self.proxy} if self.proxy else {}
-        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
-            response = await async_fetch_with_retry(url, client=client)
+        log_info(f"Reading: {doc_name}")
 
-        doc_name = url.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-        pdf_reader = DocumentReader(BytesIO(response.content))
+        try:
+            pdf_reader = DocumentReader(pdf)
+        except PdfStreamError as e:
+            logger.error(f"Error reading PDF: {e}")
+            return []
 
         # Handle PDF decryption
         if not self._decrypt_pdf(pdf_reader, doc_name, password):
             return []
 
         # Read and chunk.
-        return await self._async_pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=False)
+        return await self._async_pdf_reader_to_documents(pdf_reader, doc_name, use_uuid_for_id=True)
 
 
 class PDFImageReader(BasePDFReader):
     """Reader for PDF files with text and images extraction"""
 
-    def read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
+    def read(
+        self, pdf: Union[str, Path, IO[Any]], name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         if not pdf:
             raise ValueError("No pdf provided")
 
         try:
             if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
                 doc_name = pdf.name.split(".")[0]
         except Exception:
@@ -433,13 +431,15 @@ class PDFImageReader(BasePDFReader):
         # Read and chunk.
         return self._pdf_reader_to_documents(pdf_reader, doc_name, read_images=True, use_uuid_for_id=False)
 
-    async def async_read(self, pdf: Union[str, Path, IO[Any]], password: Optional[str] = None) -> List[Document]:
+    async def async_read(
+        self, pdf: Union[str, Path, IO[Any]], name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         if not pdf:
             raise ValueError("No pdf provided")
 
         try:
             if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
+                doc_name = name or pdf.split("/")[-1].split(".")[0].replace(" ", "_")
             else:
                 doc_name = pdf.name.split(".")[0]
         except Exception:
@@ -454,58 +454,3 @@ class PDFImageReader(BasePDFReader):
 
         # Read and chunk.
         return await self._async_pdf_reader_to_documents(pdf_reader, doc_name, read_images=True, use_uuid_for_id=False)
-
-
-class PDFUrlImageReader(BasePDFReader):
-    """Reader for PDF files from URL with text and images extraction"""
-
-    def __init__(self, proxy: Optional[str] = None, password: Optional[str] = None, **kwargs):
-        super().__init__(password=password, **kwargs)
-        self.proxy = proxy
-
-    def read(self, url: str, password: Optional[str] = None) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        from io import BytesIO
-
-        import httpx
-
-        # Read the PDF from the URL
-        log_info(f"Reading: {url}")
-        response = httpx.get(url, proxy=self.proxy) if self.proxy else httpx.get(url)
-
-        doc_name = url.split("/")[-1].split(".")[0].replace(" ", "_")
-        pdf_reader = DocumentReader(BytesIO(response.content))
-
-        # Handle PDF decryption
-        if not self._decrypt_pdf(pdf_reader, doc_name, password):
-            return []
-
-        # Read and chunk.
-        return self._pdf_reader_to_documents(pdf_reader, doc_name, read_images=True, use_uuid_for_id=False)
-
-    async def async_read(self, url: str, password: Optional[str] = None) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        from io import BytesIO
-
-        import httpx
-
-        log_info(f"Reading: {url}")
-
-        client_args = {"proxy": self.proxy} if self.proxy else {}
-        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
-            response = await client.get(url)
-            response.raise_for_status()
-
-        doc_name = url.split("/")[-1].split(".")[0].replace(" ", "_")
-        pdf_reader = DocumentReader(BytesIO(response.content))
-
-        # Handle PDF decryption
-        if not self._decrypt_pdf(pdf_reader, doc_name, password):
-            return []
-
-        # Read and chunk.
-        return await self._async_pdf_reader_to_documents(pdf_reader, doc_name, read_images=True, use_uuid_for_id=False)

agno/knowledge/reader/reader_factory.py

@@ -0,0 +1,268 @@
+import os
+from typing import Any, Callable, Dict, List, Optional
+
+from agno.knowledge.reader.base import Reader
+
+
+class ReaderFactory:
+    """Factory for creating and managing document readers with lazy loading."""
+
+    # Cache for instantiated readers
+    _reader_cache: Dict[str, Reader] = {}
+
+    @classmethod
+    def _get_pdf_reader(cls, **kwargs) -> Reader:
+        """Get PDF reader instance."""
+        from agno.knowledge.reader.pdf_reader import PDFReader
+
+        config: Dict[str, Any] = {
+            "chunk": True,
+            "chunk_size": 100,
+            "description": "Processes PDF documents with OCR support for images and text extraction",
+        }
+        config.update(kwargs)
+        return PDFReader(**config)
+
+    @classmethod
+    def _get_csv_reader(cls, **kwargs) -> Reader:
+        """Get CSV reader instance."""
+        from agno.knowledge.reader.csv_reader import CSVReader
+
+        config: Dict[str, Any] = {
+            "name": "CSV Reader",
+            "description": "Parses CSV, XLSX, and XLS files with custom delimiter support",
+        }
+        config.update(kwargs)
+        return CSVReader(**config)
+
+    @classmethod
+    def _get_docx_reader(cls, **kwargs) -> Reader:
+        """Get Docx reader instance."""
+        from agno.knowledge.reader.docx_reader import DocxReader
+
+        config: Dict[str, Any] = {
+            "name": "Docx Reader",
+            "description": "Extracts text content from Microsoft Word documents (.docx and .doc formats)",
+        }
+        config.update(kwargs)
+        return DocxReader(**config)
+
+    @classmethod
+    def _get_json_reader(cls, **kwargs) -> Reader:
+        """Get JSON reader instance."""
+        from agno.knowledge.reader.json_reader import JSONReader
+
+        config: Dict[str, Any] = {
+            "name": "JSON Reader",
+            "description": "Processes JSON data structures and API responses with nested object handling",
+        }
+        config.update(kwargs)
+        return JSONReader(**config)
+
+    @classmethod
+    def _get_markdown_reader(cls, **kwargs) -> Reader:
+        """Get Markdown reader instance."""
+        from agno.knowledge.reader.markdown_reader import MarkdownReader
+
+        config: Dict[str, Any] = {
+            "name": "Markdown Reader",
+            "description": "Processes Markdown documentation with header-aware chunking and formatting preservation",
+        }
+        config.update(kwargs)
+        return MarkdownReader(**config)
+
+    @classmethod
+    def _get_text_reader(cls, **kwargs) -> Reader:
+        """Get Text reader instance."""
+        from agno.knowledge.reader.text_reader import TextReader
+
+        config: Dict[str, Any] = {
+            "name": "Text Reader",
+            "description": "Handles plain text files with customizable chunking strategies and encoding detection",
+        }
+        config.update(kwargs)
+        return TextReader(**config)
+
+    @classmethod
+    def _get_website_reader(cls, **kwargs) -> Reader:
+        """Get Website reader instance."""
+        from agno.knowledge.reader.website_reader import WebsiteReader
+
+        config: Dict[str, Any] = {
+            "name": "Website Reader",
+            "description": "Scrapes and extracts content from web pages with HTML parsing and text cleaning",
+        }
+        config.update(kwargs)
+        return WebsiteReader(**config)
+
+    @classmethod
+    def _get_firecrawl_reader(cls, **kwargs) -> Reader:
+        """Get Firecrawl reader instance."""
+        from agno.knowledge.reader.firecrawl_reader import FirecrawlReader
+
+        config: Dict[str, Any] = {
+            "api_key": kwargs.get("api_key") or os.getenv("FIRECRAWL_API_KEY"),
+            "mode": "crawl",
+            "name": "Firecrawl Reader",
+            "description": "Advanced web scraping and crawling with JavaScript rendering and structured data extraction",
+        }
+        config.update(kwargs)
+        return FirecrawlReader(**config)
+
+    @classmethod
+    def _get_youtube_reader(cls, **kwargs) -> Reader:
+        """Get YouTube reader instance."""
+        from agno.knowledge.reader.youtube_reader import YouTubeReader
+
+        config: Dict[str, Any] = {
+            "name": "YouTube Reader",
+            "description": "Extracts transcripts and metadata from YouTube videos and playlists",
+        }
+        config.update(kwargs)
+        return YouTubeReader(**config)
+
+    @classmethod
+    def _get_arxiv_reader(cls, **kwargs) -> Reader:
+        """Get Arxiv reader instance."""
+        from agno.knowledge.reader.arxiv_reader import ArxivReader
+
+        config: Dict[str, Any] = {
+            "name": "Arxiv Reader",
+            "description": "Downloads and processes academic papers from ArXiv with PDF parsing and metadata extraction",
+        }
+        config.update(kwargs)
+        return ArxivReader(**config)
+
+    @classmethod
+    def _get_wikipedia_reader(cls, **kwargs) -> Reader:
+        """Get Wikipedia reader instance."""
+        from agno.knowledge.reader.wikipedia_reader import WikipediaReader
+
+        config: Dict[str, Any] = {
+            "name": "Wikipedia Reader",
+            "description": "Fetches and processes Wikipedia articles with section-aware chunking and link resolution",
+        }
+        config.update(kwargs)
+        return WikipediaReader(**config)
+
+    @classmethod
+    def _get_web_search_reader(cls, **kwargs) -> Reader:
+        """Get Web Search reader instance."""
+        from agno.knowledge.reader.web_search_reader import WebSearchReader
+
+        config: Dict[str, Any] = {
+            "name": "Web Search Reader",
+            "description": "Executes web searches and processes results with relevance ranking and content extraction",
+        }
+        config.update(kwargs)
+        return WebSearchReader(**config)
+
+    @classmethod
+    def _get_reader_method(cls, reader_key: str) -> Callable[[], Reader]:
+        """Get the appropriate reader method for the given key."""
+        method_name = f"_get_{reader_key}_reader"
+        if not hasattr(cls, method_name):
+            raise ValueError(f"Unknown reader: {reader_key}")
+        return getattr(cls, method_name)
+
+    @classmethod
+    def create_reader(cls, reader_key: str, **kwargs) -> Reader:
+        """Create a reader instance with the given key and optional overrides."""
+        if reader_key in cls._reader_cache:
+            return cls._reader_cache[reader_key]
+
+        # Get the reader method and create the instance
+        reader_method = cls._get_reader_method(reader_key)
+        reader = reader_method(**kwargs)
+
+        # Cache the reader
+        cls._reader_cache[reader_key] = reader
+
+        return reader
+
+    @classmethod
+    def get_reader_for_extension(cls, extension: str) -> Reader:
+        """Get the appropriate reader for a file extension."""
+        extension = extension.lower()
+
+        if extension in [".pdf", "application/pdf"]:
+            return cls.create_reader("pdf")
+        elif extension in [".csv", "text/csv"]:
+            return cls.create_reader("csv")
+        elif extension in [".docx", ".doc"]:
+            return cls.create_reader("docx")
+        elif extension == ".json":
+            return cls.create_reader("json")
+        elif extension in [".md", ".markdown"]:
+            return cls.create_reader("markdown")
+        elif extension in [".txt", ".text"]:
+            return cls.create_reader("text")
+        else:
+            # Default to text reader for unknown extensions
+            return cls.create_reader("text")
+
+    @classmethod
+    def get_reader_for_url(cls, url: str) -> Reader:
+        """Get the appropriate reader for a URL."""
+        url_lower = url.lower()
+
+        # Check for YouTube URLs
+        if any(domain in url_lower for domain in ["youtube.com", "youtu.be"]):
+            return cls.create_reader("youtube")
+
+        # Default to URL reader
+        return cls.create_reader("url")
+
+    @classmethod
+    def get_all_reader_keys(cls) -> List[str]:
+        """Get all available reader keys in priority order."""
+        # Extract reader keys from method names
+
+        PREFIX = "_get_"
+        SUFFIX = "_reader"
+
+        reader_keys = []
+        for attr_name in dir(cls):
+            if attr_name.startswith(PREFIX) and attr_name.endswith(SUFFIX):
+                reader_key = attr_name[len(PREFIX) : -len(SUFFIX)]  # Remove "_get_" prefix and "_reader" suffix
+                reader_keys.append(reader_key)
+
+        # Define priority order for URL readers
+        url_reader_priority = ["url", "website", "firecrawl", "pdf_url", "csv_url", "youtube", "web_search"]
+
+        # Sort with URL readers in priority order, others alphabetically
+        def sort_key(reader_key):
+            if reader_key in url_reader_priority:
+                return (0, url_reader_priority.index(reader_key))
+            else:
+                return (1, reader_key)
+
+        reader_keys.sort(key=sort_key)
+        return reader_keys
+
+    @classmethod
+    def create_all_readers(cls) -> Dict[str, Reader]:
+        """Create all readers and return them as a dictionary."""
+        readers = {}
+        for reader_key in cls.get_all_reader_keys():
+            readers[reader_key] = cls.create_reader(reader_key)
+        return readers
+
+    @classmethod
+    def clear_cache(cls):
+        """Clear the reader cache."""
+        cls._reader_cache.clear()
+
+    @classmethod
+    def register_reader(
+        cls,
+        key: str,
+        reader_method,
+        name: str,
+        description: str,
+        extensions: Optional[List[str]] = None,
+    ):
+        """Register a new reader type."""
+        # Add the reader method to the class
+        method_name = f"_get_{key}_reader"
+        setattr(cls, method_name, classmethod(reader_method))