agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/knowledge/reader/reader_factory.py

@@ -0,0 +1,313 @@
+import os
+from typing import Any, Callable, Dict, List, Optional
+
+from agno.knowledge.reader.base import Reader
+
+
+class ReaderFactory:
+    """Factory for creating and managing document readers with lazy loading."""
+
+    # Cache for instantiated readers
+    _reader_cache: Dict[str, Reader] = {}
+
+    @classmethod
+    def _get_pdf_reader(cls, **kwargs) -> Reader:
+        """Get PDF reader instance."""
+        from agno.knowledge.reader.pdf_reader import PDFReader
+
+        config: Dict[str, Any] = {
+            "name": "PDF Reader",
+            "description": "Processes PDF documents with OCR support for images and text extraction",
+        }
+        config.update(kwargs)
+        return PDFReader(**config)
+
+    @classmethod
+    def _get_csv_reader(cls, **kwargs) -> Reader:
+        """Get CSV reader instance."""
+        from agno.knowledge.reader.csv_reader import CSVReader
+
+        config: Dict[str, Any] = {
+            "name": "CSV Reader",
+            "description": "Parses CSV, XLSX, and XLS files with custom delimiter support",
+        }
+        config.update(kwargs)
+        return CSVReader(**config)
+
+    @classmethod
+    def _get_field_labeled_csv_reader(cls, **kwargs) -> Reader:
+        """Get Field Labeled CSV reader instance."""
+        from agno.knowledge.reader.field_labeled_csv_reader import FieldLabeledCSVReader
+
+        config: Dict[str, Any] = {
+            "name": "Field Labeled CSV Reader",
+            "description": "Converts CSV rows to field-labeled text format for enhanced readability and context",
+        }
+        config.update(kwargs)
+        return FieldLabeledCSVReader(**config)
+
+    @classmethod
+    def _get_docx_reader(cls, **kwargs) -> Reader:
+        """Get Docx reader instance."""
+        from agno.knowledge.reader.docx_reader import DocxReader
+
+        config: Dict[str, Any] = {
+            "name": "Docx Reader",
+            "description": "Extracts text content from Microsoft Word documents (.docx and .doc formats)",
+        }
+        config.update(kwargs)
+        return DocxReader(**config)
+
+    @classmethod
+    def _get_pptx_reader(cls, **kwargs) -> Reader:
+        """Get PPTX reader instance."""
+        from agno.knowledge.reader.pptx_reader import PPTXReader
+
+        config: Dict[str, Any] = {
+            "name": "PPTX Reader",
+            "description": "Extracts text content from Microsoft PowerPoint presentations (.pptx format)",
+        }
+        config.update(kwargs)
+        return PPTXReader(**config)
+
+    @classmethod
+    def _get_json_reader(cls, **kwargs) -> Reader:
+        """Get JSON reader instance."""
+        from agno.knowledge.reader.json_reader import JSONReader
+
+        config: Dict[str, Any] = {
+            "name": "JSON Reader",
+            "description": "Processes JSON data structures and API responses with nested object handling",
+        }
+        config.update(kwargs)
+        return JSONReader(**config)
+
+    @classmethod
+    def _get_markdown_reader(cls, **kwargs) -> Reader:
+        """Get Markdown reader instance."""
+        from agno.knowledge.reader.markdown_reader import MarkdownReader
+
+        config: Dict[str, Any] = {
+            "name": "Markdown Reader",
+            "description": "Processes Markdown documentation with header-aware chunking and formatting preservation",
+        }
+        config.update(kwargs)
+        return MarkdownReader(**config)
+
+    @classmethod
+    def _get_text_reader(cls, **kwargs) -> Reader:
+        """Get Text reader instance."""
+        from agno.knowledge.reader.text_reader import TextReader
+
+        config: Dict[str, Any] = {
+            "name": "Text Reader",
+            "description": "Handles plain text files with customizable chunking strategies and encoding detection",
+        }
+        config.update(kwargs)
+        return TextReader(**config)
+
+    @classmethod
+    def _get_website_reader(cls, **kwargs) -> Reader:
+        """Get Website reader instance."""
+        from agno.knowledge.reader.website_reader import WebsiteReader
+
+        config: Dict[str, Any] = {
+            "name": "Website Reader",
+            "description": "Scrapes and extracts content from web pages with HTML parsing and text cleaning",
+        }
+        config.update(kwargs)
+        return WebsiteReader(**config)
+
+    @classmethod
+    def _get_firecrawl_reader(cls, **kwargs) -> Reader:
+        """Get Firecrawl reader instance."""
+        from agno.knowledge.reader.firecrawl_reader import FirecrawlReader
+
+        config: Dict[str, Any] = {
+            "api_key": kwargs.get("api_key") or os.getenv("FIRECRAWL_API_KEY"),
+            "mode": "crawl",
+            "name": "Firecrawl Reader",
+            "description": "Advanced web scraping and crawling with JavaScript rendering and structured data extraction",
+        }
+        config.update(kwargs)
+        return FirecrawlReader(**config)
+
+    @classmethod
+    def _get_tavily_reader(cls, **kwargs) -> Reader:
+        """Get Tavily reader instance."""
+        from agno.knowledge.reader.tavily_reader import TavilyReader
+
+        config: Dict[str, Any] = {
+            "api_key": kwargs.get("api_key") or os.getenv("TAVILY_API_KEY"),
+            "extract_format": "markdown",
+            "extract_depth": "basic",
+            "name": "Tavily Reader",
+            "description": "Extracts content from URLs using Tavily's Extract API with markdown or text output",
+        }
+        config.update(kwargs)
+        return TavilyReader(**config)
+
+    @classmethod
+    def _get_youtube_reader(cls, **kwargs) -> Reader:
+        """Get YouTube reader instance."""
+        from agno.knowledge.reader.youtube_reader import YouTubeReader
+
+        config: Dict[str, Any] = {
+            "name": "YouTube Reader",
+            "description": "Extracts transcripts and metadata from YouTube videos and playlists",
+        }
+        config.update(kwargs)
+        return YouTubeReader(**config)
+
+    @classmethod
+    def _get_arxiv_reader(cls, **kwargs) -> Reader:
+        """Get Arxiv reader instance."""
+        from agno.knowledge.reader.arxiv_reader import ArxivReader
+
+        config: Dict[str, Any] = {
+            "name": "Arxiv Reader",
+            "description": "Downloads and processes academic papers from ArXiv with PDF parsing and metadata extraction",
+        }
+        config.update(kwargs)
+        return ArxivReader(**config)
+
+    @classmethod
+    def _get_wikipedia_reader(cls, **kwargs) -> Reader:
+        """Get Wikipedia reader instance."""
+        from agno.knowledge.reader.wikipedia_reader import WikipediaReader
+
+        config: Dict[str, Any] = {
+            "name": "Wikipedia Reader",
+            "description": "Fetches and processes Wikipedia articles with section-aware chunking and link resolution",
+        }
+        config.update(kwargs)
+        return WikipediaReader(**config)
+
+    @classmethod
+    def _get_web_search_reader(cls, **kwargs) -> Reader:
+        """Get Web Search reader instance."""
+        from agno.knowledge.reader.web_search_reader import WebSearchReader
+
+        config: Dict[str, Any] = {
+            "name": "Web Search Reader",
+            "description": "Executes web searches and processes results with relevance ranking and content extraction",
+        }
+        config.update(kwargs)
+        return WebSearchReader(**config)
+
+    @classmethod
+    def _get_reader_method(cls, reader_key: str) -> Callable[[], Reader]:
+        """Get the appropriate reader method for the given key."""
+        method_name = f"_get_{reader_key}_reader"
+        if not hasattr(cls, method_name):
+            raise ValueError(f"Unknown reader: {reader_key}")
+        return getattr(cls, method_name)
+
+    @classmethod
+    def create_reader(cls, reader_key: str, **kwargs) -> Reader:
+        """Create a reader instance with the given key and optional overrides."""
+        if reader_key in cls._reader_cache:
+            return cls._reader_cache[reader_key]
+
+        # Get the reader method and create the instance
+        reader_method = cls._get_reader_method(reader_key)
+        reader = reader_method(**kwargs)
+
+        # Cache the reader
+        cls._reader_cache[reader_key] = reader
+
+        return reader
+
+    @classmethod
+    def get_reader_for_extension(cls, extension: str) -> Reader:
+        """Get the appropriate reader for a file extension."""
+        extension = extension.lower()
+
+        if extension in [".pdf", "application/pdf"]:
+            return cls.create_reader("pdf")
+        elif extension in [".csv", "text/csv"]:
+            return cls.create_reader("csv")
+        elif extension in [".docx", ".doc", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"]:
+            return cls.create_reader("docx")
+        elif extension == ".pptx":
+            return cls.create_reader("pptx")
+        elif extension == ".json":
+            return cls.create_reader("json")
+        elif extension in [".md", ".markdown"]:
+            return cls.create_reader("markdown")
+        elif extension in [".txt", ".text"]:
+            return cls.create_reader("text")
+        else:
+            # Default to text reader for unknown extensions
+            return cls.create_reader("text")
+
+    @classmethod
+    def get_reader_for_url(cls, url: str) -> Reader:
+        """Get the appropriate reader for a URL."""
+        url_lower = url.lower()
+
+        # Check for YouTube URLs
+        if any(domain in url_lower for domain in ["youtube.com", "youtu.be"]):
+            return cls.create_reader("youtube")
+
+        # Default to website reader
+        return cls.create_reader("website")
+
+    @classmethod
+    def get_all_reader_keys(cls) -> List[str]:
+        """Get all available reader keys in priority order."""
+        # Extract reader keys from method names
+
+        PREFIX = "_get_"
+        SUFFIX = "_reader"
+
+        reader_keys = []
+        for attr_name in dir(cls):
+            if attr_name.startswith(PREFIX) and attr_name.endswith(SUFFIX):
+                reader_key = attr_name[len(PREFIX) : -len(SUFFIX)]  # Remove "_get_" prefix and "_reader" suffix
+                reader_keys.append(reader_key)
+
+        # Define priority order for URL readers
+        url_reader_priority = [
+            "website",
+            "firecrawl",
+            "tavily",
+            "youtube",
+        ]
+
+        # Sort with URL readers in priority order, others alphabetically
+        def sort_key(reader_key):
+            if reader_key in url_reader_priority:
+                return (0, url_reader_priority.index(reader_key))
+            else:
+                return (1, reader_key)
+
+        reader_keys.sort(key=sort_key)
+        return reader_keys
+
+    @classmethod
+    def create_all_readers(cls) -> Dict[str, Reader]:
+        """Create all readers and return them as a dictionary."""
+        readers = {}
+        for reader_key in cls.get_all_reader_keys():
+            readers[reader_key] = cls.create_reader(reader_key)
+        return readers
+
+    @classmethod
+    def clear_cache(cls):
+        """Clear the reader cache."""
+        cls._reader_cache.clear()
+
+    @classmethod
+    def register_reader(
+        cls,
+        key: str,
+        reader_method,
+        name: str,
+        description: str,
+        extensions: Optional[List[str]] = None,
+    ):
+        """Register a new reader type."""
+        # Add the reader method to the class
+        method_name = f"_get_{key}_reader"
+        setattr(cls, method_name, classmethod(reader_method))

agno/knowledge/reader/s3_reader.py

@@ -0,0 +1,89 @@
+import asyncio
+from io import BytesIO
+from pathlib import Path
+from typing import List, Optional
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.pdf_reader import PDFReader
+from agno.knowledge.reader.text_reader import TextReader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error
+
+try:
+    from agno.aws.resource.s3.object import S3Object  # type: ignore
+except (ModuleNotFoundError, ImportError):
+    raise ImportError("`agno-aws` not installed. Please install using `pip install agno-aws`")
+
+try:
+    import textract  # noqa: F401
+except ImportError:
+    raise ImportError("`textract` not installed. Please install it via `pip install textract`.")
+
+try:
+    from pypdf import PdfReader as DocumentReader  # noqa: F401
+except ImportError:
+    raise ImportError("`pypdf` not installed. Please install it via `pip install pypdf`.")
+
+
+class S3Reader(Reader):
+    """Reader for S3 files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for S3 readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.FILE, ContentType.URL, ContentType.TEXT]
+
+    def read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        try:
+            log_debug(f"Reading S3 file: {s3_object.uri}")
+
+            # Read PDF files
+            if s3_object.uri.endswith(".pdf"):
+                object_resource = s3_object.get_resource()
+                object_body = object_resource.get()["Body"]
+                doc_name = (
+                    s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+                    if name is None
+                    else name
+                )
+                return PDFReader().read(pdf=BytesIO(object_body.read()), name=doc_name)
+
+            # Read text files
+            else:
+                doc_name = (
+                    s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
+                    if name is None
+                    else name
+                )
+                obj_name = s3_object.name.split("/")[-1]
+                temporary_file = Path("storage").joinpath(obj_name)
+                s3_object.download(temporary_file)
+                documents = TextReader().read(file=temporary_file, name=doc_name)
+
+                temporary_file.unlink()
+                return documents
+
+        except Exception as e:
+            log_error(f"Error reading: {s3_object.uri}: {e}")
+
+        return []
+
+    async def async_read(self, name: Optional[str], s3_object: S3Object) -> List[Document]:
+        """Asynchronously read S3 files by running the synchronous read operation in a thread."""
+        return await asyncio.to_thread(self.read, name, s3_object)

agno/knowledge/reader/tavily_reader.py

@@ -0,0 +1,193 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Dict, List, Literal, Optional
+
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, logger
+
+try:
+    from tavily import TavilyClient  # type: ignore[attr-defined]
+except ImportError:
+    raise ImportError(
+        "The `tavily-python` package is not installed. Please install it via `pip install tavily-python`."
+    )
+
+
+@dataclass
+class TavilyReader(Reader):
+    api_key: Optional[str] = None
+    params: Optional[Dict] = None
+    extract_format: Literal["markdown", "text"] = "markdown"
+    extract_depth: Literal["basic", "advanced"] = "basic"
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        params: Optional[Dict] = None,
+        extract_format: Literal["markdown", "text"] = "markdown",
+        extract_depth: Literal["basic", "advanced"] = "basic",
+        chunk: bool = True,
+        chunk_size: int = 5000,
+        chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking(),
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> None:
+        """
+        Initialize TavilyReader for extracting content from URLs using Tavily's Extract API.
+
+        Args:
+            api_key: Tavily API key (or use TAVILY_API_KEY env var)
+            params: Additional parameters to pass to the extract API
+            extract_format: Output format - "markdown" or "text"
+            extract_depth: Extraction depth - "basic" (1 credit/5 URLs) or "advanced" (2 credits/5 URLs)
+            chunk: Whether to chunk the extracted content
+            chunk_size: Size of chunks when chunking is enabled
+            chunking_strategy: Strategy to use for chunking
+            name: Name of the reader
+            description: Description of the reader
+        """
+        # Initialize base Reader (handles chunk_size / strategy)
+        super().__init__(
+            chunk=chunk, chunk_size=chunk_size, chunking_strategy=chunking_strategy, name=name, description=description
+        )
+
+        # Tavily-specific attributes
+        self.api_key = api_key
+        self.params = params or {}
+        self.extract_format = extract_format
+        self.extract_depth = extract_depth
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Tavily readers."""
+        return [
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
+    def _extract(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Internal method to extract content from a URL using Tavily's Extract API.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document (defaults to URL)
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        log_debug(f"Extracting content from: {url}")
+
+        client = TavilyClient(api_key=self.api_key)
+
+        # Prepare extract parameters
+        extract_params = {
+            "urls": [url],
+            "depth": self.extract_depth,
+        }
+
+        # Add optional params if provided
+        if self.params:
+            extract_params.update(self.params)
+
+        try:
+            # Call Tavily Extract API
+            response = client.extract(**extract_params)
+
+            # Extract content from response
+            if not response or "results" not in response:
+                logger.warning(f"No results received for URL: {url}")
+                return [Document(name=name or url, id=url, content="")]
+
+            results = response.get("results", [])
+            if not results:
+                logger.warning(f"Empty results for URL: {url}")
+                return [Document(name=name or url, id=url, content="")]
+
+            # Get the first result (since we're extracting a single URL)
+            result = results[0]
+
+            # Check if extraction failed
+            if "failed_reason" in result:
+                logger.warning(f"Extraction failed for {url}: {result['failed_reason']}")
+                return [Document(name=name or url, id=url, content="")]
+
+            # Get raw content
+            content = result.get("raw_content", "")
+
+            if content is None:
+                content = ""
+                logger.warning(f"No content received for URL: {url}")
+
+            # Debug logging
+            log_debug(f"Received content type: {type(content)}")
+            log_debug(f"Content length: {len(content) if content else 0}")
+
+            # Create documents
+            documents = []
+            if self.chunk and content:
+                documents.extend(self.chunk_document(Document(name=name or url, id=url, content=content)))
+            else:
+                documents.append(Document(name=name or url, id=url, content=content))
+            return documents
+
+        except Exception as e:
+            logger.error(f"Error extracting content from {url}: {e}")
+            return [Document(name=name or url, id=url, content="")]
+
+    async def _async_extract(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Internal async method to extract content from a URL.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        log_debug(f"Async extracting content from: {url}")
+
+        # Use asyncio.to_thread to run the synchronous extract in a thread
+        return await asyncio.to_thread(self._extract, url, name)
+
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Reads content from a URL using Tavily Extract API.
+
+        This is the public API method that users should call.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        return self._extract(url, name)
+
+    async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        """
+        Asynchronously reads content from a URL using Tavily Extract API.
+
+        This is the public API method that users should call for async operations.
+
+        Args:
+            url: The URL to extract content from
+            name: Optional name for the document
+
+        Returns:
+            A list of documents containing the extracted content
+        """
+        return await self._async_extract(url, name)