agno 1.8.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

agno/knowledge/reader/__init__.py

@@ -0,0 +1,7 @@
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.reader_factory import ReaderFactory
+
+__all__ = [
+    "Reader",
+    "ReaderFactory",
+]

agno/{document → knowledge}/reader/arxiv_reader.py

@@ -1,8 +1,11 @@
 import asyncio
-from typing import List
+from typing import List, Optional
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 
 try:
     import arxiv  # noqa: F401
@@ -11,9 +14,34 @@ except ImportError:
 
 
 class ArxivReader(Reader):
-    max_results: int = 5  # Top articles
     sort_by: arxiv.SortCriterion = arxiv.SortCriterion.Relevance
 
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Arxiv readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TOPIC]
+
+    def __init__(
+        self,
+        chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(),
+        sort_by: arxiv.SortCriterion = arxiv.SortCriterion.Relevance,
+        **kwargs,
+    ) -> None:
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+        # ArxivReader-specific attributes
+        self.sort_by = sort_by
+
     def read(self, query: str) -> List[Document]:
         """
         Search a query from arXiv database

agno/knowledge/reader/base.py

@@ -0,0 +1,88 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, List, Optional
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyFactory, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.types import ContentType
+
+
+@dataclass
+class Reader:
+    """Base class for reading documents"""
+
+    chunk: bool = True
+    chunk_size: int = 5000
+    separators: List[str] = field(default_factory=lambda: ["\n", "\n\n", "\r", "\r\n", "\n\r", "\t", " ", "  "])
+    chunking_strategy: Optional[ChunkingStrategy] = None
+    name: Optional[str] = None
+    description: Optional[str] = None
+    max_results: int = 5  # Maximum number of results to return (useful for search-based readers)
+
+    def __init__(
+        self,
+        chunk: bool = True,
+        chunk_size: int = 5000,
+        separators: Optional[List[str]] = None,
+        chunking_strategy: Optional[ChunkingStrategy] = None,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        max_results: int = 5,
+        **kwargs,
+    ) -> None:
+        self.chunk = chunk
+        self.chunk_size = chunk_size
+        self.separators = (
+            separators if separators is not None else ["\n", "\n\n", "\r", "\r\n", "\n\r", "\t", " ", "  "]
+        )
+        self.chunking_strategy = chunking_strategy
+        self.name = name
+        self.description = description
+        self.max_results = max_results
+
+    def set_chunking_strategy_from_string(self, strategy_name: str, **kwargs) -> None:
+        """Set the chunking strategy from a string name."""
+        try:
+            strategy_type = ChunkingStrategyType.from_string(strategy_name)
+            self.chunking_strategy = ChunkingStrategyFactory.create_strategy(strategy_type, **kwargs)
+        except ValueError as e:
+            raise ValueError(f"Failed to set chunking strategy: {e}")
+
+    def read(self, obj: Any, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
+        raise NotImplementedError
+
+    async def async_read(self, obj: Any, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
+        raise NotImplementedError
+
+    @classmethod
+    def get_supported_chunking_strategies(cls) -> List[ChunkingStrategyType]:
+        raise NotImplementedError
+
+    @classmethod
+    def get_supported_content_types(cls) -> List[ContentType]:
+        raise NotImplementedError
+
+    def chunk_document(self, document: Document) -> List[Document]:
+        if self.chunking_strategy is None:
+            self.chunking_strategy = FixedSizeChunking(chunk_size=self.chunk_size)
+        return self.chunking_strategy.chunk(document)  # type: ignore
+
+    async def chunk_documents_async(self, documents: List[Document]) -> List[Document]:
+        """
+        Asynchronously chunk a list of documents using the instance's chunk_document method.
+
+        Args:
+            documents: List of documents to be chunked.
+
+        Returns:
+            A flattened list of chunked documents.
+        """
+
+        async def _chunk_document_async(doc: Document) -> List[Document]:
+            return await asyncio.to_thread(self.chunk_document, doc)
+
+        # Process chunking in parallel for all documents
+        chunked_lists = await asyncio.gather(*[_chunk_document_async(doc) for doc in documents])
+        # Flatten the result
+        return [chunk for sublist in chunked_lists for chunk in sublist]

agno/{document → knowledge}/reader/csv_reader.py

@@ -1,28 +1,47 @@
 import asyncio
 import csv
 import io
-import os
 from pathlib import Path
 from typing import IO, Any, List, Optional, Union
-from urllib.parse import urlparse
 from uuid import uuid4
 
-from agno.utils.http import async_fetch_with_retry, fetch_with_retry
-
 try:
     import aiofiles
 except ImportError:
     raise ImportError("`aiofiles` not installed. Please install it with `pip install aiofiles`")
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.row import RowChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import logger
 
 
 class CSVReader(Reader):
     """Reader for CSV files"""
 
-    def read(self, file: Union[Path, IO[Any]], delimiter: str = ",", quotechar: str = '"') -> List[Document]:
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = RowChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for CSV readers."""
+        return [
+            ChunkingStrategyType.ROW_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.CSV, ContentType.XLSX, ContentType.XLS]
+
+    def read(
+        self, file: Union[Path, IO[Any]], delimiter: str = ",", quotechar: str = '"', name: Optional[str] = None
+    ) -> List[Document]:
         try:
             if isinstance(file, Path):
                 if not file.exists():
@@ -30,11 +49,15 @@ class CSVReader(Reader):
                 logger.info(f"Reading: {file}")
                 file_content = file.open(newline="", mode="r", encoding="utf-8")
             else:
-                logger.info(f"Reading retrieved file: {file.name}")
+                logger.info(f"Reading retrieved file: {name or file.name}")
                 file.seek(0)
                 file_content = io.StringIO(file.read().decode("utf-8"))  # type: ignore
 
-            csv_name = Path(file.name).stem if isinstance(file, Path) else file.name.split(".")[0]
+            csv_name = name or (
+                Path(file.name).stem
+                if isinstance(file, Path)
+                else (getattr(file, "name", "csv_file").split(".")[0] if hasattr(file, "name") else "csv_file")
+            )
             csv_content = ""
             with file_content as csvfile:
                 csv_reader = csv.reader(csvfile, delimiter=delimiter, quotechar=quotechar)
@@ -55,11 +78,16 @@ class CSVReader(Reader):
                 return chunked_documents
             return documents
         except Exception as e:
-            logger.error(f"Error reading: {file.name if isinstance(file, IO) else file}: {e}")
+            logger.error(f"Error reading: {getattr(file, 'name', str(file)) if isinstance(file, IO) else file}: {e}")
             return []
 
     async def async_read(
-        self, file: Union[Path, IO[Any]], delimiter: str = ",", quotechar: str = '"', page_size: int = 1000
+        self,
+        file: Union[Path, IO[Any]],
+        delimiter: str = ",",
+        quotechar: str = '"',
+        page_size: int = 1000,
+        name: Optional[str] = None,
     ) -> List[Document]:
         """
         Read a CSV file asynchronously, processing batches of rows concurrently.
@@ -86,7 +114,11 @@ class CSVReader(Reader):
                 file.seek(0)
                 file_content_io = io.StringIO(file.read().decode("utf-8"))  # type: ignore
 
-            csv_name = Path(file.name).stem if isinstance(file, Path) else file.name.split(".")[0]
+            csv_name = name or (
+                Path(file.name).stem
+                if isinstance(file, Path)
+                else (getattr(file, "name", "csv_file").split(".")[0] if hasattr(file, "name") else "csv_file")
+            )
 
             file_content_io.seek(0)
             csv_reader = csv.reader(file_content_io, delimiter=delimiter, quotechar=quotechar)
@@ -128,57 +160,7 @@ class CSVReader(Reader):
 
             return documents
         except Exception as e:
-            logger.error(f"Error reading async: {file.name if isinstance(file, IO) else file}: {e}")
+            logger.error(
+                f"Error reading async: {getattr(file, 'name', str(file)) if isinstance(file, IO) else file}: {e}"
+            )
             return []
-
-
-class CSVUrlReader(Reader):
-    """Reader for CSV files"""
-
-    def __init__(self, proxy: Optional[str] = None, **kwargs):
-        super().__init__(**kwargs)
-        self.proxy = proxy
-
-    def read(self, url: str) -> List[Document]:
-        if not url:
-            raise ValueError("No URL provided")
-
-        logger.info(f"Reading: {url}")
-        # Retry the request up to 3 times with exponential backoff
-        response = fetch_with_retry(url, proxy=self.proxy)
-
-        parsed_url = urlparse(url)
-        filename = os.path.basename(parsed_url.path) or "data.csv"
-
-        file_obj = io.BytesIO(response.content)
-        file_obj.name = filename
-        documents = CSVReader().read(file=file_obj)
-
-        file_obj.close()
-
-        return documents
-
-    async def async_read(self, url: str) -> List[Document]:
-        if not url:
-            raise ValueError("No URL provided")
-
-        import httpx
-
-        logger.info(f"Reading async: {url}")
-
-        client_args = {"proxy": self.proxy} if self.proxy else {}
-        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
-            response = await async_fetch_with_retry(url, client=client)
-
-            parsed_url = urlparse(url)
-            filename = os.path.basename(parsed_url.path) or "data.csv"
-
-            file_obj = io.BytesIO(response.content)
-            file_obj.name = filename
-
-            # Use the async version of CSVReader
-            documents = await CSVReader().async_read(file=file_obj)
-
-            file_obj.close()
-
-            return documents

agno/knowledge/reader/docx_reader.py

@@ -0,0 +1,83 @@
+import asyncio
+from pathlib import Path
+from typing import IO, Any, List, Optional, Union
+from uuid import uuid4
+
+from agno.knowledge.chunking.document import DocumentChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_info, logger
+
+try:
+    from docx import Document as DocxDocument  # type: ignore
+except ImportError:
+    raise ImportError("The `python-docx` package is not installed. Please install it via `pip install python-docx`.")
+
+
+class DocxReader(Reader):
+    """Reader for Doc/Docx files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = DocumentChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for DOCX readers."""
+        return [
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.DOCX, ContentType.DOC]
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        """Read a docx file and return a list of documents"""
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_info(f"Reading: {file}")
+                docx_document = DocxDocument(str(file))
+                doc_name = name or file.stem
+            else:
+                log_info(f"Reading uploaded file: {getattr(file, 'name', 'docx_file')}")
+                docx_document = DocxDocument(file)
+                doc_name = name or (
+                    getattr(file, "name", "docx_file").split(".")[0] if hasattr(file, "name") else "docx_file"
+                )
+
+            doc_content = "\n\n".join([para.text for para in docx_document.paragraphs])
+
+            documents = [
+                Document(
+                    name=doc_name,
+                    id=str(uuid4()),
+                    content=doc_content,
+                )
+            ]
+
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+
+        except Exception as e:
+            logger.error(f"Error reading file: {e}")
+            return []
+
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        """Asynchronously read a docx file and return a list of documents"""
+        try:
+            return await asyncio.to_thread(self.read, file, name)
+        except Exception as e:
+            logger.error(f"Error reading file asynchronously: {e}")
+            return []

agno/{document → knowledge}/reader/firecrawl_reader.py

@@ -2,9 +2,11 @@ import asyncio
 from dataclasses import dataclass
 from typing import Dict, List, Literal, Optional
 
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_debug, logger
 
 try:
@@ -26,17 +28,36 @@ class FirecrawlReader(Reader):
         mode: Literal["scrape", "crawl"] = "scrape",
         chunk: bool = True,
         chunk_size: int = 5000,
-        chunking_strategy: Optional[ChunkingStrategy] = None,
+        chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking(),
+        name: Optional[str] = None,
+        description: Optional[str] = None,
     ) -> None:
         # Initialise base Reader (handles chunk_size / strategy)
-        super().__init__(chunk=chunk, chunk_size=chunk_size, chunking_strategy=chunking_strategy)
+        super().__init__(
+            chunk=chunk, chunk_size=chunk_size, chunking_strategy=chunking_strategy, name=name, description=description
+        )
 
         # Firecrawl-specific attributes
         self.api_key = api_key
         self.params = params
         self.mode = mode
 
-    def scrape(self, url: str) -> List[Document]:
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Firecrawl readers."""
+        return [
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
+    def scrape(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Scrapes a website and returns a list of documents.
 
@@ -71,12 +92,12 @@ class FirecrawlReader(Reader):
 
         documents = []
         if self.chunk and content:  # Only chunk if there's content
-            documents.extend(self.chunk_document(Document(name=url, id=url, content=content)))
+            documents.extend(self.chunk_document(Document(name=name or url, id=url, content=content)))
         else:
-            documents.append(Document(name=url, id=url, content=content))
+            documents.append(Document(name=name or url, id=url, content=content))
         return documents
 
-    async def async_scrape(self, url: str) -> List[Document]:
+    async def async_scrape(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Asynchronously scrapes a website and returns a list of documents.
 
@@ -89,9 +110,9 @@ class FirecrawlReader(Reader):
         log_debug(f"Async scraping: {url}")
 
         # Use asyncio.to_thread to run the synchronous scrape in a thread
-        return await asyncio.to_thread(self.scrape, url)
+        return await asyncio.to_thread(self.scrape, url, name)
 
-    def crawl(self, url: str) -> List[Document]:
+    def crawl(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Crawls a website and returns a list of documents.
 
@@ -124,13 +145,13 @@ class FirecrawlReader(Reader):
 
             if content:  # Only create document if content exists
                 if self.chunk:
-                    documents.extend(self.chunk_document(Document(name=url, id=url, content=content)))
+                    documents.extend(self.chunk_document(Document(name=name or url, id=url, content=content)))
                 else:
-                    documents.append(Document(name=url, id=url, content=content))
+                    documents.append(Document(name=name or url, id=url, content=content))
 
         return documents
 
-    async def async_crawl(self, url: str) -> List[Document]:
+    async def async_crawl(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Asynchronously crawls a website and returns a list of documents.
 
@@ -143,9 +164,9 @@ class FirecrawlReader(Reader):
         log_debug(f"Async crawling: {url}")
 
         # Use asyncio.to_thread to run the synchronous crawl in a thread
-        return await asyncio.to_thread(self.crawl, url)
+        return await asyncio.to_thread(self.crawl, url, name)
 
-    def read(self, url: str) -> List[Document]:
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Reads from a URL based on the mode setting.
 
@@ -156,13 +177,13 @@ class FirecrawlReader(Reader):
             A list of documents
         """
         if self.mode == "scrape":
-            return self.scrape(url)
+            return self.scrape(url, name)
         elif self.mode == "crawl":
-            return self.crawl(url)
+            return self.crawl(url, name)
         else:
             raise NotImplementedError(f"Mode {self.mode} not implemented")
 
-    async def async_read(self, url: str) -> List[Document]:
+    async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Asynchronously reads from a URL based on the mode setting.
 
@@ -173,8 +194,8 @@ class FirecrawlReader(Reader):
             A list of documents
         """
         if self.mode == "scrape":
-            return await self.async_scrape(url)
+            return await self.async_scrape(url, name)
         elif self.mode == "crawl":
-            return await self.async_crawl(url)
+            return await self.async_crawl(url, name)
         else:
             raise NotImplementedError(f"Mode {self.mode} not implemented")

agno/{document → knowledge}/reader/json_reader.py

@@ -2,11 +2,14 @@ import asyncio
 import json
 from io import BytesIO
 from pathlib import Path
-from typing import IO, Any, List, Union
+from typing import IO, Any, List, Optional, Union
 from uuid import uuid4
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_info
 
 
@@ -15,18 +18,36 @@ class JSONReader(Reader):
 
     chunk: bool = False
 
-    def read(self, path: Union[Path, IO[Any]]) -> List[Document]:
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for JSON readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.JSON]
+
+    def read(self, path: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         try:
             if isinstance(path, Path):
                 if not path.exists():
                     raise FileNotFoundError(f"Could not find file: {path}")
                 log_info(f"Reading: {path}")
-                json_name = path.name.split(".")[0]
+                json_name = name or path.name.split(".")[0]
                 json_contents = json.loads(path.read_text("utf-8"))
 
             elif isinstance(path, BytesIO):
-                log_info(f"Reading uploaded file: {path.name}")
-                json_name = path.name.split(".")[0]
+                json_name = name or path.name.split(".")[0]
+                log_info(f"Reading uploaded file: {json_name}")
                 path.seek(0)
                 json_contents = json.load(path)
 
@@ -54,7 +75,7 @@ class JSONReader(Reader):
         except Exception:
             raise
 
-    async def async_read(self, path: Union[Path, IO[Any]]) -> List[Document]:
+    async def async_read(self, path: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
         """Asynchronously read JSON files.
 
         Args:
@@ -63,4 +84,4 @@ class JSONReader(Reader):
         Returns:
             List[Document]: List of documents from the JSON file
         """
-        return await asyncio.to_thread(self.read, path)
+        return await asyncio.to_thread(self.read, path, name)