agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/knowledge/reader/__init__.py

@@ -0,0 +1,7 @@
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.reader_factory import ReaderFactory
+
+__all__ = [
+    "Reader",
+    "ReaderFactory",
+]

agno/knowledge/reader/arxiv_reader.py

@@ -0,0 +1,81 @@
+import asyncio
+from typing import List, Optional
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+
+try:
+    import arxiv  # noqa: F401
+except ImportError:
+    raise ImportError("The `arxiv` package is not installed. Please install it via `pip install arxiv`.")
+
+
+class ArxivReader(Reader):
+    sort_by: arxiv.SortCriterion = arxiv.SortCriterion.Relevance
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Arxiv readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TOPIC]
+
+    def __init__(
+        self,
+        chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(),
+        sort_by: arxiv.SortCriterion = arxiv.SortCriterion.Relevance,
+        **kwargs,
+    ) -> None:
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+        # ArxivReader-specific attributes
+        self.sort_by = sort_by
+
+    def read(self, query: str) -> List[Document]:
+        """
+        Search a query from arXiv database
+
+        This function gets the top_k articles based on a user's query, sorted by relevance from arxiv
+
+        @param query:
+        @return: List of documents
+        """
+
+        documents = []
+        search = arxiv.Search(query=query, max_results=self.max_results, sort_by=self.sort_by)
+
+        for result in search.results():
+            links = ", ".join([x.href for x in result.links])
+
+            documents.append(
+                Document(
+                    name=result.title,
+                    id=result.title,
+                    meta_data={"pdf_url": str(result.pdf_url), "article_links": links},
+                    content=result.summary,
+                )
+            )
+
+        return documents
+
+    async def async_read(self, query: str) -> List[Document]:
+        """
+        Search a query from arXiv database asynchronously
+
+        This function gets the top_k articles based on a user's query, sorted by relevance from arxiv
+
+        @param query: Search query string
+        @return: List of documents
+        """
+        return await asyncio.to_thread(self.read, query)

agno/knowledge/reader/base.py

@@ -0,0 +1,95 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, List, Optional
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyFactory, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.types import ContentType
+
+
+@dataclass
+class Reader:
+    """Base class for reading documents"""
+
+    chunk: bool = True
+    chunk_size: int = 5000
+    separators: List[str] = field(default_factory=lambda: ["\n", "\n\n", "\r", "\r\n", "\n\r", "\t", " ", "  "])
+    chunking_strategy: Optional[ChunkingStrategy] = None
+    name: Optional[str] = None
+    description: Optional[str] = None
+    max_results: int = 5  # Maximum number of results to return (useful for search-based readers)
+    encoding: Optional[str] = None
+
+    def __init__(
+        self,
+        chunk: bool = True,
+        chunk_size: int = 5000,
+        separators: Optional[List[str]] = None,
+        chunking_strategy: Optional[ChunkingStrategy] = None,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        max_results: int = 5,
+        encoding: Optional[str] = None,
+        **kwargs,
+    ) -> None:
+        self.chunk = chunk
+        self.chunk_size = chunk_size
+        self.separators = (
+            separators if separators is not None else ["\n", "\n\n", "\r", "\r\n", "\n\r", "\t", " ", "  "]
+        )
+        self.chunking_strategy = chunking_strategy
+        self.name = name
+        self.description = description
+        self.max_results = max_results
+        self.encoding = encoding
+
+    def set_chunking_strategy_from_string(
+        self, strategy_name: str, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> None:
+        """Set the chunking strategy from a string name."""
+        try:
+            strategy_type = ChunkingStrategyType.from_string(strategy_name)
+            self.chunking_strategy = ChunkingStrategyFactory.create_strategy(
+                strategy_type, chunk_size=chunk_size, overlap=overlap, **kwargs
+            )
+        except ValueError as e:
+            raise ValueError(f"Failed to set chunking strategy: {e}")
+
+    def read(self, obj: Any, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
+        raise NotImplementedError
+
+    async def async_read(self, obj: Any, name: Optional[str] = None, password: Optional[str] = None) -> List[Document]:
+        raise NotImplementedError
+
+    @classmethod
+    def get_supported_chunking_strategies(cls) -> List[ChunkingStrategyType]:
+        raise NotImplementedError
+
+    @classmethod
+    def get_supported_content_types(cls) -> List[ContentType]:
+        raise NotImplementedError
+
+    def chunk_document(self, document: Document) -> List[Document]:
+        if self.chunking_strategy is None:
+            self.chunking_strategy = FixedSizeChunking(chunk_size=self.chunk_size)
+        return self.chunking_strategy.chunk(document)  # type: ignore
+
+    async def chunk_documents_async(self, documents: List[Document]) -> List[Document]:
+        """
+        Asynchronously chunk a list of documents using the instance's chunk_document method.
+
+        Args:
+            documents: List of documents to be chunked.
+
+        Returns:
+            A flattened list of chunked documents.
+        """
+
+        async def _chunk_document_async(doc: Document) -> List[Document]:
+            return await asyncio.to_thread(self.chunk_document, doc)
+
+        # Process chunking in parallel for all documents
+        chunked_lists = await asyncio.gather(*[_chunk_document_async(doc) for doc in documents])
+        # Flatten the result
+        return [chunk for sublist in chunked_lists for chunk in sublist]

agno/knowledge/reader/csv_reader.py

@@ -0,0 +1,164 @@
+import asyncio
+import csv
+import io
+from pathlib import Path
+from typing import IO, Any, List, Optional, Union
+from uuid import uuid4
+
+try:
+    import aiofiles
+except ImportError:
+    raise ImportError("`aiofiles` not installed. Please install it with `pip install aiofiles`")
+
+from agno.knowledge.chunking.row import RowChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error
+
+
+class CSVReader(Reader):
+    """Reader for CSV files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = RowChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for CSV readers."""
+        return [
+            ChunkingStrategyType.ROW_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.CSV, ContentType.XLSX, ContentType.XLS]
+
+    def read(
+        self, file: Union[Path, IO[Any]], delimiter: str = ",", quotechar: str = '"', name: Optional[str] = None
+    ) -> List[Document]:
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_debug(f"Reading: {file}")
+                file_content = file.open(newline="", mode="r", encoding=self.encoding or "utf-8")
+            else:
+                log_debug(f"Reading retrieved file: {name or file.name}")
+                file.seek(0)
+                file_content = io.StringIO(file.read().decode("utf-8"))  # type: ignore
+
+            csv_name = name or (
+                Path(file.name).stem
+                if isinstance(file, Path)
+                else (getattr(file, "name", "csv_file").split(".")[0] if hasattr(file, "name") else "csv_file")
+            )
+            csv_content = ""
+            with file_content as csvfile:
+                csv_reader = csv.reader(csvfile, delimiter=delimiter, quotechar=quotechar)
+                for row in csv_reader:
+                    csv_content += ", ".join(row) + "\n"
+
+            documents = [
+                Document(
+                    name=csv_name,
+                    id=str(uuid4()),
+                    content=csv_content,
+                )
+            ]
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+        except Exception as e:
+            log_error(f"Error reading: {getattr(file, 'name', str(file)) if isinstance(file, IO) else file}: {e}")
+            return []
+
+    async def async_read(
+        self,
+        file: Union[Path, IO[Any]],
+        delimiter: str = ",",
+        quotechar: str = '"',
+        page_size: int = 1000,
+        name: Optional[str] = None,
+    ) -> List[Document]:
+        """
+        Read a CSV file asynchronously, processing batches of rows concurrently.
+
+        Args:
+            file: Path or file-like object
+            delimiter: CSV delimiter
+            quotechar: CSV quote character
+            page_size: Number of rows per page
+
+        Returns:
+            List of Document objects
+        """
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_debug(f"Reading async: {file}")
+                async with aiofiles.open(file, mode="r", encoding="utf-8", newline="") as file_content:
+                    content = await file_content.read()
+                    file_content_io = io.StringIO(content)
+            else:
+                log_debug(f"Reading retrieved file async: {getattr(file, 'name', 'BytesIO')}")
+                file.seek(0)
+                file_content_io = io.StringIO(file.read().decode("utf-8"))
+
+            csv_name = name or (
+                Path(file.name).stem
+                if isinstance(file, Path)
+                else (getattr(file, "name", "csv_file").split(".")[0] if hasattr(file, "name") else "csv_file")
+            )
+
+            file_content_io.seek(0)
+            csv_reader = csv.reader(file_content_io, delimiter=delimiter, quotechar=quotechar)
+            rows = list(csv_reader)
+            total_rows = len(rows)
+
+            if total_rows <= 10:
+                csv_content = " ".join(", ".join(row) for row in rows)
+                documents = [
+                    Document(
+                        name=csv_name,
+                        id=str(uuid4()),
+                        content=csv_content,
+                    )
+                ]
+            else:
+                pages = []
+                for i in range(0, total_rows, page_size):
+                    pages.append(rows[i : i + page_size])
+
+                async def _process_page(page_number: int, page_rows: List[List[str]]) -> Document:
+                    """Process a page of rows into a document"""
+                    start_row = (page_number - 1) * page_size + 1
+                    page_content = " ".join(", ".join(row) for row in page_rows)
+
+                    return Document(
+                        name=csv_name,
+                        id=str(uuid4()),
+                        meta_data={"page": page_number, "start_row": start_row, "rows": len(page_rows)},
+                        content=page_content,
+                    )
+
+                documents = await asyncio.gather(
+                    *[_process_page(page_number, page) for page_number, page in enumerate(pages, start=1)]
+                )
+
+            if self.chunk:
+                documents = await self.chunk_documents_async(documents)
+
+            return documents
+        except Exception as e:
+            log_error(f"Error reading async: {getattr(file, 'name', str(file)) if isinstance(file, IO) else file}: {e}")
+            return []

agno/knowledge/reader/docx_reader.py

@@ -0,0 +1,82 @@
+import asyncio
+from pathlib import Path
+from typing import IO, Any, List, Optional, Union
+from uuid import uuid4
+
+from agno.knowledge.chunking.document import DocumentChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error
+
+try:
+    from docx import Document as DocxDocument  # type: ignore
+except ImportError:
+    raise ImportError("The `python-docx` package is not installed. Please install it via `pip install python-docx`.")
+
+
+class DocxReader(Reader):
+    """Reader for Doc/Docx files"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = DocumentChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for DOCX readers."""
+        return [
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.DOCX, ContentType.DOC]
+
+    def read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        """Read a docx file and return a list of documents"""
+        try:
+            if isinstance(file, Path):
+                if not file.exists():
+                    raise FileNotFoundError(f"Could not find file: {file}")
+                log_debug(f"Reading: {file}")
+                docx_document = DocxDocument(str(file))
+                doc_name = name or file.stem
+            else:
+                log_debug(f"Reading uploaded file: {getattr(file, 'name', 'docx_file')}")
+                docx_document = DocxDocument(file)
+                doc_name = name or (
+                    getattr(file, "name", "docx_file").split(".")[0] if hasattr(file, "name") else "docx_file"
+                )
+
+            doc_content = "\n\n".join([para.text for para in docx_document.paragraphs])
+
+            documents = [
+                Document(
+                    name=doc_name,
+                    id=str(uuid4()),
+                    content=doc_content,
+                )
+            ]
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+
+        except Exception as e:
+            log_error(f"Error reading file: {e}")
+            return []
+
+    async def async_read(self, file: Union[Path, IO[Any]], name: Optional[str] = None) -> List[Document]:
+        """Asynchronously read a docx file and return a list of documents"""
+        try:
+            return await asyncio.to_thread(self.read, file, name)
+        except Exception as e:
+            log_error(f"Error reading file asynchronously: {e}")
+            return []