agno 2.0.0rc1__py3-none-any.whl → 2.0.1__py3-none-any.whl

agno/knowledge/reader/text_reader.py

@@ -21,11 +21,11 @@ class TextReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Text readers."""
         return [
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
         ]
 
     @classmethod

agno/knowledge/reader/url_reader.py

@@ -1,3 +1,6 @@
+from io import BytesIO
+from os.path import basename
+from pathlib import Path
 from typing import List, Optional
 from urllib.parse import urlparse
 
@@ -7,6 +10,8 @@ from agno.knowledge.chunking.fixed import FixedSizeChunking
 from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
 from agno.knowledge.document.base import Document
 from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.csv_reader import CSVReader
+from agno.knowledge.reader.pdf_reader import PDFReader
 from agno.knowledge.types import ContentType
 from agno.utils.http import async_fetch_with_retry, fetch_with_retry
 from agno.utils.log import log_debug
@@ -25,31 +30,43 @@ class URLReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for URL readers."""
         return [
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
         ]
 
     @classmethod
     def get_supported_content_types(self) -> List[ContentType]:
         return [ContentType.URL]
 
-    def read(self, url: str, id: Optional[str] = None, name: Optional[str] = None) -> List[Document]:
+    def read(
+        self, url: str, id: Optional[str] = None, name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         if not url:
             raise ValueError("No url provided")
 
         log_debug(f"Reading: {url}")
+
         # Retry the request up to 3 times with exponential backoff
         response = fetch_with_retry(url, proxy=self.proxy)
 
-        document = self._create_document(url, response.text, id, name)
-        if self.chunk:
-            return self.chunk_document(document)
-        return [document]
+        documents = self._create_documents(
+            url=url, text=response.text, content=response.content, id=id, name=name, password=password
+        )
+
+        if not self.chunk:
+            return documents
+
+        chunked_documents = []
+        for document in documents:
+            chunked_documents.append(self.chunk_document(document))
+        return [doc for sublist in chunked_documents for doc in sublist]
 
-    async def async_read(self, url: str, id: Optional[str] = None, name: Optional[str] = None) -> List[Document]:
+    async def async_read(
+        self, url: str, id: Optional[str] = None, name: Optional[str] = None, password: Optional[str] = None
+    ) -> List[Document]:
         """Async version of read method"""
         if not url:
             raise ValueError("No url provided")
@@ -59,26 +76,53 @@ class URLReader(Reader):
         async with httpx.AsyncClient(**client_args) as client:  # type: ignore
             response = await async_fetch_with_retry(url, client=client)
 
-        document = self._create_document(url, response.text, id, name)
-        if self.chunk:
-            return await self.chunk_documents_async([document])
-        return [document]
+        documents = self._create_documents(
+            url=url, text=response.text, content=response.content, id=id, name=name, password=password
+        )
+
+        if not self.chunk:
+            return documents
+
+        return await self.chunk_documents_async(documents)
 
-    def _create_document(
-        self, url: str, content: str, id: Optional[str] = None, name: Optional[str] = None
-    ) -> Document:
+    def _create_documents(
+        self,
+        url: str,
+        text: str,
+        content: bytes,
+        id: Optional[str] = None,
+        name: Optional[str] = None,
+        password: Optional[str] = None,
+    ) -> List[Document]:
         """Helper method to create a document from URL content"""
+
+        # Determine file extension from URL
         parsed_url = urlparse(url)
-        doc_name = name or parsed_url.path.strip("/").replace("/", "_").replace(" ", "_")
-        if not doc_name:
-            doc_name = parsed_url.netloc
-        if not doc_name:
-            doc_name = url
-
-        return Document(
-            name=doc_name,
-            id=id or doc_name,
-            meta_data={"url": url},
-            content=content,
-            size=len(content),
-        )
+        url_path = Path(parsed_url.path)  # type: ignore
+        file_extension = url_path.suffix.lower()
+
+        # Read the document using the appropriate reader
+        if file_extension == ".csv":
+            filename = basename(parsed_url.path) or "data.csv"
+            return CSVReader().read(file=BytesIO(content), name=filename)
+        elif file_extension == ".pdf":
+            if password:
+                return PDFReader().read(pdf=BytesIO(content), name=name, password=password)
+            else:
+                return PDFReader().read(pdf=BytesIO(content), name=name)
+        else:
+            doc_name = name or parsed_url.path.strip("/").replace("/", "_").replace(" ", "_")
+            if not doc_name:
+                doc_name = parsed_url.netloc
+            if not doc_name:
+                doc_name = url
+
+        return [
+            Document(
+                name=doc_name,
+                id=id or doc_name,
+                meta_data={"url": url},
+                content=text,
+                size=len(text),
+            )
+        ]

agno/knowledge/reader/web_search_reader.py

@@ -11,7 +11,6 @@ from agno.knowledge.chunking.semantic import SemanticChunking
 from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
 from agno.knowledge.document.base import Document
 from agno.knowledge.reader.base import Reader
-from agno.knowledge.reader.url_reader import URLReader
 from agno.knowledge.types import ContentType
 from agno.utils.log import log_debug, logger
 
@@ -48,30 +47,25 @@ class WebSearchReader(Reader):
 
     # Internal state
     _visited_urls: Set[str] = field(default_factory=set)
-    _url_reader: Optional[URLReader] = None
     _last_search_time: float = field(default=0.0, init=False)
 
     # Override default chunking strategy
     chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking()
 
-    def __post_init__(self):
-        """Initialize the URL reader and chunking strategy after dataclass initialization"""
-        self._url_reader = URLReader()
-
     @classmethod
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Web Search readers."""
         return [
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
         ]
 
     @classmethod
     def get_supported_content_types(self) -> List[ContentType]:
-        return [ContentType.URL, ContentType.TEXT]
+        return [ContentType.TOPIC]
 
     def _respect_rate_limits(self):
         """Ensure we don't exceed rate limits"""
@@ -328,23 +322,6 @@ class WebSearchReader(Reader):
             self._visited_urls.add(url)
 
             try:
-                # Use the URL reader for async fetching
-                if self._url_reader:
-                    docs = await self._url_reader.async_read(url)
-                    if docs:
-                        # Use the first document and add search metadata
-                        doc = docs[0]
-                        doc.meta_data.update(
-                            {
-                                "search_title": result.get("title", ""),
-                                "search_description": result.get("description", ""),
-                                "source": "web_search",
-                                "search_engine": self.search_engine,
-                            }
-                        )
-                        return doc
-
-                # Fallback to manual async fetching
                 headers = {"User-Agent": self.user_agent}
                 async with httpx.AsyncClient(timeout=self.request_timeout) as client:
                     response = await client.get(url, headers=headers, follow_redirects=True)

agno/knowledge/reader/website_reader.py

@@ -52,11 +52,11 @@ class WebsiteReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Website readers."""
         return [
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
         ]
 
     @classmethod

agno/knowledge/reader/wikipedia_reader.py

@@ -26,11 +26,11 @@ class WikipediaReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Wikipedia readers."""
         return [
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
         ]
 
     @classmethod

agno/knowledge/reader/youtube_reader.py

@@ -26,16 +26,16 @@ class YouTubeReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for YouTube readers."""
         return [
-            ChunkingStrategyType.RECURSIVE_CHUNKING,
-            ChunkingStrategyType.AGENTIC_CHUNKING,
-            ChunkingStrategyType.DOCUMENT_CHUNKING,
-            ChunkingStrategyType.SEMANTIC_CHUNKING,
-            ChunkingStrategyType.FIXED_SIZE_CHUNKING,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
         ]
 
     @classmethod
     def get_supported_content_types(self) -> List[ContentType]:
-        return [ContentType.URL, ContentType.YOUTUBE]
+        return [ContentType.YOUTUBE]
 
     def read(self, url: str, name: Optional[str] = None) -> List[Document]:
         try:

agno/knowledge/reranker/__init__.py

@@ -0,0 +1,9 @@
+from agno.knowledge.reranker.cohere import CohereReranker
+from agno.knowledge.reranker.infinity import InfinityReranker
+from agno.knowledge.reranker.sentence_transformer import SentenceTransformerReranker
+
+__all__ = [
+    "CohereReranker",
+    "InfinityReranker",
+    "SentenceTransformerReranker",
+]

agno/knowledge/utils.py

@@ -11,23 +11,23 @@ def _get_chunker_class(strategy_type):
 
     # Map strategy types to their corresponding classes
     strategy_class_mapping = {
-        ChunkingStrategyType.AGENTIC_CHUNKING: lambda: _import_class(
+        ChunkingStrategyType.AGENTIC_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.agentic", "AgenticChunking"
         ),
-        ChunkingStrategyType.DOCUMENT_CHUNKING: lambda: _import_class(
+        ChunkingStrategyType.DOCUMENT_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.document", "DocumentChunking"
         ),
-        ChunkingStrategyType.RECURSIVE_CHUNKING: lambda: _import_class(
+        ChunkingStrategyType.RECURSIVE_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.recursive", "RecursiveChunking"
         ),
-        ChunkingStrategyType.SEMANTIC_CHUNKING: lambda: _import_class(
+        ChunkingStrategyType.SEMANTIC_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.semantic", "SemanticChunking"
         ),
-        ChunkingStrategyType.FIXED_SIZE_CHUNKING: lambda: _import_class(
+        ChunkingStrategyType.FIXED_SIZE_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.fixed", "FixedSizeChunking"
         ),
-        ChunkingStrategyType.ROW_CHUNKING: lambda: _import_class("agno.knowledge.chunking.row", "RowChunking"),
-        ChunkingStrategyType.MARKDOWN_CHUNKING: lambda: _import_class(
+        ChunkingStrategyType.ROW_CHUNKER: lambda: _import_class("agno.knowledge.chunking.row", "RowChunking"),
+        ChunkingStrategyType.MARKDOWN_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.markdown", "MarkdownChunking"
         ),
     }
@@ -61,8 +61,8 @@ def get_reader_info(reader_key: str) -> Dict:
 
         return {
             "id": reader_key,
-            "name": reader_key.replace("_", " ").title() + " Reader",
-            "description": f"Reads {reader_key} files",
+            "name": "".join(word.capitalize() for word in reader_key.split("_")) + "Reader",
+            "description": reader_instance.description,
             "chunking_strategies": [
                 strategy.value for strategy in supported_strategies
             ],  # Convert enums to string values
@@ -132,7 +132,7 @@ def get_chunker_info(chunker_key: str) -> Dict:
             return {
                 "key": chunker_key,
                 "class_name": class_name,
-                "name": class_name.replace("Chunking", "").replace("Strategy", ""),
+                "name": chunker_key,
                 "description": docstring.strip(),
                 "strategy_type": strategy_type.value,
             }