agno 1.8.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

agno/{document → knowledge}/reader/website_reader.py

@@ -7,8 +7,11 @@ from urllib.parse import urljoin, urlparse
 
 import httpx
 
-from agno.document.base import Document
-from agno.document.reader.base import Reader
+from agno.knowledge.chunking.semantic import SemanticChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
 from agno.utils.log import log_debug, logger
 
 try:
@@ -28,9 +31,15 @@ class WebsiteReader(Reader):
     _urls_to_crawl: List[Tuple[str, int]] = field(default_factory=list)
 
     def __init__(
-        self, max_depth: int = 3, max_links: int = 10, timeout: int = 10, proxy: Optional[str] = None, **kwargs
+        self,
+        chunking_strategy: Optional[ChunkingStrategy] = SemanticChunking(),
+        max_depth: int = 3,
+        max_links: int = 10,
+        timeout: int = 10,
+        proxy: Optional[str] = None,
+        **kwargs,
     ):
-        super().__init__(**kwargs)
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
         self.max_depth = max_depth
         self.max_links = max_links
         self.proxy = proxy
@@ -39,6 +48,21 @@ class WebsiteReader(Reader):
         self._visited = set()
         self._urls_to_crawl = []
 
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Website readers."""
+        return [
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.URL]
+
     def delay(self, min_seconds=1, max_seconds=3):
         """
         Introduce a random delay.
@@ -316,7 +340,7 @@ class WebsiteReader(Reader):
 
         return crawler_result
 
-    def read(self, url: str) -> List[Document]:
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Reads a website and returns a list of documents.
 
@@ -338,7 +362,7 @@ class WebsiteReader(Reader):
                     documents.extend(
                         self.chunk_document(
                             Document(
-                                name=url,
+                                name=name or url,
                                 id=str(crawled_url),
                                 meta_data={"url": str(crawled_url)},
                                 content=crawled_content,
@@ -348,7 +372,7 @@ class WebsiteReader(Reader):
                 else:
                     documents.append(
                         Document(
-                            name=url,
+                            name=name or url,
                             id=str(crawled_url),
                             meta_data={"url": str(crawled_url)},
                             content=crawled_content,
@@ -359,7 +383,7 @@ class WebsiteReader(Reader):
             logger.error(f"Error reading website {url}: {e}")
             raise
 
-    async def async_read(self, url: str) -> List[Document]:
+    async def async_read(self, url: str, name: Optional[str] = None) -> List[Document]:
         """
         Asynchronously reads a website and returns a list of documents.
 
@@ -380,13 +404,16 @@ class WebsiteReader(Reader):
             async def process_document(crawled_url, crawled_content):
                 if self.chunk:
                     doc = Document(
-                        name=url, id=str(crawled_url), meta_data={"url": str(crawled_url)}, content=crawled_content
+                        name=name or url,
+                        id=str(crawled_url),
+                        meta_data={"url": str(crawled_url)},
+                        content=crawled_content,
                     )
                     return self.chunk_document(doc)
                 else:
                     return [
                         Document(
-                            name=url,
+                            name=name or url,
                             id=str(crawled_url),
                             meta_data={"url": str(crawled_url)},
                             content=crawled_content,

agno/knowledge/reader/wikipedia_reader.py

@@ -0,0 +1,59 @@
+from typing import List, Optional
+
+from agno.knowledge.chunking.fixed import FixedSizeChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_info
+
+try:
+    import wikipedia  # noqa: F401
+except ImportError:
+    raise ImportError("The `wikipedia` package is not installed. Please install it via `pip install wikipedia`.")
+
+
+class WikipediaReader(Reader):
+    auto_suggest: bool = True
+
+    def __init__(
+        self, chunking_strategy: Optional[ChunkingStrategy] = FixedSizeChunking(), auto_suggest: bool = True, **kwargs
+    ):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+        self.auto_suggest = auto_suggest
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for Wikipedia readers."""
+        return [
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.TOPIC]
+
+    def read(self, topic: str) -> List[Document]:
+        log_debug(f"Reading Wikipedia topic: {topic}")
+        summary = None
+        try:
+            summary = wikipedia.summary(topic, auto_suggest=self.auto_suggest)
+
+        except wikipedia.exceptions.PageError:
+            summary = None
+            log_info("PageError: Page not found.")
+
+        # Only create Document if we successfully got a summary
+        if summary:
+            return [
+                Document(
+                    name=topic,
+                    meta_data={"topic": topic},
+                    content=summary,
+                )
+            ]
+        return []

agno/knowledge/reader/youtube_reader.py

@@ -0,0 +1,78 @@
+import asyncio
+from typing import List, Optional
+
+from agno.knowledge.chunking.recursive import RecursiveChunking
+from agno.knowledge.chunking.strategy import ChunkingStrategy, ChunkingStrategyType
+from agno.knowledge.document.base import Document
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug, log_error, log_info
+
+try:
+    from youtube_transcript_api import YouTubeTranscriptApi
+except ImportError:
+    raise ImportError(
+        "`youtube_transcript_api` not installed. Please install it via `pip install youtube_transcript_api`."
+    )
+
+
+class YouTubeReader(Reader):
+    """Reader for YouTube video transcripts"""
+
+    def __init__(self, chunking_strategy: Optional[ChunkingStrategy] = RecursiveChunking(), **kwargs):
+        super().__init__(chunking_strategy=chunking_strategy, **kwargs)
+
+    @classmethod
+    def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
+        """Get the list of supported chunking strategies for YouTube readers."""
+        return [
+            ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.AGENTIC_CHUNKER,
+            ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.SEMANTIC_CHUNKER,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER,
+        ]
+
+    @classmethod
+    def get_supported_content_types(self) -> List[ContentType]:
+        return [ContentType.YOUTUBE]
+
+    def read(self, url: str, name: Optional[str] = None) -> List[Document]:
+        try:
+            # Extract video ID from URL
+            video_id = url.split("v=")[-1].split("&")[0]
+            log_info(f"Reading transcript for video: {video_id}")
+
+            # Get transcript
+            log_debug(f"Fetching transcript for video: {video_id}")
+            # Create an instance of YouTubeTranscriptApi
+            ytt_api = YouTubeTranscriptApi()
+            transcript_data = ytt_api.fetch(video_id)
+
+            # Combine transcript segments into full text
+            transcript_text = ""
+            for segment in transcript_data:
+                transcript_text += f"{segment.text} "
+
+            documents = [
+                Document(
+                    name=name or f"youtube_{video_id}",
+                    id=f"youtube_{video_id}",
+                    meta_data={"video_url": url, "video_id": video_id},
+                    content=transcript_text.strip(),
+                )
+            ]
+
+            if self.chunk:
+                chunked_documents = []
+                for document in documents:
+                    chunked_documents.extend(self.chunk_document(document))
+                return chunked_documents
+            return documents
+
+        except Exception as e:
+            log_error(f"Error reading transcript for {url}: {e}")
+            return []
+
+    async def async_read(self, url: str) -> List[Document]:
+        return await asyncio.get_event_loop().run_in_executor(None, self.read, url)

agno/knowledge/remote_content/remote_content.py

@@ -0,0 +1,88 @@
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from agno.cloud.aws.s3.bucket import S3Bucket
+from agno.cloud.aws.s3.object import S3Object
+
+
+@dataclass
+class S3Content:
+    def __init__(
+        self,
+        bucket_name: Optional[str] = None,
+        bucket: Optional[S3Bucket] = None,
+        key: Optional[str] = None,
+        object: Optional[S3Object] = None,
+        prefix: Optional[str] = None,
+    ):
+        self.bucket_name = bucket_name
+        self.bucket = bucket
+        self.key = key
+        self.object = object
+        self.prefix = prefix
+
+        if bucket_name is None and bucket is None:
+            raise ValueError("Either bucket_name or bucket must be provided")
+        if key is None and object is None:
+            raise ValueError("Either key or object must be provided")
+        if bucket_name is not None and bucket is not None:
+            raise ValueError("Either bucket_name or bucket must be provided, not both")
+        if key is not None and object is not None:
+            raise ValueError("Either key or object must be provided, not both")
+
+        if self.bucket_name is not None:
+            self.bucket = S3Bucket(name=self.bucket_name)
+
+    def get_config(self):
+        return {
+            "bucket_name": self.bucket_name,
+            "bucket": self.bucket,
+            "key": self.key,
+            "object": self.object,
+            "prefix": self.prefix,
+        }
+
+
+@dataclass
+class GCSContent:
+    def __init__(
+        self,
+        bucket=None,  # Type hint removed to avoid import issues
+        bucket_name: Optional[str] = None,
+        blob_name: Optional[str] = None,
+        prefix: Optional[str] = None,
+    ):
+        # Import Google Cloud Storage only when actually needed
+        try:
+            from google.cloud import storage  # type: ignore
+        except ImportError:
+            raise ImportError(
+                "The `google-cloud-storage` package is not installed. Please install it via `pip install google-cloud-storage`."
+            )
+
+        self.bucket = bucket
+        self.bucket_name = bucket_name
+        self.blob_name = blob_name
+        self.prefix = prefix
+
+        if self.bucket is None and self.bucket_name is None:
+            raise ValueError("No bucket or bucket_name provided")
+        if self.bucket is not None and self.bucket_name is not None:
+            raise ValueError("Provide either bucket or bucket_name")
+        if self.blob_name is None and self.prefix is None:
+            raise ValueError("Either blob_name or prefix must be provided")
+
+        if self.bucket is None:
+            client = storage.Client()
+            self.bucket = client.bucket(self.bucket_name)
+
+    def get_config(self):
+        return {
+            "bucket": self.bucket,
+            "bucket_name": self.bucket_name,
+            "blob_name": self.blob_name,
+            "prefix": self.prefix,
+        }
+
+
+RemoteContent = Union[S3Content, GCSContent]

agno/{reranker → knowledge/reranker}/base.py

@@ -2,7 +2,7 @@ from typing import List
 
 from pydantic import BaseModel, ConfigDict
 
-from agno.document import Document
+from agno.knowledge.document import Document
 
 
 class Reranker(BaseModel):

agno/{reranker → knowledge/reranker}/cohere.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Optional
 
-from agno.document import Document
-from agno.reranker.base import Reranker
+from agno.knowledge.document import Document
+from agno.knowledge.reranker.base import Reranker
 from agno.utils.log import logger
 
 try:

agno/{reranker → knowledge/reranker}/infinity.py

@@ -1,8 +1,8 @@
 from typing import Any, List, Optional
 from urllib.parse import urlparse
 
-from agno.document import Document
-from agno.reranker.base import Reranker
+from agno.knowledge.document import Document
+from agno.knowledge.reranker.base import Reranker
 from agno.utils.log import logger
 
 try:

agno/{reranker → knowledge/reranker}/sentence_transformer.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Optional
 
-from agno.document import Document
-from agno.reranker.base import Reranker
+from agno.knowledge.document import Document
+from agno.knowledge.reranker.base import Reranker
 from agno.utils.log import logger
 
 try:

agno/knowledge/types.py

@@ -0,0 +1,30 @@
+from enum import Enum
+
+
+class ContentType(str, Enum):
+    """Enum for content types supported by knowledge readers."""
+
+    # Generic types
+    FILE = "file"
+    URL = "url"
+    TEXT = "text"
+    TOPIC = "topic"
+    YOUTUBE = "youtube"
+
+    # Document file extensions
+    PDF = ".pdf"
+    TXT = ".txt"
+    MARKDOWN = ".md"
+    DOCX = ".docx"
+    DOC = ".doc"
+    JSON = ".json"
+
+    # Spreadsheet file extensions
+    CSV = ".csv"
+    XLSX = ".xlsx"
+    XLS = ".xls"
+
+
+def get_content_type_enum(content_type_str: str) -> ContentType:
+    """Convert a content type string to ContentType enum."""
+    return ContentType(content_type_str)

agno/knowledge/utils.py

@@ -0,0 +1,169 @@
+from typing import Dict, List
+
+from agno.knowledge.reader.reader_factory import ReaderFactory
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug
+
+
+def _get_chunker_class(strategy_type):
+    """Get the chunker class for a given strategy type without instantiation."""
+    from agno.knowledge.chunking.strategy import ChunkingStrategyType
+
+    # Map strategy types to their corresponding classes
+    strategy_class_mapping = {
+        ChunkingStrategyType.AGENTIC_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.agentic", "AgenticChunking"
+        ),
+        ChunkingStrategyType.DOCUMENT_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.document", "DocumentChunking"
+        ),
+        ChunkingStrategyType.RECURSIVE_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.recursive", "RecursiveChunking"
+        ),
+        ChunkingStrategyType.SEMANTIC_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.semantic", "SemanticChunking"
+        ),
+        ChunkingStrategyType.FIXED_SIZE_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.fixed", "FixedSizeChunking"
+        ),
+        ChunkingStrategyType.ROW_CHUNKER: lambda: _import_class("agno.knowledge.chunking.row", "RowChunking"),
+        ChunkingStrategyType.MARKDOWN_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.markdown", "MarkdownChunking"
+        ),
+    }
+
+    if strategy_type not in strategy_class_mapping:
+        raise ValueError(f"Unknown strategy type: {strategy_type}")
+
+    return strategy_class_mapping[strategy_type]()
+
+
+def _import_class(module_name: str, class_name: str):
+    """Dynamically import a class from a module."""
+    import importlib
+
+    module = importlib.import_module(module_name)
+    return getattr(module, class_name)
+
+
+def get_reader_info(reader_key: str) -> Dict:
+    """Get information about a reader without instantiating it."""
+    # Try to create the reader to get its info, but don't cache it
+    try:
+        reader_factory_method = ReaderFactory._get_reader_method(reader_key)
+
+        # Create an instance to get the class, then call class methods
+        reader_instance = reader_factory_method()
+        reader_class = reader_instance.__class__
+
+        supported_strategies = reader_class.get_supported_chunking_strategies()
+        supported_content_types = reader_class.get_supported_content_types()
+
+        return {
+            "id": reader_key,
+            "name": "".join(word.capitalize() for word in reader_key.split("_")) + "Reader",
+            "description": reader_instance.description,
+            "chunking_strategies": [
+                strategy.value for strategy in supported_strategies
+            ],  # Convert enums to string values
+            "content_types": [ct.value for ct in supported_content_types],  # Convert enums to string values
+        }
+    except ImportError as e:
+        # Skip readers with missing dependencies
+        raise ValueError(f"Reader '{reader_key}' has missing dependencies: {str(e)}")
+    except Exception as e:
+        raise ValueError(f"Unknown reader: {reader_key}. Error: {str(e)}")
+
+
+def get_all_readers_info() -> List[Dict]:
+    """Get information about all available readers."""
+    readers_info = []
+    keys = ReaderFactory.get_all_reader_keys()
+    for key in keys:
+        try:
+            reader_info = get_reader_info(key)
+            readers_info.append(reader_info)
+        except ValueError as e:
+            # Skip readers with missing dependencies or other issues
+            # Log the error but don't fail the entire request
+            log_debug(f"Skipping reader '{key}': {e}")
+            continue
+    return readers_info
+
+
+def get_content_types_to_readers_mapping() -> Dict[str, List[str]]:
+    """Get mapping of content types to list of reader IDs that support them.
+
+    Returns:
+        Dictionary mapping content type strings (ContentType enum values) to list of reader IDs.
+    """
+    content_type_mapping: Dict[str, List[str]] = {}
+    readers_info = get_all_readers_info()
+
+    for reader_info in readers_info:
+        reader_id = reader_info["id"]
+        content_types = reader_info.get("content_types", [])
+
+        for content_type in content_types:
+            if content_type not in content_type_mapping:
+                content_type_mapping[content_type] = []
+            content_type_mapping[content_type].append(reader_id)
+
+    return content_type_mapping
+
+
+def get_chunker_info(chunker_key: str) -> Dict:
+    """Get information about a chunker without instantiating it."""
+    try:
+        # Use chunking strategies directly
+        from agno.knowledge.chunking.strategy import ChunkingStrategyType
+
+        try:
+            # Use the chunker key directly as the strategy type value
+            strategy_type = ChunkingStrategyType.from_string(chunker_key)
+
+            # Get class directly without instantiation
+            chunker_class = _get_chunker_class(strategy_type)
+
+            # Extract class information
+            class_name = chunker_class.__name__
+            docstring = chunker_class.__doc__ or f"{class_name} chunking strategy"
+
+            return {
+                "key": chunker_key,
+                "class_name": class_name,
+                "name": chunker_key,
+                "description": docstring.strip(),
+                "strategy_type": strategy_type.value,
+            }
+        except ValueError:
+            raise ValueError(f"Unknown chunker key: {chunker_key}")
+
+    except ImportError as e:
+        # Skip chunkers with missing dependencies
+        raise ValueError(f"Chunker '{chunker_key}' has missing dependencies: {str(e)}")
+    except Exception as e:
+        raise ValueError(f"Unknown chunker: {chunker_key}. Error: {str(e)}")
+
+
+def get_all_content_types() -> List[ContentType]:
+    """Get all available content types as ContentType enums."""
+    return list(ContentType)
+
+
+def get_all_chunkers_info() -> List[Dict]:
+    """Get information about all available chunkers."""
+    chunkers_info = []
+
+    from agno.knowledge.chunking.strategy import ChunkingStrategyType
+
+    keys = [strategy_type.value for strategy_type in ChunkingStrategyType]
+
+    for key in keys:
+        try:
+            chunker_info = get_chunker_info(key)
+            chunkers_info.append(chunker_info)
+        except ValueError as e:
+            log_debug(f"Skipping chunker '{key}': {e}")
+            continue
+    return chunkers_info