agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/knowledge/remote_content/remote_content.py

@@ -0,0 +1,88 @@
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from agno.cloud.aws.s3.bucket import S3Bucket
+from agno.cloud.aws.s3.object import S3Object
+
+
+@dataclass
+class S3Content:
+    def __init__(
+        self,
+        bucket_name: Optional[str] = None,
+        bucket: Optional[S3Bucket] = None,
+        key: Optional[str] = None,
+        object: Optional[S3Object] = None,
+        prefix: Optional[str] = None,
+    ):
+        self.bucket_name = bucket_name
+        self.bucket = bucket
+        self.key = key
+        self.object = object
+        self.prefix = prefix
+
+        if bucket_name is None and bucket is None:
+            raise ValueError("Either bucket_name or bucket must be provided")
+        if key is None and object is None:
+            raise ValueError("Either key or object must be provided")
+        if bucket_name is not None and bucket is not None:
+            raise ValueError("Either bucket_name or bucket must be provided, not both")
+        if key is not None and object is not None:
+            raise ValueError("Either key or object must be provided, not both")
+
+        if self.bucket_name is not None:
+            self.bucket = S3Bucket(name=self.bucket_name)
+
+    def get_config(self):
+        return {
+            "bucket_name": self.bucket_name,
+            "bucket": self.bucket,
+            "key": self.key,
+            "object": self.object,
+            "prefix": self.prefix,
+        }
+
+
+@dataclass
+class GCSContent:
+    def __init__(
+        self,
+        bucket=None,  # Type hint removed to avoid import issues
+        bucket_name: Optional[str] = None,
+        blob_name: Optional[str] = None,
+        prefix: Optional[str] = None,
+    ):
+        # Import Google Cloud Storage only when actually needed
+        try:
+            from google.cloud import storage  # type: ignore
+        except ImportError:
+            raise ImportError(
+                "The `google-cloud-storage` package is not installed. Please install it via `pip install google-cloud-storage`."
+            )
+
+        self.bucket = bucket
+        self.bucket_name = bucket_name
+        self.blob_name = blob_name
+        self.prefix = prefix
+
+        if self.bucket is None and self.bucket_name is None:
+            raise ValueError("No bucket or bucket_name provided")
+        if self.bucket is not None and self.bucket_name is not None:
+            raise ValueError("Provide either bucket or bucket_name")
+        if self.blob_name is None and self.prefix is None:
+            raise ValueError("Either blob_name or prefix must be provided")
+
+        if self.bucket is None:
+            client = storage.Client()
+            self.bucket = client.bucket(self.bucket_name)
+
+    def get_config(self):
+        return {
+            "bucket": self.bucket,
+            "bucket_name": self.bucket_name,
+            "blob_name": self.blob_name,
+            "prefix": self.prefix,
+        }
+
+
+RemoteContent = Union[S3Content, GCSContent]

agno/knowledge/reranker/__init__.py

@@ -0,0 +1,3 @@
+from agno.knowledge.reranker.base import Reranker
+
+__all__ = ["Reranker"]

agno/{reranker → knowledge/reranker}/base.py

@@ -2,7 +2,7 @@ from typing import List
 
 from pydantic import BaseModel, ConfigDict
 
-from agno.document import Document
+from agno.knowledge.document import Document
 
 
 class Reranker(BaseModel):

agno/{reranker → knowledge/reranker}/cohere.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Optional
 
-from agno.document import Document
-from agno.reranker.base import Reranker
+from agno.knowledge.document import Document
+from agno.knowledge.reranker.base import Reranker
 from agno.utils.log import logger
 
 try:

agno/knowledge/reranker/infinity.py

@@ -0,0 +1,195 @@
+from typing import Any, List, Optional
+from urllib.parse import urlparse
+
+from agno.knowledge.document import Document
+from agno.knowledge.reranker.base import Reranker
+from agno.utils.log import logger
+
+try:
+    from infinity_client import AuthenticatedClient, Client
+    from infinity_client.api.default import rerank
+    from infinity_client.models import RerankInput
+except ImportError:
+    raise ImportError("infinity_client not installed, please run `pip install infinity_client`")
+
+
+class InfinityReranker(Reranker):
+    model: str = "BAAI/bge-reranker-base"
+    host: str = "localhost"
+    port: int = 7997
+    url: Optional[str] = None
+    top_n: Optional[int] = None
+    api_key: Optional[str] = None
+    verify_ssl: bool = True
+    _client: Optional[Any] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if self.url:
+            self._parse_url()
+
+    def _parse_url(self):
+        """Parse URL to extract host, port and path"""
+        if self.url:
+            parsed = urlparse(self.url)
+            if parsed.hostname:
+                self.host = parsed.hostname
+            if parsed.port:
+                self.port = parsed.port
+            # If no port specified in URL, keep default port
+
+    @property
+    def base_url(self) -> str:
+        """Construct the base URL for the Infinity server"""
+        return f"http://{self.host}:{self.port}"
+
+    @property
+    def client(self) -> Any:
+        """Get or create the infinity client"""
+        if self._client:
+            return self._client
+
+        base_url = self.base_url
+
+        if self.api_key:
+            self._client = AuthenticatedClient(base_url=base_url, token=self.api_key, verify_ssl=self.verify_ssl)
+        else:
+            self._client = Client(base_url=base_url, verify_ssl=self.verify_ssl)
+
+        return self._client
+
+    def _rerank(self, query: str, documents: List[Document]) -> List[Document]:
+        # Validate input documents and top_n
+        if not documents:
+            return []
+
+        top_n = self.top_n
+        if top_n and not (0 < top_n):
+            logger.warning(f"top_n should be a positive integer, got {self.top_n}, setting top_n to None")
+            top_n = None
+
+        compressed_docs: list[Document] = []
+
+        try:
+            # Prepare the request body for Infinity reranking
+            rerank_input = {
+                "model": self.model,
+                "query": query,
+                "documents": [doc.content for doc in documents],
+                "return_documents": False,  # We only need scores, we already have documents
+            }
+
+            # Add top_n to payload if specified
+            if top_n:
+                rerank_input["top_n"] = top_n
+
+            # Create the input object
+            body = RerankInput.from_dict(rerank_input)
+
+            # Make request to Infinity rerank endpoint using the client
+            with self.client as client:
+                result = rerank.sync(client=client, body=body)
+
+                if result is None:
+                    logger.error("Rerank request returned None")
+                    return documents
+
+                # Process the response
+                # Infinity returns results with index and relevance_score
+                if hasattr(result, "results") and result.results:
+                    for item in result.results:
+                        doc_index = item.index
+                        relevance_score = item.relevance_score
+
+                        if doc_index < len(documents):
+                            doc = documents[doc_index]
+                            doc.reranking_score = relevance_score
+                            compressed_docs.append(doc)
+
+                # Order by relevance score
+                compressed_docs.sort(
+                    key=lambda x: x.reranking_score if x.reranking_score is not None else float("-inf"),
+                    reverse=True,
+                )
+
+                # Limit to top_n if specified and not already limited by the API
+                if top_n and len(compressed_docs) > top_n:
+                    compressed_docs = compressed_docs[:top_n]
+
+        except Exception as e:
+            logger.error(f"Error connecting to Infinity server at {self.base_url}: {e}")
+            return documents
+
+        return compressed_docs
+
+    def rerank(self, query: str, documents: List[Document]) -> List[Document]:
+        try:
+            return self._rerank(query=query, documents=documents)
+        except Exception as e:
+            logger.error(f"Error reranking documents: {e}. Returning original documents")
+            return documents
+
+    async def arerank(self, query: str, documents: List[Document]) -> List[Document]:
+        """Async version of rerank"""
+        # Validate input documents and top_n
+        if not documents:
+            return []
+
+        top_n = self.top_n
+        if top_n and not (0 < top_n):
+            logger.warning(f"top_n should be a positive integer, got {self.top_n}, setting top_n to None")
+            top_n = None
+
+        compressed_docs: list[Document] = []
+
+        try:
+            # Prepare the request body for Infinity reranking
+            rerank_input = {
+                "model": self.model,
+                "query": query,
+                "documents": [doc.content for doc in documents],
+                "return_documents": False,  # We only need scores, we already have documents
+            }
+
+            # Add top_n to payload if specified
+            if top_n:
+                rerank_input["top_n"] = top_n
+
+            # Create the input object
+            body = RerankInput.from_dict(rerank_input)
+
+            # Make async request to Infinity rerank endpoint using the client
+            async with self.client as client:
+                result = await rerank.asyncio(client=client, body=body)
+
+                if result is None:
+                    logger.error("Async rerank request returned None")
+                    return documents
+
+                # Process the response
+                # Infinity returns results with index and relevance_score
+                if hasattr(result, "results") and result.results:
+                    for item in result.results:
+                        doc_index = item.index
+                        relevance_score = item.relevance_score
+
+                        if doc_index < len(documents):
+                            doc = documents[doc_index]
+                            doc.reranking_score = relevance_score
+                            compressed_docs.append(doc)
+
+                # Order by relevance score
+                compressed_docs.sort(
+                    key=lambda x: x.reranking_score if x.reranking_score is not None else float("-inf"),
+                    reverse=True,
+                )
+
+                # Limit to top_n if specified and not already limited by the API
+                if top_n and len(compressed_docs) > top_n:
+                    compressed_docs = compressed_docs[:top_n]
+
+        except Exception as e:
+            logger.error(f"Error connecting to Infinity server at {self.base_url}: {e}")
+            return documents
+
+        return compressed_docs

agno/knowledge/reranker/sentence_transformer.py

@@ -0,0 +1,54 @@
+from typing import Any, Dict, List, Optional
+
+from agno.knowledge.document import Document
+from agno.knowledge.reranker.base import Reranker
+from agno.utils.log import logger
+
+try:
+    from sentence_transformers import CrossEncoder
+except ImportError:
+    raise ImportError("`sentence-transformers` not installed, please run `pip install sentence-transformers`")
+
+
+class SentenceTransformerReranker(Reranker):
+    model: str = "BAAI/bge-reranker-v2-m3"
+    model_kwargs: Optional[Dict[str, Any]] = None
+    top_n: Optional[int] = None
+
+    def _rerank(self, query: str, documents: List[Document]) -> List[Document]:
+        if not documents:
+            return []
+
+        sentence_transformer_client = CrossEncoder(model_name_or_path=self.model, model_kwargs=self.model_kwargs)
+
+        top_n = self.top_n
+        if top_n and not (0 < top_n):
+            logger.warning(f"top_n should be a positive integer, got {self.top_n}, setting top_n to None")
+            top_n = None
+
+        compressed_docs: list[Document] = []
+
+        sentence_pairs = [[query, doc.content] for doc in documents]
+
+        scores = sentence_transformer_client.predict(sentence_pairs).tolist()
+        for index, score in enumerate(scores):
+            doc = documents[index]
+            doc.reranking_score = score
+            compressed_docs.append(doc)
+
+        compressed_docs.sort(
+            key=lambda x: x.reranking_score if x.reranking_score is not None else float("-inf"),
+            reverse=True,
+        )
+
+        if top_n:
+            compressed_docs = compressed_docs[:top_n]
+
+        return compressed_docs
+
+    def rerank(self, query: str, documents: List[Document]) -> List[Document]:
+        try:
+            return self._rerank(query=query, documents=documents)
+        except Exception as e:
+            logger.error(f"Error reranking documents: {e}. Returning original documents")
+            return documents

agno/knowledge/types.py

@@ -0,0 +1,39 @@
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class ContentType(str, Enum):
+    """Enum for content types supported by knowledge readers."""
+
+    # Generic types
+    FILE = "file"
+    URL = "url"
+    TEXT = "text"
+    TOPIC = "topic"
+    YOUTUBE = "youtube"
+
+    # Document file extensions
+    PDF = ".pdf"
+    TXT = ".txt"
+    MARKDOWN = ".md"
+    DOCX = ".docx"
+    DOC = ".doc"
+    PPTX = ".pptx"
+    JSON = ".json"
+
+    # Spreadsheet file extensions
+    CSV = ".csv"
+    XLSX = ".xlsx"
+    XLS = ".xls"
+
+
+def get_content_type_enum(content_type_str: str) -> ContentType:
+    """Convert a content type string to ContentType enum."""
+    return ContentType(content_type_str)
+
+
+class KnowledgeFilter(BaseModel):
+    key: str
+    value: Any

agno/knowledge/utils.py

@@ -0,0 +1,234 @@
+from typing import Any, Dict, List, Optional
+
+from agno.knowledge.reader.base import Reader
+from agno.knowledge.reader.reader_factory import ReaderFactory
+from agno.knowledge.types import ContentType
+from agno.utils.log import log_debug
+
+
+def _get_chunker_class(strategy_type):
+    """Get the chunker class for a given strategy type without instantiation."""
+    from agno.knowledge.chunking.strategy import ChunkingStrategyType
+
+    # Map strategy types to their corresponding classes
+    strategy_class_mapping = {
+        ChunkingStrategyType.AGENTIC_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.agentic", "AgenticChunking"
+        ),
+        ChunkingStrategyType.DOCUMENT_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.document", "DocumentChunking"
+        ),
+        ChunkingStrategyType.RECURSIVE_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.recursive", "RecursiveChunking"
+        ),
+        ChunkingStrategyType.SEMANTIC_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.semantic", "SemanticChunking"
+        ),
+        ChunkingStrategyType.FIXED_SIZE_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.fixed", "FixedSizeChunking"
+        ),
+        ChunkingStrategyType.ROW_CHUNKER: lambda: _import_class("agno.knowledge.chunking.row", "RowChunking"),
+        ChunkingStrategyType.MARKDOWN_CHUNKER: lambda: _import_class(
+            "agno.knowledge.chunking.markdown", "MarkdownChunking"
+        ),
+    }
+
+    if strategy_type not in strategy_class_mapping:
+        raise ValueError(f"Unknown strategy type: {strategy_type}")
+
+    return strategy_class_mapping[strategy_type]()
+
+
+def _import_class(module_name: str, class_name: str):
+    """Dynamically import a class from a module."""
+    import importlib
+
+    module = importlib.import_module(module_name)
+    return getattr(module, class_name)
+
+
+def get_reader_info(reader_key: str) -> Dict:
+    """Get information about a reader without instantiating it."""
+    # Try to create the reader to get its info, but don't cache it
+    try:
+        reader_factory_method = ReaderFactory._get_reader_method(reader_key)
+
+        # Create an instance to get the class, then call class methods
+        reader_instance = reader_factory_method()
+        reader_class = reader_instance.__class__
+
+        supported_strategies = reader_class.get_supported_chunking_strategies()
+        supported_content_types = reader_class.get_supported_content_types()
+
+        return {
+            "id": reader_key,
+            "name": "".join(word.capitalize() for word in reader_key.split("_")) + "Reader",
+            "description": reader_instance.description,
+            "chunking_strategies": [
+                strategy.value for strategy in supported_strategies
+            ],  # Convert enums to string values
+            "content_types": [ct.value for ct in supported_content_types],  # Convert enums to string values
+        }
+    except ImportError as e:
+        # Skip readers with missing dependencies
+        raise ValueError(f"Reader '{reader_key}' has missing dependencies: {str(e)}")
+    except Exception as e:
+        raise ValueError(f"Unknown reader: {reader_key}. Error: {str(e)}")
+
+
+def get_reader_info_from_instance(reader: Reader, reader_id: str) -> Dict:
+    """Get information about a reader instance."""
+    try:
+        reader_class = reader.__class__
+        supported_strategies = reader_class.get_supported_chunking_strategies()
+        supported_content_types = reader_class.get_supported_content_types()
+
+        return {
+            "id": reader_id,
+            "name": getattr(reader, "name", reader_class.__name__),
+            "description": getattr(reader, "description", f"Custom {reader_class.__name__}"),
+            "chunking_strategies": [strategy.value for strategy in supported_strategies],
+            "content_types": [ct.value for ct in supported_content_types],
+        }
+    except Exception as e:
+        raise ValueError(f"Failed to get info for reader '{reader_id}': {str(e)}")
+
+
+def get_all_readers_info(knowledge_instance: Optional[Any] = None) -> List[Dict]:
+    """Get information about all available readers, including custom readers from a Knowledge instance.
+
+    Args:
+        knowledge_instance: Optional Knowledge instance to include custom readers from.
+
+    Returns:
+        List of reader info dictionaries.
+    """
+    readers_info = []
+    keys = ReaderFactory.get_all_reader_keys()
+    for key in keys:
+        try:
+            reader_info = get_reader_info(key)
+            readers_info.append(reader_info)
+        except ValueError as e:
+            # Skip readers with missing dependencies or other issues
+            # Log the error but don't fail the entire request
+            log_debug(f"Skipping reader '{key}': {e}")
+            continue
+
+    # Add custom readers from knowledge instance if provided
+    if knowledge_instance is not None:
+        custom_readers = knowledge_instance.get_readers()
+        if isinstance(custom_readers, dict):
+            for reader_id, reader in custom_readers.items():
+                try:
+                    reader_info = get_reader_info_from_instance(reader, reader_id)
+                    # Only add if not already present (custom readers take precedence)
+                    if not any(r["id"] == reader_id for r in readers_info):
+                        readers_info.append(reader_info)
+                except ValueError as e:
+                    log_debug(f"Skipping custom reader '{reader_id}': {e}")
+                    continue
+
+    return readers_info
+
+
+def get_content_types_to_readers_mapping(knowledge_instance: Optional[Any] = None) -> Dict[str, List[str]]:
+    """Get mapping of content types to list of reader IDs that support them.
+
+    Args:
+        knowledge_instance: Optional Knowledge instance to include custom readers from.
+
+    Returns:
+        Dictionary mapping content type strings (ContentType enum values) to list of reader IDs.
+    """
+    content_type_mapping: Dict[str, List[str]] = {}
+    readers_info = get_all_readers_info(knowledge_instance)
+    for reader_info in readers_info:
+        reader_id = reader_info["id"]
+        content_types = reader_info.get("content_types", [])
+
+        for content_type in content_types:
+            if content_type not in content_type_mapping:
+                content_type_mapping[content_type] = []
+            # Avoid duplicates
+            if reader_id not in content_type_mapping[content_type]:
+                content_type_mapping[content_type].append(reader_id)
+
+    return content_type_mapping
+
+
+def get_chunker_info(chunker_key: str) -> Dict:
+    """Get information about a chunker without instantiating it."""
+    try:
+        # Use chunking strategies directly
+        from agno.knowledge.chunking.strategy import ChunkingStrategyType
+
+        try:
+            # Use the chunker key directly as the strategy type value
+            strategy_type = ChunkingStrategyType.from_string(chunker_key)
+
+            # Get class directly without instantiation
+            chunker_class = _get_chunker_class(strategy_type)
+
+            # Extract class information
+            class_name = chunker_class.__name__
+            docstring = chunker_class.__doc__ or f"{class_name} chunking strategy"
+
+            # Check class __init__ signature for chunk_size and overlap parameters
+            metadata = {}
+            import inspect
+
+            try:
+                sig = inspect.signature(chunker_class.__init__)
+                param_names = set(sig.parameters.keys())
+
+                # If class has chunk_size or max_chunk_size parameter, set default chunk_size
+                if "chunk_size" in param_names or "max_chunk_size" in param_names:
+                    metadata["chunk_size"] = 5000
+
+                # If class has overlap parameter, set default overlap
+                if "overlap" in param_names:
+                    metadata["chunk_overlap"] = 0
+            except Exception:
+                # If we can't inspect, skip metadata
+                pass
+
+            return {
+                "key": chunker_key,
+                "class_name": class_name,
+                "name": chunker_key,
+                "description": docstring.strip(),
+                "strategy_type": strategy_type.value,
+                "metadata": metadata,
+            }
+        except ValueError:
+            raise ValueError(f"Unknown chunker key: {chunker_key}")
+
+    except ImportError as e:
+        # Skip chunkers with missing dependencies
+        raise ValueError(f"Chunker '{chunker_key}' has missing dependencies: {str(e)}")
+    except Exception as e:
+        raise ValueError(f"Unknown chunker: {chunker_key}. Error: {str(e)}")
+
+
+def get_all_content_types() -> List[ContentType]:
+    """Get all available content types as ContentType enums."""
+    return list(ContentType)
+
+
+def get_all_chunkers_info() -> List[Dict]:
+    """Get information about all available chunkers."""
+    chunkers_info = []
+
+    from agno.knowledge.chunking.strategy import ChunkingStrategyType
+
+    keys = [strategy_type.value for strategy_type in ChunkingStrategyType]
+
+    for key in keys:
+        try:
+            chunker_info = get_chunker_info(key)
+            chunkers_info.append(chunker_info)
+        except ValueError as e:
+            log_debug(f"Skipping chunker '{key}': {e}")
+            continue
+    return chunkers_info