agno 1.8.1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/constants.py

@@ -1,13 +0,0 @@
-PYTHONPATH_ENV_VAR: str = "PYTHONPATH"
-AGNO_RUNTIME_ENV_VAR: str = "AGNO_RUNTIME"
-AGNO_API_KEY_ENV_VAR: str = "AGNO_API_KEY"
-
-WORKSPACE_ID_ENV_VAR: str = "AGNO_WORKSPACE_ID"
-WORKSPACE_NAME_ENV_VAR: str = "AGNO_WORKSPACE_NAME"
-WORKSPACE_ROOT_ENV_VAR: str = "AGNO_WORKSPACE_ROOT"
-WORKSPACE_DIR_ENV_VAR: str = "AGNO_WORKSPACE_DIR"
-REQUIREMENTS_FILE_PATH_ENV_VAR: str = "REQUIREMENTS_FILE_PATH"
-
-AWS_REGION_ENV_VAR: str = "AWS_REGION"
-AWS_DEFAULT_REGION_ENV_VAR: str = "AWS_DEFAULT_REGION"
-AWS_PROFILE_ENV_VAR: str = "AWS_PROFILE"

agno/document/__init__.py

@@ -1,5 +0,0 @@
-from agno.document.base import Document
-
-__all__ = [
-    "Document",
-]

agno/document/chunking/semantic.py

@@ -1,45 +0,0 @@
-from typing import List, Optional
-
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
-from agno.embedder.base import Embedder
-from agno.embedder.openai import OpenAIEmbedder
-
-try:
-    from chonkie import SemanticChunker
-except ImportError:
-    raise ImportError("`chonkie` is required for semantic chunking, please install using `pip install chonkie`")
-
-
-class SemanticChunking(ChunkingStrategy):
-    """Chunking strategy that splits text into semantic chunks using chonkie"""
-
-    def __init__(self, embedder: Optional[Embedder] = None, chunk_size: int = 5000, similarity_threshold: float = 0.5):
-        self.embedder = embedder or OpenAIEmbedder(id="text-embedding-3-small")  # type: ignore
-        self.chunk_size = chunk_size
-        self.similarity_threshold = similarity_threshold
-        self.chunker = SemanticChunker(
-            embedding_model=self.embedder.id,  # type: ignore
-            chunk_size=self.chunk_size,
-            threshold=self.similarity_threshold,
-        )
-
-    def chunk(self, document: Document) -> List[Document]:
-        """Split document into semantic chunks using chokie"""
-        if not document.content:
-            return [document]
-
-        # Use chonkie to split into semantic chunks
-        chunks = self.chunker.chunk(self.clean_text(document.content))
-
-        # Convert chunks to Documents
-        chunked_documents: List[Document] = []
-        for i, chunk in enumerate(chunks, 1):
-            meta_data = document.meta_data.copy()
-            meta_data["chunk"] = i
-            chunk_id = f"{document.id}_{i}" if document.id else None
-            meta_data["chunk_size"] = len(chunk.text)
-
-            chunked_documents.append(Document(id=chunk_id, name=document.name, meta_data=meta_data, content=chunk.text))
-
-        return chunked_documents

agno/document/chunking/strategy.py

@@ -1,31 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List
-
-from agno.document.base import Document
-
-
-class ChunkingStrategy(ABC):
-    """Base class for chunking strategies"""
-
-    @abstractmethod
-    def chunk(self, document: Document) -> List[Document]:
-        raise NotImplementedError
-
-    def clean_text(self, text: str) -> str:
-        """Clean the text by replacing multiple newlines with a single newline"""
-        import re
-
-        # Replace multiple newlines with a single newline
-        cleaned_text = re.sub(r"\n+", "\n", text)
-        # Replace multiple spaces with a single space
-        cleaned_text = re.sub(r"\s+", " ", cleaned_text)
-        # Replace multiple tabs with a single tab
-        cleaned_text = re.sub(r"\t+", "\t", cleaned_text)
-        # Replace multiple carriage returns with a single carriage return
-        cleaned_text = re.sub(r"\r+", "\r", cleaned_text)
-        # Replace multiple form feeds with a single form feed
-        cleaned_text = re.sub(r"\f+", "\f", cleaned_text)
-        # Replace multiple vertical tabs with a single vertical tab
-        cleaned_text = re.sub(r"\v+", "\v", cleaned_text)
-
-        return cleaned_text

agno/document/reader/__init__.py

@@ -1,5 +0,0 @@
-from agno.document.reader.base import Reader
-
-__all__ = [
-    "Reader",
-]

agno/document/reader/base.py

@@ -1,47 +0,0 @@
-import asyncio
-from dataclasses import dataclass, field
-from typing import Any, List, Optional
-
-from agno.document.base import Document
-from agno.document.chunking.fixed import FixedSizeChunking
-from agno.document.chunking.strategy import ChunkingStrategy
-
-
-@dataclass
-class Reader:
-    """Base class for reading documents"""
-
-    chunk: bool = True
-    chunk_size: int = 5000
-    separators: List[str] = field(default_factory=lambda: ["\n", "\n\n", "\r", "\r\n", "\n\r", "\t", " ", "  "])
-    chunking_strategy: Optional[ChunkingStrategy] = None
-
-    def read(self, obj: Any) -> List[Document]:
-        raise NotImplementedError
-
-    async def async_read(self, obj: Any) -> List[Document]:
-        raise NotImplementedError
-
-    def chunk_document(self, document: Document) -> List[Document]:
-        if self.chunking_strategy is None:
-            self.chunking_strategy = FixedSizeChunking(chunk_size=self.chunk_size)
-        return self.chunking_strategy.chunk(document)  # type: ignore
-
-    async def chunk_documents_async(self, documents: List[Document]) -> List[Document]:
-        """
-        Asynchronously chunk a list of documents using the instance's chunk_document method.
-
-        Args:
-            documents: List of documents to be chunked.
-
-        Returns:
-            A flattened list of chunked documents.
-        """
-
-        async def _chunk_document_async(doc: Document) -> List[Document]:
-            return await asyncio.to_thread(self.chunk_document, doc)
-
-        # Process chunking in parallel for all documents
-        chunked_lists = await asyncio.gather(*[_chunk_document_async(doc) for doc in documents])
-        # Flatten the result
-        return [chunk for sublist in chunked_lists for chunk in sublist]

agno/document/reader/docx_reader.py

@@ -1,60 +0,0 @@
-import asyncio
-from pathlib import Path
-from typing import IO, Any, List, Union
-from uuid import uuid4
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import log_info, logger
-
-try:
-    from docx import Document as DocxDocument  # type: ignore
-except ImportError:
-    raise ImportError("The `python-docx` package is not installed. Please install it via `pip install python-docx`.")
-
-
-class DocxReader(Reader):
-    """Reader for Doc/Docx files"""
-
-    def read(self, file: Union[Path, IO[Any]]) -> List[Document]:
-        """Read a docx file and return a list of documents"""
-        try:
-            if isinstance(file, Path):
-                if not file.exists():
-                    raise FileNotFoundError(f"Could not find file: {file}")
-                log_info(f"Reading: {file}")
-                docx_document = DocxDocument(str(file))
-                doc_name = file.stem
-            else:
-                log_info(f"Reading uploaded file: {file.name}")
-                docx_document = DocxDocument(file)
-                doc_name = file.name.split(".")[0]
-
-            doc_content = "\n\n".join([para.text for para in docx_document.paragraphs])
-
-            documents = [
-                Document(
-                    name=doc_name,
-                    id=str(uuid4()),
-                    content=doc_content,
-                )
-            ]
-
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-
-        except Exception as e:
-            logger.error(f"Error reading file: {e}")
-            return []
-
-    async def async_read(self, file: Union[Path, IO[Any]]) -> List[Document]:
-        """Asynchronously read a docx file and return a list of documents"""
-        try:
-            return await asyncio.to_thread(self.read, file)
-        except Exception as e:
-            logger.error(f"Error reading file asynchronously: {e}")
-            return []

agno/document/reader/gcs/pdf_reader.py

@@ -1,44 +0,0 @@
-import asyncio
-from io import BytesIO
-from typing import List
-from uuid import uuid4
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import log_info
-
-try:
-    from google.cloud import storage
-except ImportError:
-    raise ImportError("`google-cloud-storage` not installed. Please install it via `pip install google-cloud-storage`.")
-
-try:
-    from pypdf import PdfReader as DocumentReader
-except ImportError:
-    raise ImportError("`pypdf` not installed. Please install it via `pip install pypdf`.")
-
-
-class GCSPDFReader(Reader):
-    def read(self, blob: storage.Blob) -> List[Document]:
-        log_info(f"Reading: gs://{blob.bucket.name}/{blob.name}")
-        data = blob.download_as_bytes()
-        doc_name = blob.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-        doc_reader = DocumentReader(BytesIO(data))
-        documents = [
-            Document(
-                name=doc_name,
-                id=str(uuid4()),
-                meta_data={"page": page_number},
-                content=page.extract_text(),
-            )
-            for page_number, page in enumerate(doc_reader.pages, start=1)
-        ]
-        if self.chunk:
-            chunked_documents = []
-            for document in documents:
-                chunked_documents.extend(self.chunk_document(document))
-            return chunked_documents
-        return documents
-
-    async def async_read(self, blob: storage.Blob) -> List[Document]:
-        return await asyncio.to_thread(self.read, blob)

agno/document/reader/s3/pdf_reader.py

@@ -1,59 +0,0 @@
-import asyncio
-from io import BytesIO
-from typing import List
-from uuid import uuid4
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import log_info
-
-try:
-    from agno.aws.resource.s3.object import S3Object  # type: ignore
-except (ModuleNotFoundError, ImportError):
-    raise ImportError("`agno-aws` not installed. Please install using `pip install agno-aws`")
-
-try:
-    from pypdf import PdfReader as DocumentReader  # noqa: F401
-except ImportError:
-    raise ImportError("`pypdf` not installed. Please install it via `pip install pypdf`.")
-
-
-class S3PDFReader(Reader):
-    """Reader for PDF files on S3"""
-
-    def read(self, s3_object: S3Object) -> List[Document]:
-        try:
-            log_info(f"Reading: {s3_object.uri}")
-
-            object_resource = s3_object.get_resource()
-            object_body = object_resource.get()["Body"]
-            doc_name = s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-            doc_reader = DocumentReader(BytesIO(object_body.read()))
-            documents = [
-                Document(
-                    name=doc_name,
-                    id=str(uuid4()),
-                    meta_data={"page": page_number},
-                    content=page.extract_text(),
-                )
-                for page_number, page in enumerate(doc_reader.pages, start=1)
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-        except Exception:
-            raise
-
-    async def async_read(self, s3_object: S3Object) -> List[Document]:
-        """Asynchronously read PDF files from S3 by running the synchronous read operation in a thread.
-
-        Args:
-            s3_object (S3Object): The S3 object to read
-
-        Returns:
-            List[Document]: List of documents from the PDF file
-        """
-        return await asyncio.to_thread(self.read, s3_object)

agno/document/reader/s3/text_reader.py

@@ -1,63 +0,0 @@
-import asyncio
-from pathlib import Path
-from typing import List
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import log_debug, log_info, logger
-
-try:
-    from agno.aws.resource.s3.object import S3Object  # type: ignore
-except (ModuleNotFoundError, ImportError):
-    raise ImportError("`agno-aws` not installed. Please install using `pip install agno-aws`")
-
-try:
-    import textract  # noqa: F401
-except ImportError:
-    raise ImportError("`textract` not installed. Please install it via `pip install textract`.")
-
-
-class S3TextReader(Reader):
-    """Reader for text files on S3"""
-
-    def read(self, s3_object: S3Object) -> List[Document]:
-        try:
-            log_info(f"Reading: {s3_object.uri}")
-
-            obj_name = s3_object.name.split("/")[-1]
-            temporary_file = Path("storage").joinpath(obj_name)
-            s3_object.download(temporary_file)
-
-            log_info(f"Parsing: {temporary_file}")
-            doc_name = s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-            doc_content = textract.process(temporary_file)
-            documents = [
-                Document(
-                    name=doc_name,
-                    id=doc_name,
-                    content=doc_content.decode("utf-8"),
-                )
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-
-            log_debug(f"Deleting: {temporary_file}")
-            temporary_file.unlink()
-            return documents
-        except Exception as e:
-            logger.error(f"Error reading: {s3_object.uri}: {e}")
-        return []
-
-    async def async_read(self, s3_object: S3Object) -> List[Document]:
-        """Asynchronously read text files from S3 by running the synchronous read operation in a thread.
-
-        Args:
-            s3_object (S3Object): The S3 object to read
-
-        Returns:
-            List[Document]: List of documents from the text file
-        """
-        return await asyncio.to_thread(self.read, s3_object)

agno/document/reader/url_reader.py

@@ -1,59 +0,0 @@
-from typing import List, Optional
-from urllib.parse import urlparse
-
-import httpx
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.http import async_fetch_with_retry, fetch_with_retry
-from agno.utils.log import log_debug
-
-
-class URLReader(Reader):
-    """Reader for general URL content"""
-
-    def __init__(self, proxy: Optional[str] = None, **kwargs):
-        super().__init__(**kwargs)
-        self.proxy = proxy
-
-    def read(self, url: str) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        log_debug(f"Reading: {url}")
-        # Retry the request up to 3 times with exponential backoff
-        response = fetch_with_retry(url, proxy=self.proxy)
-
-        document = self._create_document(url, response.text)
-        if self.chunk:
-            return self.chunk_document(document)
-        return [document]
-
-    async def async_read(self, url: str) -> List[Document]:
-        """Async version of read method"""
-        if not url:
-            raise ValueError("No url provided")
-
-        log_debug(f"Reading async: {url}")
-        client_args = {"proxy": self.proxy} if self.proxy else {}
-        async with httpx.AsyncClient(**client_args) as client:  # type: ignore
-            response = await async_fetch_with_retry(url, client=client)
-
-        document = self._create_document(url, response.text)
-        if self.chunk:
-            return await self.chunk_documents_async([document])
-        return [document]
-
-    def _create_document(self, url: str, content: str) -> Document:
-        """Helper method to create a document from URL content"""
-        parsed_url = urlparse(url)
-        doc_name = parsed_url.path.strip("/").replace("/", "_").replace(" ", "_")
-        if not doc_name:
-            doc_name = parsed_url.netloc
-
-        return Document(
-            name=doc_name,
-            id=doc_name,
-            meta_data={"url": url},
-            content=content,
-        )

agno/document/reader/youtube_reader.py

@@ -1,58 +0,0 @@
-import asyncio
-from typing import List
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import log_debug, log_info, logger
-
-try:
-    from youtube_transcript_api import YouTubeTranscriptApi
-except ImportError:
-    raise ImportError(
-        "`youtube_transcript_api` not installed. Please install it via `pip install youtube_transcript_api`."
-    )
-
-
-class YouTubeReader(Reader):
-    """Reader for YouTube video transcripts"""
-
-    def read(self, video_url: str) -> List[Document]:
-        try:
-            # Extract video ID from URL
-            video_id = video_url.split("v=")[-1].split("&")[0]
-            log_info(f"Reading transcript for video: {video_id}")
-
-            # Get transcript
-            log_debug(f"Fetching transcript for video: {video_id}")
-            # Create an instance of YouTubeTranscriptApi
-            ytt_api = YouTubeTranscriptApi()
-            transcript_data = ytt_api.fetch(video_id)
-
-            # Combine transcript segments into full text
-            transcript_text = ""
-
-            for segment in transcript_data:
-                transcript_text += f"{segment.text} "
-
-            documents = [
-                Document(
-                    name=f"youtube_{video_id}",
-                    id=f"youtube_{video_id}",
-                    meta_data={"video_url": video_url, "video_id": video_id},
-                    content=transcript_text.strip(),
-                )
-            ]
-
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-
-        except Exception as e:
-            logger.error(f"Error reading transcript for {video_url}: {e}")
-            return []
-
-    async def async_read(self, video_url: str) -> List[Document]:
-        return await asyncio.get_event_loop().run_in_executor(None, self.read, video_url)

agno/embedder/__init__.py

@@ -1,5 +0,0 @@
-from agno.embedder.base import Embedder
-
-__all__ = [
-    "Embedder",
-]

agno/embedder/langdb.py

@@ -1,80 +0,0 @@
-from os import getenv
-from typing import Any, Dict, List, Optional, Tuple
-
-from typing_extensions import Literal
-
-from agno.embedder.base import Embedder
-from agno.utils.log import logger
-
-try:
-    from openai import OpenAI as OpenAIClient
-    from openai.types.create_embedding_response import CreateEmbeddingResponse
-except ImportError:
-    raise ImportError("`openai` not installed")
-
-
-class LangDBEmbedder(Embedder):
-    model: str = "text-embedding-ada-002"
-    dimensions: int = 1536
-    encoding_format: Literal["float", "base64"] = "float"
-    user: Optional[str] = None
-    api_key: Optional[str] = getenv("LANGDB_API_KEY")
-    project_id: Optional[str] = getenv("LANGDB_PROJECT_ID")
-    organization: Optional[str] = None
-    base_url: Optional[str] = None
-    request_params: Optional[Dict[str, Any]] = None
-    client_params: Optional[Dict[str, Any]] = None
-    openai_client: Optional[OpenAIClient] = None
-
-    @property
-    def client(self) -> OpenAIClient:
-        if self.openai_client:
-            return self.openai_client
-
-        if not self.project_id:
-            raise ValueError("LANGDB_PROJECT_ID not set in the environment")
-
-        _client_params: Dict[str, Any] = {}
-        if self.api_key:
-            _client_params["api_key"] = self.api_key
-        if self.organization:
-            _client_params["organization"] = self.organization
-
-        if not self.base_url:
-            self.base_url = f"https://api.us-east-1.langdb.ai/{self.project_id}/v1"
-        _client_params["base_url"] = self.base_url
-
-        if self.client_params:
-            _client_params.update(self.client_params)
-        return OpenAIClient(**_client_params)
-
-    def response(self, text: str) -> CreateEmbeddingResponse:
-        _request_params: Dict[str, Any] = {
-            "input": text,
-            "model": self.model,
-            "encoding_format": self.encoding_format,
-        }
-        if self.user is not None:
-            _request_params["user"] = self.user
-        if self.model.startswith("text-embedding-3"):
-            _request_params["dimensions"] = self.dimensions
-        if self.request_params:
-            _request_params.update(self.request_params)
-        return self.client.embeddings.create(**_request_params)
-
-    def get_embedding(self, text: str) -> List[float]:
-        response: CreateEmbeddingResponse = self.response(text=text)
-        try:
-            return response.data[0].embedding
-        except Exception as e:
-            logger.warning(e)
-            return []
-
-    def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Optional[Dict]]:
-        response: CreateEmbeddingResponse = self.response(text=text)
-
-        embedding = response.data[0].embedding
-        usage = response.usage
-        if usage:
-            return embedding, usage.model_dump()
-        return embedding, None

agno/embedder/mistral.py

@@ -1,82 +0,0 @@
-from dataclasses import dataclass
-from os import getenv
-from typing import Any, Dict, List, Optional, Tuple
-
-from agno.embedder.base import Embedder
-from agno.utils.log import logger
-
-try:
-    from mistralai import Mistral
-    from mistralai.models.embeddingresponse import EmbeddingResponse
-except ImportError:
-    raise ImportError("`mistralai` not installed")
-
-
-@dataclass
-class MistralEmbedder(Embedder):
-    id: str = "mistral-embed"
-    dimensions: int = 1024
-    # -*- Request parameters
-    request_params: Optional[Dict[str, Any]] = None
-    # -*- Client parameters
-    api_key: Optional[str] = getenv("MISTRAL_API_KEY")
-    endpoint: Optional[str] = None
-    max_retries: Optional[int] = None
-    timeout: Optional[int] = None
-    client_params: Optional[Dict[str, Any]] = None
-    # -*- Provide the Mistral Client manually
-    mistral_client: Optional[Mistral] = None
-
-    @property
-    def client(self) -> Mistral:
-        if self.mistral_client:
-            return self.mistral_client
-
-        _client_params: Dict[str, Any] = {
-            "api_key": self.api_key,
-            "endpoint": self.endpoint,
-            "max_retries": self.max_retries,
-            "timeout": self.timeout,
-        }
-        _client_params = {k: v for k, v in _client_params.items() if v is not None}
-
-        if self.client_params:
-            _client_params.update(self.client_params)
-
-        self.mistral_client = Mistral(**_client_params)
-
-        return self.mistral_client
-
-    def _response(self, text: str) -> EmbeddingResponse:
-        _request_params: Dict[str, Any] = {
-            "inputs": text,
-            "model": self.id,
-        }
-        if self.request_params:
-            _request_params.update(self.request_params)
-        response = self.client.embeddings.create(**_request_params)
-        if response is None:
-            raise ValueError("Failed to get embedding response")
-        return response
-
-    def get_embedding(self, text: str) -> List[float]:
-        try:
-            response: EmbeddingResponse = self._response(text=text)
-            if response.data and response.data[0].embedding:
-                return response.data[0].embedding
-            return []
-        except Exception as e:
-            logger.warning(f"Error getting embedding: {e}")
-            return []
-
-    def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Dict[str, Any]]:
-        try:
-            response: EmbeddingResponse = self._response(text=text)
-            embedding: List[float] = (
-                response.data[0].embedding if (response.data and response.data[0].embedding) else []
-            )
-            usage: Dict[str, Any] = response.usage.model_dump() if response.usage else {}
-            return embedding, usage
-        except Exception as e:
-            logger.warning(f"Error getting embedding and usage: {e}")
-            return [], {}