agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/document/reader/s3/pdf_reader.py

@@ -1,46 +0,0 @@
-from io import BytesIO
-from typing import List
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-try:
-    from agno.aws.resource.s3.object import S3Object  # type: ignore
-except (ModuleNotFoundError, ImportError):
-    raise ImportError("`agno-aws` not installed. Please install using `pip install agno-aws`")
-
-try:
-    from pypdf import PdfReader as DocumentReader  # noqa: F401
-except ImportError:
-    raise ImportError("`pypdf` not installed. Please install it via `pip install pypdf`.")
-
-
-class S3PDFReader(Reader):
-    """Reader for PDF files on S3"""
-
-    def read(self, s3_object: S3Object) -> List[Document]:
-        try:
-            logger.info(f"Reading: {s3_object.uri}")
-
-            object_resource = s3_object.get_resource()
-            object_body = object_resource.get()["Body"]
-            doc_name = s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-            doc_reader = DocumentReader(BytesIO(object_body.read()))
-            documents = [
-                Document(
-                    name=doc_name,
-                    id=f"{doc_name}_{page_number}",
-                    meta_data={"page": page_number},
-                    content=page.extract_text(),
-                )
-                for page_number, page in enumerate(doc_reader.pages, start=1)
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-        except Exception:
-            raise

agno/document/reader/s3/text_reader.py

@@ -1,51 +0,0 @@
-from pathlib import Path
-from typing import List
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-try:
-    from agno.aws.resource.s3.object import S3Object  # type: ignore
-except (ModuleNotFoundError, ImportError):
-    raise ImportError("`agno-aws` not installed. Please install using `pip install agno-aws`")
-
-try:
-    import textract  # noqa: F401
-except ImportError:
-    raise ImportError("`textract` not installed. Please install it via `pip install textract`.")
-
-
-class S3TextReader(Reader):
-    """Reader for text files on S3"""
-
-    def read(self, s3_object: S3Object) -> List[Document]:
-        try:
-            logger.info(f"Reading: {s3_object.uri}")
-
-            obj_name = s3_object.name.split("/")[-1]
-            temporary_file = Path("storage").joinpath(obj_name)
-            s3_object.download(temporary_file)
-
-            logger.info(f"Parsing: {temporary_file}")
-            doc_name = s3_object.name.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-            doc_content = textract.process(temporary_file)
-            documents = [
-                Document(
-                    name=doc_name,
-                    id=doc_name,
-                    content=doc_content.decode("utf-8"),
-                )
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-
-            logger.debug(f"Deleting: {temporary_file}")
-            temporary_file.unlink()
-            return documents
-        except Exception as e:
-            logger.error(f"Error reading: {s3_object.uri}: {e}")
-        return []

agno/document/reader/text_reader.py

@@ -1,41 +0,0 @@
-from pathlib import Path
-from typing import IO, Any, List, Union
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-
-class TextReader(Reader):
-    """Reader for Text files"""
-
-    def read(self, file: Union[Path, IO[Any]]) -> List[Document]:
-        try:
-            if isinstance(file, Path):
-                if not file.exists():
-                    raise FileNotFoundError(f"Could not find file: {file}")
-                logger.info(f"Reading: {file}")
-                file_name = file.stem
-                file_contents = file.read_text()
-            else:
-                logger.info(f"Reading uploaded file: {file.name}")
-                file_name = file.name.split(".")[0]
-                file.seek(0)
-                file_contents = file.read().decode("utf-8")
-
-            documents = [
-                Document(
-                    name=file_name,
-                    id=file_name,
-                    content=file_contents,
-                )
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-        except Exception as e:
-            logger.error(f"Error reading: {file}: {e}")
-            return []

agno/document/reader/website_reader.py

@@ -1,175 +0,0 @@
-import random
-import time
-from dataclasses import dataclass, field
-from typing import Dict, List, Set, Tuple
-from urllib.parse import urljoin, urlparse
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-try:
-    from bs4 import BeautifulSoup  # noqa: F401
-except ImportError:
-    raise ImportError("The `bs4` package is not installed. Please install it via `pip install beautifulsoup4`.")
-
-try:
-    import httpx
-except ImportError:
-    raise ImportError("`httpx` not installed. Please install it via `pip install httpx`.")
-
-
-@dataclass
-class WebsiteReader(Reader):
-    """Reader for Websites"""
-
-    max_depth: int = 3
-    max_links: int = 10
-
-    _visited: Set[str] = field(default_factory=set)
-    _urls_to_crawl: List[Tuple[str, int]] = field(default_factory=list)
-
-    def delay(self, min_seconds=1, max_seconds=3):
-        """
-        Introduce a random delay.
-
-        :param min_seconds: Minimum number of seconds to delay. Default is 1.
-        :param max_seconds: Maximum number of seconds to delay. Default is 3.
-        """
-        sleep_time = random.uniform(min_seconds, max_seconds)
-        time.sleep(sleep_time)
-
-    def _get_primary_domain(self, url: str) -> str:
-        """
-        Extract primary domain from the given URL.
-
-        :param url: The URL to extract the primary domain from.
-        :return: The primary domain.
-        """
-        domain_parts = urlparse(url).netloc.split(".")
-        # Return primary domain (excluding subdomains)
-        return ".".join(domain_parts[-2:])
-
-    def _extract_main_content(self, soup: BeautifulSoup) -> str:
-        """
-        Extracts the main content from a BeautifulSoup object.
-
-        :param soup: The BeautifulSoup object to extract the main content from.
-        :return: The main content.
-        """
-        # Try to find main content by specific tags or class names
-        for tag in ["article", "main"]:
-            element = soup.find(tag)
-            if element:
-                return element.get_text(strip=True, separator=" ")
-
-        for class_name in ["content", "main-content", "post-content"]:
-            element = soup.find(class_=class_name)
-            if element:
-                return element.get_text(strip=True, separator=" ")
-
-        return ""
-
-    def crawl(self, url: str, starting_depth: int = 1) -> Dict[str, str]:
-        """
-        Crawls a website and returns a dictionary of URLs and their corresponding content.
-
-        Parameters:
-        - url (str): The starting URL to begin the crawl.
-        - starting_depth (int, optional): The starting depth level for the crawl. Defaults to 1.
-
-        Returns:
-        - Dict[str, str]: A dictionary where each key is a URL and the corresponding value is the main
-                          content extracted from that URL.
-
-        Note:
-        The function focuses on extracting the main content by prioritizing content inside common HTML tags
-        like `<article>`, `<main>`, and `<div>` with class names such as "content", "main-content", etc.
-        The crawler will also respect the `max_depth` attribute of the WebCrawler class, ensuring it does not
-        crawl deeper than the specified depth.
-        """
-        num_links = 0
-        crawler_result: Dict[str, str] = {}
-        primary_domain = self._get_primary_domain(url)
-        # Add starting URL with its depth to the global list
-        self._urls_to_crawl.append((url, starting_depth))
-        while self._urls_to_crawl:
-            # Unpack URL and depth from the global list
-            current_url, current_depth = self._urls_to_crawl.pop(0)
-
-            # Skip if
-            # - URL is already visited
-            # - does not end with the primary domain,
-            # - exceeds max depth
-            # - exceeds max links
-            if (
-                current_url in self._visited
-                or not urlparse(current_url).netloc.endswith(primary_domain)
-                or current_depth > self.max_depth
-                or num_links >= self.max_links
-            ):
-                continue
-
-            self._visited.add(current_url)
-            self.delay()
-
-            try:
-                logger.debug(f"Crawling: {current_url}")
-                response = httpx.get(current_url, timeout=10)
-                soup = BeautifulSoup(response.content, "html.parser")
-
-                # Extract main content
-                main_content = self._extract_main_content(soup)
-                if main_content:
-                    crawler_result[current_url] = main_content
-                    num_links += 1
-
-                # Add found URLs to the global list, with incremented depth
-                for link in soup.find_all("a", href=True):
-                    full_url = urljoin(current_url, link["href"])
-                    parsed_url = urlparse(full_url)
-                    if parsed_url.netloc.endswith(primary_domain) and not any(
-                        parsed_url.path.endswith(ext) for ext in [".pdf", ".jpg", ".png"]
-                    ):
-                        if full_url not in self._visited and (full_url, current_depth + 1) not in self._urls_to_crawl:
-                            self._urls_to_crawl.append((full_url, current_depth + 1))
-
-            except Exception as e:
-                logger.debug(f"Failed to crawl: {current_url}: {e}")
-                pass
-
-        return crawler_result
-
-    def read(self, url: str) -> List[Document]:
-        """
-        Reads a website and returns a list of documents.
-
-        This function first converts the website into a dictionary of URLs and their corresponding content.
-        Then iterates through the dictionary and returns chunks of content.
-
-        :param url: The URL of the website to read.
-        :return: A list of documents.
-        """
-
-        logger.debug(f"Reading: {url}")
-        crawler_result = self.crawl(url)
-        documents = []
-        for crawled_url, crawled_content in crawler_result.items():
-            if self.chunk:
-                documents.extend(
-                    self.chunk_document(
-                        Document(
-                            name=url, id=str(crawled_url), meta_data={"url": str(crawled_url)}, content=crawled_content
-                        )
-                    )
-                )
-            else:
-                documents.append(
-                    Document(
-                        name=url,
-                        id=str(crawled_url),
-                        meta_data={"url": str(crawled_url)},
-                        content=crawled_content,
-                    )
-                )
-        return documents

agno/document/reader/youtube_reader.py

@@ -1,50 +0,0 @@
-from typing import List
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-try:
-    from youtube_transcript_api import YouTubeTranscriptApi
-except ImportError:
-    raise ImportError(
-        "`youtube_transcript_api` not installed. Please install it via `pip install youtube_transcript_api`."
-    )
-
-
-class YouTubeReader(Reader):
-    """Reader for YouTube video transcripts"""
-
-    def read(self, video_url: str) -> List[Document]:
-        try:
-            # Extract video ID from URL
-            video_id = video_url.split("v=")[-1].split("&")[0]
-            logger.info(f"Reading transcript for video: {video_id}")
-
-            # Get transcript
-            transcript_list = YouTubeTranscriptApi.get_transcript(video_id)
-
-            # Combine transcript segments into full text
-            transcript_text = ""
-            for segment in transcript_list:
-                transcript_text += f"{segment['text']} "
-
-            documents = [
-                Document(
-                    name=f"youtube_{video_id}",
-                    id=f"youtube_{video_id}",
-                    meta_data={"video_url": video_url, "video_id": video_id},
-                    content=transcript_text.strip(),
-                )
-            ]
-
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-
-        except Exception as e:
-            logger.error(f"Error reading transcript for {video_url}: {e}")
-            return []

agno/embedder/__init__.py

@@ -1 +0,0 @@
-from agno.embedder.base import Embedder

agno/embedder/azure_openai.py

@@ -1,86 +0,0 @@
-from os import getenv
-from typing import Any, Dict, List, Optional, Tuple
-
-from typing_extensions import Literal
-
-from agno.embedder.base import Embedder
-from agno.utils.log import logger
-
-try:
-    from openai import AzureOpenAI as AzureOpenAIClient
-    from openai.types.create_embedding_response import CreateEmbeddingResponse
-except ImportError:
-    raise ImportError("`openai` not installed")
-
-
-class AzureOpenAIEmbedder(Embedder):
-    id: str = "text-embedding-3-small"  # This has to match the model that you deployed at the provided URL
-
-    dimensions: int = 1536
-    encoding_format: Literal["float", "base64"] = "float"
-    user: Optional[str] = None
-    api_key: Optional[str] = getenv("AZURE_EMBEDDER_OPENAI_API_KEY")
-    api_version: str = getenv("AZURE_EMBEDDER_OPENAI_API_VERSION", "2024-10-21")
-    azure_endpoint: Optional[str] = getenv("AZURE_EMBEDDER_OPENAI_ENDPOINT")
-    azure_deployment: Optional[str] = getenv("AZURE_EMBEDDER_DEPLOYMENT")
-    base_url: Optional[str] = None
-    azure_ad_token: Optional[str] = None
-    azure_ad_token_provider: Optional[Any] = None
-    organization: Optional[str] = None
-    request_params: Optional[Dict[str, Any]] = None
-    client_params: Optional[Dict[str, Any]] = None
-    openai_client: Optional[AzureOpenAIClient] = None
-
-    @property
-    def client(self) -> AzureOpenAIClient:
-        if self.openai_client:
-            return self.openai_client
-
-        _client_params: Dict[str, Any] = {}
-        if self.api_key:
-            _client_params["api_key"] = self.api_key
-        if self.api_version:
-            _client_params["api_version"] = self.api_version
-        if self.organization:
-            _client_params["organization"] = self.organization
-        if self.azure_endpoint:
-            _client_params["azure_endpoint"] = self.azure_endpoint
-        if self.azure_deployment:
-            _client_params["azure_deployment"] = self.azure_deployment
-        if self.base_url:
-            _client_params["base_url"] = self.base_url
-        if self.azure_ad_token:
-            _client_params["azure_ad_token"] = self.azure_ad_token
-        if self.azure_ad_token_provider:
-            _client_params["azure_ad_token_provider"] = self.azure_ad_token_provider
-        return AzureOpenAIClient(**_client_params)
-
-    def _response(self, text: str) -> CreateEmbeddingResponse:
-        _request_params: Dict[str, Any] = {
-            "input": text,
-            "model": self.id,
-            "encoding_format": self.encoding_format,
-        }
-        if self.user is not None:
-            _request_params["user"] = self.user
-        if self.id.startswith("text-embedding-3"):
-            _request_params["dimensions"] = self.dimensions
-        if self.request_params:
-            _request_params.update(self.request_params)
-
-        return self.client.embeddings.create(**_request_params)
-
-    def get_embedding(self, text: str) -> List[float]:
-        response: CreateEmbeddingResponse = self._response(text=text)
-        try:
-            return response.data[0].embedding
-        except Exception as e:
-            logger.warning(e)
-            return []
-
-    def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Optional[Dict]]:
-        response: CreateEmbeddingResponse = self._response(text=text)
-
-        embedding = response.data[0].embedding
-        usage = response.usage
-        return embedding, usage.model_dump()

agno/embedder/cohere.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-from agno.embedder.base import Embedder
-from agno.utils.log import logger
-
-try:
-    from cohere import Client as CohereClient
-    from cohere.types.embed_response import EmbeddingsByTypeEmbedResponse, EmbeddingsFloatsEmbedResponse
-except ImportError:
-    raise ImportError("`cohere` not installed. Please install using `pip install cohere`.")
-
-
-@dataclass
-class CohereEmbedder(Embedder):
-    id: str = "embed-english-v3.0"
-    input_type: str = "search_query"
-    embedding_types: Optional[List[str]] = None
-    api_key: Optional[str] = None
-    request_params: Optional[Dict[str, Any]] = None
-    client_params: Optional[Dict[str, Any]] = None
-    cohere_client: Optional[CohereClient] = None
-
-    @property
-    def client(self) -> CohereClient:
-        if self.cohere_client:
-            return self.cohere_client
-        client_params: Dict[str, Any] = {}
-        if self.api_key:
-            client_params["api_key"] = self.api_key
-        return CohereClient(**client_params)
-
-    def response(self, text: str) -> Union[EmbeddingsFloatsEmbedResponse, EmbeddingsByTypeEmbedResponse]:
-        request_params: Dict[str, Any] = {}
-
-        if self.id:
-            request_params["model"] = self.id
-        if self.input_type:
-            request_params["input_type"] = self.input_type
-        if self.embedding_types:
-            request_params["embedding_types"] = self.embedding_types
-        if self.request_params:
-            request_params.update(self.request_params)
-        return self.client.embed(texts=[text], **request_params)
-
-    def get_embedding(self, text: str) -> List[float]:
-        response: Union[EmbeddingsFloatsEmbedResponse, EmbeddingsByTypeEmbedResponse] = self.response(text=text)
-        try:
-            if isinstance(response, EmbeddingsFloatsEmbedResponse):
-                return response.embeddings[0]
-            elif isinstance(response, EmbeddingsByTypeEmbedResponse):
-                return response.embeddings.float_[0] if response.embeddings.float_ else []
-            else:
-                logger.warning("No embeddings found")
-                return []
-        except Exception as e:
-            logger.warning(e)
-            return []
-
-    def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Optional[Dict[str, Any]]]:
-        response: Union[EmbeddingsFloatsEmbedResponse, EmbeddingsByTypeEmbedResponse] = self.response(text=text)
-
-        embedding: List[float] = []
-        if isinstance(response, EmbeddingsFloatsEmbedResponse):
-            embedding = response.embeddings[0]
-        elif isinstance(response, EmbeddingsByTypeEmbedResponse):
-            embedding = response.embeddings.float_[0] if response.embeddings.float_ else []
-
-        usage = response.meta.billed_units if response.meta else None
-        if usage:
-            return embedding, usage.model_dump()
-        return embedding, None

agno/embedder/fastembed.py

@@ -1,37 +0,0 @@
-from dataclasses import dataclass
-from typing import Dict, List, Optional, Tuple
-
-from agno.embedder.base import Embedder
-from agno.utils.log import logger
-
-try:
-    from fastembed import TextEmbedding  # type: ignore
-
-except ImportError:
-    raise ImportError("fastembed not installed, use pip install fastembed")
-
-
-@dataclass
-class FastEmbedEmbedder(Embedder):
-    """Using BAAI/bge-small-en-v1.5 model, more models available: https://qdrant.github.io/fastembed/examples/Supported_Models/"""
-
-    id: str = "BAAI/bge-small-en-v1.5"
-    dimensions: int = 384
-
-    def get_embedding(self, text: str) -> List[float]:
-        model = TextEmbedding(model_name=self.id)
-        embeddings = model.embed(text)
-        embedding_list = list(embeddings)
-
-        try:
-            return embedding_list
-        except Exception as e:
-            logger.warning(e)
-            return []
-
-    def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Optional[Dict]]:
-        embedding = self.get_embedding(text=text)
-        # Currently, FastEmbed does not provide usage information
-        usage = None
-
-        return embedding, usage

agno/embedder/google.py

@@ -1,73 +0,0 @@
-from dataclasses import dataclass
-from os import getenv
-from types import ModuleType
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-from agno.embedder.base import Embedder
-from agno.utils.log import logger
-
-try:
-    import google.generativeai as genai
-    from google.generativeai.types.text_types import BatchEmbeddingDict, EmbeddingDict
-except ImportError:
-    logger.error("`google-generativeai` not installed. Please install it using `pip install google-generativeai`")
-    raise
-
-
-@dataclass
-class GeminiEmbedder(Embedder):
-    id: str = "models/text-embedding-004"
-    task_type: str = "RETRIEVAL_QUERY"
-    title: Optional[str] = None
-    dimensions: Optional[int] = 768
-    api_key: Optional[str] = None
-    request_params: Optional[Dict[str, Any]] = None
-    client_params: Optional[Dict[str, Any]] = None
-    gemini_client: Optional[ModuleType] = None
-
-    @property
-    def client(self):
-        if self.gemini_client:
-            return self.gemini_client
-        _client_params: Dict[str, Any] = {}
-
-        self.api_key = self.api_key or getenv("GOOGLE_API_KEY")
-        if not self.api_key:
-            logger.error("GOOGLE_API_KEY not set. Please set the GOOGLE_API_KEY environment variable.")
-
-        if self.api_key:
-            _client_params["api_key"] = self.api_key
-        if self.client_params:
-            _client_params.update(self.client_params)
-        self.gemini_client = genai
-        self.gemini_client.configure(**_client_params)  # type: ignore
-        return self.gemini_client
-
-    def _response(self, text: str) -> Union[EmbeddingDict, BatchEmbeddingDict]:
-        _request_params: Dict[str, Any] = {
-            "content": text,
-            "model": self.id,
-            "output_dimensionality": self.dimensions,
-            "task_type": self.task_type,
-            "title": self.title,
-        }
-        if self.request_params:
-            _request_params.update(self.request_params)
-        return self.client.embed_content(**_request_params)
-
-    def get_embedding(self, text: str) -> List[float]:
-        response = self._response(text=text)
-        try:
-            return response.get("embedding", [])
-        except Exception as e:
-            logger.warning(e)
-            return []
-
-    def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Optional[Dict]]:
-        response = self._response(text=text)
-        usage = None
-        try:
-            return response.get("embedding", []), usage
-        except Exception as e:
-            logger.warning(e)
-            return [], usage