agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/constants.py

@@ -1,13 +0,0 @@
-PYTHONPATH_ENV_VAR: str = "PYTHONPATH"
-AGNO_RUNTIME_ENV_VAR: str = "AGNO_RUNTIME"
-AGNO_API_KEY_ENV_VAR: str = "AGNO_API_KEY"
-
-WORKSPACE_ID_ENV_VAR: str = "AGNO_WORKSPACE_ID"
-WORKSPACE_NAME_ENV_VAR: str = "AGNO_WORKSPACE_NAME"
-WORKSPACE_ROOT_ENV_VAR: str = "AGNO_WORKSPACE_ROOT"
-WORKSPACE_DIR_ENV_VAR: str = "AGNO_WORKSPACE_DIR"
-REQUIREMENTS_FILE_PATH_ENV_VAR: str = "REQUIREMENTS_FILE_PATH"
-
-AWS_REGION_ENV_VAR: str = "AWS_REGION"
-AWS_DEFAULT_REGION_ENV_VAR: str = "AWS_DEFAULT_REGION"
-AWS_PROFILE_ENV_VAR: str = "AWS_PROFILE"

agno/document/__init__.py

@@ -1 +0,0 @@
-from agno.document.base import Document

agno/document/chunking/semantic.py

@@ -1,47 +0,0 @@
-from typing import List, Optional
-
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
-from agno.embedder.base import Embedder
-from agno.embedder.openai import OpenAIEmbedder
-
-try:
-    from chonkie import SemanticChunker
-except ImportError:
-    raise ImportError("`chonkie` is required for semantic chunking, please install using `uv pip install chonkie`")
-
-
-class SemanticChunking(ChunkingStrategy):
-    """Chunking strategy that splits text into semantic chunks using chonkie"""
-
-    def __init__(
-        self, embedder: Optional[Embedder] = None, chunk_size: int = 5000, similarity_threshold: Optional[float] = 0.5
-    ):
-        self.embedder = embedder or OpenAIEmbedder(id="text-embedding-3-small")  # type: ignore
-        self.chunk_size = chunk_size
-        self.similarity_threshold = similarity_threshold
-        self.chunker = SemanticChunker(
-            embedding_model=self.embedder.model,  # type: ignore
-            chunk_size=self.chunk_size,
-            threshold=self.similarity_threshold,
-        )
-
-    def chunk(self, document: Document) -> List[Document]:
-        """Split document into semantic chunks using chokie"""
-        if not document.content:
-            return [document]
-
-        # Use chonkie to split into semantic chunks
-        chunks = self.chunker.chunk(self.clean_text(document.content))
-
-        # Convert chunks to Documents
-        chunked_documents: List[Document] = []
-        for i, chunk in enumerate(chunks, 1):
-            meta_data = document.meta_data.copy()
-            meta_data["chunk"] = i
-            chunk_id = f"{document.id}_{i}" if document.id else None
-            meta_data["chunk_size"] = len(chunk.text)
-
-            chunked_documents.append(Document(id=chunk_id, name=document.name, meta_data=meta_data, content=chunk.text))
-
-        return chunked_documents

agno/document/chunking/strategy.py

@@ -1,31 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List
-
-from agno.document.base import Document
-
-
-class ChunkingStrategy(ABC):
-    """Base class for chunking strategies"""
-
-    @abstractmethod
-    def chunk(self, document: Document) -> List[Document]:
-        raise NotImplementedError
-
-    def clean_text(self, text: str) -> str:
-        """Clean the text by replacing multiple newlines with a single newline"""
-        import re
-
-        # Replace multiple newlines with a single newline
-        cleaned_text = re.sub(r"\n+", "\n", text)
-        # Replace multiple spaces with a single space
-        cleaned_text = re.sub(r"\s+", " ", cleaned_text)
-        # Replace multiple tabs with a single tab
-        cleaned_text = re.sub(r"\t+", "\t", cleaned_text)
-        # Replace multiple carriage returns with a single carriage return
-        cleaned_text = re.sub(r"\r+", "\r", cleaned_text)
-        # Replace multiple form feeds with a single form feed
-        cleaned_text = re.sub(r"\f+", "\f", cleaned_text)
-        # Replace multiple vertical tabs with a single vertical tab
-        cleaned_text = re.sub(r"\v+", "\v", cleaned_text)
-
-        return cleaned_text

agno/document/reader/__init__.py

@@ -1 +0,0 @@
-from agno.document.reader.base import Reader

agno/document/reader/arxiv_reader.py

@@ -1,41 +0,0 @@
-from typing import List
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-
-try:
-    import arxiv  # noqa: F401
-except ImportError:
-    raise ImportError("The `arxiv` package is not installed. Please install it via `pip install arxiv`.")
-
-
-class ArxivReader(Reader):
-    max_results: int = 5  # Top articles
-    sort_by: arxiv.SortCriterion = arxiv.SortCriterion.Relevance
-
-    def read(self, query: str) -> List[Document]:
-        """
-        Search a query from arXiv database
-
-        This function gets the top_k articles based on a user's query, sorted by relevance from arxiv
-
-        @param query:
-        @return: List of documents
-        """
-
-        documents = []
-        search = arxiv.Search(query=query, max_results=self.max_results, sort_by=self.sort_by)
-
-        for result in search.results():
-            links = ", ".join([x.href for x in result.links])
-
-            documents.append(
-                Document(
-                    name=result.title,
-                    id=result.title,
-                    meta_data={"pdf_url": str(result.pdf_url), "article_links": links},
-                    content=result.summary,
-                )
-            )
-
-        return documents

agno/document/reader/base.py

@@ -1,22 +0,0 @@
-from dataclasses import dataclass, field
-from typing import Any, List
-
-from agno.document.base import Document
-from agno.document.chunking.fixed import FixedSizeChunking
-from agno.document.chunking.strategy import ChunkingStrategy
-
-
-@dataclass
-class Reader:
-    """Base class for reading documents"""
-
-    chunk: bool = True
-    chunk_size: int = 3000
-    separators: List[str] = field(default_factory=lambda: ["\n", "\n\n", "\r", "\r\n", "\n\r", "\t", " ", "  "])
-    chunking_strategy: ChunkingStrategy = field(default_factory=FixedSizeChunking)
-
-    def read(self, obj: Any) -> List[Document]:
-        raise NotImplementedError
-
-    def chunk_document(self, document: Document) -> List[Document]:
-        return self.chunking_strategy.chunk(document)

agno/document/reader/csv_reader.py

@@ -1,84 +0,0 @@
-import csv
-import io
-import os
-from pathlib import Path
-from typing import IO, Any, List, Union
-from urllib.parse import urlparse
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-
-class CSVReader(Reader):
-    """Reader for CSV files"""
-
-    def read(self, file: Union[Path, IO[Any]], delimiter: str = ",", quotechar: str = '"') -> List[Document]:
-        try:
-            if isinstance(file, Path):
-                if not file.exists():
-                    raise FileNotFoundError(f"Could not find file: {file}")
-                logger.info(f"Reading: {file}")
-                file_content = file.open(newline="", mode="r", encoding="utf-8")
-            else:
-                logger.info(f"Reading uploaded file: {file.name}")
-                file.seek(0)
-                file_content = io.StringIO(file.read().decode("utf-8"))  # type: ignore
-
-            csv_name = Path(file.name).stem if isinstance(file, Path) else file.name.split(".")[0]
-            csv_content = ""
-            with file_content as csvfile:
-                csv_reader = csv.reader(csvfile, delimiter=delimiter, quotechar=quotechar)
-                for row in csv_reader:
-                    csv_content += ", ".join(row) + "\n"
-
-            documents = [
-                Document(
-                    name=csv_name,
-                    id=csv_name,
-                    content=csv_content,
-                )
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-        except Exception as e:
-            logger.error(f"Error reading: {file.name if isinstance(file, IO) else file}: {e}")
-            return []
-
-
-class CSVUrlReader(Reader):
-    """Reader for CSV files"""
-
-    def read(self, url: str) -> List[Document]:
-        if not url:
-            raise ValueError("No URL provided")
-
-        try:
-            import httpx
-        except ImportError:
-            raise ImportError("`httpx` not installed")
-
-        logger.info(f"Reading: {url}")
-        response = httpx.get(url)
-
-        try:
-            response.raise_for_status()
-        except httpx.HTTPStatusError as e:
-            logger.error(f"HTTP error occurred: {e.response.status_code} - {e.response.text}")
-            raise
-
-        parsed_url = urlparse(url)
-        filename = os.path.basename(parsed_url.path) or "data.csv"
-
-        file_obj = io.BytesIO(response.content)
-        file_obj.name = filename
-
-        documents = CSVReader().read(file=file_obj)
-
-        file_obj.close()
-
-        return documents

agno/document/reader/docx_reader.py

@@ -1,46 +0,0 @@
-import io
-from pathlib import Path
-from typing import List, Union
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-try:
-    from docx import Document as DocxDocument  # type: ignore
-except ImportError:
-    raise ImportError("The `python-docx` package is not installed. Please install it via `pip install python-docx`.")
-
-
-class DocxReader(Reader):
-    """Reader for Doc/Docx files"""
-
-    def read(self, file: Union[Path, io.BytesIO]) -> List[Document]:
-        try:
-            if isinstance(file, Path):
-                logger.info(f"Reading: {file}")
-                docx_document = DocxDocument(str(file))
-                doc_name = file.stem
-            else:  # Handle file-like object from upload
-                logger.info(f"Reading uploaded file: {file.name}")
-                docx_document = DocxDocument(file)
-                doc_name = file.name.split(".")[0]
-
-            doc_content = "\n\n".join([para.text for para in docx_document.paragraphs])
-
-            documents = [
-                Document(
-                    name=doc_name,
-                    id=doc_name,
-                    content=doc_content,
-                )
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-        except Exception as e:
-            logger.error(f"Error reading file: {e}")
-            return []

agno/document/reader/firecrawl_reader.py

@@ -1,99 +0,0 @@
-from dataclasses import dataclass
-from typing import Dict, List, Literal, Optional
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-try:
-    from firecrawl import FirecrawlApp
-except ImportError:
-    raise ImportError("The `firecrawl` package is not installed. Please install it via `pip install firecrawl-py`.")
-
-
-@dataclass
-class FirecrawlReader(Reader):
-    api_key: Optional[str] = None
-    params: Optional[Dict] = None
-    mode: Literal["scrape", "crawl"] = "scrape"
-
-    def scrape(self, url: str) -> List[Document]:
-        """
-        Scrapes a website and returns a list of documents.
-
-        Args:
-            url: The URL of the website to scrape
-
-        Returns:
-            A list of documents
-        """
-
-        logger.debug(f"Scraping: {url}")
-
-        app = FirecrawlApp(api_key=self.api_key)
-        scraped_data = app.scrape_url(url, params=self.params)
-        # print(scraped_data)
-        content = scraped_data.get("markdown", "")
-
-        # Debug logging
-        logger.debug(f"Received content type: {type(content)}")
-        logger.debug(f"Content empty: {not bool(content)}")
-
-        # Ensure content is a string
-        if content is None:
-            content = ""  # or you could use metadata to create a meaningful message
-            logger.warning(f"No content received for URL: {url}")
-
-        documents = []
-        if self.chunk and content:  # Only chunk if there's content
-            documents.extend(self.chunk_document(Document(name=url, id=url, content=content)))
-        else:
-            documents.append(Document(name=url, id=url, content=content))
-        return documents
-
-    def crawl(self, url: str) -> List[Document]:
-        """
-        Crawls a website and returns a list of documents.
-
-        Args:
-            url: The URL of the website to crawl
-
-        Returns:
-            A list of documents
-        """
-        logger.debug(f"Crawling: {url}")
-
-        app = FirecrawlApp(api_key=self.api_key)
-        crawl_result = app.crawl_url(url, params=self.params)
-        documents = []
-
-        # Extract data from crawl results
-        results_data = crawl_result.get("data", [])
-        for result in results_data:
-            # Get markdown content, default to empty string if not found
-            content = result.get("markdown", "")
-
-            if content:  # Only create document if content exists
-                if self.chunk:
-                    documents.extend(self.chunk_document(Document(name=url, id=url, content=content)))
-                else:
-                    documents.append(Document(name=url, id=url, content=content))
-
-        return documents
-
-    def read(self, url: str) -> List[Document]:
-        """
-
-        Args:
-            url: The URL of the website to scrape
-
-        Returns:
-            A list of documents
-        """
-
-        if self.mode == "scrape":
-            return self.scrape(url)
-        elif self.mode == "crawl":
-            return self.crawl(url)
-        else:
-            raise NotImplementedError(f"Mode {self.mode} not implemented")

agno/document/reader/json_reader.py

@@ -1,43 +0,0 @@
-import json
-from pathlib import Path
-from typing import List
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-
-class JSONReader(Reader):
-    """Reader for JSON files"""
-
-    chunk: bool = False
-
-    def read(self, path: Path) -> List[Document]:
-        if not path.exists():
-            raise FileNotFoundError(f"Could not find file: {path}")
-
-        try:
-            logger.info(f"Reading: {path}")
-            json_name = path.name.split(".")[0]
-            json_contents = json.loads(path.read_text("utf-8"))
-
-            if isinstance(json_contents, dict):
-                json_contents = [json_contents]
-
-            documents = [
-                Document(
-                    name=json_name,
-                    id=f"{json_name}_{page_number}",
-                    meta_data={"page": page_number},
-                    content=json.dumps(content),
-                )
-                for page_number, content in enumerate(json_contents, start=1)
-            ]
-            if self.chunk:
-                chunked_documents = []
-                for document in documents:
-                    chunked_documents.extend(self.chunk_document(document))
-                return chunked_documents
-            return documents
-        except Exception:
-            raise

agno/document/reader/pdf_reader.py

@@ -1,219 +0,0 @@
-from pathlib import Path
-from typing import IO, Any, List, Union
-
-from agno.document.base import Document
-from agno.document.reader.base import Reader
-from agno.utils.log import logger
-
-try:
-    from pypdf import PdfReader as DocumentReader  # noqa: F401
-except ImportError:
-    raise ImportError("`pypdf` not installed. Please install it via `pip install pypdf`.")
-
-
-class PDFReader(Reader):
-    """Reader for PDF files"""
-
-    def read(self, pdf: Union[str, Path, IO[Any]]) -> List[Document]:
-        doc_name = ""
-        try:
-            if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
-            else:
-                doc_name = pdf.name.split(".")[0]
-        except Exception:
-            doc_name = "pdf"
-
-        logger.info(f"Reading: {doc_name}")
-        doc_reader = DocumentReader(pdf)
-
-        documents = [
-            Document(
-                name=doc_name,
-                id=f"{doc_name}_{page_number}",
-                meta_data={"page": page_number},
-                content=page.extract_text(),
-            )
-            for page_number, page in enumerate(doc_reader.pages, start=1)
-        ]
-        if self.chunk:
-            chunked_documents = []
-            for document in documents:
-                chunked_documents.extend(self.chunk_document(document))
-            return chunked_documents
-        return documents
-
-
-class PDFUrlReader(Reader):
-    """Reader for PDF files from URL"""
-
-    def read(self, url: str) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        from io import BytesIO
-
-        try:
-            import httpx
-        except ImportError:
-            raise ImportError("`httpx` not installed. Please install it via `pip install httpx`.")
-
-        logger.info(f"Reading: {url}")
-        response = httpx.get(url)
-
-        try:
-            response.raise_for_status()
-        except httpx.HTTPStatusError as e:
-            logger.error(f"HTTP error occurred: {e.response.status_code} - {e.response.text}")
-            raise
-
-        doc_name = url.split("/")[-1].split(".")[0].replace("/", "_").replace(" ", "_")
-        doc_reader = DocumentReader(BytesIO(response.content))
-
-        documents = [
-            Document(
-                name=doc_name,
-                id=f"{doc_name}_{page_number}",
-                meta_data={"page": page_number},
-                content=page.extract_text(),
-            )
-            for page_number, page in enumerate(doc_reader.pages, start=1)
-        ]
-        if self.chunk:
-            chunked_documents = []
-            for document in documents:
-                chunked_documents.extend(self.chunk_document(document))
-            return chunked_documents
-        return documents
-
-
-class PDFImageReader(Reader):
-    """Reader for PDF files with text and images extraction"""
-
-    def read(self, pdf: Union[str, Path, IO[Any]]) -> List[Document]:
-        if not pdf:
-            raise ValueError("No pdf provided")
-
-        try:
-            import rapidocr_onnxruntime as rapidocr
-        except ImportError:
-            raise ImportError(
-                "`rapidocr_onnxruntime` not installed. Please install it via `pip install rapidocr_onnxruntime`."
-            )
-
-        doc_name = ""
-        try:
-            if isinstance(pdf, str):
-                doc_name = pdf.split("/")[-1].split(".")[0].replace(" ", "_")
-            else:
-                doc_name = pdf.name.split(".")[0]
-        except Exception:
-            doc_name = "pdf"
-
-        logger.info(f"Reading: {doc_name}")
-        doc_reader = DocumentReader(pdf)
-
-        # Initialize RapidOCR
-        ocr = rapidocr.RapidOCR()
-
-        documents = []
-        for page_number, page in enumerate(doc_reader.pages, start=1):
-            page_text = page.extract_text() or ""
-            images_text_list: List = []
-
-            for image_object in page.images:
-                image_data = image_object.data
-
-                # Perform OCR on the image
-                ocr_result, elapse = ocr(image_data)
-
-                # Extract text from OCR result
-                if ocr_result:
-                    images_text_list += [item[1] for item in ocr_result]
-
-            images_text: str = "\n".join(images_text_list)
-            content = page_text + "\n" + images_text
-
-            documents.append(
-                Document(
-                    name=doc_name,
-                    id=f"{doc_name}_{page_number}",
-                    meta_data={"page": page_number},
-                    content=content,
-                )
-            )
-
-        if self.chunk:
-            chunked_documents = []
-            for document in documents:
-                chunked_documents.extend(self.chunk_document(document))
-            return chunked_documents
-
-        return documents
-
-
-class PDFUrlImageReader(Reader):
-    """Reader for PDF files from URL with text and images extraction"""
-
-    def read(self, url: str) -> List[Document]:
-        if not url:
-            raise ValueError("No url provided")
-
-        from io import BytesIO
-
-        try:
-            import httpx
-            import rapidocr_onnxruntime as rapidocr
-        except ImportError:
-            raise ImportError(
-                "`httpx`, `rapidocr_onnxruntime` not installed. Please install it via `pip install httpx rapidocr_onnxruntime`."
-            )
-
-        # Read the PDF from the URL
-        logger.info(f"Reading: {url}")
-        response = httpx.get(url)
-
-        doc_name = url.split("/")[-1].split(".")[0].replace(" ", "_")
-        doc_reader = DocumentReader(BytesIO(response.content))
-
-        # Initialize RapidOCR
-        ocr = rapidocr.RapidOCR()
-
-        # Process each page of the PDF
-        documents = []
-        for page_number, page in enumerate(doc_reader.pages, start=1):
-            page_text = page.extract_text() or ""
-            images_text_list = []
-
-            # Extract and process images
-            for image_object in page.images:
-                image_data = image_object.data
-
-                # Perform OCR on the image
-                ocr_result, elapse = ocr(image_data)
-
-                # Extract text from OCR result
-                if ocr_result:
-                    images_text_list += [item[1] for item in ocr_result]
-
-            images_text = "\n".join(images_text_list)
-            content = page_text + "\n" + images_text
-
-            # Append the document
-            documents.append(
-                Document(
-                    name=doc_name,
-                    id=f"{doc_name}_{page_number}",
-                    meta_data={"page": page_number},
-                    content=content,
-                )
-            )
-
-        # Optionally chunk documents
-        if self.chunk:
-            chunked_documents = []
-            for document in documents:
-                chunked_documents.extend(self.chunk_document(document))
-            return chunked_documents
-
-        return documents