agno 1.8.0__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/knowledge/pdf_bytes.py

@@ -1,42 +0,0 @@
-import io
-from typing import IO, AsyncIterator, Iterator, List, Union
-
-from pydantic import Field
-
-from agno.document import Document
-from agno.document.reader.pdf_reader import PDFImageReader, PDFReader
-from agno.knowledge.agent import AgentKnowledge
-
-
-class PDFBytesKnowledgeBase(AgentKnowledge):
-    pdfs: Union[List[bytes], List[IO]]
-
-    exclude_files: List[str] = Field(default_factory=list)
-
-    reader: Union[PDFReader, PDFImageReader] = PDFReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over PDFs bytes and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-
-        for pdf in self.pdfs:
-            _pdf = io.BytesIO(pdf) if isinstance(pdf, bytes) else pdf
-            yield self.reader.read(pdf=_pdf)
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over PDFs bytes and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-
-        for pdf in self.pdfs:
-            _pdf = io.BytesIO(pdf) if isinstance(pdf, bytes) else pdf
-            yield await self.reader.async_read(pdf=_pdf)

agno/knowledge/pdf_url.py

@@ -1,148 +0,0 @@
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
-
-from agno.document import Document
-from agno.document.reader.pdf_reader import PDFUrlImageReader, PDFUrlReader
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_info, logger
-
-
-class PDFUrlKnowledgeBase(AgentKnowledge):
-    urls: Optional[Union[List[str], List[Dict[str, Union[str, Dict[str, Any]]]]]] = None
-    formats: List[str] = [".pdf"]
-    reader: Union[PDFUrlReader, PDFUrlImageReader] = PDFUrlReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over PDF URLs and yield lists of documents."""
-        if self.urls is None:
-            raise ValueError("URLs are not set")
-
-        for item in self.urls:
-            if isinstance(item, dict) and "url" in item:
-                # Handle URL with metadata/password
-                url = item["url"]
-                config = item.get("metadata", {})
-                pdf_password = item.get("password")
-                if pdf_password is not None and not isinstance(pdf_password, str):
-                    pdf_password = None
-
-                if self._is_valid_url(url):  # type: ignore
-                    documents = self.reader.read(url=url, password=pdf_password)  # type: ignore
-                    if config:
-                        for doc in documents:
-                            log_info(f"Adding metadata {config} to document from URL: {url}")
-                            doc.meta_data.update(config)  # type: ignore
-                    yield documents
-            else:
-                # Handle simple URL - no password
-                if self._is_valid_url(item):  # type: ignore
-                    yield self.reader.read(url=item)  # type: ignore
-
-    def _is_valid_url(self, url: str) -> bool:
-        """Helper to check if URL is valid."""
-        if not url.endswith(".pdf"):
-            logger.error(f"Unsupported URL: {url}")
-            return False
-        return True
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over PDF URLs and yield lists of documents asynchronously."""
-        if not self.urls:
-            raise ValueError("URLs are not set")
-
-        for item in self.urls:
-            if isinstance(item, dict) and "url" in item:
-                # Handle URL with metadata/password
-                url = item["url"]
-                config = item.get("metadata", {})
-                pdf_password = item.get("password")
-                if pdf_password is not None and not isinstance(pdf_password, str):
-                    pdf_password = None
-
-                if self._is_valid_url(url):  # type: ignore
-                    documents = await self.reader.async_read(url=url, password=pdf_password)  # type: ignore
-                    if config:
-                        for doc in documents:
-                            log_info(f"Adding metadata {config} to document from URL: {url}")
-                            doc.meta_data.update(config)  # type: ignore
-                    yield documents
-            else:
-                # Handle simple URL
-                if self._is_valid_url(item):  # type: ignore
-                    yield await self.reader.async_read(url=item)  # type: ignore
-
-    def load_document(
-        self,
-        url: str,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load documents from a single URL with specific metadata into the vector DB.
-        Args:
-            url (str): URL of the PDF to load.
-            metadata (Optional[Dict[str, Any]]): Metadata to associate with documents from this URL.
-                                                This will be merged into the 'meta_data' field of each document.
-            recreate (bool): If True, drops and recreates the collection before loading. Defaults to False.
-            upsert (bool): If True, upserts documents (insert or update). Requires vector_db support. Defaults to False.
-            skip_existing (bool): If True and not upserting, skips documents that already exist based on content hash. Defaults to True.
-        """
-        if not url.endswith(".pdf"):
-            logger.error(f"Unsupported URL provided to load_url: {url}")
-            return
-
-        if not self.prepare_load(url, self.formats, metadata, recreate, is_url=True):  # type: ignore
-            return
-
-        try:
-            documents = self.reader.read(url=url)
-            if not documents:
-                logger.warning(f"No documents were read from URL: {url}")
-                return
-        except Exception as e:
-            logger.exception(f"Failed to read documents from URL {url}: {e}")
-            return
-
-        # Process documents
-        self.process_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(url),
-        )
-
-    async def aload_document(
-        self,
-        url: str,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        if not url.endswith(".pdf"):
-            logger.error(f"Unsupported URL provided to load_url: {url}")
-            return
-
-        if not await self.aprepare_load(url, self.formats, metadata, recreate, is_url=True):  # type: ignore
-            return
-
-        try:
-            documents = await self.reader.async_read(url=url)
-            if not documents:
-                logger.warning(f"No documents were read from URL: {url}")
-                return
-        except Exception as e:
-            logger.exception(f"Failed to read documents from URL {url}: {e}")
-            return
-
-        # Process documents
-        await self.aprocess_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(url),
-        )

agno/knowledge/s3/base.py

@@ -1,64 +0,0 @@
-from typing import AsyncIterator, Iterator, List, Optional
-
-from agno.aws.resource.s3.bucket import S3Bucket  # type: ignore
-from agno.aws.resource.s3.object import S3Object  # type: ignore
-from agno.document import Document
-from agno.knowledge.agent import AgentKnowledge
-
-
-class S3KnowledgeBase(AgentKnowledge):
-    # Provide either bucket or bucket_name
-    bucket: Optional[S3Bucket] = None
-    bucket_name: Optional[str] = None
-
-    # Provide either object or key
-    key: Optional[str] = None
-    object: Optional[S3Object] = None
-
-    # Filter objects by prefix
-    # Ignored if object or key is provided
-    prefix: Optional[str] = None
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        raise NotImplementedError
-
-    @property
-    def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        raise NotImplementedError
-
-    @property
-    def s3_objects(self) -> List[S3Object]:
-        """Iterate over PDFs in a s3 bucket and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-
-        s3_objects_to_read: List[S3Object] = []
-
-        if self.bucket is None and self.bucket_name is None:
-            raise ValueError("No bucket or bucket_name provided")
-
-        if self.bucket is not None and self.bucket_name is not None:
-            raise ValueError("Provide either bucket or bucket_name")
-
-        if self.object is not None and self.key is not None:
-            raise ValueError("Provide either object or key")
-
-        if self.bucket_name is not None:
-            self.bucket = S3Bucket(name=self.bucket_name)
-
-        if self.bucket is not None:
-            if self.key is not None:
-                _object = S3Object(bucket_name=self.bucket.name, name=self.key)
-                s3_objects_to_read.append(_object)
-            elif self.object is not None:
-                s3_objects_to_read.append(self.object)
-            elif self.prefix is not None:
-                s3_objects_to_read.extend(self.bucket.get_objects(prefix=self.prefix))
-            else:
-                s3_objects_to_read.extend(self.bucket.get_objects())
-
-        return s3_objects_to_read

agno/knowledge/s3/pdf.py

@@ -1,33 +0,0 @@
-from typing import AsyncIterator, Iterator, List
-
-from agno.document import Document
-from agno.document.reader.s3.pdf_reader import S3PDFReader
-from agno.knowledge.s3.base import S3KnowledgeBase
-
-
-class S3PDFKnowledgeBase(S3KnowledgeBase):
-    reader: S3PDFReader = S3PDFReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over PDFs in a s3 bucket and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-        for s3_object in self.s3_objects:
-            if s3_object.name.endswith(".pdf"):
-                yield self.reader.read(s3_object=s3_object)
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over PDFs in a s3 bucket and yield lists of documents asynchronously.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            AsyncIterator[List[Document]]: Async iterator yielding list of documents
-        """
-        for s3_object in self.s3_objects:
-            if s3_object.name.endswith(".pdf"):
-                yield await self.reader.async_read(s3_object=s3_object)

agno/knowledge/s3/text.py

@@ -1,34 +0,0 @@
-from typing import AsyncIterator, Iterator, List
-
-from agno.document import Document
-from agno.document.reader.s3.text_reader import S3TextReader
-from agno.knowledge.s3.base import S3KnowledgeBase
-
-
-class S3TextKnowledgeBase(S3KnowledgeBase):
-    formats: List[str] = [".doc", ".docx"]
-    reader: S3TextReader = S3TextReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over text files in a s3 bucket and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-        for s3_object in self.s3_objects:
-            if s3_object.name.endswith(tuple(self.formats)):
-                yield self.reader.read(s3_object=s3_object)
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over text files in a s3 bucket and yield lists of documents asynchronously.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            AsyncIterator[List[Document]]: Async iterator yielding list of documents
-        """
-        for s3_object in self.s3_objects:
-            if s3_object.name.endswith(tuple(self.formats)):
-                yield await self.reader.async_read(s3_object=s3_object)

agno/knowledge/text.py

@@ -1,141 +0,0 @@
-from pathlib import Path
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
-
-from agno.document import Document
-from agno.document.reader.text_reader import TextReader
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_info, logger
-
-
-class TextKnowledgeBase(AgentKnowledge):
-    path: Optional[Union[str, Path, List[Dict[str, Union[str, Dict[str, Any]]]]]] = None
-    formats: List[str] = [".txt"]
-    reader: TextReader = TextReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over text files and yield lists of documents."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _file_path = Path(file_path)  # type: ignore
-                    if self._is_valid_text(_file_path):
-                        documents = self.reader.read(file=_file_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _file_path = Path(self.path)
-            if _file_path.is_dir():
-                for _file in _file_path.glob("**/*"):
-                    if self._is_valid_text(_file):
-                        yield self.reader.read(file=_file)
-            elif self._is_valid_text(_file_path):
-                yield self.reader.read(file=_file_path)
-
-    def _is_valid_text(self, path: Path) -> bool:
-        """Helper to check if path is a valid text file."""
-        return path.exists() and path.is_file() and path.suffix in self.formats
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over text files and yield lists of documents asynchronously."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _file_path = Path(file_path)  # type: ignore
-                    if self._is_valid_text(_file_path):
-                        documents = await self.reader.async_read(file=_file_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _file_path = Path(self.path)
-            if _file_path.is_dir():
-                for _file in _file_path.glob("**/*"):
-                    if self._is_valid_text(_file):
-                        yield await self.reader.async_read(file=_file)
-            elif self._is_valid_text(_file_path):
-                yield await self.reader.async_read(file=_file_path)
-
-    def load_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load documents from a single text file with specific metadata into the vector DB."""
-
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not self.prepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = self.reader.read(file=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        self.process_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )
-
-    async def aload_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load documents from a single text file with specific metadata into the vector DB."""
-
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not await self.aprepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = await self.reader.async_read(file=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        await self.aprocess_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )

agno/knowledge/url.py

@@ -1,46 +0,0 @@
-import asyncio
-from typing import AsyncIterator, Iterator, List
-
-from agno.document import Document
-from agno.document.reader.url_reader import URLReader
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import logger
-
-
-class UrlKnowledge(AgentKnowledge):
-    urls: List[str] = []
-    reader: URLReader = URLReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over URLs and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-        for url in self.urls:
-            try:
-                yield self.reader.read(url=url)
-            except Exception as e:
-                logger.error(f"Error reading URL {url}: {str(e)}")
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Async version of document_lists"""
-
-        async def process_url(url: str) -> List[Document]:
-            try:
-                return await self.reader.async_read(url=url)
-            except Exception as e:
-                logger.error(f"Error reading URL {url}: {str(e)}")
-                return []
-
-        # Process all URLs concurrently
-        tasks = [process_url(url) for url in self.urls]
-        results = await asyncio.gather(*tasks)
-
-        # Yield each result
-        for docs in results:
-            if docs:
-                yield docs