agno 1.8.0__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/knowledge/docx.py

@@ -1,137 +0,0 @@
-from pathlib import Path
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
-
-from agno.document import Document
-from agno.document.reader.docx_reader import DocxReader
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_info, logger
-
-
-class DocxKnowledgeBase(AgentKnowledge):
-    path: Optional[Union[str, Path, List[Dict[str, Union[str, Dict[str, Any]]]]]] = None
-    formats: List[str] = [".doc", ".docx"]
-    reader: DocxReader = DocxReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over doc/docx files and yield lists of documents."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _file_path = Path(file_path)  # type: ignore
-                    if self._is_valid_docx(_file_path):
-                        documents = self.reader.read(file=_file_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _file_path = Path(self.path)
-            if _file_path.is_dir():
-                for _file in _file_path.glob("**/*"):
-                    if self._is_valid_docx(_file):
-                        yield self.reader.read(file=_file)
-            elif self._is_valid_docx(_file_path):
-                yield self.reader.read(file=_file_path)
-
-    def _is_valid_docx(self, path: Path) -> bool:
-        """Helper to check if path is a valid doc/docx file."""
-        return path.exists() and path.is_file() and path.suffix in self.formats
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over doc/docx files and yield lists of documents asynchronously."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _file_path = Path(file_path)  # type: ignore
-                    if self._is_valid_docx(_file_path):
-                        documents = await self.reader.async_read(file=_file_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _file_path = Path(self.path)
-            if _file_path.is_dir():
-                for _file in _file_path.glob("**/*"):
-                    if self._is_valid_docx(_file):
-                        yield await self.reader.async_read(file=_file)
-            elif self._is_valid_docx(_file_path):
-                yield await self.reader.async_read(file=_file_path)
-
-    def load_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not self.prepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = self.reader.read(file=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        self.process_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )
-
-    async def aload_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not await self.aprepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = await self.reader.async_read(file=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        await self.aprocess_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )

agno/knowledge/firecrawl.py

@@ -1,34 +0,0 @@
-from typing import AsyncIterator, Iterator, List
-
-from agno.document import Document
-from agno.document.reader.firecrawl_reader import FirecrawlReader
-from agno.knowledge.agent import AgentKnowledge
-
-
-class FireCrawlKnowledgeBase(AgentKnowledge):
-    urls: List[str] = []
-    reader: FirecrawlReader = FirecrawlReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Scrape urls using FireCrawl and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-        for url in self.urls:
-            yield self.reader.read(url=url)
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Asynchronously scrape urls using FireCrawl and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            AsyncIterator[List[Document]]: Async iterator yielding list of documents
-        """
-        for url in self.urls:
-            documents = await self.reader.async_read(url=url)
-            if documents:
-                yield documents

agno/knowledge/gcs/base.py

@@ -1,39 +0,0 @@
-from typing import AsyncIterator, Iterator, List, Optional
-
-from google.cloud import storage
-
-from agno.document import Document
-from agno.knowledge.agent import AgentKnowledge
-
-
-class GCSKnowledgeBase(AgentKnowledge):
-    bucket: Optional[storage.Bucket] = None
-    bucket_name: Optional[str] = None
-    blob_name: Optional[str] = None
-    prefix: Optional[str] = None
-
-    @property
-    def gcs_blobs(self) -> List[storage.Blob]:
-        if self.bucket is None and self.bucket_name is None:
-            raise ValueError("No bucket or bucket_name provided")
-        if self.bucket is not None and self.bucket_name is not None:
-            raise ValueError("Provide either bucket or bucket_name")
-        if self.bucket_name is not None:
-            client = storage.Client()
-            self.bucket = client.bucket(self.bucket_name)
-        blobs_to_read = []
-        if self.blob_name is not None:
-            blobs_to_read.append(self.bucket.blob(self.blob_name))  # type: ignore
-        elif self.prefix is not None:
-            blobs_to_read.extend(self.bucket.list_blobs(prefix=self.prefix))  # type: ignore
-        else:
-            blobs_to_read.extend(self.bucket.list_blobs())  # type: ignore
-        return list(blobs_to_read)
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        raise NotImplementedError
-
-    @property
-    def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        raise NotImplementedError

agno/knowledge/gcs/pdf.py

@@ -1,125 +0,0 @@
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional
-
-from agno.document import Document
-from agno.document.reader.gcs.pdf_reader import GCSPDFReader
-from agno.knowledge.gcs.base import GCSKnowledgeBase
-from agno.utils.log import log_debug, log_info
-
-
-class GCSPDFKnowledgeBase(GCSKnowledgeBase):
-    reader: GCSPDFReader = GCSPDFReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        for blob in self.gcs_blobs:
-            if blob.name.endswith(".pdf"):
-                yield self.reader.read(blob=blob)
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        for blob in self.gcs_blobs:
-            if blob.name.endswith(".pdf"):
-                yield await self.reader.async_read(blob=blob)
-
-    def load(
-        self,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load the knowledge base to the vector db
-        Args:
-            recreate (bool): If True, recreates the collection in the vector db. Defaults to False.
-            upsert (bool): If True, upserts documents to the vector db. Defaults to False.
-            skip_existing (bool): If True, skips documents which already exist in the vector db when inserting. Defaults to True.
-        """
-        self._load_init(recreate, upsert)
-        if self.vector_db is None:
-            return
-
-        log_info("Loading knowledge base")
-        num_documents = 0
-        for document_list in self.document_lists:
-            documents_to_load = document_list
-
-            # Track metadata for filtering capabilities and collect metadata for filters
-            filters_metadata: Optional[Dict[str, Any]] = None
-            for doc in document_list:
-                if doc.meta_data:
-                    self._track_metadata_structure(doc.meta_data)
-                    # Use the first non-None metadata for filters
-                    if filters_metadata is None:
-                        filters_metadata = doc.meta_data
-
-            # Skip processing if no documents in this batch
-            if not documents_to_load:
-                log_debug("Skipping empty document batch")
-                continue
-
-            # Upsert documents if upsert is True and vector db supports upsert
-            if upsert and self.vector_db.upsert_available():
-                self.vector_db.upsert(documents=documents_to_load, filters=filters_metadata)
-            # Insert documents
-            else:
-                # Filter out documents which already exist in the vector db
-                if skip_existing:
-                    log_debug("Filtering out existing documents before insertion.")
-                    documents_to_load = self.filter_existing_documents(document_list)
-
-                if documents_to_load:
-                    self.vector_db.insert(documents=documents_to_load, filters=filters_metadata)
-
-            num_documents += len(documents_to_load)
-        log_info(f"Added {num_documents} documents to knowledge base")
-
-    async def aload(
-        self,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load the knowledge base to the vector db asynchronously
-        Args:
-            recreate (bool): If True, recreates the collection in the vector db. Defaults to False.
-            upsert (bool): If True, upserts documents to the vector db. Defaults to False.
-            skip_existing (bool): If True, skips documents which already exist in the vector db when inserting. Defaults to True.
-        """
-        await self._aload_init(recreate, upsert)
-        if self.vector_db is None:
-            return
-
-        log_info("Loading knowledge base")
-        num_documents = 0
-        document_iterator = self.async_document_lists
-        async for document_list in document_iterator:  # type: ignore
-            documents_to_load = document_list
-
-            # Track metadata for filtering capabilities and collect metadata for filters
-            filters_metadata: Optional[Dict[str, Any]] = None
-            for doc in document_list:
-                if doc.meta_data:
-                    self._track_metadata_structure(doc.meta_data)
-                    # Use the first non-None metadata for filters
-                    if filters_metadata is None:
-                        filters_metadata = doc.meta_data
-
-            # Skip processing if no documents in this batch
-            if not documents_to_load:
-                log_debug("Skipping empty document batch")
-                continue
-
-            # Upsert documents if upsert is True and vector db supports upsert
-            if upsert and self.vector_db.upsert_available():
-                await self.vector_db.async_upsert(documents=documents_to_load, filters=filters_metadata)
-            # Insert documents
-            else:
-                # Filter out documents which already exist in the vector db
-                if skip_existing:
-                    log_debug("Filtering out existing documents before insertion.")
-                    documents_to_load = await self.async_filter_existing_documents(document_list)
-
-                if documents_to_load:
-                    await self.vector_db.async_insert(documents=documents_to_load, filters=filters_metadata)
-
-            num_documents += len(documents_to_load)
-        log_info(f"Added {num_documents} documents to knowledge base")

agno/knowledge/json.py

@@ -1,137 +0,0 @@
-from pathlib import Path
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
-
-from agno.document import Document
-from agno.document.reader.json_reader import JSONReader
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_info, logger
-
-
-class JSONKnowledgeBase(AgentKnowledge):
-    path: Optional[Union[str, Path, List[Dict[str, Union[str, Dict[str, Any]]]]]] = None
-    reader: JSONReader = JSONReader()
-    formats: List[str] = [".json"]
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over JSON files and yield lists of documents."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _file_path = Path(file_path)  # type: ignore
-                    if self._is_valid_json(_file_path):
-                        documents = self.reader.read(path=_file_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _file_path = Path(self.path)
-            if _file_path.is_dir():
-                for _file in _file_path.glob("**/*"):
-                    if self._is_valid_json(_file):
-                        yield self.reader.read(path=_file)
-            elif self._is_valid_json(_file_path):
-                yield self.reader.read(path=_file_path)
-
-    def _is_valid_json(self, path: Path) -> bool:
-        """Helper to check if path is a valid JSON file."""
-        return path.exists() and path.is_file() and path.suffix in self.formats
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over JSON files and yield lists of documents asynchronously."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _file_path = Path(file_path)  # type: ignore
-                    if self._is_valid_json(_file_path):
-                        documents = await self.reader.async_read(path=_file_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _file_path = Path(self.path)
-            if _file_path.is_dir():
-                for _file in _file_path.glob("**/*"):
-                    if self._is_valid_json(_file):
-                        yield await self.reader.async_read(path=_file)
-            elif self._is_valid_json(_file_path):
-                yield await self.reader.async_read(path=_file_path)
-
-    def load_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not self.prepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = self.reader.read(path=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        self.process_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )
-
-    async def aload_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not await self.aprepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = await self.reader.async_read(path=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        await self.aprocess_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )

agno/knowledge/langchain.py

@@ -1,71 +0,0 @@
-from typing import Any, Callable, Dict, List, Optional
-
-from agno.document import Document
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_debug, logger
-
-
-class LangChainKnowledgeBase(AgentKnowledge):
-    loader: Optional[Callable] = None
-
-    vectorstore: Optional[Any] = None
-    search_kwargs: Optional[dict] = None
-
-    retriever: Optional[Any] = None
-
-    def search(
-        self, query: str, num_documents: Optional[int] = None, filters: Optional[Dict[str, Any]] = None
-    ) -> List[Document]:
-        """Returns relevant documents matching the query"""
-
-        try:
-            from langchain_core.documents import Document as LangChainDocument
-            from langchain_core.retrievers import BaseRetriever
-        except ImportError:
-            raise ImportError(
-                "The `langchain` package is not installed. Please install it via `pip install langchain`."
-            )
-
-        if self.vectorstore is not None and self.retriever is None:
-            log_debug("Creating retriever")
-            if self.search_kwargs is None:
-                self.search_kwargs = {"k": self.num_documents}
-            if filters is not None:
-                self.search_kwargs.update(filters)
-            self.retriever = self.vectorstore.as_retriever(search_kwargs=self.search_kwargs)
-
-        if self.retriever is None:
-            logger.error("No retriever provided")
-            return []
-
-        if not isinstance(self.retriever, BaseRetriever):
-            raise ValueError(f"Retriever is not of type BaseRetriever: {self.retriever}")
-
-        _num_documents = num_documents or self.num_documents
-        log_debug(f"Getting {_num_documents} relevant documents for query: {query}")
-        lc_documents: List[LangChainDocument] = self.retriever.invoke(input=query)
-        documents = []
-        for lc_doc in lc_documents:
-            documents.append(
-                Document(
-                    content=lc_doc.page_content,
-                    meta_data=lc_doc.metadata,
-                )
-            )
-        return documents
-
-    def load(
-        self,
-        recreate: bool = False,
-        upsert: bool = True,
-        skip_existing: bool = True,
-        filters: Optional[Dict[str, Any]] = None,
-    ) -> None:
-        if self.loader is None:
-            logger.error("No loader provided for LangChainKnowledgeBase")
-            return
-        self.loader()
-
-    def exists(self) -> bool:
-        logger.warning("LangChainKnowledgeBase.exists() not supported - please check the vectorstore manually.")
-        return True