agno 1.8.1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/knowledge/arxiv.py

@@ -1,33 +0,0 @@
-from typing import AsyncIterator, Iterator, List
-
-from agno.document import Document
-from agno.document.reader.arxiv_reader import ArxivReader
-from agno.knowledge.agent import AgentKnowledge
-
-
-class ArxivKnowledgeBase(AgentKnowledge):
-    queries: List[str] = []
-    reader: ArxivReader = ArxivReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over urls and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-
-        for _query in self.queries:
-            yield self.reader.read(query=_query)
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over queries and yield lists of documents asynchronously.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            AsyncIterator[List[Document]]: Async iterator yielding list of documents
-        """
-        for _query in self.queries:
-            yield await self.reader.async_read(query=_query)

agno/knowledge/combined.py

@@ -1,36 +0,0 @@
-from typing import AsyncIterator, Iterator, List
-
-from agno.document import Document
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_debug
-
-
-class CombinedKnowledgeBase(AgentKnowledge):
-    sources: List[AgentKnowledge] = []
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over knowledge bases and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-
-        for kb in self.sources:
-            log_debug(f"Loading documents from {kb.__class__.__name__}")
-            yield from kb.document_lists
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over knowledge bases and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-
-        for kb in self.sources:
-            log_debug(f"Loading documents from {kb.__class__.__name__}")
-            async for document in kb.async_document_lists:  # type: ignore
-                yield document

agno/knowledge/csv.py

@@ -1,144 +0,0 @@
-from pathlib import Path
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
-
-from pydantic import Field
-
-from agno.document import Document
-from agno.document.reader.csv_reader import CSVReader
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_info, logger
-
-
-class CSVKnowledgeBase(AgentKnowledge):
-    path: Optional[Union[str, Path, List[Dict[str, Union[str, Dict[str, Any]]]]]] = None
-    formats: List[str] = [".csv"]
-    exclude_files: List[str] = Field(default_factory=list)
-    reader: CSVReader = CSVReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over CSV files and yield lists of documents."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _csv_path = Path(file_path)  # type: ignore
-                    if self._is_valid_csv(_csv_path):
-                        documents = self.reader.read(file=_csv_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _csv_path = Path(self.path)
-            if _csv_path.is_dir():
-                for _csv in _csv_path.glob("**/*.csv"):
-                    if _csv.name not in self.exclude_files:
-                        yield self.reader.read(file=_csv)
-            elif self._is_valid_csv(_csv_path):
-                yield self.reader.read(file=_csv_path)
-
-    def _is_valid_csv(self, path: Path) -> bool:
-        """Helper to check if path is a valid CSV file."""
-        return path.exists() and path.is_file() and path.suffix == ".csv" and path.name not in self.exclude_files
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over CSV files and yield lists of documents asynchronously."""
-        if self.path is None:
-            raise ValueError("Path is not set")
-
-        if isinstance(self.path, list):
-            for item in self.path:
-                if isinstance(item, dict) and "path" in item:
-                    # Handle path with metadata
-                    file_path = item["path"]
-                    config = item.get("metadata", {})
-                    _csv_path = Path(file_path)  # type: ignore
-                    if self._is_valid_csv(_csv_path):
-                        documents = await self.reader.async_read(file=_csv_path)
-                        if config:
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-        else:
-            # Handle single path
-            _csv_path = Path(self.path)
-            if _csv_path.is_dir():
-                for _csv in _csv_path.glob("**/*.csv"):
-                    if _csv.name not in self.exclude_files:
-                        yield await self.reader.async_read(file=_csv)
-            elif self._is_valid_csv(_csv_path):
-                yield await self.reader.async_read(file=_csv_path)
-
-    def load_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load documents from a single CSV file with specific metadata into the vector DB."""
-
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not self.prepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = self.reader.read(file=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        self.process_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )
-
-    async def aload_document(
-        self,
-        path: Union[str, Path],
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load documents from a single CSV file with specific metadata into the vector DB."""
-
-        _file_path = Path(path) if isinstance(path, str) else path
-
-        # Validate file and prepare collection in one step
-        if not await self.aprepare_load(_file_path, self.formats, metadata, recreate):
-            return
-
-        # Read documents
-        try:
-            documents = await self.reader.async_read(file=_file_path)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from file {_file_path}: {e}")
-            return
-
-        # Process documents
-        await self.aprocess_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=str(_file_path),
-        )

agno/knowledge/csv_url.py

@@ -1,124 +0,0 @@
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
-
-from agno.document import Document
-from agno.document.reader.csv_reader import CSVUrlReader
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_info, logger
-
-
-class CSVUrlKnowledgeBase(AgentKnowledge):
-    urls: Optional[Union[List[str], List[Dict[str, Union[str, Dict[str, Any]]]]]] = None
-    formats: List[str] = [".csv"]
-    reader: CSVUrlReader = CSVUrlReader()
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over CSV URLs and yield lists of documents."""
-        if self.urls is None:
-            raise ValueError("URLs are not set")
-
-        for item in self.urls:
-            if isinstance(item, dict) and "url" in item:
-                # Handle URL with metadata
-                url = item["url"]
-                if isinstance(url, str):  # Type guard
-                    config = item.get("metadata", {})
-                    if self._is_valid_csv_url(url):
-                        documents = self.reader.read(url=url)
-                        if config and isinstance(config, dict):
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-            elif isinstance(item, str):
-                # Handle plain URL string
-                if self._is_valid_csv_url(item):
-                    yield self.reader.read(url=item)
-
-    def _is_valid_csv_url(self, url: str) -> bool:
-        """Helper to check if URL is a valid CSV URL."""
-        return url.endswith(".csv")
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over CSV URLs and yield lists of documents asynchronously."""
-        if self.urls is None:
-            raise ValueError("URLs are not set")
-
-        for item in self.urls:
-            if isinstance(item, dict) and "url" in item:
-                # Handle URL with metadata
-                url = item["url"]
-                if isinstance(url, str):  # Type guard
-                    config = item.get("metadata", {})
-                    if self._is_valid_csv_url(url):
-                        documents = await self.reader.async_read(url=url)
-                        if config and isinstance(config, dict):
-                            for doc in documents:
-                                log_info(f"Adding metadata {config} to document: {doc.name}")
-                                doc.meta_data.update(config)  # type: ignore
-                        yield documents
-            elif isinstance(item, str):
-                # Handle plain URL string
-                if self._is_valid_csv_url(item):
-                    yield await self.reader.async_read(url=item)
-
-    def load_document(
-        self,
-        url: str,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load documents from a single CSV URL with specific metadata into the vector DB."""
-
-        # Validate URL and prepare collection in one step
-        if not self.prepare_load(url, self.formats, metadata, recreate, is_url=True):  # type: ignore
-            return
-
-        # Read documents
-        try:
-            documents = self.reader.read(url=url)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from URL {url}: {e}")
-            return
-
-        # Process documents
-        self.process_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=url,
-        )
-
-    async def aload_document(
-        self,
-        url: str,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load documents from a single CSV URL with specific metadata into the vector DB."""
-
-        # Validate URL and prepare collection in one step
-        if not await self.aprepare_load(url, self.formats, metadata, recreate, is_url=True):  # type: ignore
-            return
-
-        # Read documents
-        try:
-            documents = await self.reader.async_read(url=url)
-        except Exception as e:
-            logger.exception(f"Failed to read documents from URL {url}: {e}")
-            return
-
-        # Process documents
-        await self.aprocess_documents(
-            documents=documents,
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=url,
-        )

agno/knowledge/document.py

@@ -1,223 +0,0 @@
-from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
-
-from agno.document import Document
-from agno.knowledge.agent import AgentKnowledge
-from agno.utils.log import log_info, logger
-
-
-class DocumentKnowledgeBase(AgentKnowledge):
-    documents: Optional[Union[List[Document], List[Dict[str, Any]]]] = None
-
-    @property
-    def document_lists(self) -> Iterator[List[Document]]:
-        """Iterate over documents and yield lists of documents.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            Iterator[List[Document]]: Iterator yielding list of documents
-        """
-        if self.documents is None:
-            # Return empty iterator when no documents are set
-            return
-
-        for item in self.documents:
-            if isinstance(item, dict) and "document" in item:
-                # Handle document with metadata
-                document: Document = item["document"]
-                config = item.get("metadata", {})
-                if config:
-                    log_info(f"Adding metadata {config} to document: {document.name}")
-                    # Create a copy of the document with updated metadata
-                    updated_document = Document(
-                        content=document.content,
-                        id=document.id,
-                        name=document.name,
-                        meta_data={**document.meta_data, **config},
-                        embedder=document.embedder,
-                        embedding=document.embedding,
-                        usage=document.usage,
-                        reranking_score=document.reranking_score,
-                    )
-                    yield [updated_document]
-                else:
-                    yield [document]
-            elif isinstance(item, Document):
-                # Handle direct document
-                yield [item]
-            else:
-                raise ValueError(f"Invalid document format: {type(item)}")
-
-    @property
-    async def async_document_lists(self) -> AsyncIterator[List[Document]]:
-        """Iterate over documents and yield lists of documents asynchronously.
-        Each object yielded by the iterator is a list of documents.
-
-        Returns:
-            AsyncIterator[List[Document]]: Iterator yielding list of documents
-        """
-        if self.documents is None:
-            # Return empty iterator when no documents are set
-            return
-
-        for item in self.documents:
-            if isinstance(item, dict) and "document" in item:
-                # Handle document with metadata
-                document: Document = item["document"]
-                config = item.get("metadata", {})
-                if config:
-                    log_info(f"Adding metadata {config} to document: {document.name}")
-                    # Create a copy of the document with updated metadata
-                    updated_document = Document(
-                        content=document.content,
-                        id=document.id,
-                        name=document.name,
-                        meta_data={**document.meta_data, **config},
-                        embedder=document.embedder,
-                        embedding=document.embedding,
-                        usage=document.usage,
-                        reranking_score=document.reranking_score,
-                    )
-                    yield [updated_document]
-                else:
-                    yield [document]
-            elif isinstance(item, Document):
-                # Handle direct document
-                yield [item]
-            else:
-                raise ValueError(f"Invalid document format: {type(item)}")
-
-    def _prepare_document_load(
-        self,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-    ) -> bool:
-        """Prepare collection for loading documents (no file validation needed).
-        Args:
-            metadata (Optional[Dict[str, Any]]): Metadata to track
-            recreate (bool): Whether to recreate the collection
-        Returns:
-            bool: True if preparation succeeded, False otherwise
-        """
-        # 1. Track metadata
-        if metadata:
-            self._track_metadata_structure(metadata)
-
-        # 2. Prepare vector DB
-        if self.vector_db is None:
-            logger.warning("Cannot load document: No vector db provided.")
-            return False
-
-        # Recreate collection if requested
-        if recreate:
-            self.vector_db.drop()
-
-        # Create collection if it doesn't exist
-        if not self.vector_db.exists():
-            self.vector_db.create()
-
-        return True
-
-    async def _aprepare_document_load(
-        self,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-    ) -> bool:
-        """Prepare collection for loading documents asynchronously (no file validation needed).
-        Args:
-            metadata (Optional[Dict[str, Any]]): Metadata to track
-            recreate (bool): Whether to recreate the collection
-        Returns:
-            bool: True if preparation succeeded, False otherwise
-        """
-        # 1. Track metadata
-        if metadata:
-            self._track_metadata_structure(metadata)
-
-        # 2. Prepare vector DB
-        if self.vector_db is None:
-            logger.warning("Cannot load document: No vector db provided.")
-            return False
-
-        # Recreate collection if requested
-        if recreate:
-            await self.vector_db.async_drop()
-
-        # Create collection if it doesn't exist
-        if not await self.vector_db.async_exists():
-            await self.vector_db.async_create()
-
-        return True
-
-    def load_document(
-        self,
-        document: Document,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load a single document with specific metadata into the vector DB."""
-
-        # Use our document-specific preparation method
-        if not self._prepare_document_load(metadata, recreate):
-            return
-
-        # Apply metadata if provided
-        if metadata:
-            # Create a copy of the document with updated metadata
-            document = Document(
-                content=document.content,
-                id=document.id,
-                name=document.name,
-                meta_data={**document.meta_data, **metadata},
-                embedder=document.embedder,
-                embedding=document.embedding,
-                usage=document.usage,
-                reranking_score=document.reranking_score,
-            )
-
-        # Process documents
-        self.process_documents(
-            documents=[document],
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=f"document: {document.name or document.id}",
-        )
-
-    async def aload_document(
-        self,
-        document: Document,
-        metadata: Optional[Dict[str, Any]] = None,
-        recreate: bool = False,
-        upsert: bool = False,
-        skip_existing: bool = True,
-    ) -> None:
-        """Load a single document with specific metadata into the vector DB asynchronously."""
-
-        # Use our document-specific preparation method
-        if not await self._aprepare_document_load(metadata, recreate):
-            return
-
-        # Apply metadata if provided
-        if metadata:
-            # Create a copy of the document with updated metadata
-            document = Document(
-                content=document.content,
-                id=document.id,
-                name=document.name,
-                meta_data={**document.meta_data, **metadata},
-                embedder=document.embedder,
-                embedding=document.embedding,
-                usage=document.usage,
-                reranking_score=document.reranking_score,
-            )
-
-        # Process documents
-        await self.aprocess_documents(
-            documents=[document],
-            metadata=metadata,
-            upsert=upsert,
-            skip_existing=skip_existing,
-            source_info=f"document: {document.name or document.id}",
-        )