agno 2.0.11__py3-none-any.whl → 2.1.0__py3-none-any.whl

agno/knowledge/embedder/mistral.py

@@ -3,13 +3,13 @@ from os import getenv
 from typing import Any, Dict, List, Optional, Tuple
 
 from agno.knowledge.embedder.base import Embedder
-from agno.utils.log import logger
+from agno.utils.log import log_error, log_info, log_warning
 
 try:
     from mistralai import Mistral  # type: ignore
     from mistralai.models.embeddingresponse import EmbeddingResponse  # type: ignore
 except ImportError:
-    logger.error("`mistralai` not installed")
+    log_error("`mistralai` not installed")
     raise
 
 
@@ -50,7 +50,7 @@ class MistralEmbedder(Embedder):
 
     def _response(self, text: str) -> EmbeddingResponse:
         _request_params: Dict[str, Any] = {
-            "inputs": text,
+            "inputs": [text],  # Mistral API expects a list
             "model": self.id,
         }
         if self.request_params:
@@ -67,7 +67,7 @@ class MistralEmbedder(Embedder):
                 return response.data[0].embedding
             return []
         except Exception as e:
-            logger.warning(f"Error getting embedding: {e}")
+            log_warning(f"Error getting embedding: {e}")
             return []
 
     def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Dict[str, Any]]:
@@ -79,7 +79,7 @@ class MistralEmbedder(Embedder):
             usage: Dict[str, Any] = response.usage.model_dump() if response.usage else {}
             return embedding, usage
         except Exception as e:
-            logger.warning(f"Error getting embedding and usage: {e}")
+            log_warning(f"Error getting embedding and usage: {e}")
             return [], {}
 
     async def async_get_embedding(self, text: str) -> List[float]:
@@ -88,7 +88,7 @@ class MistralEmbedder(Embedder):
             # Check if the client has an async version of embeddings.create
             if hasattr(self.client.embeddings, "create_async"):
                 response: EmbeddingResponse = await self.client.embeddings.create_async(
-                    inputs=text, model=self.id, **self.request_params if self.request_params else {}
+                    inputs=[text], model=self.id, **self.request_params if self.request_params else {}
                 )
             else:
                 # Fallback to running sync method in thread executor
@@ -98,7 +98,7 @@ class MistralEmbedder(Embedder):
                 response: EmbeddingResponse = await loop.run_in_executor(  # type: ignore
                     None,
                     lambda: self.client.embeddings.create(
-                        inputs=text, model=self.id, **self.request_params if self.request_params else {}
+                        inputs=[text], model=self.id, **self.request_params if self.request_params else {}
                     ),
                 )
 
@@ -106,7 +106,7 @@ class MistralEmbedder(Embedder):
                 return response.data[0].embedding
             return []
         except Exception as e:
-            logger.warning(f"Error getting embedding: {e}")
+            log_warning(f"Error getting embedding: {e}")
             return []
 
     async def async_get_embedding_and_usage(self, text: str) -> Tuple[List[float], Dict[str, Any]]:
@@ -115,7 +115,7 @@ class MistralEmbedder(Embedder):
             # Check if the client has an async version of embeddings.create
             if hasattr(self.client.embeddings, "create_async"):
                 response: EmbeddingResponse = await self.client.embeddings.create_async(
-                    inputs=text, model=self.id, **self.request_params if self.request_params else {}
+                    inputs=[text], model=self.id, **self.request_params if self.request_params else {}
                 )
             else:
                 # Fallback to running sync method in thread executor
@@ -125,7 +125,7 @@ class MistralEmbedder(Embedder):
                 response: EmbeddingResponse = await loop.run_in_executor(  # type: ignore
                     None,
                     lambda: self.client.embeddings.create(
-                        inputs=text, model=self.id, **self.request_params if self.request_params else {}
+                        inputs=[text], model=self.id, **self.request_params if self.request_params else {}
                     ),
                 )
 
@@ -135,5 +135,72 @@ class MistralEmbedder(Embedder):
             usage: Dict[str, Any] = response.usage.model_dump() if response.usage else {}
             return embedding, usage
         except Exception as e:
-            logger.warning(f"Error getting embedding and usage: {e}")
+            log_warning(f"Error getting embedding and usage: {e}")
             return [], {}
+
+    async def async_get_embeddings_batch_and_usage(
+        self, texts: List[str]
+    ) -> Tuple[List[List[float]], List[Optional[Dict[str, Any]]]]:
+        """
+        Get embeddings and usage for multiple texts in batches.
+
+        Args:
+            texts: List of text strings to embed
+
+        Returns:
+            Tuple of (List of embedding vectors, List of usage dictionaries)
+        """
+        all_embeddings = []
+        all_usage = []
+        log_info(f"Getting embeddings and usage for {len(texts)} texts in batches of {self.batch_size}")
+
+        for i in range(0, len(texts), self.batch_size):
+            batch_texts = texts[i : i + self.batch_size]
+
+            _request_params: Dict[str, Any] = {
+                "inputs": batch_texts,  # Mistral API expects a list for batch processing
+                "model": self.id,
+            }
+            if self.request_params:
+                _request_params.update(self.request_params)
+
+            try:
+                # Check if the client has an async version of embeddings.create
+                if hasattr(self.client.embeddings, "create_async"):
+                    response: EmbeddingResponse = await self.client.embeddings.create_async(**_request_params)
+                else:
+                    # Fallback to running sync method in thread executor
+                    import asyncio
+
+                    loop = asyncio.get_running_loop()
+                    response: EmbeddingResponse = await loop.run_in_executor(  # type: ignore
+                        None, lambda: self.client.embeddings.create(**_request_params)
+                    )
+
+                # Extract embeddings from batch response
+                if response.data:
+                    batch_embeddings = [data.embedding for data in response.data if data.embedding]
+                    all_embeddings.extend(batch_embeddings)
+                else:
+                    # If no embeddings, add empty lists for each text in batch
+                    all_embeddings.extend([[] for _ in batch_texts])
+
+                # Extract usage information
+                usage_dict = response.usage.model_dump() if response.usage else None
+                # Add same usage info for each embedding in the batch
+                all_usage.extend([usage_dict] * len(batch_texts))
+
+            except Exception as e:
+                log_warning(f"Error in async batch embedding: {e}")
+                # Fallback to individual calls for this batch
+                for text in batch_texts:
+                    try:
+                        embedding, usage = await self.async_get_embedding_and_usage(text)
+                        all_embeddings.append(embedding)
+                        all_usage.append(usage)
+                    except Exception as e2:
+                        log_warning(f"Error in individual async embedding fallback: {e2}")
+                        all_embeddings.append([])
+                        all_usage.append(None)
+
+        return all_embeddings, all_usage

agno/knowledge/embedder/ollama.py

@@ -45,6 +45,11 @@ class OllamaEmbedder(Embedder):
     ollama_client: Optional[OllamaClient] = None
     async_client: Optional[AsyncOllamaClient] = None
 
+    def __post_init__(self):
+        if self.enable_batch:
+            logger.warning("OllamaEmbedder does not support batch embeddings, setting enable_batch to False")
+            self.enable_batch = False
+
     @property
     def client(self) -> OllamaClient:
         if self.ollama_client:

agno/knowledge/embedder/openai.py

@@ -140,66 +140,24 @@ class OpenAIEmbedder(Embedder):
             logger.warning(e)
             return [], None
 
-    def get_embeddings_batch(self, texts: List[str], batch_size: int = 100) -> List[List[float]]:
+    async def async_get_embeddings_batch_and_usage(
+        self, texts: List[str]
+    ) -> Tuple[List[List[float]], List[Optional[Dict]]]:
         """
-        Get embeddings for multiple texts in batches.
+        Get embeddings and usage for multiple texts in batches (async version).
 
         Args:
             texts: List of text strings to embed
-            batch_size: Number of texts to process in each API call (max ~2048)
 
         Returns:
-            List of embedding vectors
+            Tuple of (List of embedding vectors, List of usage dictionaries)
         """
         all_embeddings = []
+        all_usage = []
+        logger.info(f"Getting embeddings and usage for {len(texts)} texts in batches of {self.batch_size} (async)")
 
-        for i in range(0, len(texts), batch_size):
-            batch_texts = texts[i : i + batch_size]
-
-            req: Dict[str, Any] = {
-                "input": batch_texts,
-                "model": self.id,
-                "encoding_format": self.encoding_format,
-            }
-            if self.user is not None:
-                req["user"] = self.user
-            if self.id.startswith("text-embedding-3"):
-                req["dimensions"] = self.dimensions
-            if self.request_params:
-                req.update(self.request_params)
-
-            try:
-                response: CreateEmbeddingResponse = self.client.embeddings.create(**req)
-                batch_embeddings = [data.embedding for data in response.data]
-                all_embeddings.extend(batch_embeddings)
-            except Exception as e:
-                logger.warning(f"Error in batch embedding: {e}")
-                # Fallback to individual calls for this batch
-                for text in batch_texts:
-                    try:
-                        embedding = self.get_embedding(text)
-                        all_embeddings.append(embedding)
-                    except Exception as e2:
-                        logger.warning(f"Error in individual embedding fallback: {e2}")
-                        all_embeddings.append([])
-
-        return all_embeddings
-
-    async def async_get_embeddings_batch(self, texts: List[str], batch_size: int = 100) -> List[List[float]]:
-        """
-        Get embeddings for multiple texts in batches (async version).
-
-        Args:
-            texts: List of text strings to embed
-            batch_size: Number of texts to process in each API call (max ~2048)
-
-        Returns:
-            List of embedding vectors
-        """
-        all_embeddings = []
-
-        for i in range(0, len(texts), batch_size):
-            batch_texts = texts[i : i + batch_size]
+        for i in range(0, len(texts), self.batch_size):
+            batch_texts = texts[i : i + self.batch_size]
 
             req: Dict[str, Any] = {
                 "input": batch_texts,
@@ -217,15 +175,21 @@ class OpenAIEmbedder(Embedder):
                 response: CreateEmbeddingResponse = await self.aclient.embeddings.create(**req)
                 batch_embeddings = [data.embedding for data in response.data]
                 all_embeddings.extend(batch_embeddings)
+
+                # For each embedding in the batch, add the same usage information
+                usage_dict = response.usage.model_dump() if response.usage else None
+                all_usage.extend([usage_dict] * len(batch_embeddings))
             except Exception as e:
                 logger.warning(f"Error in async batch embedding: {e}")
-                # Fallback to individual async calls for this batch
+                # Fallback to individual calls for this batch
                 for text in batch_texts:
                     try:
-                        embedding = await self.async_get_embedding(text)
+                        embedding, usage = await self.async_get_embedding_and_usage(text)
                         all_embeddings.append(embedding)
+                        all_usage.append(usage)
                     except Exception as e2:
                         logger.warning(f"Error in individual async embedding fallback: {e2}")
                         all_embeddings.append([])
+                        all_usage.append(None)
 
-        return all_embeddings
+        return all_embeddings, all_usage

agno/knowledge/embedder/voyageai.py

@@ -30,12 +30,13 @@ class VoyageAIEmbedder(Embedder):
         if self.voyage_client:
             return self.voyage_client
 
-        _client_params = {
-            "api_key": self.api_key,
-            "max_retries": self.max_retries,
-            "timeout": self.timeout,
-        }
-        _client_params = {k: v for k, v in _client_params.items() if v is not None}
+        _client_params: Dict[str, Any] = {}
+        if self.api_key is not None:
+            _client_params["api_key"] = self.api_key
+        if self.max_retries is not None:
+            _client_params["max_retries"] = self.max_retries
+        if self.timeout is not None:
+            _client_params["timeout"] = self.timeout
         if self.client_params:
             _client_params.update(self.client_params)
         self.voyage_client = VoyageClient(**_client_params)
@@ -46,12 +47,13 @@ class VoyageAIEmbedder(Embedder):
         if self.async_client:
             return self.async_client
 
-        _client_params = {
-            "api_key": self.api_key,
-            "max_retries": self.max_retries,
-            "timeout": self.timeout,
-        }
-        _client_params = {k: v for k, v in _client_params.items() if v is not None}
+        _client_params: Dict[str, Any] = {}
+        if self.api_key is not None:
+            _client_params["api_key"] = self.api_key
+        if self.max_retries is not None:
+            _client_params["max_retries"] = self.max_retries
+        if self.timeout is not None:
+            _client_params["timeout"] = self.timeout
         if self.client_params:
             _client_params.update(self.client_params)
         self.async_client = AsyncVoyageClient(**_client_params)
@@ -69,7 +71,8 @@ class VoyageAIEmbedder(Embedder):
     def get_embedding(self, text: str) -> List[float]:
         response: EmbeddingsObject = self._response(text=text)
         try:
-            return response.embeddings[0]
+            embedding = response.embeddings[0]
+            return [float(x) for x in embedding]  # Ensure all values are float
         except Exception as e:
             logger.warning(e)
             return []
@@ -79,7 +82,7 @@ class VoyageAIEmbedder(Embedder):
 
         embedding = response.embeddings[0]
         usage = {"total_tokens": response.total_tokens}
-        return embedding, usage
+        return [float(x) for x in embedding], usage
 
     async def _async_response(self, text: str) -> EmbeddingsObject:
         """Async version of _response using AsyncVoyageClient."""
@@ -95,7 +98,8 @@ class VoyageAIEmbedder(Embedder):
         """Async version of get_embedding."""
         try:
             response: EmbeddingsObject = await self._async_response(text=text)
-            return response.embeddings[0]
+            embedding = response.embeddings[0]
+            return [float(x) for x in embedding]  # Ensure all values are float
         except Exception as e:
             logger.warning(f"Error getting embedding: {e}")
             return []
@@ -106,7 +110,56 @@ class VoyageAIEmbedder(Embedder):
             response: EmbeddingsObject = await self._async_response(text=text)
             embedding = response.embeddings[0]
             usage = {"total_tokens": response.total_tokens}
-            return embedding, usage
+            return [float(x) for x in embedding], usage
         except Exception as e:
             logger.warning(f"Error getting embedding and usage: {e}")
             return [], None
+
+    async def async_get_embeddings_batch_and_usage(
+        self, texts: List[str]
+    ) -> Tuple[List[List[float]], List[Optional[Dict]]]:
+        """
+        Get embeddings and usage for multiple texts in batches.
+
+        Args:
+            texts: List of text strings to embed
+
+        Returns:
+            Tuple of (List of embedding vectors, List of usage dictionaries)
+        """
+        all_embeddings: List[List[float]] = []
+        all_usage: List[Optional[Dict]] = []
+        logger.info(f"Getting embeddings and usage for {len(texts)} texts in batches of {self.batch_size}")
+
+        for i in range(0, len(texts), self.batch_size):
+            batch_texts = texts[i : i + self.batch_size]
+
+            req: Dict[str, Any] = {
+                "texts": batch_texts,
+                "model": self.id,
+            }
+            if self.request_params:
+                req.update(self.request_params)
+
+            try:
+                response: EmbeddingsObject = await self.aclient.embed(**req)
+                batch_embeddings = [[float(x) for x in emb] for emb in response.embeddings]
+                all_embeddings.extend(batch_embeddings)
+
+                # For each embedding in the batch, add the same usage information
+                usage_dict = {"total_tokens": response.total_tokens}
+                all_usage.extend([usage_dict] * len(batch_embeddings))
+            except Exception as e:
+                logger.warning(f"Error in async batch embedding: {e}")
+                # Fallback to individual calls for this batch
+                for text in batch_texts:
+                    try:
+                        embedding, usage = await self.async_get_embedding_and_usage(text)
+                        all_embeddings.append(embedding)
+                        all_usage.append(usage)
+                    except Exception as e2:
+                        logger.warning(f"Error in individual async embedding fallback: {e2}")
+                        all_embeddings.append([])
+                        all_usage.append(None)
+
+        return all_embeddings, all_usage

agno/knowledge/knowledge.py

@@ -89,7 +89,7 @@ class Knowledge:
                     url=argument.get("url"),
                     metadata=argument.get("metadata"),
                     topics=argument.get("topics"),
-                    text_contents=argument.get("text_contents"),
+                    text_content=argument.get("text_content"),
                     reader=argument.get("reader"),
                     include=argument.get("include"),
                     exclude=argument.get("exclude"),
@@ -251,7 +251,9 @@ class Knowledge:
     ) -> None:
         # Validation: At least one of the parameters must be provided
         if all(argument is None for argument in [path, url, text_content, topics, remote_content]):
-            log_info("At least one of 'path', 'url', 'text_content', 'topics', or 'remote_content' must be provided.")
+            log_warning(
+                "At least one of 'path', 'url', 'text_content', 'topics', or 'remote_content' must be provided."
+            )
             return
 
         if not skip_if_exists:
@@ -534,7 +536,6 @@ class Knowledge:
         reader = content.reader
         name = content.name if content.name else content.url
         # Else select based on file extension
-
         if reader is None:
             if file_extension == ".csv":
                 name = basename(parsed_url.path) or "data.csv"
@@ -570,6 +571,7 @@ class Knowledge:
                         read_documents = reader.read(bytes_content, name=name)
                     else:
                         read_documents = reader.read(content.url, name=name)
+
         except Exception as e:
             log_error(f"Error reading URL: {content.url} - {str(e)}")
             content.status = ContentStatus.FAILED
@@ -580,7 +582,6 @@ class Knowledge:
         # 6. Chunk documents if needed
         if reader and not reader.chunk:
             read_documents = await reader.chunk_documents_async(read_documents)
-
         # 7. Prepare and insert the content in the vector database
         file_size = 0
         if read_documents:

agno/knowledge/reader/pdf_reader.py

@@ -117,6 +117,10 @@ def _clean_page_numbers(
     page_numbers = [find_page_number(content) for content in page_content_list]
     if all(x is None or x > 5 for x in page_numbers):
         # This approach won't work reliably for higher page numbers.
+        page_content_list = [
+            f"\n{page_content_list[i]}\n{extra_content[i]}" if extra_content else page_content_list[i]
+            for i in range(len(page_content_list))
+        ]
         return page_content_list, None
 
     # Possible range shifts to detect page numbering
@@ -261,7 +265,6 @@ class BasePDFReader(Reader):
 
         if self.chunk:
             return self._build_chunked_documents(documents)
-
         return documents
 
     def _pdf_reader_to_documents(
@@ -339,8 +342,6 @@ class PDFReader(BasePDFReader):
         except Exception:
             doc_name = "pdf"
 
-        log_info(f"Reading: {doc_name}")
-
         try:
             DocumentReader(pdf)
         except PdfStreamError as e:

agno/knowledge/reader/website_reader.py

@@ -112,7 +112,8 @@ class WebsiteReader(Reader):
             if tag.name in ["article", "main", "section"]:
                 return True
 
-            classes = tag.get("class", [])
+            classes_attr = tag.get("class")
+            classes: List[str] = classes_attr if isinstance(classes_attr, list) else []
             content_classes = ["content", "main-content", "post-content", "entry-content", "article-body"]
             if any(cls in content_classes for cls in classes):
                 return True
@@ -126,7 +127,7 @@ class WebsiteReader(Reader):
 
         # Try to find main content element
         element = soup.find(match)
-        if element:
+        if element and hasattr(element, "find_all"):
             # Remove common unwanted elements from the found content
             for unwanted in element.find_all(["script", "style", "nav", "header", "footer"]):
                 unwanted.decompose()