agno 2.3.10__py3-none-any.whl → 2.3.12__py3-none-any.whl

agno/knowledge/knowledge.py

@@ -548,7 +548,7 @@ class Knowledge:
         else:
             return self.text_reader
 
-    def _read_with_reader(
+    def _read(
         self,
         reader: Reader,
         source: Union[Path, str, BytesIO],
@@ -581,6 +581,36 @@ class Knowledge:
             else:
                 return reader.read(source, name=name)
 
+    async def _read_async(
+        self,
+        reader: Reader,
+        source: Union[Path, str, BytesIO],
+        name: Optional[str] = None,
+        password: Optional[str] = None,
+    ) -> List[Document]:
+        """
+        Read content using a reader's async_read method with optional password handling.
+
+        Args:
+            reader: Reader to use
+            source: Source to read from (Path, URL string, or BytesIO)
+            name: Optional name for the document
+            password: Optional password for protected files
+
+        Returns:
+            List of documents read
+        """
+        import inspect
+
+        read_signature = inspect.signature(reader.async_read)
+        if password and "password" in read_signature.parameters:
+            return await reader.async_read(source, name=name, password=password)
+        else:
+            if isinstance(source, BytesIO):
+                return await reader.async_read(source, name=name)
+            else:
+                return await reader.async_read(source, name=name)
+
     def _prepare_documents_for_insert(
         self,
         documents: List[Document],
@@ -665,7 +695,7 @@ class Knowledge:
 
                 if reader:
                     password = content.auth.password if content.auth and content.auth.password else None
-                    read_documents = self._read_with_reader(
+                    read_documents = await self._read_async(
                         reader, path, name=content.name or path.name, password=password
                     )
                 else:
@@ -855,7 +885,6 @@ class Knowledge:
             content.status_message = f"Invalid URL: {content.url} - {str(e)}"
             await self._aupdate_content(content)
             log_warning(f"Invalid URL: {content.url} - {str(e)}")
-
         # 3. Fetch and load content if file has an extension
         url_path = Path(parsed_url.path)
         file_extension = url_path.suffix.lower()
@@ -874,18 +903,17 @@ class Knowledge:
                 name = basename(parsed_url.path) or default_name
         else:
             reader = content.reader or self.website_reader
-
         # 5. Read content
         try:
             read_documents = []
             if reader is not None:
                 # Special handling for YouTubeReader
                 if reader.__class__.__name__ == "YouTubeReader":
-                    read_documents = reader.read(content.url, name=name)
+                    read_documents = await reader.async_read(content.url, name=name)
                 else:
                     password = content.auth.password if content.auth and content.auth.password else None
                     source = bytes_content if bytes_content else content.url
-                    read_documents = self._read_with_reader(reader, source, name=name, password=password)
+                    read_documents = await self._read_async(reader, source, name=name, password=password)
 
         except Exception as e:
             log_error(f"Error reading URL: {content.url} - {str(e)}")
@@ -983,7 +1011,7 @@ class Knowledge:
                 else:
                     password = content.auth.password if content.auth and content.auth.password else None
                     source = bytes_content if bytes_content else content.url
-                    read_documents = self._read_with_reader(reader, source, name=name, password=password)
+                    read_documents = self._read(reader, source, name=name, password=password)
 
         except Exception as e:
             log_error(f"Error reading URL: {content.url} - {str(e)}")
@@ -1051,11 +1079,11 @@ class Knowledge:
 
             if content.reader:
                 log_debug(f"Using reader: {content.reader.__class__.__name__} to read content")
-                read_documents = content.reader.read(content_io, name=name)
+                read_documents = await content.reader.async_read(content_io, name=name)
             else:
                 text_reader = self.text_reader
                 if text_reader:
-                    read_documents = text_reader.read(content_io, name=name)
+                    read_documents = await text_reader.async_read(content_io, name=name)
                 else:
                     content.status = ContentStatus.FAILED
                     content.status_message = "Text reader not available"
@@ -1079,7 +1107,7 @@ class Knowledge:
                 else:
                     reader = self._select_reader(content.file_data.type)
                 name = content.name if content.name else f"content_{content.file_data.type}"
-                read_documents = reader.read(content_io, name=name)
+                read_documents = await reader.async_read(content_io, name=name)
                 if not content.id:
                     content.id = generate_id(content.content_hash or "")
                 self._prepare_documents_for_insert(read_documents, content.id, metadata=content.metadata)
@@ -1246,7 +1274,7 @@ class Knowledge:
                 await self._aupdate_content(content)
                 continue
 
-            read_documents = content.reader.read(topic)
+            read_documents = await content.reader.async_read(topic)
             if len(read_documents) > 0:
                 self._prepare_documents_for_insert(read_documents, content.id, calculate_sizes=True)
             else:
@@ -1405,7 +1433,7 @@ class Knowledge:
                 s3_object.download(readable_content)  # type: ignore
 
             # 6. Read the content
-            read_documents = reader.read(readable_content, name=obj_name)
+            read_documents = await reader.async_read(readable_content, name=obj_name)
 
             # 7. Prepare and insert the content in the vector database
             if not content.id:
@@ -1467,7 +1495,7 @@ class Knowledge:
             readable_content = BytesIO(gcs_object.download_as_bytes())
 
             # 6. Read the content
-            read_documents = reader.read(readable_content, name=name)
+            read_documents = await reader.async_read(readable_content, name=name)
 
             # 7. Prepare and insert the content in the vector database
             if not content.id:
@@ -1762,19 +1790,51 @@ class Knowledge:
     def _build_content_hash(self, content: Content) -> str:
         """
         Build the content hash from the content.
+
+        For URLs and paths, includes the name and description in the hash if provided
+        to ensure unique content with the same URL/path but different names/descriptions
+        get different hashes.
+
+        Hash format:
+        - URL with name and description: hash("{name}:{description}:{url}")
+        - URL with name only: hash("{name}:{url}")
+        - URL with description only: hash("{description}:{url}")
+        - URL without name/description: hash("{url}") (backward compatible)
+        - Same logic applies to paths
         """
+        hash_parts = []
+        if content.name:
+            hash_parts.append(content.name)
+        if content.description:
+            hash_parts.append(content.description)
+
         if content.path:
-            return hashlib.sha256(str(content.path).encode()).hexdigest()
+            hash_parts.append(str(content.path))
         elif content.url:
-            hash = hashlib.sha256(content.url.encode()).hexdigest()
-            return hash
+            hash_parts.append(content.url)
         elif content.file_data and content.file_data.content:
-            name = content.name or "content"
-            return hashlib.sha256(name.encode()).hexdigest()
+            # For file_data, always add filename, type, size, or content for uniqueness
+            if content.file_data.filename:
+                hash_parts.append(content.file_data.filename)
+            elif content.file_data.type:
+                hash_parts.append(content.file_data.type)
+            elif content.file_data.size is not None:
+                hash_parts.append(str(content.file_data.size))
+            else:
+                # Fallback: use the content for uniqueness
+                # Include type information to distinguish str vs bytes
+                content_type = "str" if isinstance(content.file_data.content, str) else "bytes"
+                content_bytes = (
+                    content.file_data.content.encode()
+                    if isinstance(content.file_data.content, str)
+                    else content.file_data.content
+                )
+                content_hash = hashlib.sha256(content_bytes).hexdigest()[:16]  # Use first 16 chars
+                hash_parts.append(f"{content_type}:{content_hash}")
         elif content.topics and len(content.topics) > 0:
             topic = content.topics[0]
             reader = type(content.reader).__name__ if content.reader else "unknown"
-            return hashlib.sha256(f"{topic}-{reader}".encode()).hexdigest()
+            hash_parts.append(f"{topic}-{reader}")
         else:
             # Fallback for edge cases
             import random
@@ -1785,7 +1845,10 @@ class Knowledge:
                 or content.id
                 or ("unknown_content" + "".join(random.choices(string.ascii_lowercase + string.digits, k=6)))
             )
-            return hashlib.sha256(fallback.encode()).hexdigest()
+            hash_parts.append(fallback)
+
+        hash_input = ":".join(hash_parts)
+        return hashlib.sha256(hash_input.encode()).hexdigest()
 
     def _ensure_string_field(self, value: Any, field_name: str, default: str = "") -> str:
         """
@@ -1960,8 +2023,8 @@ class Knowledge:
             content_row.updated_at = int(time.time())
             self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
 
-            if self.vector_db and content.metadata:
-                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata)
+            if self.vector_db:
+                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata or {})
 
             return content_row.to_dict()
 
@@ -2006,8 +2069,8 @@ class Knowledge:
             else:
                 self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
 
-            if self.vector_db and content.metadata:
-                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata)
+            if self.vector_db:
+                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata or {})
 
             return content_row.to_dict()
 
@@ -2783,6 +2846,24 @@ class Knowledge:
         """Get all currently loaded readers (only returns readers that have been used)."""
         if self.readers is None:
             self.readers = {}
+        elif not isinstance(self.readers, dict):
+            # Defensive check: if readers is not a dict (e.g., was set to a list), convert it
+            if isinstance(self.readers, list):
+                readers_dict: Dict[str, Reader] = {}
+                for reader in self.readers:
+                    if isinstance(reader, Reader):
+                        reader_key = self._generate_reader_key(reader)
+                        # Handle potential duplicate keys by appending index if needed
+                        original_key = reader_key
+                        counter = 1
+                        while reader_key in readers_dict:
+                            reader_key = f"{original_key}_{counter}"
+                            counter += 1
+                        readers_dict[reader_key] = reader
+                self.readers = readers_dict
+            else:
+                # For any other unexpected type, reset to empty dict
+                self.readers = {}
 
         return self.readers
 

agno/knowledge/reader/csv_reader.py

@@ -110,9 +110,9 @@ class CSVReader(Reader):
                     content = await file_content.read()
                     file_content_io = io.StringIO(content)
             else:
-                log_debug(f"Reading retrieved file async: {file.name}")
+                log_debug(f"Reading retrieved file async: {getattr(file, 'name', 'BytesIO')}")
                 file.seek(0)
-                file_content_io = io.StringIO(file.read().decode("utf-8"))  # type: ignore
+                file_content_io = io.StringIO(file.read().decode("utf-8"))
 
             csv_name = name or (
                 Path(file.name).stem

agno/knowledge/reader/text_reader.py

@@ -41,7 +41,13 @@ class TextReader(Reader):
                 file_name = name or file.stem
                 file_contents = file.read_text(self.encoding or "utf-8")
             else:
-                file_name = name or file.name.split(".")[0]
+                # Handle BytesIO and other file-like objects that may not have a name attribute
+                if name:
+                    file_name = name
+                elif hasattr(file, "name") and file.name is not None:
+                    file_name = file.name.split(".")[0]
+                else:
+                    file_name = "text_file"
                 log_debug(f"Reading uploaded file: {file_name}")
                 file.seek(0)
                 file_contents = file.read().decode(self.encoding or "utf-8")
@@ -81,8 +87,14 @@ class TextReader(Reader):
                     log_warning("aiofiles not installed, using synchronous file I/O")
                     file_contents = file.read_text(self.encoding or "utf-8")
             else:
-                log_debug(f"Reading uploaded file asynchronously: {file.name}")
-                file_name = name or file.name.split(".")[0]
+                # Handle BytesIO and other file-like objects that may not have a name attribute
+                if name:
+                    file_name = name
+                elif hasattr(file, "name") and file.name is not None:
+                    file_name = file.name.split(".")[0]
+                else:
+                    file_name = "text_file"
+                log_debug(f"Reading uploaded file asynchronously: {file_name}")
                 file.seek(0)
                 file_contents = file.read().decode(self.encoding or "utf-8")
 

agno/knowledge/reader/wikipedia_reader.py

@@ -1,3 +1,4 @@
+import asyncio
 from typing import List, Optional
 
 from agno.knowledge.chunking.fixed import FixedSizeChunking
@@ -45,7 +46,38 @@ class WikipediaReader(Reader):
 
         except wikipedia.exceptions.PageError:
             summary = None
-            log_info("PageError: Page not found.")
+            log_info("Wikipedia Error: Page not found.")
+
+        # Only create Document if we successfully got a summary
+        if summary:
+            return [
+                Document(
+                    name=topic,
+                    meta_data={"topic": topic},
+                    content=summary,
+                )
+            ]
+        return []
+
+    async def async_read(self, topic: str) -> List[Document]:
+        """
+        Asynchronously read content from Wikipedia.
+
+        Args:
+            topic: The Wikipedia topic to read
+
+        Returns:
+            A list of documents containing the Wikipedia summary
+        """
+        log_debug(f"Async reading Wikipedia topic: {topic}")
+        summary = None
+        try:
+            # Run the synchronous wikipedia API call in a thread pool
+            summary = await asyncio.to_thread(wikipedia.summary, topic, auto_suggest=self.auto_suggest)
+
+        except wikipedia.exceptions.PageError:
+            summary = None
+            log_info("Wikipedia Error: Page not found.")
 
         # Only create Document if we successfully got a summary
         if summary:

agno/knowledge/utils.py

@@ -1,5 +1,6 @@
-from typing import Dict, List
+from typing import Any, Dict, List, Optional
 
+from agno.knowledge.reader.base import Reader
 from agno.knowledge.reader.reader_factory import ReaderFactory
 from agno.knowledge.types import ContentType
 from agno.utils.log import log_debug
@@ -75,8 +76,33 @@ def get_reader_info(reader_key: str) -> Dict:
         raise ValueError(f"Unknown reader: {reader_key}. Error: {str(e)}")
 
 
-def get_all_readers_info() -> List[Dict]:
-    """Get information about all available readers."""
+def get_reader_info_from_instance(reader: Reader, reader_id: str) -> Dict:
+    """Get information about a reader instance."""
+    try:
+        reader_class = reader.__class__
+        supported_strategies = reader_class.get_supported_chunking_strategies()
+        supported_content_types = reader_class.get_supported_content_types()
+
+        return {
+            "id": reader_id,
+            "name": getattr(reader, "name", reader_class.__name__),
+            "description": getattr(reader, "description", f"Custom {reader_class.__name__}"),
+            "chunking_strategies": [strategy.value for strategy in supported_strategies],
+            "content_types": [ct.value for ct in supported_content_types],
+        }
+    except Exception as e:
+        raise ValueError(f"Failed to get info for reader '{reader_id}': {str(e)}")
+
+
+def get_all_readers_info(knowledge_instance: Optional[Any] = None) -> List[Dict]:
+    """Get information about all available readers, including custom readers from a Knowledge instance.
+
+    Args:
+        knowledge_instance: Optional Knowledge instance to include custom readers from.
+
+    Returns:
+        List of reader info dictionaries.
+    """
     readers_info = []
     keys = ReaderFactory.get_all_reader_keys()
     for key in keys:
@@ -88,18 +114,35 @@ def get_all_readers_info() -> List[Dict]:
             # Log the error but don't fail the entire request
             log_debug(f"Skipping reader '{key}': {e}")
             continue
+
+    # Add custom readers from knowledge instance if provided
+    if knowledge_instance is not None:
+        custom_readers = knowledge_instance.get_readers()
+        if isinstance(custom_readers, dict):
+            for reader_id, reader in custom_readers.items():
+                try:
+                    reader_info = get_reader_info_from_instance(reader, reader_id)
+                    # Only add if not already present (custom readers take precedence)
+                    if not any(r["id"] == reader_id for r in readers_info):
+                        readers_info.append(reader_info)
+                except ValueError as e:
+                    log_debug(f"Skipping custom reader '{reader_id}': {e}")
+                    continue
+
     return readers_info
 
 
-def get_content_types_to_readers_mapping() -> Dict[str, List[str]]:
+def get_content_types_to_readers_mapping(knowledge_instance: Optional[Any] = None) -> Dict[str, List[str]]:
     """Get mapping of content types to list of reader IDs that support them.
 
+    Args:
+        knowledge_instance: Optional Knowledge instance to include custom readers from.
+
     Returns:
         Dictionary mapping content type strings (ContentType enum values) to list of reader IDs.
     """
     content_type_mapping: Dict[str, List[str]] = {}
-    readers_info = get_all_readers_info()
-
+    readers_info = get_all_readers_info(knowledge_instance)
     for reader_info in readers_info:
         reader_id = reader_info["id"]
         content_types = reader_info.get("content_types", [])
@@ -107,7 +150,9 @@ def get_content_types_to_readers_mapping() -> Dict[str, List[str]]:
         for content_type in content_types:
             if content_type not in content_type_mapping:
                 content_type_mapping[content_type] = []
-            content_type_mapping[content_type].append(reader_id)
+            # Avoid duplicates
+            if reader_id not in content_type_mapping[content_type]:
+                content_type_mapping[content_type].append(reader_id)
 
     return content_type_mapping
 

agno/memory/strategies/base.py

@@ -3,7 +3,7 @@ from typing import List
 
 from agno.db.schemas import UserMemory
 from agno.models.base import Model
-from agno.utils.tokens import count_tokens as count_text_tokens
+from agno.utils.tokens import count_text_tokens
 
 
 class MemoryOptimizationStrategy(ABC):
@@ -60,8 +60,7 @@ class MemoryOptimizationStrategy(ABC):
 
         Args:
             memories: List of UserMemory objects
-
         Returns:
-            Total token count using tiktoken (or fallback estimation)
+            Total token count
         """
-        return sum(count_text_tokens(mem.memory or "") for mem in memories)
+        return sum(count_text_tokens(m.memory or "") for m in memories)

agno/models/anthropic/claude.py

@@ -13,9 +13,11 @@ from agno.models.message import Citations, DocumentCitation, Message, UrlCitatio
 from agno.models.metrics import Metrics
 from agno.models.response import ModelResponse
 from agno.run.agent import RunOutput
+from agno.tools.function import Function
 from agno.utils.http import get_default_async_client, get_default_sync_client
 from agno.utils.log import log_debug, log_error, log_warning
 from agno.utils.models.claude import MCPServerConfiguration, format_messages, format_tools_for_model
+from agno.utils.tokens import count_schema_tokens
 
 try:
     from anthropic import Anthropic as AnthropicClient
@@ -399,6 +401,48 @@ class Claude(Model):
         self.async_client = AsyncAnthropicClient(**_client_params)
         return self.async_client
 
+    def count_tokens(
+        self,
+        messages: List[Message],
+        tools: Optional[List[Union[Function, Dict[str, Any]]]] = None,
+        response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
+    ) -> int:
+        anthropic_messages, system_prompt = format_messages(messages, compress_tool_results=True)
+        anthropic_tools = None
+        if tools:
+            formatted_tools = self._format_tools(tools)
+            anthropic_tools = format_tools_for_model(formatted_tools)
+
+        kwargs: Dict[str, Any] = {"messages": anthropic_messages, "model": self.id}
+        if system_prompt:
+            kwargs["system"] = system_prompt
+        if anthropic_tools:
+            kwargs["tools"] = anthropic_tools
+
+        response = self.get_client().messages.count_tokens(**kwargs)
+        return response.input_tokens + count_schema_tokens(response_format, self.id)
+
+    async def acount_tokens(
+        self,
+        messages: List[Message],
+        tools: Optional[List[Union[Function, Dict[str, Any]]]] = None,
+        response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
+    ) -> int:
+        anthropic_messages, system_prompt = format_messages(messages, compress_tool_results=True)
+        anthropic_tools = None
+        if tools:
+            formatted_tools = self._format_tools(tools)
+            anthropic_tools = format_tools_for_model(formatted_tools)
+
+        kwargs: Dict[str, Any] = {"messages": anthropic_messages, "model": self.id}
+        if system_prompt:
+            kwargs["system"] = system_prompt
+        if anthropic_tools:
+            kwargs["tools"] = anthropic_tools
+
+        response = await self.get_async_client().messages.count_tokens(**kwargs)
+        return response.input_tokens + count_schema_tokens(response_format, self.id)
+
     def get_request_params(
         self,
         response_format: Optional[Union[Dict, Type[BaseModel]]] = None,

agno/models/aws/bedrock.py

@@ -12,6 +12,7 @@ from agno.models.metrics import Metrics
 from agno.models.response import ModelResponse
 from agno.run.agent import RunOutput
 from agno.utils.log import log_debug, log_error, log_warning
+from agno.utils.tokens import count_schema_tokens
 
 try:
     from boto3 import client as AwsClient
@@ -357,6 +358,65 @@ class AwsBedrock(Model):
         # TODO: Add caching: https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-call.html
         return formatted_messages, system_message
 
+    def count_tokens(
+        self,
+        messages: List[Message],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        output_schema: Optional[Union[Dict, Type[BaseModel]]] = None,
+    ) -> int:
+        try:
+            formatted_messages, system_message = self._format_messages(messages, compress_tool_results=True)
+            converse_input: Dict[str, Any] = {"messages": formatted_messages}
+            if system_message:
+                converse_input["system"] = system_message
+
+            response = self.get_client().count_tokens(modelId=self.id, input={"converse": converse_input})
+            tokens = response.get("inputTokens", 0)
+
+            # Count tool tokens
+            if tools:
+                from agno.utils.tokens import count_tool_tokens
+
+                tokens += count_tool_tokens(tools, self.id)
+
+            # Count schema tokens
+            tokens += count_schema_tokens(output_schema, self.id)
+
+            return tokens
+        except Exception as e:
+            log_warning(f"Failed to count tokens via Bedrock API: {e}")
+            return super().count_tokens(messages, tools, output_schema)
+
+    async def acount_tokens(
+        self,
+        messages: List[Message],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        output_schema: Optional[Union[Dict, Type[BaseModel]]] = None,
+    ) -> int:
+        try:
+            formatted_messages, system_message = self._format_messages(messages, compress_tool_results=True)
+            converse_input: Dict[str, Any] = {"messages": formatted_messages}
+            if system_message:
+                converse_input["system"] = system_message
+
+            async with self.get_async_client() as client:
+                response = await client.count_tokens(modelId=self.id, input={"converse": converse_input})
+            tokens = response.get("inputTokens", 0)
+
+            # Count tool tokens
+            if tools:
+                from agno.utils.tokens import count_tool_tokens
+
+                tokens += count_tool_tokens(tools, self.id)
+
+            # Count schema tokens
+            tokens += count_schema_tokens(output_schema, self.id)
+
+            return tokens
+        except Exception as e:
+            log_warning(f"Failed to count tokens via Bedrock API: {e}")
+            return await super().acount_tokens(messages, tools, output_schema)
+
     def invoke(
         self,
         messages: List[Message],