agno 2.2.13__py3-none-any.whl

agno/knowledge/knowledge.py

@@ -0,0 +1,1988 @@
+import asyncio
+import hashlib
+import io
+import time
+from dataclasses import dataclass
+from enum import Enum
+from io import BytesIO
+from os.path import basename
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload
+
+from httpx import AsyncClient
+
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.db.schemas.knowledge import KnowledgeRow
+from agno.filters import FilterExpr
+from agno.knowledge.content import Content, ContentAuth, ContentStatus, FileData
+from agno.knowledge.document import Document
+from agno.knowledge.reader import Reader, ReaderFactory
+from agno.knowledge.remote_content.remote_content import GCSContent, RemoteContent, S3Content
+from agno.utils.http import async_fetch_with_retry
+from agno.utils.log import log_debug, log_error, log_info, log_warning
+from agno.utils.string import generate_id
+
+ContentDict = Dict[str, Union[str, Dict[str, str]]]
+
+
+class KnowledgeContentOrigin(Enum):
+    PATH = "path"
+    URL = "url"
+    TOPIC = "topic"
+    CONTENT = "content"
+
+
+@dataclass
+class Knowledge:
+    """Knowledge class"""
+
+    name: Optional[str] = None
+    description: Optional[str] = None
+    vector_db: Optional[Any] = None
+    contents_db: Optional[Union[BaseDb, AsyncBaseDb]] = None
+    max_results: int = 10
+    readers: Optional[Dict[str, Reader]] = None
+
+    def __post_init__(self):
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if self.vector_db and not self.vector_db.exists():
+            self.vector_db.create()
+
+        self.construct_readers()
+        self.valid_metadata_filters = set()
+
+    # --- SDK Specific Methods ---
+
+    # --- Add Contents ---
+    @overload
+    async def add_contents_async(self, contents: List[ContentDict]) -> None: ...
+
+    @overload
+    async def add_contents_async(
+        self,
+        *,
+        paths: Optional[List[str]] = None,
+        urls: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        topics: Optional[List[str]] = None,
+        text_contents: Optional[List[str]] = None,
+        reader: Optional[Reader] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+        upsert: bool = True,
+        skip_if_exists: bool = False,
+        remote_content: Optional[RemoteContent] = None,
+    ) -> None: ...
+
+    async def add_contents_async(self, *args, **kwargs) -> None:
+        if args and isinstance(args[0], list):
+            arguments = args[0]
+            upsert = kwargs.get("upsert", True)
+            skip_if_exists = kwargs.get("skip_if_exists", False)
+            for argument in arguments:
+                await self.add_content_async(
+                    name=argument.get("name"),
+                    description=argument.get("description"),
+                    path=argument.get("path"),
+                    url=argument.get("url"),
+                    metadata=argument.get("metadata"),
+                    topics=argument.get("topics"),
+                    text_content=argument.get("text_content"),
+                    reader=argument.get("reader"),
+                    include=argument.get("include"),
+                    exclude=argument.get("exclude"),
+                    upsert=argument.get("upsert", upsert),
+                    skip_if_exists=argument.get("skip_if_exists", skip_if_exists),
+                    remote_content=argument.get("remote_content", None),
+                )
+
+        elif kwargs:
+            name = kwargs.get("name", [])
+            metadata = kwargs.get("metadata", {})
+            description = kwargs.get("description", [])
+            topics = kwargs.get("topics", [])
+            reader = kwargs.get("reader", None)
+            paths = kwargs.get("paths", [])
+            urls = kwargs.get("urls", [])
+            text_contents = kwargs.get("text_contents", [])
+            include = kwargs.get("include")
+            exclude = kwargs.get("exclude")
+            upsert = kwargs.get("upsert", True)
+            skip_if_exists = kwargs.get("skip_if_exists", False)
+            remote_content = kwargs.get("remote_content", None)
+            for path in paths:
+                await self.add_content_async(
+                    name=name,
+                    description=description,
+                    path=path,
+                    metadata=metadata,
+                    include=include,
+                    exclude=exclude,
+                    upsert=upsert,
+                    skip_if_exists=skip_if_exists,
+                    reader=reader,
+                )
+            for url in urls:
+                await self.add_content_async(
+                    name=name,
+                    description=description,
+                    url=url,
+                    metadata=metadata,
+                    include=include,
+                    exclude=exclude,
+                    upsert=upsert,
+                    skip_if_exists=skip_if_exists,
+                    reader=reader,
+                )
+            for i, text_content in enumerate(text_contents):
+                content_name = f"{name}_{i}" if name else f"text_content_{i}"
+                log_debug(f"Adding text content: {content_name}")
+                await self.add_content_async(
+                    name=content_name,
+                    description=description,
+                    text_content=text_content,
+                    metadata=metadata,
+                    include=include,
+                    exclude=exclude,
+                    upsert=upsert,
+                    skip_if_exists=skip_if_exists,
+                    reader=reader,
+                )
+            if topics:
+                await self.add_content_async(
+                    name=name,
+                    description=description,
+                    topics=topics,
+                    metadata=metadata,
+                    include=include,
+                    exclude=exclude,
+                    upsert=upsert,
+                    skip_if_exists=skip_if_exists,
+                    reader=reader,
+                )
+
+            if remote_content:
+                await self.add_content_async(
+                    name=name,
+                    metadata=metadata,
+                    description=description,
+                    remote_content=remote_content,
+                    upsert=upsert,
+                    skip_if_exists=skip_if_exists,
+                    reader=reader,
+                )
+
+        else:
+            raise ValueError("Invalid usage of add_contents.")
+
+    @overload
+    def add_contents(self, contents: List[ContentDict]) -> None: ...
+
+    @overload
+    def add_contents(
+        self,
+        *,
+        paths: Optional[List[str]] = None,
+        urls: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        topics: Optional[List[str]] = None,
+        text_contents: Optional[List[str]] = None,
+        reader: Optional[Reader] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+        upsert: bool = True,
+        skip_if_exists: bool = False,
+        remote_content: Optional[RemoteContent] = None,
+    ) -> None: ...
+
+    def add_contents(self, *args, **kwargs) -> None:
+        """
+        Synchronously add multiple content items to the knowledge base.
+
+        This method wraps the asynchronous add_contents method
+
+        Supports two usage patterns:
+        1. Pass a list of content dictionaries as first argument
+        2. Pass keyword arguments with paths, urls, metadata, etc.
+
+        Args:
+            contents: List of content dictionaries (when used as first overload)
+            paths: Optional list of file paths to load content from
+            urls: Optional list of URLs to load content from
+            metadata: Optional metadata dictionary to apply to all content
+            topics: Optional list of topics to add
+            text_contents: Optional list of text content strings to add
+            reader: Optional reader to use for processing content
+            include: Optional list of file patterns to include
+            exclude: Optional list of file patterns to exclude
+            upsert: Whether to update existing content if it already exists
+            skip_if_exists: Whether to skip adding content if it already exists
+            remote_content: Optional remote content (S3, GCS, etc.) to add
+        """
+        asyncio.run(self.add_contents_async(*args, **kwargs))
+
+    # --- Add Content ---
+
+    @overload
+    async def add_content_async(
+        self,
+        *,
+        path: Optional[str] = None,
+        url: Optional[str] = None,
+        text_content: Optional[str] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+        upsert: bool = True,
+        skip_if_exists: bool = False,
+        reader: Optional[Reader] = None,
+        auth: Optional[ContentAuth] = None,
+    ) -> None: ...
+
+    @overload
+    async def add_content_async(self, *args, **kwargs) -> None: ...
+
+    async def add_content_async(
+        self,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        path: Optional[str] = None,
+        url: Optional[str] = None,
+        text_content: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        topics: Optional[List[str]] = None,
+        remote_content: Optional[RemoteContent] = None,
+        reader: Optional[Reader] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+        upsert: bool = True,
+        skip_if_exists: bool = True,
+        auth: Optional[ContentAuth] = None,
+    ) -> None:
+        # Validation: At least one of the parameters must be provided
+        if all(argument is None for argument in [path, url, text_content, topics, remote_content]):
+            log_warning(
+                "At least one of 'path', 'url', 'text_content', 'topics', or 'remote_content' must be provided."
+            )
+            return
+
+        if not skip_if_exists:
+            log_info("skip_if_exists is disabled, disabling upsert")
+            upsert = False
+
+        content = None
+        file_data = None
+        if text_content:
+            file_data = FileData(content=text_content, type="Text")
+
+        content = Content(
+            name=name,
+            description=description,
+            path=path,
+            url=url,
+            file_data=file_data if file_data else None,
+            metadata=metadata,
+            topics=topics,
+            remote_content=remote_content,
+            reader=reader,
+            auth=auth,
+        )
+        content.content_hash = self._build_content_hash(content)
+        content.id = generate_id(content.content_hash)
+
+        await self._load_content(content, upsert, skip_if_exists, include, exclude)
+
+    @overload
+    def add_content(
+        self,
+        *,
+        path: Optional[str] = None,
+        url: Optional[str] = None,
+        text_content: Optional[str] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+        upsert: bool = True,
+        skip_if_exists: bool = False,
+        reader: Optional[Reader] = None,
+        auth: Optional[ContentAuth] = None,
+    ) -> None: ...
+
+    @overload
+    def add_content(self, *args, **kwargs) -> None: ...
+
+    def add_content(
+        self,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        path: Optional[str] = None,
+        url: Optional[str] = None,
+        text_content: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        topics: Optional[List[str]] = None,
+        remote_content: Optional[RemoteContent] = None,
+        reader: Optional[Reader] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+        upsert: bool = True,
+        skip_if_exists: bool = False,
+        auth: Optional[ContentAuth] = None,
+    ) -> None:
+        """
+        Synchronously add content to the knowledge base.
+
+        Args:
+            name: Optional name for the content
+            description: Optional description for the content
+            path: Optional file path to load content from
+            url: Optional URL to load content from
+            text_content: Optional text content to add directly
+            metadata: Optional metadata dictionary
+            topics: Optional list of topics
+            remote_content: Optional cloud storage configuration
+            reader: Optional custom reader for processing the content
+            include: Optional list of file patterns to include
+            exclude: Optional list of file patterns to exclude
+            upsert: Whether to update existing content if it already exists
+            skip_if_exists: Whether to skip adding content if it already exists
+        """
+        asyncio.run(
+            self.add_content_async(
+                name=name,
+                description=description,
+                path=path,
+                url=url,
+                text_content=text_content,
+                metadata=metadata,
+                topics=topics,
+                remote_content=remote_content,
+                reader=reader,
+                include=include,
+                exclude=exclude,
+                upsert=upsert,
+                skip_if_exists=skip_if_exists,
+                auth=auth,
+            )
+        )
+
+    def _should_skip(self, content_hash: str, skip_if_exists: bool) -> bool:
+        """
+        Handle the skip_if_exists logic for content that already exists in the vector database.
+
+        Args:
+            content_hash: The content hash string to check for existence
+            skip_if_exists: Whether to skip if content already exists
+
+        Returns:
+            bool: True if should skip processing, False if should continue
+        """
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if self.vector_db and self.vector_db.content_hash_exists(content_hash) and skip_if_exists:
+            log_debug(f"Content already exists: {content_hash}, skipping...")
+            return True
+
+        return False
+
+    async def _load_from_path(
+        self,
+        content: Content,
+        upsert: bool,
+        skip_if_exists: bool,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+    ):
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+
+        log_info(f"Adding content from path, {content.id}, {content.name}, {content.path}, {content.description}")
+        path = Path(content.path)  # type: ignore
+
+        if path.is_file():
+            if self._should_include_file(str(path), include, exclude):
+                log_debug(f"Adding file {path} due to include/exclude filters")
+
+                await self._add_to_contents_db(content)
+                if self._should_skip(content.content_hash, skip_if_exists):  # type: ignore[arg-type]
+                    content.status = ContentStatus.COMPLETED
+                    await self._aupdate_content(content)
+                    return
+
+                # Handle LightRAG special case - read file and upload directly
+                if self.vector_db.__class__.__name__ == "LightRag":
+                    await self._process_lightrag_content(content, KnowledgeContentOrigin.PATH)
+                    return
+
+                if content.reader:
+                    # TODO: We will refactor this to eventually pass authorization to all readers
+                    import inspect
+
+                    read_signature = inspect.signature(content.reader.read)
+                    if "password" in read_signature.parameters and content.auth and content.auth.password:
+                        read_documents = content.reader.read(
+                            path, name=content.name or path.name, password=content.auth.password
+                        )
+                    else:
+                        read_documents = content.reader.read(path, name=content.name or path.name)
+
+                else:
+                    reader = ReaderFactory.get_reader_for_extension(path.suffix)
+                    log_info(f"Using Reader: {reader.__class__.__name__}")
+                    if reader:
+                        # TODO: We will refactor this to eventually pass authorization to all readers
+                        import inspect
+
+                        read_signature = inspect.signature(reader.read)
+                        if "password" in read_signature.parameters and content.auth and content.auth.password:
+                            read_documents = reader.read(
+                                path, name=content.name or path.name, password=content.auth.password
+                            )
+                        else:
+                            read_documents = reader.read(path, name=content.name or path.name)
+
+                if not content.file_type:
+                    content.file_type = path.suffix
+
+                if not content.size and content.file_data:
+                    content.size = len(content.file_data.content)  # type: ignore
+                if not content.size:
+                    try:
+                        content.size = path.stat().st_size
+                    except (OSError, IOError) as e:
+                        log_warning(f"Could not get file size for {path}: {e}")
+                        content.size = 0
+
+                for read_document in read_documents:
+                    read_document.content_id = content.id
+
+                await self._handle_vector_db_insert(content, read_documents, upsert)
+
+        elif path.is_dir():
+            for file_path in path.iterdir():
+                # Apply include/exclude filtering
+                if not self._should_include_file(str(file_path), include, exclude):
+                    log_debug(f"Skipping file {file_path} due to include/exclude filters")
+                    continue
+
+                file_content = Content(
+                    name=content.name,
+                    path=str(file_path),
+                    metadata=content.metadata,
+                    description=content.description,
+                    reader=content.reader,
+                )
+                file_content.content_hash = self._build_content_hash(file_content)
+                file_content.id = generate_id(file_content.content_hash)
+
+                await self._load_from_path(file_content, upsert, skip_if_exists, include, exclude)
+        else:
+            log_warning(f"Invalid path: {path}")
+
+    async def _load_from_url(
+        self,
+        content: Content,
+        upsert: bool,
+        skip_if_exists: bool,
+    ):
+        """Load the content in the contextual URL
+
+        1. Set content hash
+        2. Validate the URL
+        3. Read the content
+        4. Prepare and insert the content in the vector database
+        """
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+
+        log_info(f"Adding content from URL {content.url}")
+        content.file_type = "url"
+
+        if not content.url:
+            raise ValueError("No url provided")
+
+        # 1. Add content to contents database
+        await self._add_to_contents_db(content)
+        if self._should_skip(content.content_hash, skip_if_exists):  # type: ignore[arg-type]
+            content.status = ContentStatus.COMPLETED
+            await self._aupdate_content(content)
+            return
+
+        if self.vector_db.__class__.__name__ == "LightRag":
+            await self._process_lightrag_content(content, KnowledgeContentOrigin.URL)
+            return
+
+        # 2. Validate URL
+        try:
+            from urllib.parse import urlparse
+
+            parsed_url = urlparse(content.url)
+            if not all([parsed_url.scheme, parsed_url.netloc]):
+                content.status = ContentStatus.FAILED
+                content.status_message = f"Invalid URL format: {content.url}"
+                await self._aupdate_content(content)
+                log_warning(f"Invalid URL format: {content.url}")
+        except Exception as e:
+            content.status = ContentStatus.FAILED
+            content.status_message = f"Invalid URL: {content.url} - {str(e)}"
+            await self._aupdate_content(content)
+            log_warning(f"Invalid URL: {content.url} - {str(e)}")
+
+        # 3. Fetch and load content if file has an extension
+        url_path = Path(parsed_url.path)
+        file_extension = url_path.suffix.lower()
+
+        bytes_content = None
+        if file_extension:
+            async with AsyncClient() as client:
+                response = await async_fetch_with_retry(content.url, client=client)
+            bytes_content = BytesIO(response.content)
+
+        # 4. Select reader
+        # If a reader was provided by the user, use it
+        reader = content.reader
+        name = content.name if content.name else content.url
+        # Else select based on file extension
+        if reader is None:
+            if file_extension == ".csv":
+                name = basename(parsed_url.path) or "data.csv"
+                reader = self.csv_reader
+            elif file_extension == ".pdf":
+                reader = self.pdf_reader
+            elif file_extension == ".docx":
+                reader = self.docx_reader
+            elif file_extension == ".pptx":
+                reader = self.pptx_reader
+            elif file_extension == ".json":
+                reader = self.json_reader
+            elif file_extension == ".markdown":
+                reader = self.markdown_reader
+            else:
+                reader = self.text_reader
+
+        # 5. Read content
+        try:
+            read_documents = []
+            if reader is not None:
+                # TODO: We will refactor this to eventually pass authorization to all readers
+                import inspect
+
+                read_signature = inspect.signature(reader.read)
+                if reader.__class__.__name__ == "YouTubeReader":
+                    read_documents = reader.read(content.url, name=name)
+                elif "password" in read_signature.parameters and content.auth and content.auth.password:
+                    if bytes_content:
+                        read_documents = reader.read(bytes_content, name=name, password=content.auth.password)
+                    else:
+                        read_documents = reader.read(content.url, name=name, password=content.auth.password)
+                else:
+                    if bytes_content:
+                        read_documents = reader.read(bytes_content, name=name)
+                    else:
+                        read_documents = reader.read(content.url, name=name)
+
+        except Exception as e:
+            log_error(f"Error reading URL: {content.url} - {str(e)}")
+            content.status = ContentStatus.FAILED
+            content.status_message = f"Error reading URL: {content.url} - {str(e)}"
+            await self._aupdate_content(content)
+            return
+
+        # 6. Chunk documents if needed
+        if reader and not reader.chunk:
+            read_documents = await reader.chunk_documents_async(read_documents)
+        # 7. Prepare and insert the content in the vector database
+        file_size = 0
+        if read_documents:
+            for read_document in read_documents:
+                if read_document.size:
+                    file_size += read_document.size
+                read_document.content_id = content.id
+        await self._handle_vector_db_insert(content, read_documents, upsert)
+
+    async def _load_from_content(
+        self,
+        content: Content,
+        upsert: bool = True,
+        skip_if_exists: bool = False,
+    ):
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+
+        if content.name:
+            name = content.name
+        elif content.file_data and content.file_data.content:
+            if isinstance(content.file_data.content, bytes):
+                name = content.file_data.content[:10].decode("utf-8", errors="ignore")
+            elif isinstance(content.file_data.content, str):
+                name = (
+                    content.file_data.content[:10]
+                    if len(content.file_data.content) >= 10
+                    else content.file_data.content
+                )
+            else:
+                name = str(content.file_data.content)[:10]
+        else:
+            name = None
+
+        if name is not None:
+            content.name = name
+
+        log_info(f"Adding content from {content.name}")
+
+        await self._add_to_contents_db(content)
+        if self._should_skip(content.content_hash, skip_if_exists):  # type: ignore[arg-type]
+            content.status = ContentStatus.COMPLETED
+            await self._aupdate_content(content)
+            return
+
+        if content.file_data and self.vector_db.__class__.__name__ == "LightRag":
+            await self._process_lightrag_content(content, KnowledgeContentOrigin.CONTENT)
+            return
+
+        read_documents = []
+
+        if isinstance(content.file_data, str):
+            content_bytes = content.file_data.encode("utf-8", errors="replace")
+            content_io = io.BytesIO(content_bytes)
+
+            if content.reader:
+                log_info(f"Using reader: {content.reader.__class__.__name__} to read content")
+                read_documents = content.reader.read(content_io, name=name)
+            else:
+                text_reader = self.text_reader
+                if text_reader:
+                    read_documents = text_reader.read(content_io, name=name)
+                else:
+                    content.status = ContentStatus.FAILED
+                    content.status_message = "Text reader not available"
+                    await self._aupdate_content(content)
+                    return
+
+        elif isinstance(content.file_data, FileData):
+            if content.file_data.type:
+                if isinstance(content.file_data.content, bytes):
+                    content_io = io.BytesIO(content.file_data.content)
+                elif isinstance(content.file_data.content, str):
+                    content_bytes = content.file_data.content.encode("utf-8", errors="replace")
+                    content_io = io.BytesIO(content_bytes)
+                else:
+                    content_io = content.file_data.content  # type: ignore
+
+                # Respect an explicitly provided reader; otherwise select based on file type
+                if content.reader:
+                    log_info(f"Using reader: {content.reader.__class__.__name__} to read content")
+                    reader = content.reader
+                else:
+                    reader = self._select_reader(content.file_data.type)
+                name = content.name if content.name else f"content_{content.file_data.type}"
+                read_documents = reader.read(content_io, name=name)
+                for read_document in read_documents:
+                    if content.metadata:
+                        read_document.meta_data.update(content.metadata)
+                    read_document.content_id = content.id
+
+                if len(read_documents) == 0:
+                    content.status = ContentStatus.FAILED
+                    content.status_message = "Content could not be read"
+                    await self._aupdate_content(content)
+                    return
+
+        else:
+            content.status = ContentStatus.FAILED
+            content.status_message = "No content provided"
+            await self._aupdate_content(content)
+            return
+
+        await self._handle_vector_db_insert(content, read_documents, upsert)
+
+    async def _load_from_topics(
+        self,
+        content: Content,
+        upsert: bool,
+        skip_if_exists: bool,
+    ):
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        log_info(f"Adding content from topics: {content.topics}")
+
+        if content.topics is None:
+            log_warning("No topics provided for content")
+            return
+
+        for topic in content.topics:
+            content = Content(
+                name=topic,
+                metadata=content.metadata,
+                reader=content.reader,
+                status=ContentStatus.PROCESSING if content.reader else ContentStatus.FAILED,
+                file_data=FileData(
+                    type="Topic",
+                ),
+                topics=[topic],
+            )
+            content.content_hash = self._build_content_hash(content)
+            content.id = generate_id(content.content_hash)
+
+            await self._add_to_contents_db(content)
+            if self._should_skip(content.content_hash, skip_if_exists):
+                content.status = ContentStatus.COMPLETED
+                await self._aupdate_content(content)
+                return
+
+            if self.vector_db.__class__.__name__ == "LightRag":
+                await self._process_lightrag_content(content, KnowledgeContentOrigin.TOPIC)
+                return
+
+            if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
+                log_info(f"Content {content.content_hash} already exists, skipping")
+                continue
+
+            await self._add_to_contents_db(content)
+            if content.reader is None:
+                log_error(f"No reader available for topic: {topic}")
+                content.status = ContentStatus.FAILED
+                content.status_message = "No reader available for topic"
+                await self._aupdate_content(content)
+                continue
+
+            read_documents = content.reader.read(topic)
+            if len(read_documents) > 0:
+                for read_document in read_documents:
+                    read_document.content_id = content.id
+                    if read_document.content:
+                        read_document.size = len(read_document.content.encode("utf-8"))
+            else:
+                content.status = ContentStatus.FAILED
+                content.status_message = "No content found for topic"
+                await self._aupdate_content(content)
+
+            await self._handle_vector_db_insert(content, read_documents, upsert)
+
+    async def _load_from_remote_content(
+        self,
+        content: Content,
+        upsert: bool,
+        skip_if_exists: bool,
+    ):
+        if content.remote_content is None:
+            log_warning("No remote content provided for content")
+            return
+
+        remote_content = content.remote_content
+
+        if isinstance(remote_content, S3Content):
+            await self._load_from_s3(content, upsert, skip_if_exists)
+
+        elif isinstance(remote_content, GCSContent):
+            await self._load_from_gcs(content, upsert, skip_if_exists)
+
+        else:
+            log_warning(f"Unsupported remote content type: {type(remote_content)}")
+
+    async def _load_from_s3(self, content: Content, upsert: bool, skip_if_exists: bool):
+        """Load the contextual S3 content.
+
+        1. Identify objects to read
+        2. Setup Content object
+        3. Hash content and add it to the contents database
+        4. Select reader
+        5. Fetch and load the content
+        6. Read the content
+        7. Prepare and insert the content in the vector database
+        8. Remove temporary file if needed
+        """
+        from agno.cloud.aws.s3.object import S3Object
+
+        remote_content: S3Content = cast(S3Content, content.remote_content)
+
+        # 1. Identify objects to read
+        objects_to_read: List[S3Object] = []
+        if remote_content.bucket is not None:
+            if remote_content.key is not None:
+                _object = S3Object(bucket_name=remote_content.bucket.name, name=remote_content.key)
+                objects_to_read.append(_object)
+            elif remote_content.object is not None:
+                objects_to_read.append(remote_content.object)
+            elif remote_content.prefix is not None:
+                objects_to_read.extend(remote_content.bucket.get_objects(prefix=remote_content.prefix))
+            else:
+                objects_to_read.extend(remote_content.bucket.get_objects())
+
+        for s3_object in objects_to_read:
+            # 2. Setup Content object
+            content_name = content.name or ""
+            content_name += "_" + (s3_object.name or "")
+            content_entry = Content(
+                name=content_name,
+                description=content.description,
+                status=ContentStatus.PROCESSING,
+                metadata=content.metadata,
+                file_type="s3",
+            )
+
+            # 3. Hash content and add it to the contents database
+            content_entry.content_hash = self._build_content_hash(content_entry)
+            content_entry.id = generate_id(content_entry.content_hash)
+            await self._add_to_contents_db(content_entry)
+            if self._should_skip(content_entry.content_hash, skip_if_exists):
+                content_entry.status = ContentStatus.COMPLETED
+                await self._aupdate_content(content_entry)
+                return
+
+            # 4. Select reader
+            reader = content.reader
+            if reader is None:
+                if s3_object.uri.endswith(".pdf"):
+                    reader = self.pdf_reader
+                elif s3_object.uri.endswith(".csv"):
+                    reader = self.csv_reader
+                elif s3_object.uri.endswith(".docx"):
+                    reader = self.docx_reader
+                elif s3_object.uri.endswith(".pptx"):
+                    reader = self.pptx_reader
+                elif s3_object.uri.endswith(".json"):
+                    reader = self.json_reader
+                elif s3_object.uri.endswith(".markdown"):
+                    reader = self.markdown_reader
+                else:
+                    reader = self.text_reader
+            reader = cast(Reader, reader)
+
+            # 5. Fetch and load the content
+            temporary_file = None
+            obj_name = content_name or s3_object.name.split("/")[-1]
+            readable_content: Optional[Union[BytesIO, Path]] = None
+            if s3_object.uri.endswith(".pdf"):
+                readable_content = BytesIO(s3_object.get_resource().get()["Body"].read())
+            else:
+                temporary_file = Path("storage").joinpath(obj_name)
+                readable_content = temporary_file
+                s3_object.download(readable_content)  # type: ignore
+
+            # 6. Read the content
+            read_documents = reader.read(readable_content, name=obj_name)
+
+            # 7. Prepare and insert the content in the vector database
+            for read_document in read_documents:
+                read_document.content_id = content.id
+            await self._handle_vector_db_insert(content_entry, read_documents, upsert)
+
+            # 8. Remove temporary file if needed
+            if temporary_file:
+                temporary_file.unlink()
+
+    async def _load_from_gcs(self, content: Content, upsert: bool, skip_if_exists: bool):
+        """Load the contextual GCS content.
+
+        1. Identify objects to read
+        2. Setup Content object
+        3. Hash content and add it to the contents database
+        4. Select reader
+        5. Fetch and load the content
+        6. Read the content
+        7. Prepare and insert the content in the vector database
+        """
+        remote_content: GCSContent = cast(GCSContent, content.remote_content)
+
+        # 1. Identify objects to read
+        objects_to_read = []
+        if remote_content.blob_name is not None:
+            objects_to_read.append(remote_content.bucket.blob(remote_content.blob_name))  # type: ignore
+        elif remote_content.prefix is not None:
+            objects_to_read.extend(remote_content.bucket.list_blobs(prefix=remote_content.prefix))  # type: ignore
+        else:
+            objects_to_read.extend(remote_content.bucket.list_blobs())  # type: ignore
+
+        for gcs_object in objects_to_read:
+            # 2. Setup Content object
+            name = (content.name or "content") + "_" + gcs_object.name
+            content_entry = Content(
+                name=name,
+                description=content.description,
+                status=ContentStatus.PROCESSING,
+                metadata=content.metadata,
+                file_type="gcs",
+            )
+
+            # 3. Hash content and add it to the contents database
+            content_entry.content_hash = self._build_content_hash(content_entry)
+            content_entry.id = generate_id(content_entry.content_hash)
+            await self._add_to_contents_db(content_entry)
+            if self._should_skip(content_entry.content_hash, skip_if_exists):
+                content_entry.status = ContentStatus.COMPLETED
+                await self._aupdate_content(content_entry)
+                return
+
+            # 4. Select reader
+            reader = content.reader
+            if reader is None:
+                if gcs_object.name.endswith(".pdf"):
+                    reader = self.pdf_reader
+                elif gcs_object.name.endswith(".csv"):
+                    reader = self.csv_reader
+                elif gcs_object.name.endswith(".docx"):
+                    reader = self.docx_reader
+                elif gcs_object.name.endswith(".pptx"):
+                    reader = self.pptx_reader
+                elif gcs_object.name.endswith(".json"):
+                    reader = self.json_reader
+                elif gcs_object.name.endswith(".markdown"):
+                    reader = self.markdown_reader
+                else:
+                    reader = self.text_reader
+            reader = cast(Reader, reader)
+
+            # 5. Fetch and load the content
+            readable_content = BytesIO(gcs_object.download_as_bytes())
+
+            # 6. Read the content
+            read_documents = reader.read(readable_content, name=name)
+
+            # 7. Prepare and insert the content in the vector database
+            for read_document in read_documents:
+                read_document.content_id = content.id
+            await self._handle_vector_db_insert(content_entry, read_documents, upsert)
+
+    async def _handle_vector_db_insert(self, content: Content, read_documents, upsert):
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+
+        if not self.vector_db:
+            log_error("No vector database configured")
+            content.status = ContentStatus.FAILED
+            content.status_message = "No vector database configured"
+            await self._aupdate_content(content)
+            return
+
+        if self.vector_db.upsert_available() and upsert:
+            try:
+                await self.vector_db.async_upsert(content.content_hash, read_documents, content.metadata)  # type: ignore[arg-type]
+            except Exception as e:
+                log_error(f"Error upserting document: {e}")
+                content.status = ContentStatus.FAILED
+                content.status_message = "Could not upsert embedding"
+                await self._aupdate_content(content)
+                return
+        else:
+            try:
+                await self.vector_db.async_insert(
+                    content.content_hash,  # type: ignore[arg-type]
+                    documents=read_documents,
+                    filters=content.metadata,  # type: ignore[arg-type]
+                )
+            except Exception as e:
+                log_error(f"Error inserting document: {e}")
+                content.status = ContentStatus.FAILED
+                content.status_message = "Could not insert embedding"
+                await self._aupdate_content(content)
+                return
+
+        content.status = ContentStatus.COMPLETED
+        await self._aupdate_content(content)
+
+    async def _load_content(
+        self,
+        content: Content,
+        upsert: bool,
+        skip_if_exists: bool,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+    ) -> None:
+        log_info(f"Loading content: {content.id}")
+
+        if content.metadata:
+            self.add_filters(content.metadata)
+
+        if content.path:
+            await self._load_from_path(content, upsert, skip_if_exists, include, exclude)
+
+        if content.url:
+            await self._load_from_url(content, upsert, skip_if_exists)
+
+        if content.file_data:
+            await self._load_from_content(content, upsert, skip_if_exists)
+
+        if content.topics:
+            await self._load_from_topics(content, upsert, skip_if_exists)
+
+        if content.remote_content:
+            await self._load_from_remote_content(content, upsert, skip_if_exists)
+
+    def _build_content_hash(self, content: Content) -> str:
+        """
+        Build the content hash from the content.
+        """
+        if content.path:
+            return hashlib.sha256(str(content.path).encode()).hexdigest()
+        elif content.url:
+            hash = hashlib.sha256(content.url.encode()).hexdigest()
+            return hash
+        elif content.file_data and content.file_data.content:
+            name = content.name or "content"
+            return hashlib.sha256(name.encode()).hexdigest()
+        elif content.topics and len(content.topics) > 0:
+            topic = content.topics[0]
+            reader = type(content.reader).__name__ if content.reader else "unknown"
+            return hashlib.sha256(f"{topic}-{reader}".encode()).hexdigest()
+        else:
+            # Fallback for edge cases
+            import random
+            import string
+
+            fallback = (
+                content.name
+                or content.id
+                or ("unknown_content" + "".join(random.choices(string.ascii_lowercase + string.digits, k=6)))
+            )
+            return hashlib.sha256(fallback.encode()).hexdigest()
+
+    def _ensure_string_field(self, value: Any, field_name: str, default: str = "") -> str:
+        """
+        Safely ensure a field is a string, handling various edge cases.
+
+        Args:
+            value: The value to convert to string
+            field_name: Name of the field for logging purposes
+            default: Default string value if conversion fails
+
+        Returns:
+            str: A safe string value
+        """
+        # Handle None/falsy values
+        if value is None or value == "":
+            return default
+
+        # Handle unexpected list types (the root cause of our Pydantic warning)
+        if isinstance(value, list):
+            if len(value) == 0:
+                log_debug(f"Empty list found for {field_name}, using default: '{default}'")
+                return default
+            elif len(value) == 1:
+                # Single item list, extract the item
+                log_debug(f"Single-item list found for {field_name}, extracting: '{value[0]}'")
+                return str(value[0]) if value[0] is not None else default
+            else:
+                # Multiple items, join them
+                log_debug(f"Multi-item list found for {field_name}, joining: {value}")
+                return " | ".join(str(item) for item in value if item is not None)
+
+        # Handle other unexpected types
+        if not isinstance(value, str):
+            log_debug(f"Non-string type {type(value)} found for {field_name}, converting: '{value}'")
+            try:
+                return str(value)
+            except Exception as e:
+                log_warning(f"Failed to convert {field_name} to string: {e}, using default")
+                return default
+
+        # Already a string, return as-is
+        return value
+
+    async def _add_to_contents_db(self, content: Content):
+        if self.contents_db:
+            created_at = content.created_at if content.created_at else int(time.time())
+            updated_at = content.updated_at if content.updated_at else int(time.time())
+
+            file_type = (
+                content.file_type
+                if content.file_type
+                else content.file_data.type
+                if content.file_data and content.file_data.type
+                else None
+            )
+            # Safely handle string fields with proper type checking
+            safe_name = self._ensure_string_field(content.name, "content.name", default="")
+            safe_description = self._ensure_string_field(content.description, "content.description", default="")
+            safe_linked_to = self._ensure_string_field(self.name, "knowledge.name", default="")
+            safe_status_message = self._ensure_string_field(
+                content.status_message, "content.status_message", default=""
+            )
+
+            content_row = KnowledgeRow(
+                id=content.id,
+                name=safe_name,
+                description=safe_description,
+                metadata=content.metadata,
+                type=file_type,
+                size=content.size
+                if content.size
+                else len(content.file_data.content)
+                if content.file_data and content.file_data.content
+                else None,
+                linked_to=safe_linked_to,
+                access_count=0,
+                status=content.status if content.status else ContentStatus.PROCESSING,
+                status_message=safe_status_message,
+                created_at=created_at,
+                updated_at=updated_at,
+            )
+            if isinstance(self.contents_db, AsyncBaseDb):
+                await self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
+            else:
+                self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
+
+    def _update_content(self, content: Content) -> Optional[Dict[str, Any]]:
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if self.contents_db:
+            if isinstance(self.contents_db, AsyncBaseDb):
+                raise ValueError(
+                    "update_content() is not supported with an async DB. Please use aupdate_content() instead."
+                )
+
+            if not content.id:
+                log_warning("Content id is required to update Knowledge content")
+                return None
+
+            # TODO: we shouldn't check for content here, we should trust the upsert method to handle conflicts
+            content_row = self.contents_db.get_knowledge_content(content.id)
+            if content_row is None:
+                log_warning(f"Content row not found for id: {content.id}, cannot update status")
+                return None
+
+            # Apply safe string handling for updates as well
+            if content.name is not None:
+                content_row.name = self._ensure_string_field(content.name, "content.name", default="")
+            if content.description is not None:
+                content_row.description = self._ensure_string_field(
+                    content.description, "content.description", default=""
+                )
+            if content.metadata is not None:
+                content_row.metadata = content.metadata
+            if content.status is not None:
+                content_row.status = content.status
+            if content.status_message is not None:
+                content_row.status_message = self._ensure_string_field(
+                    content.status_message, "content.status_message", default=""
+                )
+            if content.external_id is not None:
+                content_row.external_id = self._ensure_string_field(
+                    content.external_id, "content.external_id", default=""
+                )
+            content_row.updated_at = int(time.time())
+            self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
+
+            if self.vector_db and content.metadata:
+                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata)
+
+            if content.metadata:
+                self.add_filters(content.metadata)
+
+            return content_row.to_dict()
+
+        else:
+            if self.name:
+                log_warning(f"Contents DB not found for knowledge base: {self.name}")
+            else:
+                log_warning("Contents DB not found for knowledge base")
+            return None
+
+    async def _aupdate_content(self, content: Content) -> Optional[Dict[str, Any]]:
+        if self.contents_db:
+            if not content.id:
+                log_warning("Content id is required to update Knowledge content")
+                return None
+
+            # TODO: we shouldn't check for content here, we should trust the upsert method to handle conflicts
+            if isinstance(self.contents_db, AsyncBaseDb):
+                content_row = await self.contents_db.get_knowledge_content(content.id)
+            else:
+                content_row = self.contents_db.get_knowledge_content(content.id)
+            if content_row is None:
+                log_warning(f"Content row not found for id: {content.id}, cannot update status")
+                return None
+
+            if content.name is not None:
+                content_row.name = content.name
+            if content.description is not None:
+                content_row.description = content.description
+            if content.metadata is not None:
+                content_row.metadata = content.metadata
+            if content.status is not None:
+                content_row.status = content.status
+            if content.status_message is not None:
+                content_row.status_message = content.status_message if content.status_message else ""
+            if content.external_id is not None:
+                content_row.external_id = content.external_id
+
+            content_row.updated_at = int(time.time())
+            if isinstance(self.contents_db, AsyncBaseDb):
+                await self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
+            else:
+                self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
+
+            if self.vector_db and content.metadata:
+                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata)
+
+            if content.metadata:
+                self.add_filters(content.metadata)
+
+            return content_row.to_dict()
+
+        else:
+            log_warning(f"Contents DB not found for knowledge base: {self.name}")
+            return None
+
+    async def _process_lightrag_content(self, content: Content, content_type: KnowledgeContentOrigin) -> None:
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+
+        await self._add_to_contents_db(content)
+        if content_type == KnowledgeContentOrigin.PATH:
+            if content.file_data is None:
+                log_warning("No file data provided")
+
+            if content.path is None:
+                log_error("No path provided for content")
+                return
+
+            path = Path(content.path)
+
+            log_info(f"Uploading file to LightRAG from path: {path}")
+            try:
+                # Read the file content from path
+                with open(path, "rb") as f:
+                    file_content = f.read()
+
+                # Get file type from extension or content.file_type
+                file_type = content.file_type or path.suffix
+
+                if self.vector_db and hasattr(self.vector_db, "insert_file_bytes"):
+                    result = await self.vector_db.insert_file_bytes(
+                        file_content=file_content,
+                        filename=path.name,  # Use the original filename with extension
+                        content_type=file_type,
+                        send_metadata=True,  # Enable metadata so server knows the file type
+                    )
+
+                else:
+                    log_error("Vector database does not support file insertion")
+                    content.status = ContentStatus.FAILED
+                    await self._aupdate_content(content)
+                    return
+                content.external_id = result
+                content.status = ContentStatus.COMPLETED
+                await self._aupdate_content(content)
+                return
+
+            except Exception as e:
+                log_error(f"Error uploading file to LightRAG: {e}")
+                content.status = ContentStatus.FAILED
+                content.status_message = f"Could not upload to LightRAG: {str(e)}"
+                await self._aupdate_content(content)
+                return
+
+        elif content_type == KnowledgeContentOrigin.URL:
+            log_info(f"Uploading file to LightRAG from URL: {content.url}")
+            try:
+                reader = content.reader or self.website_reader
+                if reader is None:
+                    log_error("No URL reader available")
+                    content.status = ContentStatus.FAILED
+                    await self._aupdate_content(content)
+                    return
+
+                reader.chunk = False
+                read_documents = reader.read(content.url, name=content.name)
+
+                for read_document in read_documents:
+                    read_document.content_id = content.id
+
+                if not read_documents:
+                    log_error("No documents read from URL")
+                    content.status = ContentStatus.FAILED
+                    await self._aupdate_content(content)
+                    return
+
+                if self.vector_db and hasattr(self.vector_db, "insert_text"):
+                    result = await self.vector_db.insert_text(
+                        file_source=content.url,
+                        text=read_documents[0].content,
+                    )
+                else:
+                    log_error("Vector database does not support text insertion")
+                    content.status = ContentStatus.FAILED
+                    await self._aupdate_content(content)
+                    return
+
+                content.external_id = result
+                content.status = ContentStatus.COMPLETED
+                await self._aupdate_content(content)
+                return
+
+            except Exception as e:
+                log_error(f"Error uploading file to LightRAG: {e}")
+                content.status = ContentStatus.FAILED
+                content.status_message = f"Could not upload to LightRAG: {str(e)}"
+                await self._aupdate_content(content)
+                return
+
+        elif content_type == KnowledgeContentOrigin.CONTENT:
+            filename = (
+                content.file_data.filename if content.file_data and content.file_data.filename else "uploaded_file"
+            )
+            log_info(f"Uploading file to LightRAG: {filename}")
+
+            # Use the content from file_data
+            if content.file_data and content.file_data.content:
+                if self.vector_db and hasattr(self.vector_db, "insert_file_bytes"):
+                    result = await self.vector_db.insert_file_bytes(
+                        file_content=content.file_data.content,
+                        filename=filename,
+                        content_type=content.file_data.type,
+                        send_metadata=True,  # Enable metadata so server knows the file type
+                    )
+                else:
+                    log_error("Vector database does not support file insertion")
+                    content.status = ContentStatus.FAILED
+                    await self._aupdate_content(content)
+                    return
+                content.external_id = result
+                content.status = ContentStatus.COMPLETED
+                await self._aupdate_content(content)
+            else:
+                log_warning(f"No file data available for LightRAG upload: {content.name}")
+            return
+
+        elif content_type == KnowledgeContentOrigin.TOPIC:
+            log_info(f"Uploading file to LightRAG: {content.name}")
+
+            if content.reader is None:
+                log_error("No reader available for topic content")
+                content.status = ContentStatus.FAILED
+                await self._aupdate_content(content)
+                return
+
+            if not content.topics:
+                log_error("No topics available for content")
+                content.status = ContentStatus.FAILED
+                await self._aupdate_content(content)
+                return
+
+            read_documents = content.reader.read(content.topics)
+            if len(read_documents) > 0:
+                if self.vector_db and hasattr(self.vector_db, "insert_text"):
+                    result = await self.vector_db.insert_text(
+                        file_source=content.topics[0],
+                        text=read_documents[0].content,
+                    )
+                else:
+                    log_error("Vector database does not support text insertion")
+                    content.status = ContentStatus.FAILED
+                    await self._aupdate_content(content)
+                    return
+                content.external_id = result
+                content.status = ContentStatus.COMPLETED
+                await self._aupdate_content(content)
+                return
+            else:
+                log_warning(f"No documents found for LightRAG upload: {content.name}")
+                return
+
+    def search(
+        self,
+        query: str,
+        max_results: Optional[int] = None,
+        filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None,
+        search_type: Optional[str] = None,
+    ) -> List[Document]:
+        """Returns relevant documents matching a query"""
+        from agno.vectordb import VectorDb
+        from agno.vectordb.search import SearchType
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+
+        if (
+            hasattr(self.vector_db, "search_type")
+            and isinstance(self.vector_db.search_type, SearchType)
+            and search_type
+        ):
+            self.vector_db.search_type = SearchType(search_type)
+        try:
+            if self.vector_db is None:
+                log_warning("No vector db provided")
+                return []
+
+            _max_results = max_results or self.max_results
+            log_debug(f"Getting {_max_results} relevant documents for query: {query}")
+            return self.vector_db.search(query=query, limit=_max_results, filters=filters)
+        except Exception as e:
+            log_error(f"Error searching for documents: {e}")
+            return []
+
+    async def async_search(
+        self,
+        query: str,
+        max_results: Optional[int] = None,
+        filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None,
+        search_type: Optional[str] = None,
+    ) -> List[Document]:
+        """Returns relevant documents matching a query"""
+        from agno.vectordb import VectorDb
+        from agno.vectordb.search import SearchType
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if (
+            hasattr(self.vector_db, "search_type")
+            and isinstance(self.vector_db.search_type, SearchType)
+            and search_type
+        ):
+            self.vector_db.search_type = SearchType(search_type)
+        try:
+            if self.vector_db is None:
+                log_warning("No vector db provided")
+                return []
+
+            _max_results = max_results or self.max_results
+            log_debug(f"Getting {_max_results} relevant documents for query: {query}")
+            try:
+                return await self.vector_db.async_search(query=query, limit=_max_results, filters=filters)
+            except NotImplementedError:
+                log_info("Vector db does not support async search")
+                return self.search(query=query, max_results=_max_results, filters=filters)
+        except Exception as e:
+            log_error(f"Error searching for documents: {e}")
+            return []
+
+    def get_valid_filters(self) -> Set[str]:
+        if self.valid_metadata_filters is None:
+            self.valid_metadata_filters = set()
+        self.valid_metadata_filters.update(self._get_filters_from_db())
+        return self.valid_metadata_filters
+
+    async def aget_valid_filters(self) -> Set[str]:
+        if self.valid_metadata_filters is None:
+            self.valid_metadata_filters = set()
+        self.valid_metadata_filters.update(await self._aget_filters_from_db())
+        return self.valid_metadata_filters
+
+    def _validate_filters(self, filters: Optional[Union[Dict[str, Any], List[FilterExpr]]]) -> Tuple[Any, List[str]]:
+        """Internal method to validate filters against known metadata keys."""
+        if not filters:
+            return None, []
+
+        valid_filters: Optional[Dict[str, Any]] = None
+        invalid_keys = []
+
+        if isinstance(filters, dict):
+            # If no metadata filters tracked yet, all keys are considered invalid
+            if self.valid_metadata_filters is None:
+                invalid_keys = list(filters.keys())
+                log_debug(f"No valid metadata filters tracked yet. All filter keys considered invalid: {invalid_keys}")
+                return None, invalid_keys
+
+            valid_filters = {}
+            for key, value in filters.items():
+                # Handle both normal keys and prefixed keys like meta_data.key
+                base_key = key.split(".")[-1] if "." in key else key
+                if base_key in self.valid_metadata_filters or key in self.valid_metadata_filters:
+                    valid_filters[key] = value
+                else:
+                    invalid_keys.append(key)
+                    log_debug(f"Invalid filter key: {key} - not present in knowledge base")
+
+        elif isinstance(filters, List):
+            # Validate that list contains FilterExpr instances
+            for i, filter_item in enumerate(filters):
+                if not isinstance(filter_item, FilterExpr):
+                    log_warning(
+                        f"Invalid filter at index {i}: expected FilterExpr instance, "
+                        f"got {type(filter_item).__name__}. "
+                        f"Use filter expressions like EQ('key', 'value'), IN('key', [values]), "
+                        f"AND(...), OR(...), NOT(...) from agno.filters"
+                    )
+
+            # Filter expressions are already validated, return empty dict/list
+            # The actual filtering happens in the vector_db layer
+            return filters, []
+
+        return valid_filters, invalid_keys
+
+    def validate_filters(self, filters: Optional[Union[Dict[str, Any], List[FilterExpr]]]) -> Tuple[Any, List[str]]:
+        if self.valid_metadata_filters is None:
+            self.valid_metadata_filters = set()
+        self.valid_metadata_filters.update(self._get_filters_from_db())
+
+        return self._validate_filters(filters)
+
+    async def async_validate_filters(
+        self, filters: Optional[Union[Dict[str, Any], List[FilterExpr]]]
+    ) -> Tuple[Any, List[str]]:
+        if self.valid_metadata_filters is None:
+            self.valid_metadata_filters = set()
+        self.valid_metadata_filters.update(await self._aget_filters_from_db())
+
+        return self._validate_filters(filters)
+
+    def add_filters(self, metadata: Dict[str, Any]) -> None:
+        if self.valid_metadata_filters is None:
+            self.valid_metadata_filters = set()
+
+        if metadata is not None:
+            for key in metadata.keys():
+                self.valid_metadata_filters.add(key)
+
+    def _get_filters_from_db(self) -> Set[str]:
+        if self.contents_db is None:
+            return set()
+        contents, _ = self.get_content()
+        valid_filters: Set[str] = set()
+        for content in contents:
+            if content.metadata:
+                valid_filters.update(content.metadata.keys())
+        return valid_filters
+
+    async def _aget_filters_from_db(self) -> Set[str]:
+        if self.contents_db is None:
+            return set()
+        contents, _ = await self.aget_content()
+        valid_filters: Set[str] = set()
+        for content in contents:
+            if content.metadata:
+                valid_filters.update(content.metadata.keys())
+        return valid_filters
+
+    def remove_vector_by_id(self, id: str) -> bool:
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if self.vector_db is None:
+            log_warning("No vector DB provided")
+            return False
+        return self.vector_db.delete_by_id(id)
+
+    def remove_vectors_by_name(self, name: str) -> bool:
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if self.vector_db is None:
+            log_warning("No vector DB provided")
+            return False
+        return self.vector_db.delete_by_name(name)
+
+    def remove_vectors_by_metadata(self, metadata: Dict[str, Any]) -> bool:
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if self.vector_db is None:
+            log_warning("No vector DB provided")
+            return False
+        return self.vector_db.delete_by_metadata(metadata)
+
+    # --- API Only Methods ---
+
+    def patch_content(self, content: Content) -> Optional[Dict[str, Any]]:
+        return self._update_content(content)
+
+    async def apatch_content(self, content: Content) -> Optional[Dict[str, Any]]:
+        return await self._aupdate_content(content)
+
+    def get_content_by_id(self, content_id: str) -> Optional[Content]:
+        if self.contents_db is None:
+            raise ValueError("No contents db provided")
+
+        if isinstance(self.contents_db, AsyncBaseDb):
+            raise ValueError(
+                "get_content_by_id() is not supported for async databases. Please use aget_content_by_id() instead."
+            )
+
+        content_row = self.contents_db.get_knowledge_content(content_id)
+
+        if content_row is None:
+            return None
+        content = Content(
+            id=content_row.id,
+            name=content_row.name,
+            description=content_row.description,
+            metadata=content_row.metadata,
+            file_type=content_row.type,
+            size=content_row.size,
+            status=ContentStatus(content_row.status) if content_row.status else None,
+            status_message=content_row.status_message,
+            created_at=content_row.created_at,
+            updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
+            external_id=content_row.external_id,
+        )
+        return content
+
+    async def aget_content_by_id(self, content_id: str) -> Optional[Content]:
+        if self.contents_db is None:
+            raise ValueError("No contents db provided")
+
+        if isinstance(self.contents_db, AsyncBaseDb):
+            content_row = await self.contents_db.get_knowledge_content(content_id)
+        else:
+            content_row = self.contents_db.get_knowledge_content(content_id)
+
+        if content_row is None:
+            return None
+        content = Content(
+            id=content_row.id,
+            name=content_row.name,
+            description=content_row.description,
+            metadata=content_row.metadata,
+            file_type=content_row.type,
+            size=content_row.size,
+            status=ContentStatus(content_row.status) if content_row.status else None,
+            status_message=content_row.status_message,
+            created_at=content_row.created_at,
+            updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
+            external_id=content_row.external_id,
+        )
+        return content
+
+    def get_content(
+        self,
+        limit: Optional[int] = None,
+        page: Optional[int] = None,
+        sort_by: Optional[str] = None,
+        sort_order: Optional[str] = None,
+    ) -> Tuple[List[Content], int]:
+        if self.contents_db is None:
+            raise ValueError("No contents db provided")
+
+        if isinstance(self.contents_db, AsyncBaseDb):
+            raise ValueError("get_content() is not supported for async databases. Please use aget_content() instead.")
+
+        contents, count = self.contents_db.get_knowledge_contents(
+            limit=limit, page=page, sort_by=sort_by, sort_order=sort_order
+        )
+
+        result = []
+        for content_row in contents:
+            # Create Content from database row
+            content = Content(
+                id=content_row.id,
+                name=content_row.name,
+                description=content_row.description,
+                metadata=content_row.metadata,
+                size=content_row.size,
+                file_type=content_row.type,
+                status=ContentStatus(content_row.status) if content_row.status else None,
+                status_message=content_row.status_message,
+                created_at=content_row.created_at,
+                updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
+                external_id=content_row.external_id,
+            )
+            result.append(content)
+        return result, count
+
+    async def aget_content(
+        self,
+        limit: Optional[int] = None,
+        page: Optional[int] = None,
+        sort_by: Optional[str] = None,
+        sort_order: Optional[str] = None,
+    ) -> Tuple[List[Content], int]:
+        if self.contents_db is None:
+            raise ValueError("No contents db provided")
+
+        if isinstance(self.contents_db, AsyncBaseDb):
+            contents, count = await self.contents_db.get_knowledge_contents(
+                limit=limit, page=page, sort_by=sort_by, sort_order=sort_order
+            )
+        else:
+            contents, count = self.contents_db.get_knowledge_contents(
+                limit=limit, page=page, sort_by=sort_by, sort_order=sort_order
+            )
+
+        result = []
+        for content_row in contents:
+            # Create Content from database row
+            content = Content(
+                id=content_row.id,
+                name=content_row.name,
+                description=content_row.description,
+                metadata=content_row.metadata,
+                size=content_row.size,
+                file_type=content_row.type,
+                status=ContentStatus(content_row.status) if content_row.status else None,
+                status_message=content_row.status_message,
+                created_at=content_row.created_at,
+                updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
+                external_id=content_row.external_id,
+            )
+            result.append(content)
+        return result, count
+
+    def get_content_status(self, content_id: str) -> Tuple[Optional[ContentStatus], Optional[str]]:
+        if self.contents_db is None:
+            raise ValueError("No contents db provided")
+
+        if isinstance(self.contents_db, AsyncBaseDb):
+            raise ValueError(
+                "get_content_status() is not supported for async databases. Please use aget_content_status() instead."
+            )
+
+        content_row = self.contents_db.get_knowledge_content(content_id)
+        if content_row is None:
+            return None, "Content not found"
+
+        # Convert string status to enum, defaulting to PROCESSING if unknown
+        status_str = content_row.status
+        try:
+            status = ContentStatus(status_str.lower()) if status_str else ContentStatus.PROCESSING
+        except ValueError:
+            # Handle legacy or unknown statuses
+            if status_str and "failed" in status_str.lower():
+                status = ContentStatus.FAILED
+            elif status_str and "completed" in status_str.lower():
+                status = ContentStatus.COMPLETED
+            else:
+                status = ContentStatus.PROCESSING
+
+        return status, content_row.status_message
+
+    async def aget_content_status(self, content_id: str) -> Tuple[Optional[ContentStatus], Optional[str]]:
+        if self.contents_db is None:
+            raise ValueError("No contents db provided")
+
+        if isinstance(self.contents_db, AsyncBaseDb):
+            content_row = await self.contents_db.get_knowledge_content(content_id)
+        else:
+            content_row = self.contents_db.get_knowledge_content(content_id)
+
+        if content_row is None:
+            return None, "Content not found"
+
+        # Convert string status to enum, defaulting to PROCESSING if unknown
+        status_str = content_row.status
+        try:
+            status = ContentStatus(status_str.lower()) if status_str else ContentStatus.PROCESSING
+        except ValueError:
+            # Handle legacy or unknown statuses
+            if status_str and "failed" in status_str.lower():
+                status = ContentStatus.FAILED
+            elif status_str and "completed" in status_str.lower():
+                status = ContentStatus.COMPLETED
+            else:
+                status = ContentStatus.PROCESSING
+
+        return status, content_row.status_message
+
+    def remove_content_by_id(self, content_id: str):
+        from agno.vectordb import VectorDb
+
+        self.vector_db = cast(VectorDb, self.vector_db)
+        if self.vector_db is not None:
+            if self.vector_db.__class__.__name__ == "LightRag":
+                # For LightRAG, get the content first to find the external_id
+                content = self.get_content_by_id(content_id)
+                if content and content.external_id:
+                    self.vector_db.delete_by_external_id(content.external_id)  # type: ignore
+                else:
+                    log_warning(f"No external_id found for content {content_id}, cannot delete from LightRAG")
+            else:
+                self.vector_db.delete_by_content_id(content_id)
+
+        if self.contents_db is not None:
+            self.contents_db.delete_knowledge_content(content_id)
+
+    async def aremove_content_by_id(self, content_id: str):
+        if self.vector_db is not None:
+            if self.vector_db.__class__.__name__ == "LightRag":
+                # For LightRAG, get the content first to find the external_id
+                content = await self.aget_content_by_id(content_id)
+                if content and content.external_id:
+                    self.vector_db.delete_by_external_id(content.external_id)  # type: ignore
+                else:
+                    log_warning(f"No external_id found for content {content_id}, cannot delete from LightRAG")
+            else:
+                self.vector_db.delete_by_content_id(content_id)
+
+        if self.contents_db is not None:
+            if isinstance(self.contents_db, AsyncBaseDb):
+                await self.contents_db.delete_knowledge_content(content_id)
+            else:
+                self.contents_db.delete_knowledge_content(content_id)
+
+    def remove_all_content(self):
+        contents, _ = self.get_content()
+        for content in contents:
+            if content.id is not None:
+                self.remove_content_by_id(content.id)
+
+    async def aremove_all_content(self):
+        contents, _ = await self.aget_content()
+        for content in contents:
+            if content.id is not None:
+                await self.aremove_content_by_id(content.id)
+
+    # --- Reader Factory Integration ---
+
+    def construct_readers(self):
+        """Initialize readers dictionary for lazy loading."""
+        # Initialize empty readers dict - readers will be created on-demand
+        if self.readers is None:
+            self.readers = {}
+
+    def add_reader(self, reader: Reader):
+        """Add a custom reader to the knowledge base."""
+        if self.readers is None:
+            self.readers = {}
+
+        # Generate a key for the reader
+        reader_key = self._generate_reader_key(reader)
+        self.readers[reader_key] = reader
+        return reader
+
+    def get_readers(self) -> Dict[str, Reader]:
+        """Get all currently loaded readers (only returns readers that have been used)."""
+        if self.readers is None:
+            self.readers = {}
+
+        return self.readers
+
+    def _generate_reader_key(self, reader: Reader) -> str:
+        """Generate a key for a reader instance."""
+        if reader.name:
+            return f"{reader.name.lower().replace(' ', '_')}"
+        else:
+            return f"{reader.__class__.__name__.lower().replace(' ', '_')}"
+
+    def _select_reader(self, extension: str) -> Reader:
+        """Select the appropriate reader for a file extension."""
+        log_info(f"Selecting reader for extension: {extension}")
+        return ReaderFactory.get_reader_for_extension(extension)
+
+    def get_filters(self) -> List[str]:
+        return [
+            "filter_tag_1",
+            "filter_tag2",
+        ]
+
+    # --- Convenience Properties for Backward Compatibility ---
+
+    def _is_text_mime_type(self, mime_type: str) -> bool:
+        """
+        Check if a MIME type represents text content that can be safely encoded as UTF-8.
+
+        Args:
+            mime_type: The MIME type to check
+
+        Returns:
+            bool: True if it's a text type, False if binary
+        """
+        if not mime_type:
+            return False
+
+        text_types = [
+            "text/",
+            "application/json",
+            "application/xml",
+            "application/javascript",
+            "application/csv",
+            "application/sql",
+        ]
+
+        return any(mime_type.startswith(t) for t in text_types)
+
+    def _should_include_file(self, file_path: str, include: Optional[List[str]], exclude: Optional[List[str]]) -> bool:
+        """
+        Determine if a file should be included based on include/exclude patterns.
+
+        Logic:
+        1. If include is specified, file must match at least one include pattern
+        2. If exclude is specified, file must not match any exclude pattern
+        3. If neither specified, include all files
+
+        Args:
+            file_path: Path to the file to check
+            include: Optional list of include patterns (glob-style)
+            exclude: Optional list of exclude patterns (glob-style)
+
+        Returns:
+            bool: True if file should be included, False otherwise
+        """
+        import fnmatch
+
+        # If include patterns specified, file must match at least one
+        if include:
+            if not any(fnmatch.fnmatch(file_path, pattern) for pattern in include):
+                return False
+
+        # If exclude patterns specified, file must not match any
+        if exclude:
+            if any(fnmatch.fnmatch(file_path, pattern) for pattern in exclude):
+                return False
+
+        return True
+
+    def _get_reader(self, reader_type: str) -> Optional[Reader]:
+        """Get a cached reader or create it if not cached, handling missing dependencies gracefully."""
+        if self.readers is None:
+            self.readers = {}
+
+        if reader_type not in self.readers:
+            try:
+                reader = ReaderFactory.create_reader(reader_type)
+                if reader:
+                    self.readers[reader_type] = reader
+                else:
+                    return None
+
+            except Exception as e:
+                log_warning(f"Cannot create {reader_type} reader {e}")
+                return None
+
+        return self.readers.get(reader_type)
+
+    @property
+    def pdf_reader(self) -> Optional[Reader]:
+        """PDF reader - lazy loaded via factory."""
+        return self._get_reader("pdf")
+
+    @property
+    def csv_reader(self) -> Optional[Reader]:
+        """CSV reader - lazy loaded via factory."""
+        return self._get_reader("csv")
+
+    @property
+    def docx_reader(self) -> Optional[Reader]:
+        """Docx reader - lazy loaded via factory."""
+        return self._get_reader("docx")
+
+    @property
+    def pptx_reader(self) -> Optional[Reader]:
+        """PPTX reader - lazy loaded via factory."""
+        return self._get_reader("pptx")
+
+    @property
+    def json_reader(self) -> Optional[Reader]:
+        """JSON reader - lazy loaded via factory."""
+        return self._get_reader("json")
+
+    @property
+    def markdown_reader(self) -> Optional[Reader]:
+        """Markdown reader - lazy loaded via factory."""
+        return self._get_reader("markdown")
+
+    @property
+    def text_reader(self) -> Optional[Reader]:
+        """Text reader - lazy loaded via factory."""
+        return self._get_reader("text")
+
+    @property
+    def website_reader(self) -> Optional[Reader]:
+        """Website reader - lazy loaded via factory."""
+        return self._get_reader("website")
+
+    @property
+    def firecrawl_reader(self) -> Optional[Reader]:
+        """Firecrawl reader - lazy loaded via factory."""
+        return self._get_reader("firecrawl")
+
+    @property
+    def youtube_reader(self) -> Optional[Reader]:
+        """YouTube reader - lazy loaded via factory."""
+        return self._get_reader("youtube")