agno 2.0.0rc2__py3-none-any.whl → 2.3.0__py3-none-any.whl

agno/os/routers/knowledge/knowledge.py

@@ -2,7 +2,6 @@ import json
 import logging
 import math
 from typing import Dict, List, Optional
-from uuid import uuid4
 
 from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, Path, Query, UploadFile
 
@@ -20,6 +19,9 @@ from agno.os.routers.knowledge.schemas import (
     ContentStatusResponse,
     ContentUpdateSchema,
     ReaderSchema,
+    VectorDbSchema,
+    VectorSearchRequestSchema,
+    VectorSearchResult,
 )
 from agno.os.schema import (
     BadRequestResponse,
@@ -34,6 +36,7 @@ from agno.os.schema import (
 from agno.os.settings import AgnoAPISettings
 from agno.os.utils import get_knowledge_instance_by_db_id
 from agno.utils.log import log_debug, log_info
+from agno.utils.string import generate_id
 
 logger = logging.getLogger(__name__)
 
@@ -99,11 +102,12 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
         text_content: Optional[str] = Form(None, description="Raw text content to process"),
         reader_id: Optional[str] = Form(None, description="ID of the reader to use for content processing"),
         chunker: Optional[str] = Form(None, description="Chunking strategy to apply during processing"),
+        chunk_size: Optional[int] = Form(None, description="Chunk size to use for processing"),
+        chunk_overlap: Optional[int] = Form(None, description="Chunk overlap to use for processing"),
         db_id: Optional[str] = Query(default=None, description="Database ID to use for content storage"),
     ):
         knowledge = get_knowledge_instance_by_db_id(knowledge_instances, db_id)
-        content_id = str(uuid4())
-        log_info(f"Adding content: {name}, {description}, {url}, {metadata} with ID: {content_id}")
+        log_info(f"Adding content: {name}, {description}, {url}, {metadata}")
 
         parsed_metadata = None
         if metadata:
@@ -166,10 +170,14 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
             file_data=file_data,
             size=file.size if file else None if text_content else None,
         )
-        background_tasks.add_task(process_content, knowledge, content_id, content, reader_id, chunker)
+        content_hash = knowledge._build_content_hash(content)
+        content.content_hash = content_hash
+        content.id = generate_id(content_hash)
+
+        background_tasks.add_task(process_content, knowledge, content, reader_id, chunker, chunk_size, chunk_overlap)
 
         response = ContentResponseSchema(
-            id=content_id,
+            id=content.id,
             name=name,
             description=description,
             metadata=parsed_metadata,
@@ -300,7 +308,7 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
             }
         },
     )
-    def get_content(
+    async def get_content(
         limit: Optional[int] = Query(default=20, description="Number of content entries to return"),
         page: Optional[int] = Query(default=1, description="Page number"),
         sort_by: Optional[str] = Query(default="created_at", description="Field to sort by"),
@@ -308,7 +316,7 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
     ) -> PaginatedResponse[ContentResponseSchema]:
         knowledge = get_knowledge_instance_by_db_id(knowledge_instances, db_id)
-        contents, count = knowledge.get_content(limit=limit, page=page, sort_by=sort_by, sort_order=sort_order)
+        contents, count = await knowledge.aget_content(limit=limit, page=page, sort_by=sort_by, sort_order=sort_order)
 
         return PaginatedResponse(
             data=[
@@ -368,13 +376,13 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
             404: {"description": "Content not found", "model": NotFoundResponse},
         },
     )
-    def get_content_by_id(
+    async def get_content_by_id(
         content_id: str,
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
     ) -> ContentResponseSchema:
         log_info(f"Getting content by id: {content_id}")
         knowledge = get_knowledge_instance_by_db_id(knowledge_instances, db_id)
-        content = knowledge.get_content_by_id(content_id=content_id)
+        content = await knowledge.aget_content_by_id(content_id=content_id)
         if not content:
             raise HTTPException(status_code=404, detail=f"Content not found: {content_id}")
         response = ContentResponseSchema.from_dict(
@@ -408,12 +416,12 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
             500: {"description": "Failed to delete content", "model": InternalServerErrorResponse},
         },
     )
-    def delete_content_by_id(
+    async def delete_content_by_id(
         content_id: str,
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
     ) -> ContentResponseSchema:
         knowledge = get_knowledge_instance_by_db_id(knowledge_instances, db_id)
-        knowledge.remove_content_by_id(content_id=content_id)
+        await knowledge.aremove_content_by_id(content_id=content_id)
         log_info(f"Deleting content by id: {content_id}")
 
         return ContentResponseSchema(
@@ -440,7 +448,6 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
         knowledge = get_knowledge_instance_by_db_id(knowledge_instances, db_id)
         log_info("Deleting all content")
         knowledge.remove_all_content()
-
         return "success"
 
     @router.get(
@@ -473,13 +480,13 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
             404: {"description": "Content not found", "model": NotFoundResponse},
         },
     )
-    def get_content_status(
+    async def get_content_status(
         content_id: str,
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
     ) -> ContentStatusResponse:
         log_info(f"Getting content status: {content_id}")
         knowledge = get_knowledge_instance_by_db_id(knowledge_instances, db_id)
-        knowledge_status, status_message = knowledge.get_content_status(content_id=content_id)
+        knowledge_status, status_message = await knowledge.aget_content_status(content_id=content_id)
 
         # Handle the case where content is not found
         if knowledge_status is None:
@@ -510,11 +517,107 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
 
         return ContentStatusResponse(status=status, status_message=status_message or "")
 
+    @router.post(
+        "/knowledge/search",
+        status_code=200,
+        operation_id="search_knowledge",
+        summary="Search Knowledge",
+        description="Search the knowledge base for relevant documents using query, filters and search type.",
+        response_model=PaginatedResponse[VectorSearchResult],
+        responses={
+            200: {
+                "description": "Search results retrieved successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "data": [
+                                {
+                                    "id": "doc_123",
+                                    "content": "Jordan Mitchell - Software Engineer with skills in JavaScript, React, Python",
+                                    "name": "cv_1",
+                                    "meta_data": {"page": 1, "chunk": 1},
+                                    "usage": {"total_tokens": 14},
+                                    "reranking_score": 0.95,
+                                    "content_id": "content_456",
+                                }
+                            ],
+                            "meta": {"page": 1, "limit": 20, "total_pages": 2, "total_count": 35},
+                        }
+                    }
+                },
+            },
+            400: {"description": "Invalid search parameters"},
+            404: {"description": "No documents found"},
+        },
+    )
+    def search_knowledge(request: VectorSearchRequestSchema) -> PaginatedResponse[VectorSearchResult]:
+        import time
+
+        start_time = time.time()
+
+        knowledge = get_knowledge_instance_by_db_id(knowledge_instances, request.db_id)
+
+        # For now, validate the vector db ids exist in the knowledge base
+        # We will add more logic around this once we have multi vectordb support
+        # If vector db ids are provided, check if any of them match the knowledge's vector db
+        if request.vector_db_ids:
+            if knowledge.vector_db and knowledge.vector_db.id:
+                if knowledge.vector_db.id not in request.vector_db_ids:
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"None of the provided Vector DB IDs {request.vector_db_ids} match the knowledge base Vector DB ID {knowledge.vector_db.id}",
+                    )
+            else:
+                raise HTTPException(status_code=400, detail="Knowledge base has no vector database configured")
+
+        # Calculate pagination parameters
+        meta = request.meta
+        limit = meta.limit if meta and meta.limit is not None else 20
+        page = meta.page if meta and meta.page is not None else 1
+
+        # Use max_results if specified, otherwise use a higher limit for search then paginate
+        search_limit = request.max_results
+
+        results = knowledge.search(
+            query=request.query, max_results=search_limit, filters=request.filters, search_type=request.search_type
+        )
+
+        # Calculate pagination
+        total_results = len(results)
+        start_idx = (page - 1) * limit
+
+        # Ensure start_idx doesn't exceed the total results
+        if start_idx >= total_results and total_results > 0:
+            # If page is beyond available results, return empty results
+            paginated_results = []
+        else:
+            end_idx = min(start_idx + limit, total_results)
+            paginated_results = results[start_idx:end_idx]
+
+        search_time_ms = (time.time() - start_time) * 1000
+
+        # Convert Document objects to serializable format
+        document_results = [VectorSearchResult.from_document(doc) for doc in paginated_results]
+
+        # Calculate pagination info
+        total_pages = (total_results + limit - 1) // limit  # Ceiling division
+
+        return PaginatedResponse(
+            data=document_results,
+            meta=PaginationInfo(
+                page=page,
+                limit=limit,
+                total_pages=total_pages,
+                total_count=total_results,
+                search_time_ms=search_time_ms,
+            ),
+        )
+
     @router.get(
         "/knowledge/config",
         status_code=200,
         operation_id="get_knowledge_config",
-        summary="Get Knowledge Configuration",
+        summary="Get Config",
         description=(
             "Retrieve available readers, chunkers, and configuration options for content processing. "
             "This endpoint provides metadata about supported file types, processing strategies, and filters."
@@ -700,38 +803,65 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
                                     "key": "AgenticChunker",
                                     "name": "AgenticChunker",
                                     "description": "Chunking strategy that uses an LLM to determine natural breakpoints in the text",
+                                    "metadata": {"chunk_size": 5000},
                                 },
                                 "DocumentChunker": {
                                     "key": "DocumentChunker",
                                     "name": "DocumentChunker",
                                     "description": "A chunking strategy that splits text based on document structure like paragraphs and sections",
-                                },
-                                "RecursiveChunker": {
-                                    "key": "RecursiveChunker",
-                                    "name": "RecursiveChunker",
-                                    "description": "Chunking strategy that recursively splits text into chunks by finding natural break points",
-                                },
-                                "SemanticChunker": {
-                                    "key": "SemanticChunker",
-                                    "name": "SemanticChunker",
-                                    "description": "Chunking strategy that splits text into semantic chunks using chonkie",
+                                    "metadata": {
+                                        "chunk_size": 5000,
+                                        "chunk_overlap": 0,
+                                    },
                                 },
                                 "FixedSizeChunker": {
                                     "key": "FixedSizeChunker",
                                     "name": "FixedSizeChunker",
                                     "description": "Chunking strategy that splits text into fixed-size chunks with optional overlap",
+                                    "metadata": {
+                                        "chunk_size": 5000,
+                                        "chunk_overlap": 0,
+                                    },
+                                },
+                                "MarkdownChunker": {
+                                    "key": "MarkdownChunker",
+                                    "name": "MarkdownChunker",
+                                    "description": "A chunking strategy that splits markdown based on structure like headers, paragraphs and sections",
+                                    "metadata": {
+                                        "chunk_size": 5000,
+                                        "chunk_overlap": 0,
+                                    },
+                                },
+                                "RecursiveChunker": {
+                                    "key": "RecursiveChunker",
+                                    "name": "RecursiveChunker",
+                                    "description": "Chunking strategy that recursively splits text into chunks by finding natural break points",
+                                    "metadata": {
+                                        "chunk_size": 5000,
+                                        "chunk_overlap": 0,
+                                    },
                                 },
                                 "RowChunker": {
                                     "key": "RowChunker",
                                     "name": "RowChunker",
                                     "description": "RowChunking chunking strategy",
+                                    "metadata": {},
                                 },
-                                "MarkdownChunker": {
-                                    "key": "MarkdownChunker",
-                                    "name": "MarkdownChunker",
-                                    "description": "A chunking strategy that splits markdown based on structure like headers, paragraphs and sections",
+                                "SemanticChunker": {
+                                    "key": "SemanticChunker",
+                                    "name": "SemanticChunker",
+                                    "description": "Chunking strategy that splits text into semantic chunks using chonkie",
+                                    "metadata": {"chunk_size": 5000},
                                 },
                             },
+                            "vector_dbs": [
+                                {
+                                    "id": "vector_db_1",
+                                    "name": "Vector DB 1",
+                                    "description": "Vector DB 1 description",
+                                    "search_types": ["vector", "keyword", "hybrid"],
+                                }
+                            ],
                             "filters": ["filter_tag_1", "filter_tag2"],
                         }
                     }
@@ -787,14 +917,32 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
             chunker_key = chunker_info.get("key")
             if chunker_key:
                 chunkers_dict[chunker_key] = ChunkerSchema(
-                    key=chunker_key, name=chunker_info.get("name"), description=chunker_info.get("description")
+                    key=chunker_key,
+                    name=chunker_info.get("name"),
+                    description=chunker_info.get("description"),
+                    metadata=chunker_info.get("metadata", {}),
                 )
 
+        vector_dbs = []
+        if knowledge.vector_db:
+            search_types = knowledge.vector_db.get_supported_search_types()
+            name = knowledge.vector_db.name
+            db_id = knowledge.vector_db.id
+            vector_dbs.append(
+                VectorDbSchema(
+                    id=db_id,
+                    name=name,
+                    description=knowledge.vector_db.description,
+                    search_types=search_types,
+                )
+            )
+
         return ConfigResponseSchema(
             readers=reader_schemas,
+            vector_dbs=vector_dbs,
             readersForType=types_of_readers,
             chunkers=chunkers_dict,
-            filters=knowledge.get_filters(),
+            filters=knowledge.get_valid_filters(),
         )
 
     return router
@@ -802,15 +950,15 @@ def attach_routes(router: APIRouter, knowledge_instances: List[Knowledge]) -> AP
 
 async def process_content(
     knowledge: Knowledge,
-    content_id: str,
     content: Content,
     reader_id: Optional[str] = None,
     chunker: Optional[str] = None,
+    chunk_size: Optional[int] = None,
+    chunk_overlap: Optional[int] = None,
 ):
     """Background task to process the content"""
-    log_info(f"Processing content {content_id}")
+
     try:
-        content.id = content_id
         if reader_id:
             reader = None
             if knowledge.readers and reader_id in knowledge.readers:
@@ -829,21 +977,20 @@ async def process_content(
                 content.reader = reader
         if chunker and content.reader:
             # Set the chunker name on the reader - let the reader handle it internally
-            content.reader.set_chunking_strategy_from_string(chunker)
+            content.reader.set_chunking_strategy_from_string(chunker, chunk_size=chunk_size, overlap=chunk_overlap)
             log_debug(f"Set chunking strategy: {chunker}")
 
         log_debug(f"Using reader: {content.reader.__class__.__name__}")
         await knowledge._load_content(content, upsert=False, skip_if_exists=True)
-        log_info(f"Content {content_id} processed successfully")
+        log_info(f"Content {content.id} processed successfully")
     except Exception as e:
-        log_info(f"Error processing content {content_id}: {e}")
+        log_info(f"Error processing content: {e}")
         # Mark content as failed in the contents DB
         try:
             from agno.knowledge.content import ContentStatus as KnowledgeContentStatus
 
             content.status = KnowledgeContentStatus.FAILED
             content.status_message = str(e)
-            content.id = content_id
             knowledge.patch_content(content)
         except Exception:
             # Swallow any secondary errors to avoid crashing the background task

agno/os/routers/knowledge/schemas.py

@@ -16,23 +16,23 @@ class ContentStatus(str, Enum):
 class ContentStatusResponse(BaseModel):
     """Response model for content status endpoint."""
 
-    status: ContentStatus
-    status_message: str = ""
+    status: ContentStatus = Field(..., description="Current processing status of the content")
+    status_message: str = Field("", description="Status message or error details")
 
 
 class ContentResponseSchema(BaseModel):
-    id: str
-    name: Optional[str] = None
-    description: Optional[str] = None
-    type: Optional[str] = None
-    size: Optional[str] = None
-    linked_to: Optional[str] = None
-    metadata: Optional[dict] = None
-    access_count: Optional[int] = None
-    status: Optional[ContentStatus] = None
-    status_message: Optional[str] = None
-    created_at: Optional[datetime] = None
-    updated_at: Optional[datetime] = None
+    id: str = Field(..., description="Unique identifier for the content")
+    name: Optional[str] = Field(None, description="Name of the content")
+    description: Optional[str] = Field(None, description="Description of the content")
+    type: Optional[str] = Field(None, description="MIME type of the content")
+    size: Optional[str] = Field(None, description="Size of the content in bytes")
+    linked_to: Optional[str] = Field(None, description="ID of related content if linked")
+    metadata: Optional[dict] = Field(None, description="Additional metadata as key-value pairs")
+    access_count: Optional[int] = Field(None, description="Number of times content has been accessed", ge=0)
+    status: Optional[ContentStatus] = Field(None, description="Processing status of the content")
+    status_message: Optional[str] = Field(None, description="Status message or error details")
+    created_at: Optional[datetime] = Field(None, description="Timestamp when content was created")
+    updated_at: Optional[datetime] = Field(None, description="Timestamp when content was last updated")
 
     @classmethod
     def from_dict(cls, content: Dict[str, Any]) -> "ContentResponseSchema":
@@ -99,20 +99,80 @@ class ContentUpdateSchema(BaseModel):
 
 
 class ReaderSchema(BaseModel):
-    id: str
-    name: Optional[str] = None
-    description: Optional[str] = None
-    chunkers: Optional[List[str]] = None
+    id: str = Field(..., description="Unique identifier for the reader")
+    name: Optional[str] = Field(None, description="Name of the reader")
+    description: Optional[str] = Field(None, description="Description of the reader's capabilities")
+    chunkers: Optional[List[str]] = Field(None, description="List of supported chunking strategies")
 
 
 class ChunkerSchema(BaseModel):
     key: str
     name: Optional[str] = None
     description: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+
+
+class VectorDbSchema(BaseModel):
+    id: str = Field(..., description="Unique identifier for the vector database")
+    name: Optional[str] = Field(None, description="Name of the vector database")
+    description: Optional[str] = Field(None, description="Description of the vector database")
+    search_types: Optional[List[str]] = Field(
+        None, description="List of supported search types (vector, keyword, hybrid)"
+    )
+
+
+class VectorSearchResult(BaseModel):
+    """Schema for search result documents."""
+
+    id: str = Field(..., description="Unique identifier for the search result document")
+    content: str = Field(..., description="Content text of the document")
+    name: Optional[str] = Field(None, description="Name of the document")
+    meta_data: Optional[Dict[str, Any]] = Field(None, description="Metadata associated with the document")
+    usage: Optional[Dict[str, Any]] = Field(None, description="Usage statistics (e.g., token counts)")
+    reranking_score: Optional[float] = Field(None, description="Reranking score for relevance", ge=0.0, le=1.0)
+    content_id: Optional[str] = Field(None, description="ID of the source content")
+    content_origin: Optional[str] = Field(None, description="Origin URL or source of the content")
+    size: Optional[int] = Field(None, description="Size of the content in bytes", ge=0)
+
+    @classmethod
+    def from_document(cls, document) -> "VectorSearchResult":
+        """Convert a Document object to a serializable VectorSearchResult."""
+        return cls(
+            id=document.id,
+            content=document.content,
+            name=getattr(document, "name", None),
+            meta_data=getattr(document, "meta_data", None),
+            usage=getattr(document, "usage", None),
+            reranking_score=getattr(document, "reranking_score", None),
+            content_id=getattr(document, "content_id", None),
+            content_origin=getattr(document, "content_origin", None),
+            size=getattr(document, "size", None),
+        )
+
+
+class VectorSearchRequestSchema(BaseModel):
+    """Schema for vector search request."""
+
+    class Meta(BaseModel):
+        """Inline metadata schema for pagination."""
+
+        limit: int = Field(20, description="Number of results per page", ge=1, le=100)
+        page: int = Field(1, description="Page number", ge=1)
+
+    query: str = Field(..., description="The search query text")
+    db_id: Optional[str] = Field(None, description="The content database ID to search in")
+    vector_db_ids: Optional[List[str]] = Field(None, description="List of vector database IDs to search in")
+    search_type: Optional[str] = Field(None, description="The type of search to perform (vector, keyword, hybrid)")
+    max_results: Optional[int] = Field(None, description="The maximum number of results to return", ge=1, le=1000)
+    filters: Optional[Dict[str, Any]] = Field(None, description="Filters to apply to the search results")
+    meta: Optional[Meta] = Field(
+        None, description="Pagination metadata. Limit and page number to return a subset of results."
+    )
 
 
 class ConfigResponseSchema(BaseModel):
-    readers: Optional[Dict[str, ReaderSchema]] = None
-    readersForType: Optional[Dict[str, List[str]]] = None
-    chunkers: Optional[Dict[str, ChunkerSchema]] = None
-    filters: Optional[List[str]] = None
+    readers: Optional[Dict[str, ReaderSchema]] = Field(None, description="Available content readers")
+    readersForType: Optional[Dict[str, List[str]]] = Field(None, description="Mapping of content types to reader IDs")
+    chunkers: Optional[Dict[str, ChunkerSchema]] = Field(None, description="Available chunking strategies")
+    filters: Optional[List[str]] = Field(None, description="Available filter tags")
+    vector_dbs: Optional[List[VectorDbSchema]] = Field(None, description="Configured vector databases")