agent-brain-rag 1.2.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

agent_brain_server/api/routers/health.py

@@ -1,4 +1,4 @@
-"""Health check endpoints."""
+"""Health check endpoints with non-blocking queue status."""
 
 from datetime import datetime, timezone
 from typing import Literal
@@ -20,6 +20,8 @@ router = APIRouter()
 async def health_check(request: Request) -> HealthStatus:
     """Check server health status.
 
+    This endpoint never blocks and always returns quickly.
+
     Returns:
         HealthStatus with current status:
         - healthy: Server is running and ready for queries
@@ -27,20 +29,35 @@ async def health_check(request: Request) -> HealthStatus:
         - degraded: Server is up but some services are unavailable
         - unhealthy: Server is not operational
     """
-    indexing_service = request.app.state.indexing_service
     vector_store = request.app.state.vector_store
+    job_service = getattr(request.app.state, "job_service", None)
 
-    # Determine status
+    # Determine status using queue service (non-blocking)
     status: Literal["healthy", "indexing", "degraded", "unhealthy"]
-    if indexing_service.is_indexing:
+    message: str
+
+    # Check queue status (non-blocking)
+    is_indexing = False
+    current_folder = None
+    if job_service:
+        try:
+            queue_stats = await job_service.get_queue_stats()
+            is_indexing = queue_stats.running > 0
+            if is_indexing and queue_stats.current_job_id:
+                # Get current job details for message
+                current_job = await job_service.get_job(queue_stats.current_job_id)
+                if current_job:
+                    current_folder = current_job.folder_path
+        except Exception:
+            # Non-blocking: don't fail health check if queue service errors
+            pass
+
+    if is_indexing:
         status = "indexing"
-        message = f"Indexing in progress: {indexing_service.state.folder_path}"
+        message = f"Indexing in progress: {current_folder or 'unknown'}"
     elif not vector_store.is_initialized:
         status = "degraded"
         message = "Vector store not initialized"
-    elif indexing_service.state.error:
-        status = "degraded"
-        message = f"Last indexing failed: {indexing_service.state.error}"
     else:
         status = "healthy"
         message = "Server is running and ready for queries"
@@ -67,35 +84,82 @@ async def health_check(request: Request) -> HealthStatus:
     "/status",
     response_model=IndexingStatus,
     summary="Indexing Status",
-    description="Returns detailed indexing status information.",
+    description="Returns detailed indexing status information. Never blocks.",
 )
 async def indexing_status(request: Request) -> IndexingStatus:
     """Get detailed indexing status.
 
+    This endpoint never blocks and always returns quickly, even during indexing.
+
     Returns:
         IndexingStatus with:
         - total_documents: Number of documents indexed
         - total_chunks: Number of chunks in vector store
         - indexing_in_progress: Boolean indicating active indexing
+        - queue_pending: Number of pending jobs
+        - queue_running: Number of running jobs (0 or 1)
+        - current_job_running_time_ms: How long current job has been running
         - last_indexed_at: Timestamp of last indexing operation
         - indexed_folders: List of folders that have been indexed
     """
     indexing_service = request.app.state.indexing_service
-    status = await indexing_service.get_status()
+    vector_store = request.app.state.vector_store
+    job_service = getattr(request.app.state, "job_service", None)
+
+    # Get vector store count (non-blocking read)
+    try:
+        total_chunks = (
+            await vector_store.get_count() if vector_store.is_initialized else 0
+        )
+    except Exception:
+        total_chunks = 0
+
+    # Get queue status (non-blocking)
+    queue_pending = 0
+    queue_running = 0
+    current_job_id = None
+    current_job_running_time_ms = None
+    progress_percent = 0.0
+
+    if job_service:
+        try:
+            queue_stats = await job_service.get_queue_stats()
+            queue_pending = queue_stats.pending
+            queue_running = queue_stats.running
+            current_job_id = queue_stats.current_job_id
+            current_job_running_time_ms = queue_stats.current_job_running_time_ms
+
+            # Get progress from current job
+            if current_job_id:
+                current_job = await job_service.get_job(current_job_id)
+                if current_job and current_job.progress:
+                    progress_percent = current_job.progress.percent_complete
+        except Exception:
+            # Non-blocking: don't fail status if queue service errors
+            pass
+
+    # Get indexing service status for historical data
+    # This is read-only and non-blocking
+    service_status = await indexing_service.get_status()
 
     return IndexingStatus(
-        total_documents=status["total_documents"],
-        total_chunks=status["total_chunks"],
-        total_doc_chunks=status.get("total_doc_chunks", 0),
-        total_code_chunks=status.get("total_code_chunks", 0),
-        indexing_in_progress=status["is_indexing"],
-        current_job_id=status["current_job_id"],
-        progress_percent=status["progress_percent"],
+        total_documents=service_status.get("total_documents", 0),
+        total_chunks=total_chunks,
+        total_doc_chunks=service_status.get("total_doc_chunks", 0),
+        total_code_chunks=service_status.get("total_code_chunks", 0),
+        indexing_in_progress=queue_running > 0,
+        current_job_id=current_job_id,
+        progress_percent=progress_percent,
         last_indexed_at=(
-            datetime.fromisoformat(status["completed_at"])
-            if status["completed_at"]
+            datetime.fromisoformat(service_status["completed_at"])
+            if service_status.get("completed_at")
             else None
         ),
-        indexed_folders=status["indexed_folders"],
-        supported_languages=status.get("supported_languages", []),
+        indexed_folders=service_status.get("indexed_folders", []),
+        supported_languages=service_status.get("supported_languages", []),
+        graph_index=service_status.get("graph_index"),
+        # Queue status (Feature 115)
+        queue_pending=queue_pending,
+        queue_running=queue_running,
+        current_job_running_time_ms=current_job_running_time_ms,
     )

agent_brain_server/api/routers/index.py

@@ -1,40 +1,52 @@
-"""Indexing endpoints for document processing."""
+"""Indexing endpoints for document processing with job queue support."""
 
 import os
 from pathlib import Path
 
-from fastapi import APIRouter, HTTPException, Request, status
+from fastapi import APIRouter, HTTPException, Query, Request, status
 
+from agent_brain_server.config import settings
 from agent_brain_server.models import IndexRequest, IndexResponse
 
 router = APIRouter()
 
+# Maximum queue length for backpressure
+MAX_QUEUE_LENGTH = settings.AGENT_BRAIN_MAX_QUEUE
+
 
 @router.post(
     "/",
     response_model=IndexResponse,
     status_code=status.HTTP_202_ACCEPTED,
     summary="Index Documents",
-    description="Start indexing documents from a folder.",
+    description="Enqueue a job to index documents from a folder.",
 )
 async def index_documents(
-    request_body: IndexRequest, request: Request
+    request_body: IndexRequest,
+    request: Request,
+    force: bool = Query(False, description="Bypass deduplication and force a new job"),
+    allow_external: bool = Query(
+        False, description="Allow paths outside the project directory"
+    ),
 ) -> IndexResponse:
-    """Start indexing documents from the specified folder.
+    """Enqueue an indexing job for documents from the specified folder.
 
-    This endpoint initiates a background indexing job and returns immediately.
-    Use the /health/status endpoint to monitor progress.
+    This endpoint accepts the request and returns immediately with a job ID.
+    The job is processed asynchronously by a background worker.
+    Use the /index/jobs/{job_id} endpoint to monitor progress.
 
     Args:
         request_body: IndexRequest with folder_path and optional configuration.
         request: FastAPI request for accessing app state.
+        force: If True, bypass deduplication and create a new job.
+        allow_external: If True, allow indexing paths outside the project.
 
     Returns:
         IndexResponse with job_id and status.
 
     Raises:
-        400: Invalid folder path
-        409: Indexing already in progress
+        400: Invalid folder path or path outside project (without allow_external)
+        429: Queue is full (backpressure)
     """
     # Validate folder path
     folder_path = Path(request_body.folder_path).expanduser().resolve()
@@ -57,17 +69,20 @@ async def index_documents(
             detail=f"Cannot read folder: {request_body.folder_path}",
         )
 
-    # Get indexing service from app state
-    indexing_service = request.app.state.indexing_service
+    # Get job service from app state
+    job_service = request.app.state.job_service
 
-    # Check if already indexing
-    if indexing_service.is_indexing:
+    # Backpressure check (pending + running to prevent overflow)
+    stats = await job_service.get_queue_stats()
+    active_jobs = stats.pending + stats.running
+    if active_jobs >= MAX_QUEUE_LENGTH:
         raise HTTPException(
-            status_code=status.HTTP_409_CONFLICT,
-            detail="Indexing already in progress. Please wait for completion.",
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail=f"Queue full ({stats.pending} pending, {stats.running} running). "
+            "Try again later.",
         )
 
-    # Start indexing
+    # Enqueue the job
     try:
         # Update request with resolved path
         resolved_request = IndexRequest(
@@ -82,17 +97,37 @@ async def index_documents(
             exclude_patterns=request_body.exclude_patterns,
             generate_summaries=request_body.generate_summaries,
         )
-        job_id = await indexing_service.start_indexing(resolved_request)
+
+        result = await job_service.enqueue_job(
+            request=resolved_request,
+            operation="index",
+            force=force,
+            allow_external=allow_external,
+        )
+    except ValueError as e:
+        # Path validation error (outside project)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        ) from e
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to start indexing: {str(e)}",
+            detail=f"Failed to enqueue indexing job: {str(e)}",
         ) from e
 
+    # Build response message
+    if result.dedupe_hit:
+        message = (
+            f"Duplicate detected - existing job {result.job_id} is {result.status}"
+        )
+    else:
+        message = f"Job queued for {request_body.folder_path}"
+
     return IndexResponse(
-        job_id=job_id,
-        status="started",
-        message=f"Indexing started for {request_body.folder_path}",
+        job_id=result.job_id,
+        status=result.status,
+        message=message,
     )
 
 
@@ -101,10 +136,17 @@ async def index_documents(
     response_model=IndexResponse,
     status_code=status.HTTP_202_ACCEPTED,
     summary="Add Documents",
-    description="Add documents from another folder to the existing index.",
+    description="Enqueue a job to add documents from another folder.",
 )
-async def add_documents(request_body: IndexRequest, request: Request) -> IndexResponse:
-    """Add documents from a new folder to the existing index.
+async def add_documents(
+    request_body: IndexRequest,
+    request: Request,
+    force: bool = Query(False, description="Bypass deduplication and force a new job"),
+    allow_external: bool = Query(
+        False, description="Allow paths outside the project directory"
+    ),
+) -> IndexResponse:
+    """Enqueue a job to add documents from a new folder to the existing index.
 
     This is similar to the index endpoint but adds to the existing
     vector store instead of replacing it.
@@ -112,6 +154,8 @@ async def add_documents(request_body: IndexRequest, request: Request) -> IndexRe
     Args:
         request_body: IndexRequest with folder_path and optional configuration.
         request: FastAPI request for accessing app state.
+        force: If True, bypass deduplication and create a new job.
+        allow_external: If True, allow indexing paths outside the project.
 
     Returns:
         IndexResponse with job_id and status.
@@ -131,12 +175,17 @@ async def add_documents(request_body: IndexRequest, request: Request) -> IndexRe
             detail=f"Path is not a directory: {request_body.folder_path}",
         )
 
-    indexing_service = request.app.state.indexing_service
+    # Get job service from app state
+    job_service = request.app.state.job_service
 
-    if indexing_service.is_indexing:
+    # Backpressure check (pending + running to prevent overflow)
+    stats = await job_service.get_queue_stats()
+    active_jobs = stats.pending + stats.running
+    if active_jobs >= MAX_QUEUE_LENGTH:
         raise HTTPException(
-            status_code=status.HTTP_409_CONFLICT,
-            detail="Indexing already in progress. Please wait for completion.",
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail=f"Queue full ({stats.pending} pending, {stats.running} running). "
+            "Try again later.",
         )
 
     try:
@@ -151,17 +200,36 @@ async def add_documents(request_body: IndexRequest, request: Request) -> IndexRe
             include_patterns=request_body.include_patterns,
             exclude_patterns=request_body.exclude_patterns,
         )
-        job_id = await indexing_service.start_indexing(resolved_request)
+
+        result = await job_service.enqueue_job(
+            request=resolved_request,
+            operation="add",
+            force=force,
+            allow_external=allow_external,
+        )
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        ) from e
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to add documents: {str(e)}",
+            detail=f"Failed to enqueue add job: {str(e)}",
         ) from e
 
+    # Build response message
+    if result.dedupe_hit:
+        message = (
+            f"Duplicate detected - existing job {result.job_id} is {result.status}"
+        )
+    else:
+        message = f"Job queued to add documents from {request_body.folder_path}"
+
     return IndexResponse(
-        job_id=job_id,
-        status="started",
-        message=f"Adding documents from {request_body.folder_path}",
+        job_id=result.job_id,
+        status=result.status,
+        message=message,
     )
 
 
@@ -175,6 +243,7 @@ async def reset_index(request: Request) -> IndexResponse:
     """Reset the index by deleting all stored documents.
 
     Warning: This permanently removes all indexed content.
+    Cannot be performed while jobs are running.
 
     Args:
         request: FastAPI request for accessing app state.
@@ -183,14 +252,17 @@ async def reset_index(request: Request) -> IndexResponse:
         IndexResponse confirming the reset.
 
     Raises:
-        409: Indexing in progress
+        409: Jobs in progress
     """
+    job_service = request.app.state.job_service
     indexing_service = request.app.state.indexing_service
 
-    if indexing_service.is_indexing:
+    # Check if any jobs are running
+    stats = await job_service.get_queue_stats()
+    if stats.running > 0:
         raise HTTPException(
             status_code=status.HTTP_409_CONFLICT,
-            detail="Cannot reset while indexing is in progress.",
+            detail="Cannot reset while indexing jobs are in progress.",
         )
 
     try:

agent_brain_server/api/routers/jobs.py

@@ -0,0 +1,111 @@
+"""Job management endpoints for indexing job queue."""
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query, Request, status
+
+from agent_brain_server.job_queue.job_service import JobQueueService
+from agent_brain_server.models.job import JobDetailResponse, JobListResponse
+
+router = APIRouter()
+
+
+@router.get(
+    "/",
+    response_model=JobListResponse,
+    summary="List Jobs",
+    description="List all indexing jobs with pagination.",
+)
+async def list_jobs(
+    request: Request,
+    limit: int = Query(
+        50, ge=1, le=100, description="Maximum number of jobs to return"
+    ),
+    offset: int = Query(0, ge=0, description="Number of jobs to skip"),
+) -> JobListResponse:
+    """List all jobs with pagination.
+
+    Returns a paginated list of jobs with summary information and queue statistics.
+
+    Args:
+        request: FastAPI request for accessing app state.
+        limit: Maximum number of jobs to return (1-100, default 50).
+        offset: Number of jobs to skip for pagination (default 0).
+
+    Returns:
+        JobListResponse with list of job summaries and queue statistics.
+    """
+    job_service: JobQueueService = request.app.state.job_service
+    return await job_service.list_jobs(limit=limit, offset=offset)
+
+
+@router.get(
+    "/{job_id}",
+    response_model=JobDetailResponse,
+    summary="Get Job Details",
+    description="Get detailed information about a specific job.",
+)
+async def get_job(job_id: str, request: Request) -> JobDetailResponse:
+    """Get details for a specific job.
+
+    Returns full job information including progress, timestamps, and results.
+
+    Args:
+        job_id: The unique job identifier.
+        request: FastAPI request for accessing app state.
+
+    Returns:
+        JobDetailResponse with full job details.
+
+    Raises:
+        404: Job not found.
+    """
+    job_service: JobQueueService = request.app.state.job_service
+    job = await job_service.get_job(job_id)
+    if not job:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found",
+        )
+    return job
+
+
+@router.delete(
+    "/{job_id}",
+    summary="Cancel Job",
+    description="Cancel a pending or running job.",
+)
+async def cancel_job(job_id: str, request: Request) -> dict[str, Any]:
+    """Cancel a job.
+
+    Cancellation behavior depends on job status:
+    - PENDING jobs are cancelled immediately
+    - RUNNING jobs have cancel_requested flag set; worker will stop at next checkpoint
+    - Completed/Failed/Cancelled jobs return 409 Conflict
+
+    Args:
+        job_id: The unique job identifier.
+        request: FastAPI request for accessing app state.
+
+    Returns:
+        Dictionary with cancellation status and message.
+
+    Raises:
+        404: Job not found.
+        409: Job cannot be cancelled (already completed, failed, or cancelled).
+    """
+    job_service: JobQueueService = request.app.state.job_service
+
+    try:
+        result = await job_service.cancel_job(job_id)
+        return result
+    except KeyError as e:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=str(e),
+        ) from e
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=str(e),
+        ) from e