agent-brain-rag 2.0.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

agent_brain_server/api/routers/index.py

@@ -1,40 +1,52 @@
-"""Indexing endpoints for document processing."""
+"""Indexing endpoints for document processing with job queue support."""
 
 import os
 from pathlib import Path
 
-from fastapi import APIRouter, HTTPException, Request, status
+from fastapi import APIRouter, HTTPException, Query, Request, status
 
+from agent_brain_server.config import settings
 from agent_brain_server.models import IndexRequest, IndexResponse
 
 router = APIRouter()
 
+# Maximum queue length for backpressure
+MAX_QUEUE_LENGTH = settings.AGENT_BRAIN_MAX_QUEUE
+
 
 @router.post(
     "/",
     response_model=IndexResponse,
     status_code=status.HTTP_202_ACCEPTED,
     summary="Index Documents",
-    description="Start indexing documents from a folder.",
+    description="Enqueue a job to index documents from a folder.",
 )
 async def index_documents(
-    request_body: IndexRequest, request: Request
+    request_body: IndexRequest,
+    request: Request,
+    force: bool = Query(False, description="Bypass deduplication and force a new job"),
+    allow_external: bool = Query(
+        False, description="Allow paths outside the project directory"
+    ),
 ) -> IndexResponse:
-    """Start indexing documents from the specified folder.
+    """Enqueue an indexing job for documents from the specified folder.
 
-    This endpoint initiates a background indexing job and returns immediately.
-    Use the /health/status endpoint to monitor progress.
+    This endpoint accepts the request and returns immediately with a job ID.
+    The job is processed asynchronously by a background worker.
+    Use the /index/jobs/{job_id} endpoint to monitor progress.
 
     Args:
         request_body: IndexRequest with folder_path and optional configuration.
         request: FastAPI request for accessing app state.
+        force: If True, bypass deduplication and create a new job.
+        allow_external: If True, allow indexing paths outside the project.
 
     Returns:
         IndexResponse with job_id and status.
 
     Raises:
-        400: Invalid folder path
-        409: Indexing already in progress
+        400: Invalid folder path or path outside project (without allow_external)
+        429: Queue is full (backpressure)
     """
     # Validate folder path
     folder_path = Path(request_body.folder_path).expanduser().resolve()
@@ -57,17 +69,20 @@ async def index_documents(
             detail=f"Cannot read folder: {request_body.folder_path}",
         )
 
-    # Get indexing service from app state
-    indexing_service = request.app.state.indexing_service
+    # Get job service from app state
+    job_service = request.app.state.job_service
 
-    # Check if already indexing
-    if indexing_service.is_indexing:
+    # Backpressure check (pending + running to prevent overflow)
+    stats = await job_service.get_queue_stats()
+    active_jobs = stats.pending + stats.running
+    if active_jobs >= MAX_QUEUE_LENGTH:
         raise HTTPException(
-            status_code=status.HTTP_409_CONFLICT,
-            detail="Indexing already in progress. Please wait for completion.",
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail=f"Queue full ({stats.pending} pending, {stats.running} running). "
+            "Try again later.",
         )
 
-    # Start indexing
+    # Enqueue the job
     try:
         # Update request with resolved path
         resolved_request = IndexRequest(
@@ -82,17 +97,37 @@ async def index_documents(
             exclude_patterns=request_body.exclude_patterns,
             generate_summaries=request_body.generate_summaries,
         )
-        job_id = await indexing_service.start_indexing(resolved_request)
+
+        result = await job_service.enqueue_job(
+            request=resolved_request,
+            operation="index",
+            force=force,
+            allow_external=allow_external,
+        )
+    except ValueError as e:
+        # Path validation error (outside project)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        ) from e
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to start indexing: {str(e)}",
+            detail=f"Failed to enqueue indexing job: {str(e)}",
         ) from e
 
+    # Build response message
+    if result.dedupe_hit:
+        message = (
+            f"Duplicate detected - existing job {result.job_id} is {result.status}"
+        )
+    else:
+        message = f"Job queued for {request_body.folder_path}"
+
     return IndexResponse(
-        job_id=job_id,
-        status="started",
-        message=f"Indexing started for {request_body.folder_path}",
+        job_id=result.job_id,
+        status=result.status,
+        message=message,
     )
 
 
@@ -101,10 +136,17 @@ async def index_documents(
     response_model=IndexResponse,
     status_code=status.HTTP_202_ACCEPTED,
     summary="Add Documents",
-    description="Add documents from another folder to the existing index.",
+    description="Enqueue a job to add documents from another folder.",
 )
-async def add_documents(request_body: IndexRequest, request: Request) -> IndexResponse:
-    """Add documents from a new folder to the existing index.
+async def add_documents(
+    request_body: IndexRequest,
+    request: Request,
+    force: bool = Query(False, description="Bypass deduplication and force a new job"),
+    allow_external: bool = Query(
+        False, description="Allow paths outside the project directory"
+    ),
+) -> IndexResponse:
+    """Enqueue a job to add documents from a new folder to the existing index.
 
     This is similar to the index endpoint but adds to the existing
     vector store instead of replacing it.
@@ -112,6 +154,8 @@ async def add_documents(request_body: IndexRequest, request: Request) -> IndexRe
     Args:
         request_body: IndexRequest with folder_path and optional configuration.
         request: FastAPI request for accessing app state.
+        force: If True, bypass deduplication and create a new job.
+        allow_external: If True, allow indexing paths outside the project.
 
     Returns:
         IndexResponse with job_id and status.
@@ -131,12 +175,17 @@ async def add_documents(request_body: IndexRequest, request: Request) -> IndexRe
             detail=f"Path is not a directory: {request_body.folder_path}",
         )
 
-    indexing_service = request.app.state.indexing_service
+    # Get job service from app state
+    job_service = request.app.state.job_service
 
-    if indexing_service.is_indexing:
+    # Backpressure check (pending + running to prevent overflow)
+    stats = await job_service.get_queue_stats()
+    active_jobs = stats.pending + stats.running
+    if active_jobs >= MAX_QUEUE_LENGTH:
         raise HTTPException(
-            status_code=status.HTTP_409_CONFLICT,
-            detail="Indexing already in progress. Please wait for completion.",
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail=f"Queue full ({stats.pending} pending, {stats.running} running). "
+            "Try again later.",
         )
 
     try:
@@ -151,17 +200,36 @@ async def add_documents(request_body: IndexRequest, request: Request) -> IndexRe
             include_patterns=request_body.include_patterns,
             exclude_patterns=request_body.exclude_patterns,
         )
-        job_id = await indexing_service.start_indexing(resolved_request)
+
+        result = await job_service.enqueue_job(
+            request=resolved_request,
+            operation="add",
+            force=force,
+            allow_external=allow_external,
+        )
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        ) from e
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to add documents: {str(e)}",
+            detail=f"Failed to enqueue add job: {str(e)}",
         ) from e
 
+    # Build response message
+    if result.dedupe_hit:
+        message = (
+            f"Duplicate detected - existing job {result.job_id} is {result.status}"
+        )
+    else:
+        message = f"Job queued to add documents from {request_body.folder_path}"
+
     return IndexResponse(
-        job_id=job_id,
-        status="started",
-        message=f"Adding documents from {request_body.folder_path}",
+        job_id=result.job_id,
+        status=result.status,
+        message=message,
     )
 
 
@@ -175,6 +243,7 @@ async def reset_index(request: Request) -> IndexResponse:
     """Reset the index by deleting all stored documents.
 
     Warning: This permanently removes all indexed content.
+    Cannot be performed while jobs are running.
 
     Args:
         request: FastAPI request for accessing app state.
@@ -183,14 +252,17 @@ async def reset_index(request: Request) -> IndexResponse:
         IndexResponse confirming the reset.
 
     Raises:
-        409: Indexing in progress
+        409: Jobs in progress
     """
+    job_service = request.app.state.job_service
     indexing_service = request.app.state.indexing_service
 
-    if indexing_service.is_indexing:
+    # Check if any jobs are running
+    stats = await job_service.get_queue_stats()
+    if stats.running > 0:
         raise HTTPException(
             status_code=status.HTTP_409_CONFLICT,
-            detail="Cannot reset while indexing is in progress.",
+            detail="Cannot reset while indexing jobs are in progress.",
         )
 
     try:

agent_brain_server/api/routers/jobs.py

@@ -0,0 +1,111 @@
+"""Job management endpoints for indexing job queue."""
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query, Request, status
+
+from agent_brain_server.job_queue.job_service import JobQueueService
+from agent_brain_server.models.job import JobDetailResponse, JobListResponse
+
+router = APIRouter()
+
+
+@router.get(
+    "/",
+    response_model=JobListResponse,
+    summary="List Jobs",
+    description="List all indexing jobs with pagination.",
+)
+async def list_jobs(
+    request: Request,
+    limit: int = Query(
+        50, ge=1, le=100, description="Maximum number of jobs to return"
+    ),
+    offset: int = Query(0, ge=0, description="Number of jobs to skip"),
+) -> JobListResponse:
+    """List all jobs with pagination.
+
+    Returns a paginated list of jobs with summary information and queue statistics.
+
+    Args:
+        request: FastAPI request for accessing app state.
+        limit: Maximum number of jobs to return (1-100, default 50).
+        offset: Number of jobs to skip for pagination (default 0).
+
+    Returns:
+        JobListResponse with list of job summaries and queue statistics.
+    """
+    job_service: JobQueueService = request.app.state.job_service
+    return await job_service.list_jobs(limit=limit, offset=offset)
+
+
+@router.get(
+    "/{job_id}",
+    response_model=JobDetailResponse,
+    summary="Get Job Details",
+    description="Get detailed information about a specific job.",
+)
+async def get_job(job_id: str, request: Request) -> JobDetailResponse:
+    """Get details for a specific job.
+
+    Returns full job information including progress, timestamps, and results.
+
+    Args:
+        job_id: The unique job identifier.
+        request: FastAPI request for accessing app state.
+
+    Returns:
+        JobDetailResponse with full job details.
+
+    Raises:
+        404: Job not found.
+    """
+    job_service: JobQueueService = request.app.state.job_service
+    job = await job_service.get_job(job_id)
+    if not job:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found",
+        )
+    return job
+
+
+@router.delete(
+    "/{job_id}",
+    summary="Cancel Job",
+    description="Cancel a pending or running job.",
+)
+async def cancel_job(job_id: str, request: Request) -> dict[str, Any]:
+    """Cancel a job.
+
+    Cancellation behavior depends on job status:
+    - PENDING jobs are cancelled immediately
+    - RUNNING jobs have cancel_requested flag set; worker will stop at next checkpoint
+    - Completed/Failed/Cancelled jobs return 409 Conflict
+
+    Args:
+        job_id: The unique job identifier.
+        request: FastAPI request for accessing app state.
+
+    Returns:
+        Dictionary with cancellation status and message.
+
+    Raises:
+        404: Job not found.
+        409: Job cannot be cancelled (already completed, failed, or cancelled).
+    """
+    job_service: JobQueueService = request.app.state.job_service
+
+    try:
+        result = await job_service.cancel_job(job_id)
+        return result
+    except KeyError as e:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=str(e),
+        ) from e
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=str(e),
+        ) from e

agent_brain_server/config/provider_config.py

@@ -32,6 +32,10 @@ class EmbeddingConfig(BaseModel):
         default="text-embedding-3-large",
         description="Model name for embeddings",
     )
+    api_key: Optional[str] = Field(
+        default=None,
+        description="API key (alternative to api_key_env for local config files)",
+    )
     api_key_env: Optional[str] = Field(
         default="OPENAI_API_KEY",
         description="Environment variable name containing API key",
@@ -58,13 +62,21 @@ class EmbeddingConfig(BaseModel):
         return EmbeddingProviderType(v)
 
     def get_api_key(self) -> Optional[str]:
-        """Resolve API key from environment variable.
+        """Resolve API key from config or environment variable.
+
+        Resolution order:
+        1. api_key field in config (direct value)
+        2. Environment variable specified by api_key_env
 
         Returns:
             API key value or None if not found/not needed
         """
         if self.provider == EmbeddingProviderType.OLLAMA:
             return None  # Ollama doesn't need API key
+        # Check direct api_key first
+        if self.api_key:
+            return self.api_key
+        # Fall back to environment variable
         if self.api_key_env:
             return os.getenv(self.api_key_env)
         return None
@@ -93,6 +105,10 @@ class SummarizationConfig(BaseModel):
         default="claude-haiku-4-5-20251001",
         description="Model name for summarization",
     )
+    api_key: Optional[str] = Field(
+        default=None,
+        description="API key (alternative to api_key_env for local config files)",
+    )
     api_key_env: Optional[str] = Field(
         default="ANTHROPIC_API_KEY",
         description="Environment variable name containing API key",
@@ -119,13 +135,21 @@ class SummarizationConfig(BaseModel):
         return SummarizationProviderType(v)
 
     def get_api_key(self) -> Optional[str]:
-        """Resolve API key from environment variable.
+        """Resolve API key from config or environment variable.
+
+        Resolution order:
+        1. api_key field in config (direct value)
+        2. Environment variable specified by api_key_env
 
         Returns:
             API key value or None if not found/not needed
         """
         if self.provider == SummarizationProviderType.OLLAMA:
             return None  # Ollama doesn't need API key
+        # Check direct api_key first
+        if self.api_key:
+            return self.api_key
+        # Fall back to environment variable
         if self.api_key_env:
             return os.getenv(self.api_key_env)
         return None
@@ -162,40 +186,60 @@ def _find_config_file() -> Optional[Path]:
     """Find the configuration file in standard locations.
 
     Search order:
-    1. DOC_SERVE_CONFIG environment variable
-    2. Current directory config.yaml
-    3. State directory config.yaml (if DOC_SERVE_STATE_DIR set)
-    4. Project root config.yaml
+    1. AGENT_BRAIN_CONFIG environment variable
+    2. State directory config.yaml (if AGENT_BRAIN_STATE_DIR or DOC_SERVE_STATE_DIR set)
+    3. Current directory config.yaml
+    4. Walk up from CWD looking for .claude/agent-brain/config.yaml
+    5. User home ~/.agent-brain/config.yaml
+    6. XDG config ~/.config/agent-brain/config.yaml
 
     Returns:
         Path to config file or None if not found
     """
     # 1. Environment variable override
-    env_config = os.getenv("DOC_SERVE_CONFIG")
+    env_config = os.getenv("AGENT_BRAIN_CONFIG")
     if env_config:
         path = Path(env_config)
         if path.exists():
+            logger.debug(f"Found config via AGENT_BRAIN_CONFIG: {path}")
             return path
-        logger.warning(f"DOC_SERVE_CONFIG points to non-existent file: {env_config}")
-
-    # 2. Current directory
-    cwd_config = Path.cwd() / "config.yaml"
-    if cwd_config.exists():
-        return cwd_config
+        logger.warning(f"AGENT_BRAIN_CONFIG points to non-existent file: {env_config}")
 
-    # 3. State directory
-    state_dir = os.getenv("DOC_SERVE_STATE_DIR")
+    # 2. State directory (check both new and legacy env vars)
+    state_dir = os.getenv("AGENT_BRAIN_STATE_DIR") or os.getenv("DOC_SERVE_STATE_DIR")
     if state_dir:
         state_config = Path(state_dir) / "config.yaml"
         if state_config.exists():
+            logger.debug(f"Found config in state directory: {state_config}")
             return state_config
 
-    # 4. .claude/doc-serve directory (project root pattern)
-    claude_dir = Path.cwd() / ".claude" / "doc-serve"
-    if claude_dir.exists():
-        claude_config = claude_dir / "config.yaml"
+    # 3. Current directory
+    cwd_config = Path.cwd() / "config.yaml"
+    if cwd_config.exists():
+        logger.debug(f"Found config in current directory: {cwd_config}")
+        return cwd_config
+
+    # 4. Walk up from CWD looking for .claude/agent-brain/config.yaml
+    current = Path.cwd()
+    root = Path(current.anchor)
+    while current != root:
+        claude_config = current / ".claude" / "agent-brain" / "config.yaml"
         if claude_config.exists():
+            logger.debug(f"Found config walking up from CWD: {claude_config}")
             return claude_config
+        current = current.parent
+
+    # 5. User home directory ~/.agent-brain/config.yaml
+    home_config = Path.home() / ".agent-brain" / "config.yaml"
+    if home_config.exists():
+        logger.debug(f"Found config in home directory: {home_config}")
+        return home_config
+
+    # 6. XDG config directory ~/.config/agent-brain/config.yaml
+    xdg_config = Path.home() / ".config" / "agent-brain" / "config.yaml"
+    if xdg_config.exists():
+        logger.debug(f"Found config in XDG config directory: {xdg_config}")
+        return xdg_config
 
     return None
 

agent_brain_server/config/settings.py

@@ -31,7 +31,7 @@ class Settings(BaseSettings):
     # Chroma Configuration
     CHROMA_PERSIST_DIR: str = "./chroma_db"
     BM25_INDEX_PATH: str = "./bm25_index"
-    COLLECTION_NAME: str = "doc_serve_collection"
+    COLLECTION_NAME: str = "agent_brain_collection"
 
     # Chunking Configuration
     DEFAULT_CHUNK_SIZE: int = 512
@@ -48,8 +48,8 @@ class Settings(BaseSettings):
     EMBEDDING_BATCH_SIZE: int = 100
 
     # Multi-instance Configuration
-    DOC_SERVE_STATE_DIR: Optional[str] = None  # Override state directory
-    DOC_SERVE_MODE: str = "project"  # "project" or "shared"
+    AGENT_BRAIN_STATE_DIR: Optional[str] = None  # Override state directory
+    AGENT_BRAIN_MODE: str = "project"  # "project" or "shared"
 
     # GraphRAG Configuration (Feature 113)
     ENABLE_GRAPH_INDEX: bool = False  # Master switch for graph indexing
@@ -62,11 +62,17 @@ class Settings(BaseSettings):
     GRAPH_TRAVERSAL_DEPTH: int = 2  # Depth for graph traversal in queries
     GRAPH_RRF_K: int = 60  # Reciprocal Rank Fusion constant for multi-retrieval
 
+    # Job Queue Configuration (Feature 115)
+    AGENT_BRAIN_MAX_QUEUE: int = 100  # Max pending jobs in queue
+    AGENT_BRAIN_JOB_TIMEOUT: int = 7200  # Job timeout in seconds (2 hours)
+    AGENT_BRAIN_MAX_RETRIES: int = 3  # Max retries for failed jobs
+    AGENT_BRAIN_CHECKPOINT_INTERVAL: int = 50  # Progress checkpoint every N files
+
     model_config = SettingsConfigDict(
         env_file=[
             ".env",  # Current directory
             Path(__file__).parent.parent.parent / ".env",  # Project root
-            Path(__file__).parent.parent / ".env",  # doc-serve-server directory
+            Path(__file__).parent.parent / ".env",  # agent-brain-server directory
         ],
         env_file_encoding="utf-8",
         case_sensitive=True,

agent_brain_server/indexing/bm25_index.py

@@ -89,10 +89,23 @@ class BM25IndexManager:
         if not self._retriever:
             raise RuntimeError("BM25 index not initialized")
 
-        # BM25Retriever similarity_top_k is usually set during initialization.
-        self._retriever.similarity_top_k = top_k
+        # Cap top_k to corpus size to avoid bm25s "k larger than available scores" error
+        corpus_size = len(self._retriever.corpus) if self._retriever.corpus else 0
+        if corpus_size > 0:
+            effective_top_k = min(top_k, corpus_size)
+        else:
+            effective_top_k = top_k
+
+        self._retriever.similarity_top_k = effective_top_k
         return self._retriever
 
+    @property
+    def corpus_size(self) -> int:
+        """Get the number of documents in the BM25 index."""
+        if not self._retriever or not self._retriever.corpus:
+            return 0
+        return len(self._retriever.corpus)
+
     async def search_with_filters(
         self,
         query: str,

agent_brain_server/indexing/document_loader.py

@@ -1,5 +1,6 @@
 """Document loading from various file formats using LlamaIndex."""
 
+import asyncio
 import logging
 import re
 from dataclasses import dataclass, field
@@ -272,9 +273,30 @@ class DocumentLoader:
 
     SUPPORTED_EXTENSIONS: set[str] = DOCUMENT_EXTENSIONS | CODE_EXTENSIONS
 
+    # Default directories to exclude from indexing
+    DEFAULT_EXCLUDE_PATTERNS: list[str] = [
+        "**/node_modules/**",
+        "**/__pycache__/**",
+        "**/.venv/**",
+        "**/venv/**",
+        "**/.git/**",
+        "**/dist/**",
+        "**/build/**",
+        "**/target/**",
+        "**/.next/**",
+        "**/.nuxt/**",
+        "**/coverage/**",
+        "**/.pytest_cache/**",
+        "**/.mypy_cache/**",
+        "**/.tox/**",
+        "**/egg-info/**",
+        "**/*.egg-info/**",
+    ]
+
     def __init__(
         self,
         supported_extensions: Optional[set[str]] = None,
+        exclude_patterns: Optional[list[str]] = None,
     ):
         """
         Initialize the document loader.
@@ -282,8 +304,15 @@ class DocumentLoader:
         Args:
             supported_extensions: Set of file extensions to load.
                                   Defaults to SUPPORTED_EXTENSIONS.
+            exclude_patterns: List of glob patterns to exclude.
+                              Defaults to DEFAULT_EXCLUDE_PATTERNS.
         """
         self.extensions = supported_extensions or self.SUPPORTED_EXTENSIONS
+        self.exclude_patterns = (
+            exclude_patterns
+            if exclude_patterns is not None
+            else self.DEFAULT_EXCLUDE_PATTERNS
+        )
 
     async def load_from_folder(
         self,
@@ -313,16 +342,24 @@ class DocumentLoader:
             raise ValueError(f"Path is not a directory: {folder_path}")
 
         logger.info(f"Loading documents from: {folder_path} (recursive={recursive})")
+        if self.exclude_patterns:
+            logger.info(
+                f"Excluding patterns: {self.exclude_patterns[:3]}... "
+                f"({len(self.exclude_patterns)} total)"
+            )
 
         # Use LlamaIndex's SimpleDirectoryReader
+        # Run in thread pool to avoid blocking the event loop
         try:
             reader = SimpleDirectoryReader(
                 input_dir=str(path),
                 recursive=recursive,
                 required_exts=list(self.extensions),
+                exclude=self.exclude_patterns,
                 filename_as_id=True,
             )
-            llama_documents: list[Document] = reader.load_data()
+            # reader.load_data() is blocking I/O - run in thread pool
+            llama_documents: list[Document] = await asyncio.to_thread(reader.load_data)
         except Exception as e:
             logger.error(f"Failed to load documents: {e}")
             raise
@@ -398,7 +435,8 @@ class DocumentLoader:
             input_files=[str(path)],
             filename_as_id=True,
         )
-        docs = reader.load_data()
+        # Run in thread pool to avoid blocking the event loop
+        docs = await asyncio.to_thread(reader.load_data)
 
         if not docs:
             raise ValueError(f"No content loaded from file: {file_path}")
@@ -456,8 +494,11 @@ class DocumentLoader:
             # Use only document extensions
             effective_extensions = self.DOCUMENT_EXTENSIONS
 
-        # Create a temporary loader with the effective extensions
-        temp_loader = DocumentLoader(supported_extensions=effective_extensions)
+        # Create a temporary loader with the effective extensions and exclude patterns
+        temp_loader = DocumentLoader(
+            supported_extensions=effective_extensions,
+            exclude_patterns=self.exclude_patterns,
+        )
 
         # Load files using the configured extensions
         loaded_docs = await temp_loader.load_from_folder(folder_path, recursive)