agent-brain-rag 2.0.0__tar.gz → 3.0.0__tar.gz

{agent_brain_rag-2.0.0 → agent_brain_rag-3.0.0}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: agent-brain-rag
-Version: 2.0.0
+Version: 3.0.0
 Summary: Agent Brain RAG - Intelligent document indexing and semantic search server that gives AI agents long-term memory
+Home-page: https://github.com/SpillwaveSolutions/agent-brain
 License: MIT
 Keywords: agent-brain,rag,semantic-search,ai-memory,llm-memory,documentation,indexing,llama-index,chromadb,ai-agent,claude-code,agent-memory
 Author: Spillwave Solutions
@@ -13,7 +14,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Documentation
 Classifier: Topic :: Text Processing :: Indexing
 Provides-Extra: graphrag
@@ -42,7 +42,6 @@ Requires-Dist: tiktoken (>=0.8.0,<0.9.0)
 Requires-Dist: tree-sitter-language-pack (>=0.7.3,<0.8.0)
 Requires-Dist: uvicorn[standard] (>=0.32.0,<0.33.0)
 Project-URL: Documentation, https://github.com/SpillwaveSolutions/agent-brain/wiki
-Project-URL: Homepage, https://github.com/SpillwaveSolutions/agent-brain
 Project-URL: Repository, https://github.com/SpillwaveSolutions/agent-brain
 Description-Content-Type: text/markdown
 

{agent_brain_rag-2.0.0 → agent_brain_rag-3.0.0}/agent_brain_server/__init__.py

@@ -1,3 +1,3 @@
 """Doc-Serve Server - RAG-based document indexing and query service."""
 
-__version__ = "2.0.0"
+__version__ = "3.0.0"

{agent_brain_rag-2.0.0 → agent_brain_rag-3.0.0}/agent_brain_server/api/main.py

@@ -1,15 +1,16 @@
 """FastAPI application entry point.
 
 This module provides the Agent Brain RAG server, a FastAPI application
-for document indexing and semantic search. The primary entry point is
-`agent-brain-serve`, with `doc-serve` provided for backward compatibility.
+for document indexing and semantic search.
+
+Note: This server assumes a single uvicorn worker process. If running
+multiple workers, ensure only one worker handles indexing jobs by using
+the single-worker model or a separate job processor service.
 """
 
 import logging
 import os
 import socket
-import sys
-import warnings
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 from pathlib import Path
@@ -23,10 +24,12 @@ from fastapi.middleware.cors import CORSMiddleware
 from agent_brain_server import __version__
 from agent_brain_server.config import settings
 from agent_brain_server.config.provider_config import (
+    clear_settings_cache,
     load_provider_settings,
     validate_provider_config,
 )
 from agent_brain_server.indexing.bm25_index import BM25IndexManager
+from agent_brain_server.job_queue import JobQueueService, JobQueueStore, JobWorker
 from agent_brain_server.locking import (
     acquire_lock,
     cleanup_stale,
@@ -39,7 +42,7 @@ from agent_brain_server.services import IndexingService, QueryService
 from agent_brain_server.storage import VectorStoreManager
 from agent_brain_server.storage_paths import resolve_state_dir, resolve_storage_paths
 
-from .routers import health_router, index_router, query_router
+from .routers import health_router, index_router, jobs_router, query_router
 
 # Configure logging
 logging.basicConfig(
@@ -52,6 +55,9 @@ logger = logging.getLogger(__name__)
 _runtime_state: Optional[RuntimeState] = None
 _state_dir: Optional[Path] = None
 
+# Module-level reference to job worker for cleanup
+_job_worker: Optional[JobWorker] = None
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncIterator[None]:
@@ -64,13 +70,16 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     - Resolves project root and state directory
     - Acquires lock (with stale detection)
     - Writes runtime.json with server info
+    - Initializes job queue system
     - Cleans up on shutdown
     """
-    global _runtime_state, _state_dir
+    global _runtime_state, _state_dir, _job_worker
 
     logger.info("Starting Agent Brain RAG server...")
 
     # Load and validate provider configuration
+    # Clear cache first to ensure we pick up env vars set by CLI
+    clear_settings_cache()
     try:
         provider_settings = load_provider_settings()
         validation_errors = validate_provider_config(provider_settings)
@@ -98,8 +107,14 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         os.environ["OPENAI_API_KEY"] = settings.OPENAI_API_KEY
 
     # Determine mode and resolve paths
-    mode = settings.DOC_SERVE_MODE
-    state_dir = _state_dir  # May be set by CLI
+    mode = settings.AGENT_BRAIN_MODE
+    state_dir = _state_dir  # May be set by run() function
+
+    # If not set via run(), check environment variable (set by CLI subprocess)
+    if state_dir is None and settings.AGENT_BRAIN_STATE_DIR:
+        state_dir = Path(settings.AGENT_BRAIN_STATE_DIR).resolve()
+        logger.info(f"Using state directory from environment: {state_dir}")
+
     storage_paths: Optional[dict[str, Path]] = None
 
     if state_dir is not None:
@@ -114,13 +129,19 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         # Acquire exclusive lock
         if not acquire_lock(state_dir):
             raise RuntimeError(
-                f"Another doc-serve instance is already running for {state_dir}"
+                f"Another Agent Brain instance is already running for {state_dir}"
             )
 
         # Resolve storage paths (creates directories)
         storage_paths = resolve_storage_paths(state_dir)
         logger.info(f"State directory: {state_dir}")
 
+    # Determine project root for path validation
+    project_root: Optional[Path] = None
+    if state_dir is not None:
+        # Project root is 3 levels up from .claude/agent-brain
+        project_root = state_dir.parent.parent.parent
+
     try:
         # Determine persistence directories
         chroma_dir = (
@@ -149,10 +170,28 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         app.state.bm25_manager = bm25_manager
         logger.info("BM25 index manager initialized")
 
+        # Load project config for exclude patterns
+        exclude_patterns = None
+        if state_dir:
+            from agent_brain_server.config.settings import load_project_config
+
+            project_config = load_project_config(state_dir)
+            exclude_patterns = project_config.get("exclude_patterns")
+            if exclude_patterns:
+                logger.info(
+                    f"Using exclude patterns from config: {exclude_patterns[:3]}..."
+                )
+
+        # Create document loader with exclude patterns
+        from agent_brain_server.indexing import DocumentLoader
+
+        document_loader = DocumentLoader(exclude_patterns=exclude_patterns)
+
         # Create indexing service with injected deps
         indexing_service = IndexingService(
             vector_store=vector_store,
             bm25_manager=bm25_manager,
+            document_loader=document_loader,
         )
         app.state.indexing_service = indexing_service
 
@@ -163,6 +202,57 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         )
         app.state.query_service = query_service
 
+        # Initialize job queue system (Feature 115)
+        if state_dir is not None:
+            # Initialize job queue store
+            job_store = JobQueueStore(state_dir)
+            await job_store.initialize()
+            logger.info("Job queue store initialized")
+
+            # Initialize job queue service
+            job_service = JobQueueService(
+                store=job_store,
+                project_root=project_root,
+            )
+            app.state.job_service = job_service
+            logger.info("Job queue service initialized")
+
+            # Initialize and start job worker
+            _job_worker = JobWorker(
+                job_store=job_store,
+                indexing_service=indexing_service,
+                max_runtime_seconds=settings.AGENT_BRAIN_JOB_TIMEOUT,
+                progress_checkpoint_interval=settings.AGENT_BRAIN_CHECKPOINT_INTERVAL,
+            )
+            await _job_worker.start()
+            logger.info("Job worker started")
+        else:
+            # No state directory - create minimal job service for backward compat
+            # Jobs will not be persisted in this mode
+            logger.warning(
+                "No state directory configured - job queue persistence disabled"
+            )
+            # Create in-memory store with temp directory
+            import tempfile
+
+            temp_dir = Path(tempfile.mkdtemp(prefix="agent-brain-"))
+            job_store = JobQueueStore(temp_dir)
+            await job_store.initialize()
+
+            job_service = JobQueueService(
+                store=job_store,
+                project_root=project_root,
+            )
+            app.state.job_service = job_service
+
+            _job_worker = JobWorker(
+                job_store=job_store,
+                indexing_service=indexing_service,
+                max_runtime_seconds=settings.AGENT_BRAIN_JOB_TIMEOUT,
+                progress_checkpoint_interval=settings.AGENT_BRAIN_CHECKPOINT_INTERVAL,
+            )
+            await _job_worker.start()
+
         # Set multi-instance metadata on app.state for health endpoint
         app.state.mode = mode
         app.state.instance_id = _runtime_state.instance_id if _runtime_state else None
@@ -180,6 +270,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 
     logger.info("Shutting down Agent Brain RAG server...")
 
+    # Stop job worker gracefully
+    if _job_worker is not None:
+        await _job_worker.stop()
+        logger.info("Job worker stopped")
+        _job_worker = None
+
     # Cleanup for per-project mode
     if state_dir is not None:
         delete_runtime(state_dir)
@@ -194,7 +290,7 @@ app = FastAPI(
         "RAG-based document indexing and semantic search API. "
         "Index documents from folders and query them using natural language."
     ),
-    version="2.0.0",
+    version=__version__,
     lifespan=lifespan,
     docs_url="/docs",
     redoc_url="/redoc",
@@ -213,6 +309,7 @@ app.add_middleware(
 # Include routers
 app.include_router(health_router, prefix="/health", tags=["Health"])
 app.include_router(index_router, prefix="/index", tags=["Indexing"])
+app.include_router(jobs_router, prefix="/index/jobs", tags=["Jobs"])
 app.include_router(query_router, prefix="/query", tags=["Querying"])
 
 
@@ -221,7 +318,7 @@ async def root() -> dict[str, str]:
     """Root endpoint redirects to docs."""
     return {
         "name": "Agent Brain RAG API",
-        "version": "2.0.0",
+        "version": __version__,
         "docs": "/docs",
         "health": "/health",
     }
@@ -271,7 +368,7 @@ def run(
         # Create runtime state
         _runtime_state = RuntimeState(
             mode="project",
-            project_root=str(_state_dir.parent.parent.parent),  # .claude/doc-serve
+            project_root=str(_state_dir.parent.parent.parent),  # .claude/agent-brain
             bind_host=resolved_host,
             port=resolved_port,
             pid=os.getpid(),
@@ -323,7 +420,7 @@ def run(
     "--project-dir",
     "-d",
     default=None,
-    help="Project directory (auto-resolves state-dir to .claude/doc-serve)",
+    help="Project directory (auto-resolves state-dir to .claude/agent-brain)",
 )
 def cli(
     host: Optional[str],
@@ -344,15 +441,15 @@ def cli(
       agent-brain-serve --host 0.0.0.0            # Bind to all interfaces
       agent-brain-serve --reload                  # Enable auto-reload
       agent-brain-serve --project-dir /my/project # Per-project mode
-      agent-brain-serve --state-dir /path/.claude/doc-serve  # Explicit state dir
+      agent-brain-serve --state-dir /path/.claude/agent-brain  # Explicit state dir
 
     \b
     Environment Variables:
-      API_HOST              Server host (default: 127.0.0.1)
-      API_PORT              Server port (default: 8000)
-      DEBUG                 Enable debug mode (default: false)
-      DOC_SERVE_STATE_DIR   Override state directory
-      DOC_SERVE_MODE        Instance mode: 'project' or 'shared'
+      API_HOST                Server host (default: 127.0.0.1)
+      API_PORT                Server port (default: 8000)
+      DEBUG                   Enable debug mode (default: false)
+      AGENT_BRAIN_STATE_DIR   Override state directory
+      AGENT_BRAIN_MODE        Instance mode: 'project' or 'shared'
     """
     # Resolve state directory from options
     resolved_state_dir = state_dir
@@ -361,36 +458,12 @@ def cli(
         # Auto-resolve state-dir from project directory
         project_root = resolve_project_root(Path(project_dir))
         resolved_state_dir = str(resolve_state_dir(project_root))
-    elif settings.DOC_SERVE_STATE_DIR and not state_dir:
+    elif settings.AGENT_BRAIN_STATE_DIR and not state_dir:
         # Use environment variable if set
-        resolved_state_dir = settings.DOC_SERVE_STATE_DIR
+        resolved_state_dir = settings.AGENT_BRAIN_STATE_DIR
 
     run(host=host, port=port, reload=reload, state_dir=resolved_state_dir)
 
 
-def cli_deprecated() -> None:
-    """Deprecated entry point for doc-serve command.
-
-    Shows a deprecation warning and then runs the main CLI.
-    """
-    warnings.warn(
-        "\n"
-        "WARNING: 'doc-serve' is deprecated and will be removed in v2.0.\n"
-        "Please use 'agent-brain-serve' instead.\n"
-        "\n"
-        "Migration guide: docs/MIGRATION.md\n"
-        "Online: https://github.com/SpillwaveSolutions/agent-brain/blob/main/docs/MIGRATION.md\n",
-        DeprecationWarning,
-        stacklevel=1,
-    )
-    # Print to stderr for visibility since warnings may be filtered
-    print(
-        "\033[93mWARNING: 'doc-serve' is deprecated. "
-        "Use 'agent-brain-serve' instead. See docs/MIGRATION.md\033[0m",
-        file=sys.stderr,
-    )
-    cli()
-
-
 if __name__ == "__main__":
     cli()

{agent_brain_rag-2.0.0 → agent_brain_rag-3.0.0}/agent_brain_server/api/routers/__init__.py

@@ -2,10 +2,12 @@
 
 from .health import router as health_router
 from .index import router as index_router
+from .jobs import router as jobs_router
 from .query import router as query_router
 
 __all__ = [
     "health_router",
     "index_router",
+    "jobs_router",
     "query_router",
 ]

agent_brain_rag-3.0.0/agent_brain_server/api/routers/health.py

@@ -0,0 +1,165 @@
+"""Health check endpoints with non-blocking queue status."""
+
+from datetime import datetime, timezone
+from typing import Literal
+
+from fastapi import APIRouter, Request
+
+from agent_brain_server import __version__
+from agent_brain_server.models import HealthStatus, IndexingStatus
+
+router = APIRouter()
+
+
+@router.get(
+    "/",
+    response_model=HealthStatus,
+    summary="Health Check",
+    description="Returns the current server health status.",
+)
+async def health_check(request: Request) -> HealthStatus:
+    """Check server health status.
+
+    This endpoint never blocks and always returns quickly.
+
+    Returns:
+        HealthStatus with current status:
+        - healthy: Server is running and ready for queries
+        - indexing: Server is currently indexing documents
+        - degraded: Server is up but some services are unavailable
+        - unhealthy: Server is not operational
+    """
+    vector_store = request.app.state.vector_store
+    job_service = getattr(request.app.state, "job_service", None)
+
+    # Determine status using queue service (non-blocking)
+    status: Literal["healthy", "indexing", "degraded", "unhealthy"]
+    message: str
+
+    # Check queue status (non-blocking)
+    is_indexing = False
+    current_folder = None
+    if job_service:
+        try:
+            queue_stats = await job_service.get_queue_stats()
+            is_indexing = queue_stats.running > 0
+            if is_indexing and queue_stats.current_job_id:
+                # Get current job details for message
+                current_job = await job_service.get_job(queue_stats.current_job_id)
+                if current_job:
+                    current_folder = current_job.folder_path
+        except Exception:
+            # Non-blocking: don't fail health check if queue service errors
+            pass
+
+    if is_indexing:
+        status = "indexing"
+        message = f"Indexing in progress: {current_folder or 'unknown'}"
+    elif not vector_store.is_initialized:
+        status = "degraded"
+        message = "Vector store not initialized"
+    else:
+        status = "healthy"
+        message = "Server is running and ready for queries"
+
+    # Multi-instance metadata
+    mode = getattr(request.app.state, "mode", "project")
+    instance_id = getattr(request.app.state, "instance_id", None)
+    project_id = getattr(request.app.state, "project_id", None)
+    active_projects = getattr(request.app.state, "active_projects", None)
+
+    return HealthStatus(
+        status=status,
+        message=message,
+        timestamp=datetime.now(timezone.utc),
+        version=__version__,
+        mode=mode,
+        instance_id=instance_id,
+        project_id=project_id,
+        active_projects=active_projects,
+    )
+
+
+@router.get(
+    "/status",
+    response_model=IndexingStatus,
+    summary="Indexing Status",
+    description="Returns detailed indexing status information. Never blocks.",
+)
+async def indexing_status(request: Request) -> IndexingStatus:
+    """Get detailed indexing status.
+
+    This endpoint never blocks and always returns quickly, even during indexing.
+
+    Returns:
+        IndexingStatus with:
+        - total_documents: Number of documents indexed
+        - total_chunks: Number of chunks in vector store
+        - indexing_in_progress: Boolean indicating active indexing
+        - queue_pending: Number of pending jobs
+        - queue_running: Number of running jobs (0 or 1)
+        - current_job_running_time_ms: How long current job has been running
+        - last_indexed_at: Timestamp of last indexing operation
+        - indexed_folders: List of folders that have been indexed
+    """
+    indexing_service = request.app.state.indexing_service
+    vector_store = request.app.state.vector_store
+    job_service = getattr(request.app.state, "job_service", None)
+
+    # Get vector store count (non-blocking read)
+    try:
+        total_chunks = (
+            await vector_store.get_count() if vector_store.is_initialized else 0
+        )
+    except Exception:
+        total_chunks = 0
+
+    # Get queue status (non-blocking)
+    queue_pending = 0
+    queue_running = 0
+    current_job_id = None
+    current_job_running_time_ms = None
+    progress_percent = 0.0
+
+    if job_service:
+        try:
+            queue_stats = await job_service.get_queue_stats()
+            queue_pending = queue_stats.pending
+            queue_running = queue_stats.running
+            current_job_id = queue_stats.current_job_id
+            current_job_running_time_ms = queue_stats.current_job_running_time_ms
+
+            # Get progress from current job
+            if current_job_id:
+                current_job = await job_service.get_job(current_job_id)
+                if current_job and current_job.progress:
+                    progress_percent = current_job.progress.percent_complete
+        except Exception:
+            # Non-blocking: don't fail status if queue service errors
+            pass
+
+    # Get indexing service status for historical data
+    # This is read-only and non-blocking
+    service_status = await indexing_service.get_status()
+
+    return IndexingStatus(
+        total_documents=service_status.get("total_documents", 0),
+        total_chunks=total_chunks,
+        total_doc_chunks=service_status.get("total_doc_chunks", 0),
+        total_code_chunks=service_status.get("total_code_chunks", 0),
+        indexing_in_progress=queue_running > 0,
+        current_job_id=current_job_id,
+        progress_percent=progress_percent,
+        last_indexed_at=(
+            datetime.fromisoformat(service_status["completed_at"])
+            if service_status.get("completed_at")
+            else None
+        ),
+        indexed_folders=service_status.get("indexed_folders", []),
+        supported_languages=service_status.get("supported_languages", []),
+        graph_index=service_status.get("graph_index"),
+        # Queue status (Feature 115)
+        queue_pending=queue_pending,
+        queue_running=queue_running,
+        current_job_running_time_ms=current_job_running_time_ms,
+    )