evalvault 1.63.1__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/adapters/inbound/api/main.py

@@ -2,14 +2,59 @@
 
 from __future__ import annotations
 
+import hashlib
+import logging
+import time
+from collections import defaultdict, deque
 from contextlib import asynccontextmanager
 from typing import Annotated
 
-from fastapi import Depends, FastAPI, Request
+from fastapi import Depends, FastAPI, HTTPException, Request, Security
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from starlette.responses import JSONResponse
 
 from evalvault.adapters.inbound.api.adapter import WebUIAdapter, create_adapter
-from evalvault.config.settings import get_settings
+from evalvault.config.settings import Settings, get_settings, is_production_profile
+
+logger = logging.getLogger(__name__)
+
+
+class RateLimiter:
+    def __init__(self) -> None:
+        self._requests: dict[str, deque[float]] = defaultdict(deque)
+        self._blocked_counts: dict[str, int] = defaultdict(int)
+
+    def check(self, key: str, limit: int, window_seconds: int) -> tuple[bool, int | None, int]:
+        now = time.monotonic()
+        window = max(window_seconds, 1)
+        queue = self._requests[key]
+        while queue and now - queue[0] >= window:
+            queue.popleft()
+        if len(queue) >= limit:
+            self._blocked_counts[key] += 1
+            retry_after = int(window - (now - queue[0])) if queue else window
+            return False, max(retry_after, 1), self._blocked_counts[key]
+        queue.append(now)
+        return True, None, self._blocked_counts[key]
+
+
+rate_limiter = RateLimiter()
+
+
+def _hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode("utf-8")).hexdigest()[:8]
+
+
+def _rate_limit_key(request: Request) -> str:
+    auth_header = request.headers.get("Authorization", "")
+    if auth_header.lower().startswith("bearer "):
+        token = auth_header[7:].strip()
+        if token:
+            return f"token:{_hash_token(token)}"
+    client = request.client
+    host = client.host if client else "unknown"
+    return f"ip:{host}"
 
 
 @asynccontextmanager
@@ -23,6 +68,31 @@ async def lifespan(app: FastAPI):
     pass
 
 
+auth_scheme = HTTPBearer(auto_error=False)
+
+
+def _normalize_api_tokens(raw_tokens: str | None) -> set[str]:
+    if not raw_tokens:
+        return set()
+    return {token.strip() for token in raw_tokens.split(",") if token.strip()}
+
+
+def require_api_token(
+    credentials: Annotated[HTTPAuthorizationCredentials | None, Security(auth_scheme)],
+    settings: Settings = Depends(get_settings),
+) -> str | None:
+    tokens = _normalize_api_tokens(settings.api_auth_tokens)
+    if not tokens:
+        return None
+    if credentials is None or credentials.credentials not in tokens:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid or missing API token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    return credentials.credentials
+
+
 def create_app() -> FastAPI:
     """Create and configure the FastAPI application."""
     app = FastAPI(
@@ -32,10 +102,46 @@ def create_app() -> FastAPI:
         lifespan=lifespan,
     )
 
+    @app.middleware("http")
+    async def rate_limit_middleware(request: Request, call_next):
+        settings = get_settings()
+        if not settings.rate_limit_enabled:
+            return await call_next(request)
+        if not request.url.path.startswith("/api/"):
+            return await call_next(request)
+        limit = max(settings.rate_limit_requests, 1)
+        window_seconds = max(settings.rate_limit_window_seconds, 1)
+        key = _rate_limit_key(request)
+        allowed, retry_after, blocked_count = rate_limiter.check(
+            key,
+            limit,
+            window_seconds,
+        )
+        if not allowed:
+            if blocked_count >= settings.rate_limit_block_threshold:
+                logger.warning(
+                    "Rate limit blocked request",
+                    extra={
+                        "rate_limit_key": key,
+                        "blocked_count": blocked_count,
+                    },
+                )
+            headers = {"Retry-After": str(retry_after)} if retry_after else None
+            return JSONResponse(
+                status_code=429,
+                content={"detail": "Rate limit exceeded"},
+                headers=headers,
+            )
+        return await call_next(request)
+
     settings = get_settings()
     cors_origins = [
         origin.strip() for origin in (settings.cors_origins or "").split(",") if origin.strip()
-    ] or ["http://localhost:5173"]
+    ]
+    if not cors_origins:
+        if is_production_profile(settings.evalvault_profile):
+            raise RuntimeError("CORS_ORIGINS must be set for production profile.")
+        cors_origins = ["http://localhost:5173"]
 
     # Configure CORS
     app.add_middleware(
@@ -48,12 +154,44 @@ def create_app() -> FastAPI:
 
     from .routers import benchmark, config, domain, knowledge, pipeline, runs
 
-    app.include_router(runs.router, prefix="/api/v1/runs", tags=["runs"])
-    app.include_router(benchmark.router, prefix="/api/v1/benchmarks", tags=["benchmarks"])
-    app.include_router(knowledge.router, prefix="/api/v1/knowledge", tags=["knowledge"])
-    app.include_router(pipeline.router, prefix="/api/v1/pipeline", tags=["pipeline"])
-    app.include_router(domain.router, prefix="/api/v1/domain", tags=["domain"])
-    app.include_router(config.router, prefix="/api/v1/config", tags=["config"])
+    auth_dependencies = [Depends(require_api_token)]
+
+    app.include_router(
+        runs.router,
+        prefix="/api/v1/runs",
+        tags=["runs"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        benchmark.router,
+        prefix="/api/v1/benchmarks",
+        tags=["benchmarks"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        knowledge.router,
+        prefix="/api/v1/knowledge",
+        tags=["knowledge"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        pipeline.router,
+        prefix="/api/v1/pipeline",
+        tags=["pipeline"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        domain.router,
+        prefix="/api/v1/domain",
+        tags=["domain"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        config.router,
+        prefix="/api/v1/config",
+        tags=["config"],
+        dependencies=auth_dependencies,
+    )
 
     @app.get("/health")
     def health_check():

evalvault/adapters/inbound/api/routers/config.py

@@ -28,6 +28,9 @@ def get_config():
             "phoenix_api_token",
             "postgres_password",
             "postgres_connection_string",
+            "api_auth_tokens",
+            "knowledge_read_tokens",
+            "knowledge_write_tokens",
         }
     )
 
@@ -80,7 +83,6 @@ def update_config(
     payload: ConfigUpdateRequest,
     adapter: AdapterDep,
 ):
-    """Update runtime configuration (non-secret fields only)."""
     updates = payload.model_dump(exclude_unset=True)
     if not updates:
         return get_config()
@@ -96,6 +98,9 @@ def update_config(
             "phoenix_api_token",
             "postgres_password",
             "postgres_connection_string",
+            "api_auth_tokens",
+            "knowledge_read_tokens",
+            "knowledge_write_tokens",
         }
     )
 

evalvault/adapters/inbound/api/routers/knowledge.py

@@ -2,10 +2,11 @@ import shutil
 from pathlib import Path
 from typing import Any
 
-from fastapi import APIRouter, BackgroundTasks, File, HTTPException, UploadFile
+from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Request, UploadFile
 from pydantic import BaseModel
 
 from evalvault.adapters.outbound.kg.parallel_kg_builder import ParallelKGBuilder
+from evalvault.config.settings import Settings, get_settings
 
 router = APIRouter(tags=["knowledge"])
 
@@ -18,6 +19,47 @@ KG_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
 KG_JOBS: dict[str, dict[str, Any]] = {}
 
 
+def _normalize_tokens(raw_tokens: str | None) -> set[str]:
+    if not raw_tokens:
+        return set()
+    return {token.strip() for token in raw_tokens.split(",") if token.strip()}
+
+
+def _extract_bearer_token(request: Request) -> str | None:
+    auth_header = request.headers.get("Authorization", "")
+    if not auth_header:
+        return None
+    prefix = "bearer "
+    if auth_header.lower().startswith(prefix):
+        return auth_header[len(prefix) :].strip()
+    return None
+
+
+def _require_knowledge_read_token(
+    request: Request,
+    settings: Settings = Depends(get_settings),
+) -> None:
+    read_tokens = _normalize_tokens(settings.knowledge_read_tokens)
+    write_tokens = _normalize_tokens(settings.knowledge_write_tokens)
+    if not read_tokens and not write_tokens:
+        return
+    token = _extract_bearer_token(request)
+    if token is None or token not in (read_tokens | write_tokens):
+        raise HTTPException(status_code=403, detail="Invalid or missing knowledge read token")
+
+
+def _require_knowledge_write_token(
+    request: Request,
+    settings: Settings = Depends(get_settings),
+) -> None:
+    write_tokens = _normalize_tokens(settings.knowledge_write_tokens)
+    if not write_tokens:
+        return
+    token = _extract_bearer_token(request)
+    if token is None or token not in write_tokens:
+        raise HTTPException(status_code=403, detail="Invalid or missing knowledge write token")
+
+
 class BuildKGRequest(BaseModel):
     workers: int = 4
     batch_size: int = 32
@@ -26,7 +68,10 @@ class BuildKGRequest(BaseModel):
 
 
 @router.post("/upload")
-async def upload_files(files: list[UploadFile] = File(...)):
+async def upload_files(
+    files: list[UploadFile] = File(...),
+    _: None = Depends(_require_knowledge_write_token),
+):
     """Upload documents for Knowledge Graph building."""
     uploaded = []
     for file in files:
@@ -40,7 +85,9 @@ async def upload_files(files: list[UploadFile] = File(...)):
 
 
 @router.get("/files")
-def list_files():
+def list_files(
+    _: None = Depends(_require_knowledge_read_token),
+):
     """List uploaded files."""
     files = []
     if DATA_DIR.exists():
@@ -49,7 +96,11 @@ def list_files():
 
 
 @router.post("/build", status_code=202)
-async def build_knowledge_graph(request: BuildKGRequest, background_tasks: BackgroundTasks):
+async def build_knowledge_graph(
+    request: BuildKGRequest,
+    background_tasks: BackgroundTasks,
+    _: None = Depends(_require_knowledge_write_token),
+):
     """Trigger background Knowledge Graph construction."""
     job_id = f"kg_build_{len(KG_JOBS) + 1}"
     KG_JOBS[job_id] = {"status": "pending", "progress": "0%", "details": "Queued"}
@@ -121,7 +172,10 @@ async def build_knowledge_graph(request: BuildKGRequest, background_tasks: Backg
 
 
 @router.get("/jobs/{job_id}")
-def get_job_status(job_id: str):
+def get_job_status(
+    job_id: str,
+    _: None = Depends(_require_knowledge_read_token),
+):
     job = KG_JOBS.get(job_id)
     if not job:
         raise HTTPException(status_code=404, detail="Job not found")
@@ -129,7 +183,9 @@ def get_job_status(job_id: str):
 
 
 @router.get("/stats")
-def get_graph_stats():
+def get_graph_stats(
+    _: None = Depends(_require_knowledge_read_token),
+):
     """Get statistics of the built Knowledge Graph."""
     # Try to load from memory DB or default output JSON
     # For now, we'll try to load the JSON if it exists, or just return empty

evalvault/adapters/inbound/cli/commands/__init__.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from collections.abc import Callable
 from dataclasses import dataclass
-from typing import Any, Protocol
+from typing import Any
 
 import typer
 from rich.console import Console
@@ -12,8 +12,11 @@ from rich.console import Console
 from .agent import register_agent_commands
 from .analyze import register_analyze_commands
 from .api import register_api_command
+from .artifacts import create_artifacts_app
 from .benchmark import create_benchmark_app
 from .calibrate import register_calibrate_commands
+from .calibrate_judge import register_calibrate_judge_commands
+from .compare import register_compare_commands
 from .config import register_config_commands
 from .debug import create_debug_app
 from .domain import create_domain_app
@@ -25,19 +28,17 @@ from .init import register_init_command
 from .kg import create_kg_app
 from .langfuse import register_langfuse_commands
 from .method import create_method_app
+from .ops import create_ops_app
 from .phoenix import create_phoenix_app
 from .pipeline import register_pipeline_commands
+from .profile_difficulty import register_profile_difficulty_commands
 from .prompts import create_prompts_app
+from .regress import register_regress_commands
 from .run import register_run_commands
 from .stage import create_stage_app
 
 CommandFactory = Callable[[Console], typer.Typer]
-
-
-class CommandRegistrar(Protocol):
-    """Callable protocol for Typer command registrars."""
-
-    def __call__(self, app: typer.Typer, console: Console, **kwargs: Any) -> None: ...
+CommandRegistrar = Callable[..., Any]
 
 
 @dataclass(frozen=True)
@@ -61,10 +62,14 @@ COMMAND_MODULES: tuple[CommandModule, ...] = (
     CommandModule(register_run_commands, needs_metrics=True),
     CommandModule(register_pipeline_commands),
     CommandModule(register_history_commands),
+    CommandModule(register_compare_commands),
     CommandModule(register_analyze_commands),
     CommandModule(register_calibrate_commands),
+    CommandModule(register_calibrate_judge_commands),
     CommandModule(register_generate_commands),
     CommandModule(register_gate_commands),
+    CommandModule(register_profile_difficulty_commands, needs_metrics=True),
+    CommandModule(register_regress_commands),
     CommandModule(register_agent_commands),
     CommandModule(register_experiment_commands),
     CommandModule(register_config_commands),
@@ -78,9 +83,11 @@ SUB_APPLICATIONS: tuple[SubAppModule, ...] = (
     SubAppModule("domain", create_domain_app),
     SubAppModule("benchmark", create_benchmark_app),
     SubAppModule("method", create_method_app),
+    SubAppModule("ops", create_ops_app),
     SubAppModule("phoenix", create_phoenix_app),
     SubAppModule("prompts", create_prompts_app),
     SubAppModule("stage", create_stage_app),
+    SubAppModule("artifacts", create_artifacts_app),
     SubAppModule("debug", create_debug_app),
 )
 

evalvault/adapters/inbound/cli/commands/artifacts.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from evalvault.adapters.inbound.cli.utils.console import print_cli_error
+from evalvault.adapters.inbound.cli.utils.validators import validate_choice
+from evalvault.adapters.outbound.artifact_fs import LocalArtifactFileSystemAdapter
+from evalvault.domain.services.artifact_lint_service import ArtifactLintService
+
+logger = logging.getLogger(__name__)
+
+
+def create_artifacts_app(console: Console) -> typer.Typer:
+    artifacts_app = typer.Typer(name="artifacts", help="Artifact utilities.")
+
+    @artifacts_app.command("lint")
+    def lint(
+        artifacts_dir: Path = typer.Argument(..., help="Artifacts directory."),
+        strict: bool = typer.Option(False, "--strict", help="Fail on missing files."),
+        output_format: str = typer.Option(
+            "json",
+            "--format",
+            "-f",
+            help="Output format (json).",
+        ),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="Output file path for lint result.",
+        ),
+        parallel: bool = typer.Option(
+            True,
+            "--parallel/--no-parallel",
+            help="Enable parallel validation (placeholder).",
+        ),
+        concurrency: int = typer.Option(
+            8,
+            "--concurrency",
+            min=1,
+            help="Parallel validation concurrency (placeholder).",
+        ),
+    ) -> None:
+        validate_choice(output_format, ["json"], console, value_label="format")
+
+        logger.info("Artifacts lint command started: %s", artifacts_dir)
+        fs_adapter = LocalArtifactFileSystemAdapter()
+        service = ArtifactLintService(fs_adapter)
+        summary = service.lint(artifacts_dir, strict=strict)
+
+        payload = _build_payload(summary, parallel=parallel, concurrency=concurrency)
+        if output:
+            output.parent.mkdir(parents=True, exist_ok=True)
+            output.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+            console.print(f"[green]Lint report saved:[/green] {output}")
+        else:
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+        if summary.status == "error":
+            logger.error("Artifacts lint command failed: %s", artifacts_dir)
+            print_cli_error(console, "Artifact lint failed", details=str(artifacts_dir))
+            raise typer.Exit(1)
+
+        logger.info("Artifacts lint command finished: %s", artifacts_dir)
+
+    return artifacts_app
+
+
+def _build_payload(summary, *, parallel: bool, concurrency: int) -> dict[str, object]:
+    issues = [
+        {
+            "level": issue.level,
+            "code": issue.code,
+            "message": issue.message,
+            "path": issue.path,
+        }
+        for issue in summary.issues
+    ]
+    error_count = sum(1 for issue in summary.issues if issue.level == "error")
+    warning_count = sum(1 for issue in summary.issues if issue.level == "warning")
+    return {
+        "command": "artifacts.lint",
+        "version": 1,
+        "status": summary.status,
+        "started_at": summary.started_at.isoformat(),
+        "finished_at": summary.finished_at.isoformat(),
+        "duration_ms": summary.duration_ms,
+        "artifacts": {
+            "dir": str(summary.artifacts_dir),
+            "index": str(summary.index_path),
+        },
+        "data": {
+            "strict": summary.strict,
+            "parallel": parallel,
+            "concurrency": concurrency,
+            "issue_counts": {
+                "error": error_count,
+                "warning": warning_count,
+            },
+            "issues": issues,
+        },
+    }