evalvault 1.63.0__py3-none-any.whl → 1.64.0__py3-none-any.whl

evalvault/adapters/inbound/api/main.py

@@ -2,14 +2,59 @@
 
 from __future__ import annotations
 
+import hashlib
+import logging
+import time
+from collections import defaultdict, deque
 from contextlib import asynccontextmanager
 from typing import Annotated
 
-from fastapi import Depends, FastAPI, Request
+from fastapi import Depends, FastAPI, HTTPException, Request, Security
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from starlette.responses import JSONResponse
 
 from evalvault.adapters.inbound.api.adapter import WebUIAdapter, create_adapter
-from evalvault.config.settings import get_settings
+from evalvault.config.settings import Settings, get_settings, is_production_profile
+
+logger = logging.getLogger(__name__)
+
+
+class RateLimiter:
+    def __init__(self) -> None:
+        self._requests: dict[str, deque[float]] = defaultdict(deque)
+        self._blocked_counts: dict[str, int] = defaultdict(int)
+
+    def check(self, key: str, limit: int, window_seconds: int) -> tuple[bool, int | None, int]:
+        now = time.monotonic()
+        window = max(window_seconds, 1)
+        queue = self._requests[key]
+        while queue and now - queue[0] >= window:
+            queue.popleft()
+        if len(queue) >= limit:
+            self._blocked_counts[key] += 1
+            retry_after = int(window - (now - queue[0])) if queue else window
+            return False, max(retry_after, 1), self._blocked_counts[key]
+        queue.append(now)
+        return True, None, self._blocked_counts[key]
+
+
+rate_limiter = RateLimiter()
+
+
+def _hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode("utf-8")).hexdigest()[:8]
+
+
+def _rate_limit_key(request: Request) -> str:
+    auth_header = request.headers.get("Authorization", "")
+    if auth_header.lower().startswith("bearer "):
+        token = auth_header[7:].strip()
+        if token:
+            return f"token:{_hash_token(token)}"
+    client = request.client
+    host = client.host if client else "unknown"
+    return f"ip:{host}"
 
 
 @asynccontextmanager
@@ -23,6 +68,31 @@ async def lifespan(app: FastAPI):
     pass
 
 
+auth_scheme = HTTPBearer(auto_error=False)
+
+
+def _normalize_api_tokens(raw_tokens: str | None) -> set[str]:
+    if not raw_tokens:
+        return set()
+    return {token.strip() for token in raw_tokens.split(",") if token.strip()}
+
+
+def require_api_token(
+    credentials: Annotated[HTTPAuthorizationCredentials | None, Security(auth_scheme)],
+    settings: Settings = Depends(get_settings),
+) -> str | None:
+    tokens = _normalize_api_tokens(settings.api_auth_tokens)
+    if not tokens:
+        return None
+    if credentials is None or credentials.credentials not in tokens:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid or missing API token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    return credentials.credentials
+
+
 def create_app() -> FastAPI:
     """Create and configure the FastAPI application."""
     app = FastAPI(
@@ -32,10 +102,46 @@ def create_app() -> FastAPI:
         lifespan=lifespan,
     )
 
+    @app.middleware("http")
+    async def rate_limit_middleware(request: Request, call_next):
+        settings = get_settings()
+        if not settings.rate_limit_enabled:
+            return await call_next(request)
+        if not request.url.path.startswith("/api/"):
+            return await call_next(request)
+        limit = max(settings.rate_limit_requests, 1)
+        window_seconds = max(settings.rate_limit_window_seconds, 1)
+        key = _rate_limit_key(request)
+        allowed, retry_after, blocked_count = rate_limiter.check(
+            key,
+            limit,
+            window_seconds,
+        )
+        if not allowed:
+            if blocked_count >= settings.rate_limit_block_threshold:
+                logger.warning(
+                    "Rate limit blocked request",
+                    extra={
+                        "rate_limit_key": key,
+                        "blocked_count": blocked_count,
+                    },
+                )
+            headers = {"Retry-After": str(retry_after)} if retry_after else None
+            return JSONResponse(
+                status_code=429,
+                content={"detail": "Rate limit exceeded"},
+                headers=headers,
+            )
+        return await call_next(request)
+
     settings = get_settings()
     cors_origins = [
         origin.strip() for origin in (settings.cors_origins or "").split(",") if origin.strip()
-    ] or ["http://localhost:5173"]
+    ]
+    if not cors_origins:
+        if is_production_profile(settings.evalvault_profile):
+            raise RuntimeError("CORS_ORIGINS must be set for production profile.")
+        cors_origins = ["http://localhost:5173"]
 
     # Configure CORS
     app.add_middleware(
@@ -48,12 +154,44 @@ def create_app() -> FastAPI:
 
     from .routers import benchmark, config, domain, knowledge, pipeline, runs
 
-    app.include_router(runs.router, prefix="/api/v1/runs", tags=["runs"])
-    app.include_router(benchmark.router, prefix="/api/v1/benchmarks", tags=["benchmarks"])
-    app.include_router(knowledge.router, prefix="/api/v1/knowledge", tags=["knowledge"])
-    app.include_router(pipeline.router, prefix="/api/v1/pipeline", tags=["pipeline"])
-    app.include_router(domain.router, prefix="/api/v1/domain", tags=["domain"])
-    app.include_router(config.router, prefix="/api/v1/config", tags=["config"])
+    auth_dependencies = [Depends(require_api_token)]
+
+    app.include_router(
+        runs.router,
+        prefix="/api/v1/runs",
+        tags=["runs"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        benchmark.router,
+        prefix="/api/v1/benchmarks",
+        tags=["benchmarks"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        knowledge.router,
+        prefix="/api/v1/knowledge",
+        tags=["knowledge"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        pipeline.router,
+        prefix="/api/v1/pipeline",
+        tags=["pipeline"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        domain.router,
+        prefix="/api/v1/domain",
+        tags=["domain"],
+        dependencies=auth_dependencies,
+    )
+    app.include_router(
+        config.router,
+        prefix="/api/v1/config",
+        tags=["config"],
+        dependencies=auth_dependencies,
+    )
 
     @app.get("/health")
     def health_check():

evalvault/adapters/inbound/api/routers/config.py

@@ -28,6 +28,9 @@ def get_config():
             "phoenix_api_token",
             "postgres_password",
             "postgres_connection_string",
+            "api_auth_tokens",
+            "knowledge_read_tokens",
+            "knowledge_write_tokens",
         }
     )
 
@@ -80,7 +83,6 @@ def update_config(
     payload: ConfigUpdateRequest,
     adapter: AdapterDep,
 ):
-    """Update runtime configuration (non-secret fields only)."""
     updates = payload.model_dump(exclude_unset=True)
     if not updates:
         return get_config()
@@ -96,6 +98,9 @@ def update_config(
             "phoenix_api_token",
             "postgres_password",
             "postgres_connection_string",
+            "api_auth_tokens",
+            "knowledge_read_tokens",
+            "knowledge_write_tokens",
         }
     )
 

evalvault/adapters/inbound/api/routers/knowledge.py

@@ -2,10 +2,11 @@ import shutil
 from pathlib import Path
 from typing import Any
 
-from fastapi import APIRouter, BackgroundTasks, File, HTTPException, UploadFile
+from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Request, UploadFile
 from pydantic import BaseModel
 
 from evalvault.adapters.outbound.kg.parallel_kg_builder import ParallelKGBuilder
+from evalvault.config.settings import Settings, get_settings
 
 router = APIRouter(tags=["knowledge"])
 
@@ -18,6 +19,47 @@ KG_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
 KG_JOBS: dict[str, dict[str, Any]] = {}
 
 
+def _normalize_tokens(raw_tokens: str | None) -> set[str]:
+    if not raw_tokens:
+        return set()
+    return {token.strip() for token in raw_tokens.split(",") if token.strip()}
+
+
+def _extract_bearer_token(request: Request) -> str | None:
+    auth_header = request.headers.get("Authorization", "")
+    if not auth_header:
+        return None
+    prefix = "bearer "
+    if auth_header.lower().startswith(prefix):
+        return auth_header[len(prefix) :].strip()
+    return None
+
+
+def _require_knowledge_read_token(
+    request: Request,
+    settings: Settings = Depends(get_settings),
+) -> None:
+    read_tokens = _normalize_tokens(settings.knowledge_read_tokens)
+    write_tokens = _normalize_tokens(settings.knowledge_write_tokens)
+    if not read_tokens and not write_tokens:
+        return
+    token = _extract_bearer_token(request)
+    if token is None or token not in (read_tokens | write_tokens):
+        raise HTTPException(status_code=403, detail="Invalid or missing knowledge read token")
+
+
+def _require_knowledge_write_token(
+    request: Request,
+    settings: Settings = Depends(get_settings),
+) -> None:
+    write_tokens = _normalize_tokens(settings.knowledge_write_tokens)
+    if not write_tokens:
+        return
+    token = _extract_bearer_token(request)
+    if token is None or token not in write_tokens:
+        raise HTTPException(status_code=403, detail="Invalid or missing knowledge write token")
+
+
 class BuildKGRequest(BaseModel):
     workers: int = 4
     batch_size: int = 32
@@ -26,7 +68,10 @@ class BuildKGRequest(BaseModel):
 
 
 @router.post("/upload")
-async def upload_files(files: list[UploadFile] = File(...)):
+async def upload_files(
+    files: list[UploadFile] = File(...),
+    _: None = Depends(_require_knowledge_write_token),
+):
     """Upload documents for Knowledge Graph building."""
     uploaded = []
     for file in files:
@@ -40,7 +85,9 @@ async def upload_files(files: list[UploadFile] = File(...)):
 
 
 @router.get("/files")
-def list_files():
+def list_files(
+    _: None = Depends(_require_knowledge_read_token),
+):
     """List uploaded files."""
     files = []
     if DATA_DIR.exists():
@@ -49,7 +96,11 @@ def list_files():
 
 
 @router.post("/build", status_code=202)
-async def build_knowledge_graph(request: BuildKGRequest, background_tasks: BackgroundTasks):
+async def build_knowledge_graph(
+    request: BuildKGRequest,
+    background_tasks: BackgroundTasks,
+    _: None = Depends(_require_knowledge_write_token),
+):
     """Trigger background Knowledge Graph construction."""
     job_id = f"kg_build_{len(KG_JOBS) + 1}"
     KG_JOBS[job_id] = {"status": "pending", "progress": "0%", "details": "Queued"}
@@ -121,7 +172,10 @@ async def build_knowledge_graph(request: BuildKGRequest, background_tasks: Backg
 
 
 @router.get("/jobs/{job_id}")
-def get_job_status(job_id: str):
+def get_job_status(
+    job_id: str,
+    _: None = Depends(_require_knowledge_read_token),
+):
     job = KG_JOBS.get(job_id)
     if not job:
         raise HTTPException(status_code=404, detail="Job not found")
@@ -129,7 +183,9 @@ def get_job_status(job_id: str):
 
 
 @router.get("/stats")
-def get_graph_stats():
+def get_graph_stats(
+    _: None = Depends(_require_knowledge_read_token),
+):
     """Get statistics of the built Knowledge Graph."""
     # Try to load from memory DB or default output JSON
     # For now, we'll try to load the JSON if it exists, or just return empty

evalvault/adapters/inbound/cli/commands/run.py

@@ -213,7 +213,9 @@ def register_run_commands(
             None,
             "--output",
             "-o",
-            help="Output file for results (JSON format).",
+            help=(
+                "Output file for results (JSON format). If .xlsx/.xls, exports Excel via DB save."
+            ),
         ),
         auto_analyze: bool = typer.Option(
             False,
@@ -813,6 +815,27 @@ def register_run_commands(
         if db_path is None:
             db_path = Path(settings.evalvault_db_path)
 
+        excel_output: Path | None = None
+        if output and output.suffix.lower() in {".xlsx", ".xls"}:
+            excel_output = output
+            output = None
+            if db_path is None:
+                print_cli_error(
+                    console,
+                    "엑셀 출력은 DB 저장이 필요합니다.",
+                    fixes=["--db <sqlite_path> 옵션을 함께 지정하세요."],
+                )
+                raise typer.Exit(1)
+            print_cli_warning(
+                console,
+                "엑셀 출력은 DB 저장이 필수이며, 지정한 경로로만 저장됩니다.",
+                tips=[
+                    f"DB 저장 경로: {db_path}",
+                    "기본 DB 엑셀은 생성하지 않습니다.",
+                    "필요 시 --db로 경로를 변경하세요.",
+                ],
+            )
+
         # Override model if specified
         if model:
             if _is_oss_open_model(model) and settings.llm_provider != "vllm":
@@ -1954,8 +1977,23 @@ def register_run_commands(
                 console,
                 storage_cls=SQLiteStorageAdapter,
                 prompt_bundle=prompt_bundle,
+                export_excel=excel_output is None,
             )
             _log_duration(console, verbose, "DB 저장 완료", db_started_at)
+        if excel_output:
+            excel_started_at = datetime.now()
+            _log_timestamp(console, verbose, f"엑셀 저장 시작 ({excel_output})")
+            try:
+                storage = SQLiteStorageAdapter(db_path=db_path)
+                storage.export_run_to_excel(result.run_id, excel_output)
+                console.print(f"[green]Excel export saved: {excel_output}[/green]")
+            except Exception as exc:
+                print_cli_warning(
+                    console,
+                    "엑셀 내보내기에 실패했습니다.",
+                    tips=[str(exc)],
+                )
+            _log_duration(console, verbose, "엑셀 저장 완료", excel_started_at)
         if output:
             output_started_at = datetime.now()
             _log_timestamp(console, verbose, f"결과 저장 시작 ({output})")
@@ -2060,7 +2098,9 @@ def register_run_commands(
             None,
             "--output",
             "-o",
-            help="Output file for results (JSON format).",
+            help=(
+                "Output file for results (JSON format). If .xlsx/.xls, exports Excel via DB save."
+            ),
         ),
         auto_analyze: bool = typer.Option(
             False,
@@ -2344,7 +2384,9 @@ def register_run_commands(
             None,
             "--output",
             "-o",
-            help="Output file for results (JSON format).",
+            help=(
+                "Output file for results (JSON format). If .xlsx/.xls, exports Excel via DB save."
+            ),
         ),
         auto_analyze: bool = typer.Option(
             False,

evalvault/adapters/inbound/cli/commands/run_helpers.py

@@ -430,6 +430,7 @@ def _save_to_db(
     *,
     storage_cls: type[SQLiteStorageAdapter] = SQLiteStorageAdapter,
     prompt_bundle: PromptSetBundle | None = None,
+    export_excel: bool = True,
 ) -> None:
     """Persist evaluation run (and optional prompt set) to SQLite database."""
     with console.status(f"[bold green]Saving to database {db_path}..."):
@@ -443,16 +444,17 @@ def _save_to_db(
                     result.run_id,
                     prompt_bundle.prompt_set.prompt_set_id,
                 )
-            excel_path = db_path.parent / f"evalvault_run_{result.run_id}.xlsx"
-            try:
-                storage.export_run_to_excel(result.run_id, excel_path)
-                console.print(f"[green]Excel export saved: {excel_path}[/green]")
-            except Exception as exc:
-                print_cli_warning(
-                    console,
-                    "엑셀 내보내기에 실패했습니다.",
-                    tips=[str(exc)],
-                )
+            if export_excel:
+                excel_path = db_path.parent / f"evalvault_run_{result.run_id}.xlsx"
+                try:
+                    storage.export_run_to_excel(result.run_id, excel_path)
+                    console.print(f"[green]Excel export saved: {excel_path}[/green]")
+                except Exception as exc:
+                    print_cli_warning(
+                        console,
+                        "엑셀 내보내기에 실패했습니다.",
+                        tips=[str(exc)],
+                    )
             console.print(f"[green]Results saved to database: {db_path}[/green]")
             console.print(f"[dim]Run ID: {result.run_id}[/dim]")
             if prompt_bundle:

evalvault/adapters/outbound/improvement/pattern_detector.py

@@ -11,7 +11,7 @@ import logging
 import re
 from collections.abc import Callable, Sequence
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 from scipy import stats

evalvault/adapters/outbound/improvement/playbook_loader.py

@@ -9,7 +9,7 @@ import logging
 from collections.abc import Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import yaml
 

evalvault/adapters/outbound/methods/external_command.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import os
 import subprocess
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
@@ -18,7 +19,9 @@ class ExternalCommandMethod(RagMethodPort):
 
     name = "external_command"
     version = "0.1.0"
-    description = "Execute a method in a separate process."
+    description = (
+        "Execute a method in a separate process (shell=True requires a trusted command string)."
+    )
     tags = ("external", "isolation")
 
     def __init__(
@@ -67,6 +70,7 @@ class ExternalCommandMethod(RagMethodPort):
         )
 
         command = self._build_command(runtime)
+        self._validate_shell_usage(command)
         result = subprocess.run(  # noqa: S603 - user-controlled command by design
             command,
             cwd=self._workdir,
@@ -104,6 +108,23 @@ class ExternalCommandMethod(RagMethodPort):
         except KeyError as exc:
             raise ValueError(f"Unknown command placeholder: {exc}") from exc
 
+    def _validate_shell_usage(self, command: list[str] | str) -> None:
+        if not self._shell:
+            return
+        if not isinstance(command, str):
+            raise ValueError(
+                "shell=True requires a single command string; list arguments are rejected."
+            )
+        if not command.strip():
+            raise ValueError("shell=True requires a non-empty command string.")
+        if "\n" in command or "\r" in command:
+            raise ValueError("shell=True command must not contain newlines.")
+        warnings.warn(
+            "shell=True executes through the system shell. Use only trusted commands.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
     @staticmethod
     def _load_payload(path: Path) -> Any:
         if not path.exists():

evalvault/adapters/outbound/storage/base_sql.py

@@ -622,7 +622,8 @@ class BaseSQLStorageAdapter(ABC):
         if isinstance(row, dict):
             return dict(row)
         if hasattr(row, "keys"):
-            return {key: row[key] for key in row}
+            keys = row.keys()
+            return {key: row[key] for key in keys}
         try:
             return dict(row)
         except Exception:

evalvault/adapters/outbound/tracker/langfuse_adapter.py

@@ -4,6 +4,13 @@ from typing import Any
 
 from langfuse import Langfuse
 
+from evalvault.adapters.outbound.tracker.log_sanitizer import (
+    MAX_CONTEXT_CHARS,
+    MAX_LOG_CHARS,
+    sanitize_payload,
+    sanitize_text,
+    sanitize_text_list,
+)
 from evalvault.config.phoenix_support import extract_phoenix_links
 from evalvault.domain.entities import EvaluationRun
 from evalvault.ports.outbound.tracker_port import TrackerPort
@@ -88,21 +95,31 @@ class LangfuseAdapter(TrackerPort):
             raise ValueError(f"Trace not found: {trace_id}")
 
         trace_or_span = self._traces[trace_id]
+        safe_input = (
+            sanitize_payload(input_data, max_chars=MAX_LOG_CHARS)
+            if input_data is not None
+            else None
+        )
+        safe_output = (
+            sanitize_payload(output_data, max_chars=MAX_LOG_CHARS)
+            if output_data is not None
+            else None
+        )
         # Support both old and new Langfuse API
         if hasattr(trace_or_span, "start_span"):
             # Langfuse 3.x: create nested span
             child_span = trace_or_span.start_span(
                 name=name,
-                input=input_data,
-                output=output_data,
+                input=safe_input,
+                output=safe_output,
             )
             child_span.end()
         else:
             # Langfuse 2.x: use span method on trace
             trace_or_span.span(
                 name=name,
-                input=input_data,
-                output=output_data,
+                input=safe_input,
+                output=safe_output,
             )
 
     def log_score(
@@ -377,10 +394,13 @@ class LangfuseAdapter(TrackerPort):
             # Span input: test case data (question, answer, contexts, ground_truth)
             span_input = {
                 "test_case_id": result.test_case_id,
-                "question": result.question,
-                "answer": result.answer,
-                "contexts": result.contexts,
-                "ground_truth": result.ground_truth,
+                "question": sanitize_text(result.question, max_chars=MAX_LOG_CHARS),
+                "answer": sanitize_text(result.answer, max_chars=MAX_LOG_CHARS),
+                "contexts": sanitize_text_list(
+                    result.contexts,
+                    max_chars=MAX_CONTEXT_CHARS,
+                ),
+                "ground_truth": sanitize_text(result.ground_truth, max_chars=MAX_LOG_CHARS),
             }
 
             # Span output: evaluation results

evalvault/adapters/outbound/tracker/log_sanitizer.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+MASK_TOKEN = "[REDACTED]"
+MAX_LOG_CHARS = 1000
+MAX_CONTEXT_CHARS = 500
+MAX_LIST_ITEMS = 20
+MAX_PAYLOAD_DEPTH = 2
+
+_EMAIL_PATTERN = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+_PHONE_PATTERN = re.compile(
+    r"\b(?:\+?\d{1,3}[-.\s]?)?(?:\(?\d{2,4}\)?[-.\s]?)?\d{3,4}[-.\s]?\d{4}\b"
+)
+_SSN_PATTERN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
+_CARD_PATTERN = re.compile(r"\b(?:\d[ -]*?){13,16}\b")
+
+
+def _mask_pii(text: str) -> str:
+    text = _EMAIL_PATTERN.sub(MASK_TOKEN, text)
+    text = _PHONE_PATTERN.sub(MASK_TOKEN, text)
+    text = _SSN_PATTERN.sub(MASK_TOKEN, text)
+    text = _CARD_PATTERN.sub(MASK_TOKEN, text)
+    return text
+
+
+def _truncate(text: str, max_chars: int) -> str:
+    if max_chars <= 0:
+        return ""
+    if len(text) <= max_chars:
+        return text
+    if max_chars <= 3:
+        return text[:max_chars]
+    return f"{text[: max_chars - 3]}..."
+
+
+def sanitize_text(value: str | None, *, max_chars: int = MAX_LOG_CHARS) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        value = str(value)
+    return _truncate(_mask_pii(value), max_chars)
+
+
+def sanitize_text_list(
+    values: list[str] | tuple[str, ...] | None,
+    *,
+    max_items: int = MAX_LIST_ITEMS,
+    max_chars: int = MAX_CONTEXT_CHARS,
+) -> list[str]:
+    if not values:
+        return []
+    trimmed = list(values)[:max_items]
+    return [sanitize_text(item, max_chars=max_chars) or "" for item in trimmed]
+
+
+def sanitize_payload(
+    value: Any,
+    *,
+    max_chars: int = MAX_LOG_CHARS,
+    max_items: int = MAX_LIST_ITEMS,
+    max_depth: int = MAX_PAYLOAD_DEPTH,
+) -> Any:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return sanitize_text(value, max_chars=max_chars)
+    if isinstance(value, bool | int | float):
+        return value
+    if max_depth <= 0:
+        return sanitize_text(str(value), max_chars=max_chars)
+    if isinstance(value, dict):
+        return {
+            key: sanitize_payload(
+                item,
+                max_chars=max_chars,
+                max_items=max_items,
+                max_depth=max_depth - 1,
+            )
+            for key, item in list(value.items())[:max_items]
+        }
+    if isinstance(value, list | tuple | set):
+        return [
+            sanitize_payload(
+                item,
+                max_chars=max_chars,
+                max_items=max_items,
+                max_depth=max_depth - 1,
+            )
+            for item in list(value)[:max_items]
+        ]
+    return sanitize_text(str(value), max_chars=max_chars)

evalvault/adapters/outbound/tracker/mlflow_adapter.py

@@ -4,6 +4,7 @@ import json
 import tempfile
 from typing import Any
 
+from evalvault.adapters.outbound.tracker.log_sanitizer import MAX_LOG_CHARS, sanitize_payload
 from evalvault.domain.entities import EvaluationRun
 from evalvault.ports.outbound.tracker_port import TrackerPort
 
@@ -85,8 +86,8 @@ class MLflowAdapter(TrackerPort):
         # Store span data as JSON artifact
         span_data = {
             "name": name,
-            "input": input_data,
-            "output": output_data,
+            "input": sanitize_payload(input_data, max_chars=MAX_LOG_CHARS),
+            "output": sanitize_payload(output_data, max_chars=MAX_LOG_CHARS),
         }
 
         with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:

evalvault/adapters/outbound/tracker/phoenix_adapter.py

@@ -9,6 +9,13 @@ from datetime import datetime
 from typing import TYPE_CHECKING, Any
 
 from evalvault.adapters.outbound.tracer.open_rag_trace_helpers import serialize_json
+from evalvault.adapters.outbound.tracker.log_sanitizer import (
+    MAX_CONTEXT_CHARS,
+    MAX_LOG_CHARS,
+    sanitize_payload,
+    sanitize_text,
+    sanitize_text_list,
+)
 from evalvault.domain.entities import (
     EvaluationRun,
     GenerationData,
@@ -171,9 +178,11 @@ class PhoenixAdapter(TrackerPort):
 
         with self._tracer.start_span(name, context=context) as span:
             if input_data is not None:
-                span.set_attribute("input", json.dumps(input_data, default=str))
+                safe_input = sanitize_payload(input_data, max_chars=MAX_LOG_CHARS)
+                span.set_attribute("input", json.dumps(safe_input, default=str))
             if output_data is not None:
-                span.set_attribute("output", json.dumps(output_data, default=str))
+                safe_output = sanitize_payload(output_data, max_chars=MAX_LOG_CHARS)
+                span.set_attribute("output", json.dumps(safe_output, default=str))
 
     def log_score(
         self,
@@ -368,12 +377,20 @@ class PhoenixAdapter(TrackerPort):
             context=context,
         ) as span:
             # Input data
-            span.set_attribute("input.question", result.question or "")
-            span.set_attribute("input.answer", result.answer or "")
+            safe_question = sanitize_text(result.question, max_chars=MAX_LOG_CHARS) or ""
+            safe_answer = sanitize_text(result.answer, max_chars=MAX_LOG_CHARS) or ""
+            span.set_attribute("input.question", safe_question)
+            span.set_attribute("input.answer", safe_answer)
             if result.contexts:
-                span.set_attribute("input.contexts", json.dumps(result.contexts))
+                safe_contexts = sanitize_text_list(
+                    result.contexts,
+                    max_chars=MAX_CONTEXT_CHARS,
+                )
+                span.set_attribute("input.contexts", json.dumps(safe_contexts))
             if result.ground_truth:
-                span.set_attribute("input.ground_truth", result.ground_truth)
+                safe_ground_truth = sanitize_text(result.ground_truth, max_chars=MAX_LOG_CHARS)
+                if safe_ground_truth:
+                    span.set_attribute("input.ground_truth", safe_ground_truth)
 
             # Metrics
             span.set_attribute("output.all_passed", result.all_passed)
@@ -468,8 +485,10 @@ class PhoenixAdapter(TrackerPort):
 
             # Set query
             if data.query:
-                span.set_attribute("retrieval.query", data.query)
-                span.set_attribute("input.value", data.query)
+                safe_query = sanitize_text(data.query, max_chars=MAX_LOG_CHARS)
+                if safe_query:
+                    span.set_attribute("retrieval.query", safe_query)
+                    span.set_attribute("input.value", safe_query)
 
             span.set_attribute("spec.version", "0.1")
             span.set_attribute("rag.module", "retrieve")
@@ -495,11 +514,14 @@ class PhoenixAdapter(TrackerPort):
                     event_attrs["doc.rerank_rank"] = doc.rerank_rank
                 if doc.chunk_id:
                     event_attrs["doc.chunk_id"] = doc.chunk_id
-                preview = doc.content[:200] if doc.content else ""
-                if preview:
-                    event_attrs["doc.preview"] = preview
+                safe_preview = (
+                    sanitize_text(doc.content, max_chars=MAX_CONTEXT_CHARS) if doc.content else ""
+                )
+                if safe_preview:
+                    event_attrs["doc.preview"] = safe_preview
                 if doc.metadata:
-                    event_attrs["doc.metadata"] = json.dumps(doc.metadata, default=str)
+                    safe_metadata = sanitize_payload(doc.metadata, max_chars=MAX_LOG_CHARS)
+                    event_attrs["doc.metadata"] = json.dumps(safe_metadata, default=str)
                 span.add_event(f"retrieved_doc_{i}", attributes=event_attrs)
 
     def log_generation(
@@ -544,9 +566,8 @@ class PhoenixAdapter(TrackerPort):
                 span.set_attribute(key, value)
 
             # Set prompt/response (truncate if too long)
-            max_len = 10000
-            prompt = data.prompt[:max_len] if data.prompt else ""
-            response = data.response[:max_len] if data.response else ""
+            prompt = sanitize_text(data.prompt, max_chars=MAX_LOG_CHARS) or ""
+            response = sanitize_text(data.response, max_chars=MAX_LOG_CHARS) or ""
             if prompt:
                 span.set_attribute("generation.prompt", prompt)
                 span.set_attribute("input.value", prompt)
@@ -559,24 +580,28 @@ class PhoenixAdapter(TrackerPort):
 
             # Set prompt template if available
             if data.prompt_template:
-                span.set_attribute("generation.prompt_template", data.prompt_template[:max_len])
+                safe_template = sanitize_text(data.prompt_template, max_chars=MAX_LOG_CHARS)
+                if safe_template:
+                    span.set_attribute("generation.prompt_template", safe_template)
 
     def log_rag_trace(self, data: RAGTraceData) -> str:
         """Log a full RAG trace (retrieval + generation) to Phoenix."""
 
         self._ensure_initialized()
         metadata = {"event_type": "rag_trace", "total_time_ms": data.total_time_ms}
-        if data.query:
-            metadata["query"] = data.query
+        safe_query = sanitize_text(data.query, max_chars=MAX_LOG_CHARS)
+        if safe_query:
+            metadata["query"] = safe_query
         if data.metadata:
-            metadata.update(data.metadata)
+            safe_metadata = sanitize_payload(data.metadata, max_chars=MAX_LOG_CHARS)
+            metadata.update(safe_metadata)
 
         should_end = False
         trace_id = data.trace_id
         if trace_id and trace_id in self._active_spans:
             span = self._active_spans[trace_id]
         else:
-            trace_name = f"rag-trace-{(data.query or 'run')[:12]}"
+            trace_name = f"rag-trace-{(safe_query or 'run')[:12]}"
             trace_id = self.start_trace(trace_name, metadata=metadata)
             span = self._active_spans[trace_id]
             should_end = True
@@ -589,12 +614,13 @@ class PhoenixAdapter(TrackerPort):
         if data.generation:
             self.log_generation(trace_id, data.generation)
         if data.final_answer:
-            preview = data.final_answer[:1000]
-            span.set_attribute("rag.final_answer", preview)
-            span.set_attribute("output.value", preview)
+            preview = sanitize_text(data.final_answer, max_chars=MAX_LOG_CHARS)
+            if preview:
+                span.set_attribute("rag.final_answer", preview)
+                span.set_attribute("output.value", preview)
 
-        if data.query:
-            span.set_attribute("input.value", data.query)
+        if safe_query:
+            span.set_attribute("input.value", safe_query)
 
         span.set_attribute("spec.version", "0.1")
         span.set_attribute("rag.module", "custom.pipeline")

evalvault/config/secret_manager.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import base64
+import os
+from dataclasses import dataclass
+from typing import Protocol
+
+SECRET_REF_PREFIX = "secret://"
+
+
+class SecretProvider(Protocol):
+    def get_secret(self, name: str) -> str: ...
+
+
+class SecretProviderError(RuntimeError):
+    pass
+
+
+@dataclass
+class EnvSecretProvider:
+    def get_secret(self, name: str) -> str:
+        value = os.environ.get(name)
+        if value is None:
+            raise SecretProviderError(f"Missing secret in environment: {name}")
+        return value
+
+
+@dataclass
+class AwsSecretsManagerProvider:
+    region_name: str | None = None
+
+    def get_secret(self, name: str) -> str:
+        try:
+            import boto3  # type: ignore
+        except ImportError as exc:
+            raise SecretProviderError("boto3 is required for AWS Secrets Manager") from exc
+        client = boto3.client("secretsmanager", region_name=self.region_name)
+        response = client.get_secret_value(SecretId=name)
+        if "SecretString" in response and response["SecretString"] is not None:
+            return response["SecretString"]
+        secret_binary = response.get("SecretBinary")
+        if secret_binary is None:
+            raise SecretProviderError("Empty secret value returned from AWS Secrets Manager")
+        return base64.b64decode(secret_binary).decode("utf-8")
+
+
+@dataclass
+class GcpSecretManagerProvider:
+    def get_secret(self, name: str) -> str:
+        try:
+            from google.cloud import secretmanager  # type: ignore
+        except ImportError as exc:
+            raise SecretProviderError(
+                "google-cloud-secret-manager is required for GCP Secret Manager"
+            ) from exc
+        client = secretmanager.SecretManagerServiceClient()
+        response = client.access_secret_version(request={"name": name})
+        return response.payload.data.decode("utf-8")
+
+
+@dataclass
+class VaultSecretProvider:
+    def get_secret(self, name: str) -> str:
+        try:
+            import hvac  # type: ignore
+        except ImportError as exc:
+            raise SecretProviderError("hvac is required for Vault secret access") from exc
+        client = hvac.Client()
+        if not client.is_authenticated():
+            raise SecretProviderError("Vault client authentication failed")
+        response = client.secrets.kv.v2.read_secret_version(path=name)
+        data = response.get("data", {}).get("data", {})
+        if not data:
+            raise SecretProviderError("Vault secret payload is empty")
+        if "value" in data:
+            return str(data["value"])
+        if len(data) == 1:
+            return str(next(iter(data.values())))
+        raise SecretProviderError("Vault secret has multiple keys; specify 'value' key")
+
+
+def is_secret_reference(value: str | None) -> bool:
+    return bool(value) and value.startswith(SECRET_REF_PREFIX)
+
+
+def parse_secret_reference(value: str) -> str:
+    return value.removeprefix(SECRET_REF_PREFIX).strip()
+
+
+def build_secret_provider(provider_name: str | None) -> SecretProvider:
+    provider = (provider_name or "").strip().lower()
+    if not provider:
+        raise SecretProviderError("Secret provider is not configured.")
+    if provider == "env":
+        return EnvSecretProvider()
+    if provider in {"aws", "aws-secrets-manager", "secretsmanager"}:
+        return AwsSecretsManagerProvider(region_name=os.environ.get("AWS_REGION"))
+    if provider in {"gcp", "gcp-secret-manager", "secretmanager"}:
+        return GcpSecretManagerProvider()
+    if provider in {"vault", "hashicorp-vault"}:
+        return VaultSecretProvider()
+    raise SecretProviderError(f"Unknown secret provider: {provider_name}")
+
+
+def resolve_secret_reference(
+    value: str,
+    provider: SecretProvider,
+    cache: dict[str, str] | None = None,
+) -> str:
+    secret_name = parse_secret_reference(value)
+    if not secret_name:
+        raise SecretProviderError("Secret reference must include a name.")
+    if cache is not None and secret_name in cache:
+        return cache[secret_name]
+    secret_value = provider.get_secret(secret_name)
+    if cache is not None:
+        cache[secret_name] = secret_value
+    return secret_value

evalvault/config/settings.py

@@ -3,9 +3,16 @@
 from pathlib import Path
 from typing import Any
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+from evalvault.config.secret_manager import (
+    SecretProviderError,
+    build_secret_provider,
+    is_secret_reference,
+    resolve_secret_reference,
+)
+
 
 def _detect_repo_root(start: Path, max_depth: int = 6) -> Path | None:
     current = start
@@ -38,6 +45,75 @@ def _ensure_http_scheme(url_value: str) -> str:
     return f"http://{value}"
 
 
+def is_production_profile(profile_name: str | None) -> bool:
+    return (profile_name or "").strip().lower() == "prod"
+
+
+def _parse_cors_origins(cors_origins: str | None) -> list[str]:
+    if not cors_origins:
+        return []
+    return [origin.strip() for origin in cors_origins.split(",") if origin.strip()]
+
+
+SECRET_REFERENCE_FIELDS = (
+    "api_auth_tokens",
+    "knowledge_read_tokens",
+    "knowledge_write_tokens",
+    "openai_api_key",
+    "anthropic_api_key",
+    "azure_api_key",
+    "vllm_api_key",
+    "langfuse_public_key",
+    "langfuse_secret_key",
+    "phoenix_api_token",
+    "postgres_password",
+    "postgres_connection_string",
+)
+
+
+def _validate_production_settings(settings: "Settings") -> None:
+    if not is_production_profile(settings.evalvault_profile):
+        return
+
+    missing: list[str] = []
+
+    if not settings.api_auth_tokens:
+        missing.append("API_AUTH_TOKENS")
+
+    if settings.llm_provider == "openai" and not settings.openai_api_key:
+        missing.append("OPENAI_API_KEY")
+
+    if settings.tracker_provider == "langfuse":
+        if not settings.langfuse_public_key:
+            missing.append("LANGFUSE_PUBLIC_KEY")
+        if not settings.langfuse_secret_key:
+            missing.append("LANGFUSE_SECRET_KEY")
+
+    if settings.tracker_provider == "mlflow" and not settings.mlflow_tracking_uri:
+        missing.append("MLFLOW_TRACKING_URI")
+
+    if (
+        settings.postgres_connection_string is None
+        and settings.postgres_host
+        and not settings.postgres_password
+    ):
+        missing.append("POSTGRES_PASSWORD")
+
+    cors_origins = _parse_cors_origins(settings.cors_origins)
+    if not cors_origins:
+        missing.append("CORS_ORIGINS")
+    else:
+        localhost_origins = {"localhost", "127.0.0.1"}
+        for origin in cors_origins:
+            if any(host in origin for host in localhost_origins):
+                raise ValueError("Production profile forbids localhost in CORS_ORIGINS.")
+
+    if missing:
+        raise ValueError(
+            "Missing required settings for prod profile: " + ", ".join(sorted(set(missing)))
+        )
+
+
 class Settings(BaseSettings):
     """Application configuration settings."""
 
@@ -48,6 +124,8 @@ class Settings(BaseSettings):
         extra="ignore",
     )
 
+    _secret_cache: dict[str, str] = PrivateAttr(default_factory=dict)
+
     # Profile Configuration (YAML 기반 모델 프로필)
     evalvault_profile: str | None = Field(
         default=None,
@@ -58,6 +136,45 @@ class Settings(BaseSettings):
         default="http://localhost:5173,http://127.0.0.1:5173",
         description="Comma-separated list of allowed CORS origins.",
     )
+    secret_provider: str | None = Field(
+        default=None,
+        description="Secret provider name for secret:// references (env/aws/gcp/vault).",
+    )
+    secret_cache_enabled: bool = Field(
+        default=True,
+        description="Cache resolved secret references in memory.",
+    )
+    api_auth_tokens: str | None = Field(
+        default=None,
+        description=(
+            "Comma-separated list of API bearer tokens for FastAPI auth. "
+            "Leave empty to disable authentication."
+        ),
+    )
+    knowledge_read_tokens: str | None = Field(
+        default=None,
+        description="Comma-separated read tokens for knowledge endpoints.",
+    )
+    knowledge_write_tokens: str | None = Field(
+        default=None,
+        description="Comma-separated write tokens for knowledge endpoints.",
+    )
+    rate_limit_enabled: bool = Field(
+        default=False,
+        description="Enable API rate limiting for /api routes.",
+    )
+    rate_limit_requests: int = Field(
+        default=120,
+        description="Max requests allowed within rate_limit_window_seconds.",
+    )
+    rate_limit_window_seconds: int = Field(
+        default=60,
+        description="Window size for rate limit checks in seconds.",
+    )
+    rate_limit_block_threshold: int = Field(
+        default=10,
+        description="Log suspicious activity after this many rate limit blocks.",
+    )
     evalvault_db_path: str = Field(
         default="data/db/evalvault.db",
         description="SQLite database path for API/CLI storage.",
@@ -71,6 +188,26 @@ class Settings(BaseSettings):
         self.evalvault_db_path = _resolve_storage_path(self.evalvault_db_path)
         self.evalvault_memory_db_path = _resolve_storage_path(self.evalvault_memory_db_path)
         self.ollama_base_url = _ensure_http_scheme(self.ollama_base_url)
+        self._resolve_secret_references()
+
+    def _resolve_secret_references(self) -> None:
+        secret_values = [
+            value
+            for value in (getattr(self, field, None) for field in SECRET_REFERENCE_FIELDS)
+            if isinstance(value, str)
+        ]
+        if not any(is_secret_reference(value) for value in secret_values):
+            return
+        try:
+            provider = build_secret_provider(self.secret_provider)
+        except SecretProviderError as exc:
+            raise ValueError(str(exc)) from exc
+        cache = self._secret_cache if self.secret_cache_enabled else None
+        for field in SECRET_REFERENCE_FIELDS:
+            value = getattr(self, field, None)
+            if isinstance(value, str) and is_secret_reference(value):
+                resolved = resolve_secret_reference(value, provider, cache)
+                setattr(self, field, resolved)
 
     # LLM Provider Selection
     llm_provider: str = Field(
@@ -314,6 +451,8 @@ def get_settings() -> Settings:
         if _settings.evalvault_profile:
             _settings = apply_profile(_settings, _settings.evalvault_profile)
 
+        _validate_production_settings(_settings)
+
     return _settings
 
 
@@ -346,6 +485,7 @@ def apply_runtime_overrides(overrides: dict[str, object]) -> Settings:
     updated = Settings.model_validate(payload)
     if updated.evalvault_profile:
         updated = apply_profile(updated, updated.evalvault_profile)
+    _validate_production_settings(updated)
     for key, value in updated.model_dump().items():
         setattr(settings, key, value)
 

{evalvault-1.63.0.dist-info → evalvault-1.64.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.63.0
+Version: 1.64.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme
@@ -111,6 +111,10 @@ Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == 'phoenix'
 Requires-Dist: opentelemetry-sdk>=1.20.0; extra == 'phoenix'
 Provides-Extra: postgres
 Requires-Dist: psycopg[binary]>=3.0.0; extra == 'postgres'
+Provides-Extra: secrets
+Requires-Dist: boto3; extra == 'secrets'
+Requires-Dist: google-cloud-secret-manager; extra == 'secrets'
+Requires-Dist: hvac; extra == 'secrets'
 Provides-Extra: timeseries
 Requires-Dist: aeon>=1.3.0; extra == 'timeseries'
 Requires-Dist: numba>=0.55.0; extra == 'timeseries'
@@ -175,6 +179,9 @@ uv run evalvault run --mode simple tests/fixtures/e2e/insurance_qa_korean.json \
   --auto-analyze
 ```
 
+- API 인증을 쓰려면 `.env`에 `API_AUTH_TOKENS`를 설정하세요.
+- `secret://` 참조를 쓰면 `SECRET_PROVIDER`와 `--extra secrets`가 필요합니다.
+- 레이트리밋은 `RATE_LIMIT_ENABLED`로 활성화합니다.
 - 결과는 기본 DB(`data/db/evalvault.db`)에 저장되어 `history`, Web UI, 비교 분석에서 재사용됩니다.
 - `--db`를 생략해도 기본 경로로 저장되며, 모든 데이터가 자동으로 엑셀로 내보내집니다.
 - `--auto-analyze`는 요약 리포트 + 모듈별 아티팩트를 함께 생성합니다.

{evalvault-1.63.0.dist-info → evalvault-1.64.0.dist-info}/RECORD

@@ -6,12 +6,12 @@ evalvault/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 evalvault/adapters/inbound/__init__.py,sha256=SG1svel1PwqetnqVpKFLSv612_WwGwLTbFpYgwk6FMw,166
 evalvault/adapters/inbound/api/__init__.py,sha256=LeVVttCA3tLKoHA2PO4z3y8VkfVcf3Bq8CZSzo91lf4,34
 evalvault/adapters/inbound/api/adapter.py,sha256=tYkJciUUFOK80QcSwzrqkXP1G4qUFItFV7uBYbjBGqU,68473
-evalvault/adapters/inbound/api/main.py,sha256=KdlAxKn0QfGI3UuoTrBDBbUs2xCvP8lnWOY1ce3svcU,2619
+evalvault/adapters/inbound/api/main.py,sha256=lRuyg3aBs5jIk7tq4p4d7jrRkFpV_brZypoOq8s56Rk,6896
 evalvault/adapters/inbound/api/routers/__init__.py,sha256=q07_YF9TnBl68bqcRCvhPU4-zRTyvmPoHVehwO6W7QM,19
 evalvault/adapters/inbound/api/routers/benchmark.py,sha256=yevntbZcNtMvbVODsITUBgR1Ka4pdFQrXBJJ4K4Jyr4,4477
-evalvault/adapters/inbound/api/routers/config.py,sha256=CN-FH2cn0Ive-BD3WacWY6PFfuMtZEHP5_out3fvST4,3957
+evalvault/adapters/inbound/api/routers/config.py,sha256=LygN0fVMr8NFtj5zuQXnVFhoafx56Txa98vpwtPa4Jc,4104
 evalvault/adapters/inbound/api/routers/domain.py,sha256=RsR7GIFMjccDN7vpG1uDyk9n1DnCTH18JDGAX7o4Qqc,3648
-evalvault/adapters/inbound/api/routers/knowledge.py,sha256=7mgyoUM1PepFb4X8_Ntn0vd7ZZYcNbM3_9nyD10g4Aw,5307
+evalvault/adapters/inbound/api/routers/knowledge.py,sha256=yb_e7OEPtwldOAzHTGiWe7jShHw2JdpOFnzGPMceRsg,7109
 evalvault/adapters/inbound/api/routers/pipeline.py,sha256=8UgQzNFHcuqS61s69mOrPee4OMwfxVdvRWHJ2_qYBF0,17175
 evalvault/adapters/inbound/api/routers/runs.py,sha256=rydOvwWk24QIYafu3XYS3oL_VVCE_jHDmjADhA19T1s,40059
 evalvault/adapters/inbound/cli/__init__.py,sha256=a42flC5NK-VfbdbBrE49IrUL5zAyKdXZYJVM6E3NTE0,675
@@ -36,8 +36,8 @@ evalvault/adapters/inbound/cli/commands/method.py,sha256=OWdoofhvsDJchgNKnGGjXfI
 evalvault/adapters/inbound/cli/commands/phoenix.py,sha256=LQi3KTLq1ybjjBuz92oQ6lYyBS3mHrCHk0qe-7bqB4U,15611
 evalvault/adapters/inbound/cli/commands/pipeline.py,sha256=NeqWLzO9kRDuZd0pHAIHglP3F7VzoNOU4JI0QcSZ120,7788
 evalvault/adapters/inbound/cli/commands/prompts.py,sha256=lddde5VbjYaqN_9gHPLNu6DWpg5fE-KqZzjN-XYwvJw,27153
-evalvault/adapters/inbound/cli/commands/run.py,sha256=5rWCh8dTVqRgoiKu2Kd_53PxeIh0GRIkULl3GSpoSiU,117412
-evalvault/adapters/inbound/cli/commands/run_helpers.py,sha256=cc6oZHJSBJM9cxr928zq3sGrIh73u2vD0z2j9IzbPo4,40236
+evalvault/adapters/inbound/cli/commands/run.py,sha256=X19rgXhajhvZNA4c0JMmzmPatTxhZgfapuW07bZL9xA,119265
+evalvault/adapters/inbound/cli/commands/run_helpers.py,sha256=hu2TioocitUZzGR7HUwZ6gOeEJSvt5tGNjwXOlo4Eic,40336
 evalvault/adapters/inbound/cli/commands/stage.py,sha256=oRC9c5CysLX90Iy5Ba1pc_00DaOBS78lcBvzkbdrGRM,17123
 evalvault/adapters/inbound/cli/utils/__init__.py,sha256=QPNKneZS-Z-tTnYYxtgJXgcJWY6puUlRQcKrn7Mlv1M,685
 evalvault/adapters/inbound/cli/utils/analysis_io.py,sha256=RHkKEq4e-PtbtRDlXAJWU80RYHNPw-O5V9_GujdaGfc,13393
@@ -127,8 +127,8 @@ evalvault/adapters/outbound/domain_memory/domain_memory_schema.sql,sha256=APlNhJ
 evalvault/adapters/outbound/domain_memory/sqlite_adapter.py,sha256=RWobnFgvxiItxFAr6niY89sT19O-cnExTbP0I7UAY78,85186
 evalvault/adapters/outbound/improvement/__init__.py,sha256=tXA6vaZOLvqwJpyjGMiC8WrvszMmvUPzJnHjvJhQxSI,1143
 evalvault/adapters/outbound/improvement/insight_generator.py,sha256=U16l0euCZy0_08Zb_i0eijXSjS5t-iq0iMUfttwPqgI,17636
-evalvault/adapters/outbound/improvement/pattern_detector.py,sha256=4Pc5yrsi2warhKdpWxL0Ba9Ms2sCvFeRVWU8jTeALZ8,24608
-evalvault/adapters/outbound/improvement/playbook_loader.py,sha256=zXDpiTpYWtQvVrDeo149YHKIyhF6nUP34j0FVnlBCJo,7471
+evalvault/adapters/outbound/improvement/pattern_detector.py,sha256=uFFjWNy8A4KIihw_ANtL6At73RirwNnFnN4rFsEvcXk,24602
+evalvault/adapters/outbound/improvement/playbook_loader.py,sha256=keheUoJn--cjSbdngEAUlkhrc_dYqdrlW_iZAI2R4Y4,7456
 evalvault/adapters/outbound/improvement/stage_metric_playbook_loader.py,sha256=JdmXQsackWqeWTnULE4gfTK8vAikGR27h-TVc03CGXk,1706
 evalvault/adapters/outbound/kg/__init__.py,sha256=fUCKOV080ZjiEob9s4TmXWf-IDa6GbIFQMLfH6gFCKg,567
 evalvault/adapters/outbound/kg/graph_rag_retriever.py,sha256=_6qd8p_2TpHnppv8LUQQPxUdTPoE0QTQ-rCVnY1ap6c,18658
@@ -148,7 +148,7 @@ evalvault/adapters/outbound/llm/token_aware_chat.py,sha256=yYmynaniNrYxtvXL6ejTE
 evalvault/adapters/outbound/llm/vllm_adapter.py,sha256=OKb3Nda9OLMmHdvLjvkeJcQVeXf-B8TDibmAs7PS7kg,5157
 evalvault/adapters/outbound/methods/__init__.py,sha256=3vyE9w3Ex2oMaO4ZE7Fy6xlHhJ6YQXHQNCvBiW9X2lM,345
 evalvault/adapters/outbound/methods/baseline_oracle.py,sha256=oUsF5sIiPY5vuDtrz0Ki05SnPlnVzn7APERP5v1KpPM,1308
-evalvault/adapters/outbound/methods/external_command.py,sha256=gR2mlgr-SCAO3cS3I7pYgS8hL8JE8Y-0VZIhg7USazY,5287
+evalvault/adapters/outbound/methods/external_command.py,sha256=hsWaqMG0u2JhsS736n0t8sobrGSJMNNp1tUL_M4zgyg,6118
 evalvault/adapters/outbound/methods/registry.py,sha256=Znd35eouoe8k2E0NfDpVlDBSNAAWmyQkqBhAwVWllGI,7635
 evalvault/adapters/outbound/nlp/__init__.py,sha256=9MQMIjEUU03T0ZZtG-Wjz0Bt2-esGEcfv1kT9W6_CBY,40
 evalvault/adapters/outbound/nlp/korean/__init__.py,sha256=3ZVFHDxS6jzXat-WhTvW3hnbGNaeFhhWVVN1TtEOlnE,2267
@@ -167,7 +167,7 @@ evalvault/adapters/outbound/report/dashboard_generator.py,sha256=Dcu18NTK4lS8XNK
 evalvault/adapters/outbound/report/llm_report_generator.py,sha256=HUDA_IPBbl54cyEjTTJzdKTQ6H4IoZi-1VBdVmZf0uI,26593
 evalvault/adapters/outbound/report/markdown_adapter.py,sha256=5PS72h_qe4ZtYs-umhX5TqQL2k5SuDaCUc6rRw9AKRw,16761
 evalvault/adapters/outbound/storage/__init__.py,sha256=n5R6thAPTx1leSwv6od6nBWcLWFa-UYD6cOLzN89T8I,614
-evalvault/adapters/outbound/storage/base_sql.py,sha256=kWYaiUq5D35iMx34cX3_mjhRZoEXfgQR-tSk3UhbvcE,40792
+evalvault/adapters/outbound/storage/base_sql.py,sha256=7jWtmNDBHncLDABf5ewwQJnfhFjySTfpfDJmEbPBD1w,40823
 evalvault/adapters/outbound/storage/benchmark_storage_adapter.py,sha256=Qgf9xSSIkYQRpG4uLzcUdoYO9LTQDQ4tFRkkMYer-WA,9803
 evalvault/adapters/outbound/storage/postgres_adapter.py,sha256=HLaoQ3YJDFwOxeY0S92oPIqb-7EgWSasgt89RM86vr0,47148
 evalvault/adapters/outbound/storage/postgres_schema.sql,sha256=A9MfO0pjf4kjxoRj2KPI0Gg1cbX13I2YE3oieT-PGiI,8906
@@ -180,9 +180,10 @@ evalvault/adapters/outbound/tracer/open_rag_trace_decorators.py,sha256=LFnk-3FSL
 evalvault/adapters/outbound/tracer/open_rag_trace_helpers.py,sha256=D48Mbj-ioDKztjhV9513Q5DiUNiVdO60B_2sWMFEmnI,3520
 evalvault/adapters/outbound/tracer/phoenix_tracer_adapter.py,sha256=inmTAolAVsm0IrszE9VTJoI7HSvGGAnGNZVu_vZRAGg,741
 evalvault/adapters/outbound/tracker/__init__.py,sha256=Suu5BznOK5uTuD5_jS8JMZd8RPfQNlddLxHCBvMTm_4,358
-evalvault/adapters/outbound/tracker/langfuse_adapter.py,sha256=Gejd3fOBwShfjbtjVcZK9sCJKRz6oB3OaN6KukOYN38,17782
-evalvault/adapters/outbound/tracker/mlflow_adapter.py,sha256=Wee1S7OPemPt5SoIdwBHuBdnXmLxNd3lcgQ9NNMKcDQ,7000
-evalvault/adapters/outbound/tracker/phoenix_adapter.py,sha256=TNGU1RqpWwEEw5uQfx7-ClAh4C7wITwu_-X-fyVsCgc,22888
+evalvault/adapters/outbound/tracker/langfuse_adapter.py,sha256=HmuMVUfDYjqNqHZGZMRybhrgca_EmeENuX7DfP-L5Fg,18504
+evalvault/adapters/outbound/tracker/log_sanitizer.py,sha256=ilKTTSzsHslQYc-elnWu0Z3HKNNw1D1iI0_cCvYbo1M,2653
+evalvault/adapters/outbound/tracker/mlflow_adapter.py,sha256=m4xj3XBULFYg27U3twKrldLhbLyLNefezmb2pCpHJrw,7180
+evalvault/adapters/outbound/tracker/phoenix_adapter.py,sha256=sz5TyWC67e3YbQd2y-ogU9_66rilLdf8TbC-7bN_JR0,24316
 evalvault/config/__init__.py,sha256=UCgeDx62M2gOuFvdN29wWwny2fdH4bPY_uUC3-42eDw,1297
 evalvault/config/agent_types.py,sha256=EP2Pv3ZtOzDXIvIa-Hnd1to9JIbMUtGitrlwzZtx0Ys,13418
 evalvault/config/domain_config.py,sha256=rOgNA2T8NWlDzcEFC0shdUCCww0lI1E5fUm5QrKQSZI,9264
@@ -190,7 +191,8 @@ evalvault/config/instrumentation.py,sha256=L8on9HjB6Ji8cSOJ6Pepsopfg9okDNMWF7LKZ
 evalvault/config/langfuse_support.py,sha256=DEzVMfMGGf1V45W_2oUG-NCDfsYI4UUdnYJIgBSrN2o,582
 evalvault/config/model_config.py,sha256=KlzDbGyDLeOGE7ElekFFk5YjjT5u8i6KO2B4EyZkLnI,3542
 evalvault/config/phoenix_support.py,sha256=e6RPWd6Qb7KU6Q8pLaYTpJGWULtvEEU6B0xHWyVyOH0,13604
-evalvault/config/settings.py,sha256=T92GShlYKDaVinwbsbWX2DmNfm91Cvcvh8Te8pNOTsw,12875
+evalvault/config/secret_manager.py,sha256=YjPMuNqeBrAR2BzCJvsBNUExaU4TBSFyZ8kVYZZifqA,4172
+evalvault/config/settings.py,sha256=JKJf8t20sOHYnHoCfTxqupQixNgfmWYJhChiGMNz-W0,17617
 evalvault/config/playbooks/improvement_playbook.yaml,sha256=9F9WVVCydFfz6zUuGYzZ4PKdW1LLtcBKVF36T7xT764,26965
 evalvault/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalvault/domain/entities/__init__.py,sha256=RZi_6oQcq-2-sJcydfKOSr03vFxo-mF7CGHN9Ma4Cdg,3379
@@ -304,8 +306,8 @@ evalvault/reports/__init__.py,sha256=Bb1X4871msAN8I6PM6nKGED3psPwZt88hXZBAOdH06Y
 evalvault/reports/release_notes.py,sha256=pZj0PBFT-4F_Ty-Kv5P69BuoOnmTCn4kznDcORFJd0w,4011
 evalvault/scripts/__init__.py,sha256=NwEeIFQbkX4ml2R_PhtIoNtArDSX_suuoymgG_7Kwso,89
 evalvault/scripts/regression_runner.py,sha256=SxZori5BZ8jVQ057Mf5V5FPgIVDccrV5oRONmnhuk8w,8438
-evalvault-1.63.0.dist-info/METADATA,sha256=Kscv51ExIOOosrBnBXI5S1_3V0S2t2nCZhfyssREdg4,23879
-evalvault-1.63.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evalvault-1.63.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
-evalvault-1.63.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
-evalvault-1.63.0.dist-info/RECORD,,
+evalvault-1.64.0.dist-info/METADATA,sha256=DcFREpjg4tyoNf8FXTK632rgrOsWuFjSGnVBBQ4LeQ4,24276
+evalvault-1.64.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evalvault-1.64.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
+evalvault-1.64.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
+evalvault-1.64.0.dist-info/RECORD,,