evalvault 1.59.0__py3-none-any.whl → 1.61.0__py3-none-any.whl

evalvault/adapters/inbound/__init__.py

@@ -1,5 +1,6 @@
 """Inbound adapters."""
 
 from evalvault.adapters.inbound.cli import app
+from evalvault.adapters.inbound.mcp import tools as mcp_tools
 
-__all__ = ["app"]
+__all__ = ["app", "mcp_tools"]

evalvault/adapters/inbound/api/adapter.py

@@ -15,6 +15,13 @@ from urllib.request import urlopen
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
 from evalvault.config.settings import Settings
 from evalvault.domain.entities.prompt import PromptSetBundle
+from evalvault.domain.metrics.registry import (
+    get_metric_descriptions as registry_metric_descriptions,
+)
+from evalvault.domain.metrics.registry import (
+    list_metric_names,
+    list_metric_specs,
+)
 from evalvault.domain.services.cluster_map_builder import build_cluster_map
 from evalvault.domain.services.prompt_registry import (
     PromptInput,
@@ -42,21 +49,6 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-# 지원하는 메트릭 목록
-AVAILABLE_METRICS = [
-    "faithfulness",
-    "answer_relevancy",
-    "context_precision",
-    "context_recall",
-    "factual_correctness",
-    "semantic_similarity",
-    "summary_score",
-    "summary_faithfulness",
-    "insurance_term_accuracy",
-    "entity_preservation",
-    "contextual_relevancy",
-]
-
 
 @dataclass
 class GateResult:
@@ -978,19 +970,15 @@ class WebUIAdapter:
 
     def get_available_metrics(self) -> list[str]:
         """사용 가능한 메트릭 목록 반환."""
-        return AVAILABLE_METRICS.copy()
+        return list_metric_names()
+
+    def get_metric_specs(self) -> list[dict[str, object]]:
+        """메트릭 스펙 목록 반환."""
+        return [spec.to_dict() for spec in list_metric_specs()]
 
     def get_metric_descriptions(self) -> dict[str, str]:
         """메트릭별 설명 반환."""
-        return {
-            "faithfulness": "답변이 컨텍스트에 충실한지 평가",
-            "answer_relevancy": "답변이 질문과 관련있는지 평가",
-            "context_precision": "검색된 컨텍스트의 정밀도 평가",
-            "context_recall": "필요한 정보가 검색되었는지 평가",
-            "factual_correctness": "ground_truth 대비 사실적 정확성 평가",
-            "semantic_similarity": "답변과 ground_truth 간 의미적 유사도 평가",
-            "insurance_term_accuracy": "보험 용어 정확성 평가",
-        }
+        return registry_metric_descriptions()
 
     def create_dataset_from_upload(
         self,

evalvault/adapters/inbound/api/routers/pipeline.py

@@ -11,6 +11,7 @@ from evalvault.adapters.outbound.llm import get_llm_adapter
 from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
 from evalvault.config.settings import get_settings
 from evalvault.domain.entities.analysis_pipeline import AnalysisIntent
+from evalvault.domain.metrics.analysis_registry import list_analysis_metric_specs
 from evalvault.domain.services.pipeline_orchestrator import AnalysisPipelineService
 
 router = APIRouter(tags=["pipeline"])
@@ -220,6 +221,15 @@ class PipelineResultResponse(PipelineResultSummary):
     final_output: dict[str, Any] | None = None
 
 
+class AnalysisMetricSpecResponse(BaseModel):
+    key: str
+    label: str
+    description: str
+    signal_group: str
+    module_id: str
+    output_path: list[str]
+
+
 def _serialize_payload(value: Any) -> Any:
     try:
         return jsonable_encoder(value)
@@ -366,6 +376,12 @@ async def list_intents():
         raise HTTPException(status_code=500, detail=str(exc)) from exc
 
 
+@router.get("/options/analysis-metric-specs", response_model=list[AnalysisMetricSpecResponse])
+async def list_analysis_metric_specs_endpoint():
+    """List analysis metric specs for pipeline outputs."""
+    return [spec.to_dict() for spec in list_analysis_metric_specs()]
+
+
 @router.post("/results", response_model=PipelineResultSummary)
 async def save_pipeline_result(payload: PipelineResultPayload):
     """Save a pipeline analysis result for history."""

evalvault/adapters/inbound/api/routers/runs.py

@@ -113,6 +113,16 @@ class ModelItemResponse(BaseModel):
     supports_tools: bool | None = None
 
 
+class MetricSpecResponse(BaseModel):
+    name: str
+    description: str
+    requires_ground_truth: bool
+    requires_embeddings: bool
+    source: str
+    category: str
+    signal_group: str
+
+
 class ClusterMapItemResponse(BaseModel):
     test_case_id: str
     cluster_id: str
@@ -395,6 +405,12 @@ def list_metrics(adapter: AdapterDep):
     return adapter.get_available_metrics()
 
 
+@router.get("/options/metric-specs", response_model=list[MetricSpecResponse])
+def list_metric_specs(adapter: AdapterDep):
+    """Get available metrics with metadata."""
+    return adapter.get_metric_specs()
+
+
 @router.get("/options/cluster-maps", response_model=list[ClusterMapFileResponse])
 def list_cluster_maps():
     """List available cluster map CSV files."""

evalvault/adapters/inbound/cli/app.py

@@ -14,6 +14,8 @@ import typer
 from rich import print as rprint
 from rich.console import Console
 
+from evalvault.domain.metrics.registry import list_metric_names
+
 from .commands import attach_sub_apps, register_all_commands
 
 
@@ -32,19 +34,7 @@ app = typer.Typer(
 )
 console = Console()
 
-AVAILABLE_METRICS: list[str] = [
-    "faithfulness",
-    "answer_relevancy",
-    "context_precision",
-    "context_recall",
-    "factual_correctness",
-    "semantic_similarity",
-    "summary_score",
-    "summary_faithfulness",
-    "insurance_term_accuracy",
-    "entity_preservation",
-    "contextual_relevancy",
-]
+AVAILABLE_METRICS = list_metric_names()
 
 register_all_commands(app, console, available_metrics=AVAILABLE_METRICS)
 attach_sub_apps(app, console)

evalvault/adapters/inbound/cli/commands/config.py

@@ -7,6 +7,7 @@ from rich.console import Console
 from rich.table import Table
 
 from evalvault.config.settings import Settings, apply_profile
+from evalvault.domain.metrics.registry import list_metric_specs
 
 
 def register_config_commands(app: typer.Typer, console: Console) -> None:
@@ -22,46 +23,9 @@ def register_config_commands(app: typer.Typer, console: Console) -> None:
         table.add_column("Description")
         table.add_column("Requires Ground Truth", justify="center")
 
-        table.add_row(
-            "faithfulness",
-            "Measures factual accuracy of the answer based on contexts",
-            "[red]No[/red]",
-        )
-        table.add_row(
-            "answer_relevancy",
-            "Measures how relevant the answer is to the question",
-            "[red]No[/red]",
-        )
-        table.add_row(
-            "context_precision",
-            "Measures ranking quality of retrieved contexts",
-            "[green]Yes[/green]",
-        )
-        table.add_row(
-            "context_recall",
-            "Measures if all relevant info is in retrieved contexts",
-            "[green]Yes[/green]",
-        )
-        table.add_row(
-            "summary_score",
-            "Measures summary coverage and conciseness against contexts",
-            "[red]No[/red]",
-        )
-        table.add_row(
-            "summary_faithfulness",
-            "Measures whether summary statements are grounded in contexts",
-            "[red]No[/red]",
-        )
-        table.add_row(
-            "entity_preservation",
-            "Measures preservation of key insurance entities in summaries",
-            "[red]No[/red]",
-        )
-        table.add_row(
-            "insurance_term_accuracy",
-            "Measures if insurance terms in answer are grounded in contexts",
-            "[red]No[/red]",
-        )
+        for spec in list_metric_specs():
+            needs_gt = "[green]Yes[/green]" if spec.requires_ground_truth else "[red]No[/red]"
+            table.add_row(spec.name, spec.description, needs_gt)
 
         console.print(table)
         console.print("\n[dim]Use --metrics flag with 'run' command to specify metrics.[/dim]")

evalvault/adapters/inbound/cli/commands/pipeline.py

@@ -13,6 +13,7 @@ from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdap
 from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
 from evalvault.config.settings import Settings
 
+from ..utils.analysis_io import serialize_pipeline_result
 from ..utils.options import db_option
 
 
@@ -104,15 +105,10 @@ def register_pipeline_commands(app: typer.Typer, console) -> None:
                     console.print(f"  [red]{node_id}:[/red] {node_result.error}")
 
         if output:
-            data = {
-                "query": query,
-                "intent": result.intent.value if result.intent else None,
-                "is_complete": result.is_complete,
-                "duration_ms": result.total_duration_ms,
-                "final_output": result.final_output,
-            }
+            payload = serialize_pipeline_result(result)
+            payload["query"] = query
             with open(output, "w", encoding="utf-8") as f:
-                json.dump(data, f, ensure_ascii=False, indent=2)
+                json.dump(payload, f, ensure_ascii=False, indent=2)
             console.print(f"\n[green]Results saved to {output}[/green]")
 
         console.print()

evalvault/adapters/inbound/cli/commands/run.py

@@ -886,7 +886,7 @@ def register_run_commands(
                     details=str(exc),
                     fixes=[
                         "Ollama가 실행 중인지 확인하세요: `ollama serve` (또는 데스크톱 앱 실행).",
-                        "필요 모델을 받아두세요: `ollama pull gemma3:1b`, `ollama pull qwen3-embedding:0.6b`.",
+                        "필요 모델을 받아두세요: `ollama pull gpt-oss-safeguard:20b`, `ollama pull qwen3-embedding:0.6b`.",
                         "서버 URL을 바꿨다면 .env의 `OLLAMA_BASE_URL`을 확인하세요.",
                     ],
                 )
@@ -1461,7 +1461,7 @@ def register_run_commands(
                 if provider == "ollama":
                     fixes = [
                         "Ollama 서버가 실행 중인지 확인하세요 (기본: http://localhost:11434).",
-                        "필요 모델을 받아두세요: `ollama pull gemma3:1b` 및 `ollama pull qwen3-embedding:0.6b`.",
+                        "필요 모델을 받아두세요: `ollama pull gpt-oss-safeguard:20b` 및 `ollama pull qwen3-embedding:0.6b`.",
                         "URL을 바꿨다면 .env의 `OLLAMA_BASE_URL`을 확인하세요.",
                     ]
                 elif provider == "openai":

evalvault/adapters/inbound/mcp/__init__.py

@@ -0,0 +1,51 @@
+"""MCP inbound adapter package."""
+
+from .schemas import (
+    AnalyzeCompareRequest,
+    AnalyzeCompareResponse,
+    ComparisonArtifactsPayload,
+    EvaluationArtifactsPayload,
+    GetArtifactsRequest,
+    GetArtifactsResponse,
+    GetRunSummaryRequest,
+    GetRunSummaryResponse,
+    ListRunsRequest,
+    ListRunsResponse,
+    McpError,
+    MetricsDeltaPayload,
+    RunEvaluationRequest,
+    RunEvaluationResponse,
+    RunSummaryPayload,
+)
+from .tools import (
+    analyze_compare,
+    get_artifacts,
+    get_run_summary,
+    get_tool_specs,
+    list_runs,
+    run_evaluation,
+)
+
+__all__ = [
+    "AnalyzeCompareRequest",
+    "AnalyzeCompareResponse",
+    "ComparisonArtifactsPayload",
+    "EvaluationArtifactsPayload",
+    "GetArtifactsRequest",
+    "GetArtifactsResponse",
+    "GetRunSummaryRequest",
+    "GetRunSummaryResponse",
+    "ListRunsRequest",
+    "ListRunsResponse",
+    "McpError",
+    "MetricsDeltaPayload",
+    "RunEvaluationRequest",
+    "RunEvaluationResponse",
+    "RunSummaryPayload",
+    "analyze_compare",
+    "get_artifacts",
+    "get_run_summary",
+    "get_tool_specs",
+    "list_runs",
+    "run_evaluation",
+]

evalvault/adapters/inbound/mcp/schemas.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ErrorStage(str, Enum):
+    preprocess = "preprocess"
+    evaluate = "evaluate"
+    analyze = "analyze"
+    compare = "compare"
+    storage = "storage"
+
+
+class McpError(BaseModel):
+    code: str
+    message: str
+    details: dict[str, Any] | None = None
+    retryable: bool = False
+    stage: ErrorStage | None = None
+
+
+class RunSummaryPayload(BaseModel):
+    run_id: str
+    dataset_name: str
+    model_name: str
+    pass_rate: float
+    total_test_cases: int
+    passed_test_cases: int
+    started_at: str
+    finished_at: str | None = None
+    metrics_evaluated: list[str] = Field(default_factory=list)
+    threshold_profile: str | None = None
+    run_mode: str | None = None
+    evaluation_task: str | None = None
+    project_name: str | None = None
+    avg_metric_scores: dict[str, float] | None = None
+    thresholds: dict[str, float] | None = None
+
+    model_config = ConfigDict(extra="allow")
+
+
+class ListRunsRequest(BaseModel):
+    limit: int = Field(50, ge=1, le=500)
+    dataset_name: str | None = None
+    model_name: str | None = None
+    run_mode: str | None = None
+    project_names: list[str] | None = None
+    db_path: Path | None = None
+
+
+class ListRunsResponse(BaseModel):
+    runs: list[RunSummaryPayload] = Field(default_factory=list)
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class GetRunSummaryRequest(BaseModel):
+    run_id: str
+    db_path: Path | None = None
+
+
+class GetRunSummaryResponse(BaseModel):
+    summary: RunSummaryPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class ArtifactsKind(str, Enum):
+    analysis = "analysis"
+    comparison = "comparison"
+
+
+class GetArtifactsRequest(BaseModel):
+    run_id: str
+    kind: ArtifactsKind = ArtifactsKind.analysis
+    comparison_run_id: str | None = None
+    base_dir: Path | None = None
+
+
+class ArtifactsPayload(BaseModel):
+    kind: Literal["analysis", "comparison"]
+    report_path: str | None = None
+    output_path: str | None = None
+    artifacts_dir: str | None = None
+    artifacts_index_path: str | None = None
+
+
+class GetArtifactsResponse(BaseModel):
+    run_id: str
+    artifacts: ArtifactsPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class RunEvaluationRequest(BaseModel):
+    dataset_path: Path
+    metrics: list[str]
+    profile: str | None = None
+    model_name: str | None = None
+    evaluation_task: str | None = None
+    db_path: Path | None = None
+    thresholds: dict[str, float] | None = None
+    threshold_profile: str | None = None
+    parallel: bool = True
+    batch_size: int = 5
+    auto_analyze: bool = False
+    analysis_output: Path | None = None
+    analysis_report: Path | None = None
+    analysis_dir: Path | None = None
+
+
+class EvaluationArtifactsPayload(BaseModel):
+    analysis_report_path: str | None = None
+    analysis_output_path: str | None = None
+    analysis_artifacts_dir: str | None = None
+    analysis_artifacts_index_path: str | None = None
+
+
+class RunEvaluationResponse(BaseModel):
+    run_id: str
+    metrics: dict[str, float | None] = Field(default_factory=dict)
+    thresholds: dict[str, float] | None = None
+    artifacts: EvaluationArtifactsPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class AnalyzeCompareRequest(BaseModel):
+    run_id_a: str
+    run_id_b: str
+    metrics: list[str] | None = None
+    test_type: Literal["t-test", "mann-whitney"] = "t-test"
+    profile: str | None = None
+    db_path: Path | None = None
+    output: Path | None = None
+    report: Path | None = None
+    output_dir: Path | None = None
+
+
+class MetricsDeltaPayload(BaseModel):
+    avg: dict[str, float] = Field(default_factory=dict)
+    by_metric: dict[str, float] = Field(default_factory=dict)
+    notes: list[str] | None = None
+
+
+class ComparisonArtifactsPayload(BaseModel):
+    json_path: str | None = None
+    report_path: str | None = None
+    artifacts_dir: str | None = None
+    artifacts_index_path: str | None = None
+
+
+class AnalyzeCompareResponse(BaseModel):
+    baseline_run_id: str
+    candidate_run_id: str
+    comparison_report_path: str | None = None
+    metrics_delta: MetricsDeltaPayload = Field(default_factory=MetricsDeltaPayload)
+    artifacts: ComparisonArtifactsPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)