evalvault 1.60.0__py3-none-any.whl → 1.62.0__py3-none-any.whl

evalvault/adapters/inbound/__init__.py

@@ -1,5 +1,6 @@
 """Inbound adapters."""
 
 from evalvault.adapters.inbound.cli import app
+from evalvault.adapters.inbound.mcp import tools as mcp_tools
 
-__all__ = ["app"]
+__all__ = ["app", "mcp_tools"]

evalvault/adapters/inbound/api/adapter.py

@@ -14,6 +14,11 @@ from urllib.request import urlopen
 
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
 from evalvault.config.settings import Settings
+from evalvault.domain.entities import (
+    CalibrationResult,
+    FeedbackSummary,
+    SatisfactionFeedback,
+)
 from evalvault.domain.entities.prompt import PromptSetBundle
 from evalvault.domain.metrics.registry import (
     get_metric_descriptions as registry_metric_descriptions,
@@ -29,6 +34,9 @@ from evalvault.domain.services.prompt_registry import (
     build_prompt_summary,
 )
 from evalvault.domain.services.prompt_status import extract_prompt_entries
+from evalvault.domain.services.satisfaction_calibration_service import (
+    SatisfactionCalibrationService,
+)
 from evalvault.domain.services.stage_event_builder import StageEventBuilder
 from evalvault.domain.services.stage_metric_service import StageMetricService
 from evalvault.domain.services.threshold_profiles import apply_threshold_profile
@@ -893,6 +901,27 @@ class WebUIAdapter:
             raise RuntimeError("Storage not configured")
         return self._storage.delete_run_cluster_map(run_id, map_id)
 
+    def save_feedback(self, feedback: SatisfactionFeedback) -> str:
+        if self._storage is None or not hasattr(self._storage, "save_feedback"):
+            raise RuntimeError("Storage not configured")
+        return self._storage.save_feedback(feedback)
+
+    def list_feedback(self, run_id: str) -> list[SatisfactionFeedback]:
+        if self._storage is None or not hasattr(self._storage, "list_feedback"):
+            raise RuntimeError("Storage not configured")
+        return self._storage.list_feedback(run_id)
+
+    def get_feedback_summary(self, run_id: str) -> FeedbackSummary:
+        if self._storage is None or not hasattr(self._storage, "get_feedback_summary"):
+            raise RuntimeError("Storage not configured")
+        return self._storage.get_feedback_summary(run_id)
+
+    def build_calibration(self, run_id: str, *, model: str = "both") -> CalibrationResult:
+        run = self.get_run_details(run_id)
+        feedbacks = self.list_feedback(run_id)
+        service = SatisfactionCalibrationService()
+        return service.build_calibration(run, feedbacks, model=model)
+
     def list_stage_events(self, run_id: str, *, stage_type: str | None = None) -> list[StageEvent]:
         """Stage 이벤트 목록 조회."""
         if self._storage is None or not hasattr(self._storage, "list_stage_events"):

evalvault/adapters/inbound/api/routers/runs.py

@@ -21,7 +21,11 @@ from evalvault.adapters.outbound.dataset.templates import (
 )
 from evalvault.adapters.outbound.domain_memory.sqlite_adapter import SQLiteDomainMemoryAdapter
 from evalvault.config.settings import get_settings
-from evalvault.domain.entities import EvaluationRun
+from evalvault.domain.entities import (
+    CalibrationResult,
+    EvaluationRun,
+    SatisfactionFeedback,
+)
 from evalvault.domain.services.domain_learning_hook import DomainLearningHook
 from evalvault.domain.services.ragas_prompt_overrides import (
     PromptOverrideError,
@@ -178,6 +182,31 @@ class ClusterMapDeleteResponse(BaseModel):
     deleted_count: int
 
 
+class FeedbackSaveRequest(BaseModel):
+    test_case_id: str
+    satisfaction_score: float | None = None
+    thumb_feedback: Literal["up", "down", "none"] | None = None
+    comment: str | None = None
+    rater_id: str | None = None
+
+
+class FeedbackResponse(BaseModel):
+    feedback_id: str
+    run_id: str
+    test_case_id: str
+    satisfaction_score: float | None = None
+    thumb_feedback: str | None = None
+    comment: str | None = None
+    rater_id: str | None = None
+    created_at: str | None = None
+
+
+class FeedbackSummaryResponse(BaseModel):
+    avg_satisfaction_score: float | None = None
+    thumb_up_rate: float | None = None
+    total_feedback: int
+
+
 class VisualSpaceRequest(BaseModel):
     granularity: Literal["run", "case", "cluster"] = "case"
     base_run_id: str | None = None
@@ -188,9 +217,22 @@ class VisualSpaceRequest(BaseModel):
     cluster_map: dict[str, str] | None = None
 
 
-def _serialize_run_details(run: EvaluationRun) -> dict[str, Any]:
+def _serialize_run_details(
+    run: EvaluationRun,
+    *,
+    calibration: CalibrationResult | None = None,
+) -> dict[str, Any]:
+    summary = run.to_summary_dict()
+    if calibration is not None:
+        summary.update(
+            {
+                "avg_satisfaction_score": calibration.summary.avg_satisfaction_score,
+                "thumb_up_rate": calibration.summary.thumb_up_rate,
+                "imputed_ratio": calibration.summary.imputed_ratio,
+            }
+        )
     payload = {
-        "summary": run.to_summary_dict(),
+        "summary": summary,
         "results": [
             {
                 "test_case_id": result.test_case_id,
@@ -207,6 +249,21 @@ def _serialize_run_details(run: EvaluationRun) -> dict[str, Any]:
                     }
                     for metric in result.metrics
                 ],
+                "calibrated_satisfaction": (
+                    calibration.cases[result.test_case_id].calibrated_satisfaction
+                    if calibration and result.test_case_id in calibration.cases
+                    else None
+                ),
+                "imputed": (
+                    calibration.cases[result.test_case_id].imputed
+                    if calibration and result.test_case_id in calibration.cases
+                    else False
+                ),
+                "imputation_source": (
+                    calibration.cases[result.test_case_id].imputation_source
+                    if calibration and result.test_case_id in calibration.cases
+                    else None
+                ),
             }
             for result in run.results
         ],
@@ -719,9 +776,12 @@ def compare_runs(
             }
         )
 
+    base_calibration = adapter.build_calibration(base_id)
+    target_calibration = adapter.build_calibration(target_id)
+
     return {
-        "base": _serialize_run_details(base_run),
-        "target": _serialize_run_details(target_run),
+        "base": _serialize_run_details(base_run, calibration=base_calibration),
+        "target": _serialize_run_details(target_run, calibration=target_calibration),
         "metric_deltas": metric_deltas,
         "case_counts": _build_case_counts(base_run, target_run),
         "pass_rate_delta": target_run.pass_rate - base_run.pass_rate,
@@ -898,7 +958,70 @@ def get_run_details(run_id: str, adapter: AdapterDep) -> dict[str, Any]:
     """Get detailed information for a specific run."""
     try:
         run: EvaluationRun = adapter.get_run_details(run_id)
-        return _serialize_run_details(run)
+        calibration = adapter.build_calibration(run_id)
+        return _serialize_run_details(run, calibration=calibration)
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Run not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/{run_id}/feedback", response_model=FeedbackResponse)
+def save_feedback(
+    run_id: str,
+    request: FeedbackSaveRequest,
+    adapter: AdapterDep,
+) -> dict[str, Any]:
+    try:
+        adapter.get_run_details(run_id)
+        thumb_feedback = request.thumb_feedback
+        if thumb_feedback == "none":
+            thumb_feedback = None
+        satisfaction_score = request.satisfaction_score
+        if satisfaction_score is not None:
+            satisfaction_score = max(1.0, min(5.0, satisfaction_score))
+        feedback = SatisfactionFeedback(
+            feedback_id="",
+            run_id=run_id,
+            test_case_id=request.test_case_id,
+            satisfaction_score=satisfaction_score,
+            thumb_feedback=thumb_feedback,
+            comment=request.comment,
+            rater_id=request.rater_id,
+            created_at=datetime.now(),
+        )
+        feedback_id = adapter.save_feedback(feedback)
+        saved = feedback.to_dict()
+        saved["feedback_id"] = feedback_id
+        return saved
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Run not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/{run_id}/feedback", response_model=list[FeedbackResponse])
+def list_feedback(run_id: str, adapter: AdapterDep) -> list[dict[str, Any]]:
+    try:
+        adapter.get_run_details(run_id)
+        feedbacks = adapter.list_feedback(run_id)
+        return [feedback.to_dict() for feedback in feedbacks]
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Run not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/{run_id}/feedback/summary", response_model=FeedbackSummaryResponse)
+def get_feedback_summary(run_id: str, adapter: AdapterDep) -> dict[str, Any]:
+    try:
+        adapter.get_run_details(run_id)
+        summary = adapter.get_feedback_summary(run_id)
+        return {
+            "avg_satisfaction_score": summary.avg_satisfaction_score,
+            "thumb_up_rate": summary.thumb_up_rate,
+            "total_feedback": summary.total_feedback,
+        }
     except KeyError:
         raise HTTPException(status_code=404, detail="Run not found")
     except Exception as e:

evalvault/adapters/inbound/cli/commands/__init__.py

@@ -13,6 +13,7 @@ from .agent import register_agent_commands
 from .analyze import register_analyze_commands
 from .api import register_api_command
 from .benchmark import create_benchmark_app
+from .calibrate import register_calibrate_commands
 from .config import register_config_commands
 from .debug import create_debug_app
 from .domain import create_domain_app
@@ -61,6 +62,7 @@ COMMAND_MODULES: tuple[CommandModule, ...] = (
     CommandModule(register_pipeline_commands),
     CommandModule(register_history_commands),
     CommandModule(register_analyze_commands),
+    CommandModule(register_calibrate_commands),
     CommandModule(register_generate_commands),
     CommandModule(register_gate_commands),
     CommandModule(register_agent_commands),

evalvault/adapters/inbound/cli/commands/calibrate.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings
+from evalvault.domain.services.satisfaction_calibration_service import (
+    SatisfactionCalibrationService,
+)
+
+from ..utils.options import db_option
+
+_console = Console()
+
+
+def register_calibrate_commands(app: typer.Typer, console: Console) -> None:
+    global _console
+    _console = console
+
+    @app.command()
+    def calibrate(
+        run_id: str = typer.Argument(..., help="보정 대상 Run ID"),
+        model: str = typer.Option(
+            "both", "--model", help="모델 선택 (linear|xgb|both)", show_default=True
+        ),
+        write_back: bool = typer.Option(
+            False,
+            "--write-back",
+            help="보정 결과를 메타데이터에 저장",
+            show_default=True,
+        ),
+        db_path: Path | None = db_option(help_text="DB 경로"),
+    ) -> None:
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            _console.print("[red]오류: DB 경로가 설정되지 않았습니다.[/red]")
+            raise typer.Exit(1)
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        try:
+            run = storage.get_run(run_id)
+        except KeyError:
+            _console.print("[red]오류: Run을 찾을 수 없습니다.[/red]")
+            raise typer.Exit(1)
+
+        normalized_model = model.lower()
+        if normalized_model not in {"linear", "xgb", "both"}:
+            _console.print("[red]오류: model은 linear|xgb|both 중 하나여야 합니다.[/red]")
+            raise typer.Exit(1)
+
+        feedbacks = storage.list_feedback(run_id)
+        service = SatisfactionCalibrationService()
+        calibration = service.build_calibration(run, feedbacks, model=normalized_model)
+
+        table = Table(title="보정 모델 성능 요약")
+        table.add_column("모델")
+        table.add_column("MAE", justify="right")
+        table.add_column("Pearson", justify="right")
+        table.add_column("Spearman", justify="right")
+
+        if calibration.summary.model_metrics:
+            for model_name, metrics in calibration.summary.model_metrics.items():
+                table.add_row(
+                    model_name,
+                    _format_metric(metrics.get("mae")),
+                    _format_metric(metrics.get("pearson")),
+                    _format_metric(metrics.get("spearman")),
+                )
+        else:
+            table.add_row("N/A", "-", "-", "-")
+
+        _console.print(table)
+        _console.print(
+            f"평균 만족도: {calibration.summary.avg_satisfaction_score} | "
+            f"Thumb Up 비율: {calibration.summary.thumb_up_rate} | "
+            f"보정 비율: {calibration.summary.imputed_ratio}"
+        )
+
+        if write_back:
+            metadata = run.tracker_metadata or {}
+            metadata["calibration"] = {
+                "updated_at": datetime.now().isoformat(),
+                "model": model,
+                "summary": {
+                    "avg_satisfaction_score": calibration.summary.avg_satisfaction_score,
+                    "thumb_up_rate": calibration.summary.thumb_up_rate,
+                    "imputed_ratio": calibration.summary.imputed_ratio,
+                    "model_metrics": calibration.summary.model_metrics,
+                },
+                "cases": {
+                    case_id: {
+                        "calibrated_satisfaction": case.calibrated_satisfaction,
+                        "imputed": case.imputed,
+                        "imputation_source": case.imputation_source,
+                    }
+                    for case_id, case in calibration.cases.items()
+                },
+            }
+            storage.update_run_metadata(run_id, metadata)
+            _console.print("[green]보정 결과를 메타데이터에 저장했습니다.[/green]")
+
+
+def _format_metric(value: float | None) -> str:
+    if value is None:
+        return "-"
+    return f"{value:.3f}"

evalvault/adapters/inbound/cli/commands/run.py

@@ -886,7 +886,7 @@ def register_run_commands(
                     details=str(exc),
                     fixes=[
                         "Ollama가 실행 중인지 확인하세요: `ollama serve` (또는 데스크톱 앱 실행).",
-                        "필요 모델을 받아두세요: `ollama pull gemma3:1b`, `ollama pull qwen3-embedding:0.6b`.",
+                        "필요 모델을 받아두세요: `ollama pull gpt-oss-safeguard:20b`, `ollama pull qwen3-embedding:0.6b`.",
                         "서버 URL을 바꿨다면 .env의 `OLLAMA_BASE_URL`을 확인하세요.",
                     ],
                 )
@@ -1461,7 +1461,7 @@ def register_run_commands(
                 if provider == "ollama":
                     fixes = [
                         "Ollama 서버가 실행 중인지 확인하세요 (기본: http://localhost:11434).",
-                        "필요 모델을 받아두세요: `ollama pull gemma3:1b` 및 `ollama pull qwen3-embedding:0.6b`.",
+                        "필요 모델을 받아두세요: `ollama pull gpt-oss-safeguard:20b` 및 `ollama pull qwen3-embedding:0.6b`.",
                         "URL을 바꿨다면 .env의 `OLLAMA_BASE_URL`을 확인하세요.",
                     ]
                 elif provider == "openai":

evalvault/adapters/inbound/mcp/__init__.py

@@ -0,0 +1,51 @@
+"""MCP inbound adapter package."""
+
+from .schemas import (
+    AnalyzeCompareRequest,
+    AnalyzeCompareResponse,
+    ComparisonArtifactsPayload,
+    EvaluationArtifactsPayload,
+    GetArtifactsRequest,
+    GetArtifactsResponse,
+    GetRunSummaryRequest,
+    GetRunSummaryResponse,
+    ListRunsRequest,
+    ListRunsResponse,
+    McpError,
+    MetricsDeltaPayload,
+    RunEvaluationRequest,
+    RunEvaluationResponse,
+    RunSummaryPayload,
+)
+from .tools import (
+    analyze_compare,
+    get_artifacts,
+    get_run_summary,
+    get_tool_specs,
+    list_runs,
+    run_evaluation,
+)
+
+__all__ = [
+    "AnalyzeCompareRequest",
+    "AnalyzeCompareResponse",
+    "ComparisonArtifactsPayload",
+    "EvaluationArtifactsPayload",
+    "GetArtifactsRequest",
+    "GetArtifactsResponse",
+    "GetRunSummaryRequest",
+    "GetRunSummaryResponse",
+    "ListRunsRequest",
+    "ListRunsResponse",
+    "McpError",
+    "MetricsDeltaPayload",
+    "RunEvaluationRequest",
+    "RunEvaluationResponse",
+    "RunSummaryPayload",
+    "analyze_compare",
+    "get_artifacts",
+    "get_run_summary",
+    "get_tool_specs",
+    "list_runs",
+    "run_evaluation",
+]

evalvault/adapters/inbound/mcp/schemas.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ErrorStage(str, Enum):
+    preprocess = "preprocess"
+    evaluate = "evaluate"
+    analyze = "analyze"
+    compare = "compare"
+    storage = "storage"
+
+
+class McpError(BaseModel):
+    code: str
+    message: str
+    details: dict[str, Any] | None = None
+    retryable: bool = False
+    stage: ErrorStage | None = None
+
+
+class RunSummaryPayload(BaseModel):
+    run_id: str
+    dataset_name: str
+    model_name: str
+    pass_rate: float
+    total_test_cases: int
+    passed_test_cases: int
+    started_at: str
+    finished_at: str | None = None
+    metrics_evaluated: list[str] = Field(default_factory=list)
+    threshold_profile: str | None = None
+    run_mode: str | None = None
+    evaluation_task: str | None = None
+    project_name: str | None = None
+    avg_metric_scores: dict[str, float] | None = None
+    thresholds: dict[str, float] | None = None
+
+    model_config = ConfigDict(extra="allow")
+
+
+class ListRunsRequest(BaseModel):
+    limit: int = Field(50, ge=1, le=500)
+    dataset_name: str | None = None
+    model_name: str | None = None
+    run_mode: str | None = None
+    project_names: list[str] | None = None
+    db_path: Path | None = None
+
+
+class ListRunsResponse(BaseModel):
+    runs: list[RunSummaryPayload] = Field(default_factory=list)
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class GetRunSummaryRequest(BaseModel):
+    run_id: str
+    db_path: Path | None = None
+
+
+class GetRunSummaryResponse(BaseModel):
+    summary: RunSummaryPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class ArtifactsKind(str, Enum):
+    analysis = "analysis"
+    comparison = "comparison"
+
+
+class GetArtifactsRequest(BaseModel):
+    run_id: str
+    kind: ArtifactsKind = ArtifactsKind.analysis
+    comparison_run_id: str | None = None
+    base_dir: Path | None = None
+
+
+class ArtifactsPayload(BaseModel):
+    kind: Literal["analysis", "comparison"]
+    report_path: str | None = None
+    output_path: str | None = None
+    artifacts_dir: str | None = None
+    artifacts_index_path: str | None = None
+
+
+class GetArtifactsResponse(BaseModel):
+    run_id: str
+    artifacts: ArtifactsPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class RunEvaluationRequest(BaseModel):
+    dataset_path: Path
+    metrics: list[str]
+    profile: str | None = None
+    model_name: str | None = None
+    evaluation_task: str | None = None
+    db_path: Path | None = None
+    thresholds: dict[str, float] | None = None
+    threshold_profile: str | None = None
+    parallel: bool = True
+    batch_size: int = 5
+    auto_analyze: bool = False
+    analysis_output: Path | None = None
+    analysis_report: Path | None = None
+    analysis_dir: Path | None = None
+
+
+class EvaluationArtifactsPayload(BaseModel):
+    analysis_report_path: str | None = None
+    analysis_output_path: str | None = None
+    analysis_artifacts_dir: str | None = None
+    analysis_artifacts_index_path: str | None = None
+
+
+class RunEvaluationResponse(BaseModel):
+    run_id: str
+    metrics: dict[str, float | None] = Field(default_factory=dict)
+    thresholds: dict[str, float] | None = None
+    artifacts: EvaluationArtifactsPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)
+
+
+class AnalyzeCompareRequest(BaseModel):
+    run_id_a: str
+    run_id_b: str
+    metrics: list[str] | None = None
+    test_type: Literal["t-test", "mann-whitney"] = "t-test"
+    profile: str | None = None
+    db_path: Path | None = None
+    output: Path | None = None
+    report: Path | None = None
+    output_dir: Path | None = None
+
+
+class MetricsDeltaPayload(BaseModel):
+    avg: dict[str, float] = Field(default_factory=dict)
+    by_metric: dict[str, float] = Field(default_factory=dict)
+    notes: list[str] | None = None
+
+
+class ComparisonArtifactsPayload(BaseModel):
+    json_path: str | None = None
+    report_path: str | None = None
+    artifacts_dir: str | None = None
+    artifacts_index_path: str | None = None
+
+
+class AnalyzeCompareResponse(BaseModel):
+    baseline_run_id: str
+    candidate_run_id: str
+    comparison_report_path: str | None = None
+    metrics_delta: MetricsDeltaPayload = Field(default_factory=MetricsDeltaPayload)
+    artifacts: ComparisonArtifactsPayload | None = None
+    errors: list[McpError] = Field(default_factory=list)