evalvault 1.61.0__py3-none-any.whl → 1.62.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -14,6 +14,11 @@ from urllib.request import urlopen
 
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
 from evalvault.config.settings import Settings
+from evalvault.domain.entities import (
+    CalibrationResult,
+    FeedbackSummary,
+    SatisfactionFeedback,
+)
 from evalvault.domain.entities.prompt import PromptSetBundle
 from evalvault.domain.metrics.registry import (
     get_metric_descriptions as registry_metric_descriptions,
@@ -29,6 +34,9 @@ from evalvault.domain.services.prompt_registry import (
     build_prompt_summary,
 )
 from evalvault.domain.services.prompt_status import extract_prompt_entries
+from evalvault.domain.services.satisfaction_calibration_service import (
+    SatisfactionCalibrationService,
+)
 from evalvault.domain.services.stage_event_builder import StageEventBuilder
 from evalvault.domain.services.stage_metric_service import StageMetricService
 from evalvault.domain.services.threshold_profiles import apply_threshold_profile
@@ -893,6 +901,27 @@ class WebUIAdapter:
             raise RuntimeError("Storage not configured")
         return self._storage.delete_run_cluster_map(run_id, map_id)
 
+    def save_feedback(self, feedback: SatisfactionFeedback) -> str:
+        if self._storage is None or not hasattr(self._storage, "save_feedback"):
+            raise RuntimeError("Storage not configured")
+        return self._storage.save_feedback(feedback)
+
+    def list_feedback(self, run_id: str) -> list[SatisfactionFeedback]:
+        if self._storage is None or not hasattr(self._storage, "list_feedback"):
+            raise RuntimeError("Storage not configured")
+        return self._storage.list_feedback(run_id)
+
+    def get_feedback_summary(self, run_id: str) -> FeedbackSummary:
+        if self._storage is None or not hasattr(self._storage, "get_feedback_summary"):
+            raise RuntimeError("Storage not configured")
+        return self._storage.get_feedback_summary(run_id)
+
+    def build_calibration(self, run_id: str, *, model: str = "both") -> CalibrationResult:
+        run = self.get_run_details(run_id)
+        feedbacks = self.list_feedback(run_id)
+        service = SatisfactionCalibrationService()
+        return service.build_calibration(run, feedbacks, model=model)
+
     def list_stage_events(self, run_id: str, *, stage_type: str | None = None) -> list[StageEvent]:
         """Stage 이벤트 목록 조회."""
         if self._storage is None or not hasattr(self._storage, "list_stage_events"):

evalvault/adapters/inbound/api/routers/runs.py

@@ -21,7 +21,11 @@ from evalvault.adapters.outbound.dataset.templates import (
 )
 from evalvault.adapters.outbound.domain_memory.sqlite_adapter import SQLiteDomainMemoryAdapter
 from evalvault.config.settings import get_settings
-from evalvault.domain.entities import EvaluationRun
+from evalvault.domain.entities import (
+    CalibrationResult,
+    EvaluationRun,
+    SatisfactionFeedback,
+)
 from evalvault.domain.services.domain_learning_hook import DomainLearningHook
 from evalvault.domain.services.ragas_prompt_overrides import (
     PromptOverrideError,
@@ -178,6 +182,31 @@ class ClusterMapDeleteResponse(BaseModel):
     deleted_count: int
 
 
+class FeedbackSaveRequest(BaseModel):
+    test_case_id: str
+    satisfaction_score: float | None = None
+    thumb_feedback: Literal["up", "down", "none"] | None = None
+    comment: str | None = None
+    rater_id: str | None = None
+
+
+class FeedbackResponse(BaseModel):
+    feedback_id: str
+    run_id: str
+    test_case_id: str
+    satisfaction_score: float | None = None
+    thumb_feedback: str | None = None
+    comment: str | None = None
+    rater_id: str | None = None
+    created_at: str | None = None
+
+
+class FeedbackSummaryResponse(BaseModel):
+    avg_satisfaction_score: float | None = None
+    thumb_up_rate: float | None = None
+    total_feedback: int
+
+
 class VisualSpaceRequest(BaseModel):
     granularity: Literal["run", "case", "cluster"] = "case"
     base_run_id: str | None = None
@@ -188,9 +217,22 @@ class VisualSpaceRequest(BaseModel):
     cluster_map: dict[str, str] | None = None
 
 
-def _serialize_run_details(run: EvaluationRun) -> dict[str, Any]:
+def _serialize_run_details(
+    run: EvaluationRun,
+    *,
+    calibration: CalibrationResult | None = None,
+) -> dict[str, Any]:
+    summary = run.to_summary_dict()
+    if calibration is not None:
+        summary.update(
+            {
+                "avg_satisfaction_score": calibration.summary.avg_satisfaction_score,
+                "thumb_up_rate": calibration.summary.thumb_up_rate,
+                "imputed_ratio": calibration.summary.imputed_ratio,
+            }
+        )
     payload = {
-        "summary": run.to_summary_dict(),
+        "summary": summary,
         "results": [
             {
                 "test_case_id": result.test_case_id,
@@ -207,6 +249,21 @@ def _serialize_run_details(run: EvaluationRun) -> dict[str, Any]:
                     }
                     for metric in result.metrics
                 ],
+                "calibrated_satisfaction": (
+                    calibration.cases[result.test_case_id].calibrated_satisfaction
+                    if calibration and result.test_case_id in calibration.cases
+                    else None
+                ),
+                "imputed": (
+                    calibration.cases[result.test_case_id].imputed
+                    if calibration and result.test_case_id in calibration.cases
+                    else False
+                ),
+                "imputation_source": (
+                    calibration.cases[result.test_case_id].imputation_source
+                    if calibration and result.test_case_id in calibration.cases
+                    else None
+                ),
             }
             for result in run.results
         ],
@@ -719,9 +776,12 @@ def compare_runs(
             }
         )
 
+    base_calibration = adapter.build_calibration(base_id)
+    target_calibration = adapter.build_calibration(target_id)
+
     return {
-        "base": _serialize_run_details(base_run),
-        "target": _serialize_run_details(target_run),
+        "base": _serialize_run_details(base_run, calibration=base_calibration),
+        "target": _serialize_run_details(target_run, calibration=target_calibration),
         "metric_deltas": metric_deltas,
         "case_counts": _build_case_counts(base_run, target_run),
         "pass_rate_delta": target_run.pass_rate - base_run.pass_rate,
@@ -898,7 +958,70 @@ def get_run_details(run_id: str, adapter: AdapterDep) -> dict[str, Any]:
     """Get detailed information for a specific run."""
     try:
         run: EvaluationRun = adapter.get_run_details(run_id)
-        return _serialize_run_details(run)
+        calibration = adapter.build_calibration(run_id)
+        return _serialize_run_details(run, calibration=calibration)
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Run not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/{run_id}/feedback", response_model=FeedbackResponse)
+def save_feedback(
+    run_id: str,
+    request: FeedbackSaveRequest,
+    adapter: AdapterDep,
+) -> dict[str, Any]:
+    try:
+        adapter.get_run_details(run_id)
+        thumb_feedback = request.thumb_feedback
+        if thumb_feedback == "none":
+            thumb_feedback = None
+        satisfaction_score = request.satisfaction_score
+        if satisfaction_score is not None:
+            satisfaction_score = max(1.0, min(5.0, satisfaction_score))
+        feedback = SatisfactionFeedback(
+            feedback_id="",
+            run_id=run_id,
+            test_case_id=request.test_case_id,
+            satisfaction_score=satisfaction_score,
+            thumb_feedback=thumb_feedback,
+            comment=request.comment,
+            rater_id=request.rater_id,
+            created_at=datetime.now(),
+        )
+        feedback_id = adapter.save_feedback(feedback)
+        saved = feedback.to_dict()
+        saved["feedback_id"] = feedback_id
+        return saved
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Run not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/{run_id}/feedback", response_model=list[FeedbackResponse])
+def list_feedback(run_id: str, adapter: AdapterDep) -> list[dict[str, Any]]:
+    try:
+        adapter.get_run_details(run_id)
+        feedbacks = adapter.list_feedback(run_id)
+        return [feedback.to_dict() for feedback in feedbacks]
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Run not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/{run_id}/feedback/summary", response_model=FeedbackSummaryResponse)
+def get_feedback_summary(run_id: str, adapter: AdapterDep) -> dict[str, Any]:
+    try:
+        adapter.get_run_details(run_id)
+        summary = adapter.get_feedback_summary(run_id)
+        return {
+            "avg_satisfaction_score": summary.avg_satisfaction_score,
+            "thumb_up_rate": summary.thumb_up_rate,
+            "total_feedback": summary.total_feedback,
+        }
     except KeyError:
         raise HTTPException(status_code=404, detail="Run not found")
     except Exception as e:

evalvault/adapters/inbound/cli/commands/__init__.py

@@ -13,6 +13,7 @@ from .agent import register_agent_commands
 from .analyze import register_analyze_commands
 from .api import register_api_command
 from .benchmark import create_benchmark_app
+from .calibrate import register_calibrate_commands
 from .config import register_config_commands
 from .debug import create_debug_app
 from .domain import create_domain_app
@@ -61,6 +62,7 @@ COMMAND_MODULES: tuple[CommandModule, ...] = (
     CommandModule(register_pipeline_commands),
     CommandModule(register_history_commands),
     CommandModule(register_analyze_commands),
+    CommandModule(register_calibrate_commands),
     CommandModule(register_generate_commands),
     CommandModule(register_gate_commands),
     CommandModule(register_agent_commands),

evalvault/adapters/inbound/cli/commands/calibrate.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings
+from evalvault.domain.services.satisfaction_calibration_service import (
+    SatisfactionCalibrationService,
+)
+
+from ..utils.options import db_option
+
+_console = Console()
+
+
+def register_calibrate_commands(app: typer.Typer, console: Console) -> None:
+    global _console
+    _console = console
+
+    @app.command()
+    def calibrate(
+        run_id: str = typer.Argument(..., help="보정 대상 Run ID"),
+        model: str = typer.Option(
+            "both", "--model", help="모델 선택 (linear|xgb|both)", show_default=True
+        ),
+        write_back: bool = typer.Option(
+            False,
+            "--write-back",
+            help="보정 결과를 메타데이터에 저장",
+            show_default=True,
+        ),
+        db_path: Path | None = db_option(help_text="DB 경로"),
+    ) -> None:
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            _console.print("[red]오류: DB 경로가 설정되지 않았습니다.[/red]")
+            raise typer.Exit(1)
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        try:
+            run = storage.get_run(run_id)
+        except KeyError:
+            _console.print("[red]오류: Run을 찾을 수 없습니다.[/red]")
+            raise typer.Exit(1)
+
+        normalized_model = model.lower()
+        if normalized_model not in {"linear", "xgb", "both"}:
+            _console.print("[red]오류: model은 linear|xgb|both 중 하나여야 합니다.[/red]")
+            raise typer.Exit(1)
+
+        feedbacks = storage.list_feedback(run_id)
+        service = SatisfactionCalibrationService()
+        calibration = service.build_calibration(run, feedbacks, model=normalized_model)
+
+        table = Table(title="보정 모델 성능 요약")
+        table.add_column("모델")
+        table.add_column("MAE", justify="right")
+        table.add_column("Pearson", justify="right")
+        table.add_column("Spearman", justify="right")
+
+        if calibration.summary.model_metrics:
+            for model_name, metrics in calibration.summary.model_metrics.items():
+                table.add_row(
+                    model_name,
+                    _format_metric(metrics.get("mae")),
+                    _format_metric(metrics.get("pearson")),
+                    _format_metric(metrics.get("spearman")),
+                )
+        else:
+            table.add_row("N/A", "-", "-", "-")
+
+        _console.print(table)
+        _console.print(
+            f"평균 만족도: {calibration.summary.avg_satisfaction_score} | "
+            f"Thumb Up 비율: {calibration.summary.thumb_up_rate} | "
+            f"보정 비율: {calibration.summary.imputed_ratio}"
+        )
+
+        if write_back:
+            metadata = run.tracker_metadata or {}
+            metadata["calibration"] = {
+                "updated_at": datetime.now().isoformat(),
+                "model": model,
+                "summary": {
+                    "avg_satisfaction_score": calibration.summary.avg_satisfaction_score,
+                    "thumb_up_rate": calibration.summary.thumb_up_rate,
+                    "imputed_ratio": calibration.summary.imputed_ratio,
+                    "model_metrics": calibration.summary.model_metrics,
+                },
+                "cases": {
+                    case_id: {
+                        "calibrated_satisfaction": case.calibrated_satisfaction,
+                        "imputed": case.imputed,
+                        "imputation_source": case.imputation_source,
+                    }
+                    for case_id, case in calibration.cases.items()
+                },
+            }
+            storage.update_run_metadata(run_id, metadata)
+            _console.print("[green]보정 결과를 메타데이터에 저장했습니다.[/green]")
+
+
+def _format_metric(value: float | None) -> str:
+    if value is None:
+        return "-"
+    return f"{value:.3f}"

evalvault/adapters/outbound/analysis/nlp_adapter.py

@@ -727,8 +727,52 @@ class NLPAnalysisAdapter(BaseAnalysisAdapter):
                     if values:
                         avg_scores[metric_name] = sum(values) / len(values)
 
-                # 대표 질문 선택 (처음 3개)
-                representative_questions = cluster_qs[:3]
+                representative_questions: list[str] = []
+                try:
+                    cluster_idx_list = cluster_indices[cluster_id]
+                    cluster_vectors = embedding_array[cluster_idx_list]
+                    centroid = cluster_vectors.mean(axis=0)
+                    distances = np.linalg.norm(cluster_vectors - centroid, axis=1)
+                    sorted_pairs = sorted(
+                        zip(cluster_idx_list, distances, strict=True), key=lambda x: x[1]
+                    )
+
+                    center_indices = [idx for idx, _dist in sorted_pairs[:2]]
+                    edge_far = sorted_pairs[-1][0] if sorted_pairs else None
+
+                    worst_idx = None
+                    worst_score = None
+                    for idx in cluster_idx_list:
+                        q = questions[idx]
+                        result = question_to_result.get(q)
+                        if not result or not result.metrics:
+                            continue
+                        avg_score = sum(m.score for m in result.metrics) / len(result.metrics)
+                        if worst_score is None or avg_score < worst_score:
+                            worst_score = avg_score
+                            worst_idx = idx
+
+                    edge_needed = worst_idx
+                    if edge_needed is None and len(sorted_pairs) > 1:
+                        edge_needed = sorted_pairs[-2][0]
+
+                    candidate_indices: list[int] = []
+                    candidate_indices.extend(center_indices)
+                    if edge_far is not None:
+                        candidate_indices.append(edge_far)
+                    if edge_needed is not None:
+                        candidate_indices.append(edge_needed)
+
+                    seen: set[int] = set()
+                    for idx in candidate_indices:
+                        if idx in seen:
+                            continue
+                        seen.add(idx)
+                        representative_questions.append(questions[idx])
+                        if len(representative_questions) >= 4:
+                            break
+                except Exception:
+                    representative_questions = cluster_qs[:4]
 
                 clusters.append(
                     TopicCluster(

evalvault/adapters/outbound/analysis/nlp_analyzer_module.py

@@ -216,7 +216,7 @@ class NLPAnalyzerModule(BaseAnalysisModule):
                     "keywords": list(cluster.keywords),
                     "document_count": cluster.document_count,
                     "avg_scores": cluster.avg_scores,
-                    "representative_questions": cluster.representative_questions[:3],
+                    "representative_questions": cluster.representative_questions[:4],
                 }
             )
         return serialized

evalvault/adapters/outbound/storage/base_sql.py

@@ -11,9 +11,11 @@ from typing import Any
 
 from evalvault.domain.entities import (
     EvaluationRun,
+    FeedbackSummary,
     MetricScore,
     RunClusterMap,
     RunClusterMapInfo,
+    SatisfactionFeedback,
     TestCaseResult,
 )
 
@@ -27,10 +29,12 @@ class SQLQueries:
         placeholder: str = "?",
         metric_name_column: str = "metric_name",
         test_case_returning_clause: str = "",
+        feedback_returning_clause: str = "",
     ) -> None:
         self.placeholder = placeholder
         self.metric_name_column = metric_name_column
         self._test_case_returning = test_case_returning_clause
+        self._feedback_returning = feedback_returning_clause
 
     def _values(self, count: int) -> str:
         return ", ".join([self.placeholder] * count)
@@ -75,6 +79,25 @@ class SQLQueries:
         ) VALUES ({values})
         """
 
+    def insert_feedback(self) -> str:
+        values = self._values(7)
+        query = f"""
+        INSERT INTO satisfaction_feedback (
+            run_id, test_case_id, satisfaction_score, thumb_feedback, comment, rater_id, created_at
+        ) VALUES ({values})
+        """
+        if self._feedback_returning:
+            query = f"{query.strip()} {self._feedback_returning}"
+        return query
+
+    def select_feedback_by_run(self) -> str:
+        return f"""
+        SELECT id, run_id, test_case_id, satisfaction_score, thumb_feedback, comment, rater_id, created_at
+        FROM satisfaction_feedback
+        WHERE run_id = {self.placeholder}
+        ORDER BY created_at DESC
+        """
+
     def select_run(self) -> str:
         return f"""
         SELECT run_id, dataset_name, dataset_version, model_name,
@@ -129,6 +152,13 @@ class SQLQueries:
         ORDER BY created_at DESC
         """
 
+    def update_run_metadata(self) -> str:
+        return f"""
+        UPDATE evaluation_runs
+        SET metadata = {self.placeholder}
+        WHERE run_id = {self.placeholder}
+        """
+
     def delete_run(self) -> str:
         return f"DELETE FROM evaluation_runs WHERE run_id = {self.placeholder}"
 
@@ -259,6 +289,12 @@ class BaseSQLStorageAdapter(ABC):
             conn.commit()
             return deleted
 
+    def update_run_metadata(self, run_id: str, metadata: dict[str, Any]) -> None:
+        payload = self._serialize_json(metadata)
+        with self._get_connection() as conn:
+            self._execute(conn, self.queries.update_run_metadata(), (payload, run_id))
+            conn.commit()
+
     def save_run_cluster_map(
         self,
         run_id: str,
@@ -365,6 +401,45 @@ class BaseSQLStorageAdapter(ABC):
             conn.commit()
             return deleted
 
+    def save_feedback(self, feedback: SatisfactionFeedback) -> str:
+        created_at = feedback.created_at or datetime.now()
+        with self._get_connection() as conn:
+            cursor = self._execute(
+                conn,
+                self.queries.insert_feedback(),
+                (
+                    feedback.run_id,
+                    feedback.test_case_id,
+                    feedback.satisfaction_score,
+                    feedback.thumb_feedback,
+                    feedback.comment,
+                    feedback.rater_id,
+                    self._serialize_datetime(created_at),
+                ),
+            )
+            feedback_id = self._fetch_lastrowid(cursor)
+            conn.commit()
+            return str(feedback_id)
+
+    def list_feedback(self, run_id: str) -> list[SatisfactionFeedback]:
+        with self._get_connection() as conn:
+            rows = self._execute(conn, self.queries.select_feedback_by_run(), (run_id,)).fetchall()
+            return [self._row_to_feedback(row) for row in rows]
+
+    def get_feedback_summary(self, run_id: str) -> FeedbackSummary:
+        feedbacks = self.list_feedback(run_id)
+        scores = [f.satisfaction_score for f in feedbacks if f.satisfaction_score is not None]
+        thumbs = [f.thumb_feedback for f in feedbacks if f.thumb_feedback in {"up", "down"}]
+        avg_score = sum(scores) / len(scores) if scores else None
+        thumb_up_rate = None
+        if thumbs:
+            thumb_up_rate = thumbs.count("up") / len(thumbs)
+        return FeedbackSummary(
+            avg_satisfaction_score=avg_score,
+            thumb_up_rate=thumb_up_rate,
+            total_feedback=len(feedbacks),
+        )
+
     # Serialization helpers --------------------------------------------
 
     def _run_params(self, run: EvaluationRun) -> Sequence[Any]:
@@ -428,6 +503,22 @@ class BaseSQLStorageAdapter(ABC):
             ground_truth=row["ground_truth"],
         )
 
+    def _row_to_feedback(self, row) -> SatisfactionFeedback:
+        feedback_id = self._row_value(row, "id")
+        run_id = self._row_value(row, "run_id")
+        test_case_id = self._row_value(row, "test_case_id")
+        created_at = self._deserialize_datetime(self._row_value(row, "created_at"))
+        return SatisfactionFeedback(
+            feedback_id=str(feedback_id or ""),
+            run_id=str(run_id or ""),
+            test_case_id=str(test_case_id or ""),
+            satisfaction_score=self._maybe_float(self._row_value(row, "satisfaction_score")),
+            thumb_feedback=self._row_value(row, "thumb_feedback"),
+            comment=self._row_value(row, "comment"),
+            rater_id=self._row_value(row, "rater_id"),
+            created_at=created_at,
+        )
+
     def _fetch_metric_scores(self, conn, result_id: int) -> list[MetricScore]:
         rows = self._execute(conn, self.queries.select_metric_scores(), (result_id,)).fetchall()
         metric_column = self.queries.metric_name_column

evalvault/adapters/outbound/storage/postgres_adapter.py

@@ -60,6 +60,7 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
                 placeholder="%s",
                 metric_name_column="name",
                 test_case_returning_clause="RETURNING id",
+                feedback_returning_clause="RETURNING id",
             )
         )
         if connection_string:
@@ -198,6 +199,27 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
         elif cluster_columns and "metadata" not in cluster_columns:
             conn.execute("ALTER TABLE run_cluster_maps ADD COLUMN metadata JSONB")
 
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS satisfaction_feedback (
+                id SERIAL PRIMARY KEY,
+                run_id UUID NOT NULL REFERENCES evaluation_runs(run_id) ON DELETE CASCADE,
+                test_case_id VARCHAR(255) NOT NULL,
+                satisfaction_score DECIMAL(4, 2),
+                thumb_feedback VARCHAR(10),
+                comment TEXT,
+                rater_id VARCHAR(255),
+                created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+            )
+            """
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_feedback_run_id ON satisfaction_feedback(run_id)"
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_feedback_test_case_id ON satisfaction_feedback(test_case_id)"
+        )
+
     # Prompt set methods
 
     def save_prompt_set(self, bundle: PromptSetBundle) -> None:

evalvault/adapters/outbound/storage/postgres_schema.sql

@@ -59,6 +59,20 @@ CREATE TABLE IF NOT EXISTS run_cluster_maps (
 CREATE INDEX IF NOT EXISTS idx_cluster_maps_run_id ON run_cluster_maps(run_id);
 CREATE INDEX IF NOT EXISTS idx_cluster_maps_map_id ON run_cluster_maps(map_id);
 
+CREATE TABLE IF NOT EXISTS satisfaction_feedback (
+    id SERIAL PRIMARY KEY,
+    run_id UUID NOT NULL REFERENCES evaluation_runs(run_id) ON DELETE CASCADE,
+    test_case_id VARCHAR(255) NOT NULL,
+    satisfaction_score DECIMAL(4, 2),
+    thumb_feedback VARCHAR(10),
+    comment TEXT,
+    rater_id VARCHAR(255),
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_feedback_run_id ON satisfaction_feedback(run_id);
+CREATE INDEX IF NOT EXISTS idx_feedback_test_case_id ON satisfaction_feedback(test_case_id);
+
 -- Metric scores table
 CREATE TABLE IF NOT EXISTS metric_scores (
     id SERIAL PRIMARY KEY,

evalvault/adapters/outbound/storage/schema.sql

@@ -61,6 +61,21 @@ CREATE TABLE IF NOT EXISTS run_cluster_maps (
 CREATE INDEX IF NOT EXISTS idx_cluster_maps_run_id ON run_cluster_maps(run_id);
 CREATE INDEX IF NOT EXISTS idx_cluster_maps_map_id ON run_cluster_maps(map_id);
 
+CREATE TABLE IF NOT EXISTS satisfaction_feedback (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id TEXT NOT NULL,
+    test_case_id TEXT NOT NULL,
+    satisfaction_score REAL,
+    thumb_feedback TEXT,
+    comment TEXT,
+    rater_id TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (run_id) REFERENCES evaluation_runs(run_id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_feedback_run_id ON satisfaction_feedback(run_id);
+CREATE INDEX IF NOT EXISTS idx_feedback_test_case_id ON satisfaction_feedback(test_case_id);
+
 -- Metric scores table
 CREATE TABLE IF NOT EXISTS metric_scores (
     id INTEGER PRIMARY KEY AUTOINCREMENT,

evalvault/adapters/outbound/storage/sqlite_adapter.py

@@ -140,6 +140,31 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         elif cluster_columns and "metadata" not in cluster_columns:
             conn.execute("ALTER TABLE run_cluster_maps ADD COLUMN metadata TEXT")
 
+        feedback_cursor = conn.execute("PRAGMA table_info(satisfaction_feedback)")
+        feedback_columns = {row[1] for row in feedback_cursor.fetchall()}
+        if not feedback_columns:
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS satisfaction_feedback (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    run_id TEXT NOT NULL,
+                    test_case_id TEXT NOT NULL,
+                    satisfaction_score REAL,
+                    thumb_feedback TEXT,
+                    comment TEXT,
+                    rater_id TEXT,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (run_id) REFERENCES evaluation_runs(run_id) ON DELETE CASCADE
+                )
+                """
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_feedback_run_id ON satisfaction_feedback(run_id)"
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_feedback_test_case_id ON satisfaction_feedback(test_case_id)"
+            )
+
         pipeline_cursor = conn.execute("PRAGMA table_info(pipeline_results)")
         pipeline_columns = {row[1] for row in pipeline_cursor.fetchall()}
         if pipeline_columns:

evalvault/domain/entities/__init__.py

@@ -14,6 +14,13 @@ from evalvault.domain.entities.analysis import (
 )
 from evalvault.domain.entities.dataset import Dataset, TestCase
 from evalvault.domain.entities.experiment import Experiment, ExperimentGroup
+from evalvault.domain.entities.feedback import (
+    CalibrationCaseResult,
+    CalibrationResult,
+    CalibrationSummary,
+    FeedbackSummary,
+    SatisfactionFeedback,
+)
 from evalvault.domain.entities.improvement import (
     EffortLevel,
     EvidenceSource,
@@ -74,6 +81,11 @@ __all__ = [
     # Experiment
     "Experiment",
     "ExperimentGroup",
+    "CalibrationCaseResult",
+    "CalibrationResult",
+    "CalibrationSummary",
+    "FeedbackSummary",
+    "SatisfactionFeedback",
     # Improvement
     "EffortLevel",
     "EvidenceSource",

evalvault/domain/entities/feedback.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class SatisfactionFeedback:
+    feedback_id: str
+    run_id: str
+    test_case_id: str
+    satisfaction_score: float | None = None
+    thumb_feedback: str | None = None
+    comment: str | None = None
+    rater_id: str | None = None
+    created_at: datetime | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "feedback_id": self.feedback_id,
+            "run_id": self.run_id,
+            "test_case_id": self.test_case_id,
+            "satisfaction_score": self.satisfaction_score,
+            "thumb_feedback": self.thumb_feedback,
+            "comment": self.comment,
+            "rater_id": self.rater_id,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }
+
+
+@dataclass
+class FeedbackSummary:
+    avg_satisfaction_score: float | None = None
+    thumb_up_rate: float | None = None
+    total_feedback: int = 0
+
+
+@dataclass
+class CalibrationCaseResult:
+    test_case_id: str
+    calibrated_satisfaction: float | None = None
+    imputed: bool = False
+    imputation_source: str | None = None
+
+
+@dataclass
+class CalibrationSummary:
+    avg_satisfaction_score: float | None = None
+    thumb_up_rate: float | None = None
+    imputed_ratio: float | None = None
+    model_metrics: dict[str, dict[str, float | None]] = field(default_factory=dict)
+
+
+@dataclass
+class CalibrationResult:
+    summary: CalibrationSummary
+    cases: dict[str, CalibrationCaseResult] = field(default_factory=dict)

evalvault/domain/services/satisfaction_calibration_service.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+import pandas as pd
+
+from evalvault.domain.entities import (
+    CalibrationCaseResult,
+    CalibrationResult,
+    CalibrationSummary,
+    EvaluationRun,
+    SatisfactionFeedback,
+)
+
+
+@dataclass
+class CalibrationModelResult:
+    model_name: str
+    mae: float | None
+    pearson: float | None
+    spearman: float | None
+
+
+class SatisfactionCalibrationService:
+    def __init__(self, *, thumb_mapping: dict[str, float] | None = None) -> None:
+        self._thumb_mapping = thumb_mapping or {"up": 4.0, "down": 2.0}
+
+    def build_calibration(
+        self,
+        run: EvaluationRun,
+        feedbacks: list[SatisfactionFeedback],
+        *,
+        model: str = "both",
+    ) -> CalibrationResult:
+        feedback_index = self._build_feedback_index(feedbacks)
+        feature_map = self._build_feature_matrix(run)
+        labels, label_sources = self._build_labels(run, feedback_index)
+
+        if not feedback_index:
+            summary = CalibrationSummary(
+                avg_satisfaction_score=None,
+                thumb_up_rate=None,
+                imputed_ratio=0.0,
+            )
+            return CalibrationResult(summary=summary, cases={})
+
+        model_metrics: dict[str, dict[str, float | None]] = {}
+        model_choice, predictors = self._train_models(
+            feature_map,
+            labels,
+            model=model,
+            model_metrics=model_metrics,
+        )
+
+        summary = self._build_summary(run, feedback_index)
+        cases: dict[str, CalibrationCaseResult] = {}
+        imputed_count = 0
+
+        for test_case_id, features in feature_map.items():
+            label = labels.get(test_case_id)
+            source = label_sources.get(test_case_id)
+            if label is not None:
+                calibrated = self._clip_score(label)
+                imputed = source != "label"
+                imputation_source = source
+            else:
+                calibrated = self._predict_or_fallback(
+                    predictors.get(model_choice),
+                    features,
+                    labels,
+                )
+                if calibrated is None:
+                    imputed = False
+                    imputation_source = None
+                else:
+                    imputed = True
+                    imputation_source = "model" if predictors.get(model_choice) else "fallback_mean"
+
+            if imputed:
+                imputed_count += 1
+
+            cases[test_case_id] = CalibrationCaseResult(
+                test_case_id=test_case_id,
+                calibrated_satisfaction=calibrated,
+                imputed=imputed,
+                imputation_source=imputation_source,
+            )
+
+        summary.imputed_ratio = imputed_count / len(cases) if cases else summary.imputed_ratio
+        summary.model_metrics = model_metrics
+        return CalibrationResult(summary=summary, cases=cases)
+
+    def _build_feedback_index(
+        self, feedbacks: list[SatisfactionFeedback]
+    ) -> dict[str, SatisfactionFeedback]:
+        latest: dict[str, SatisfactionFeedback] = {}
+        for feedback in feedbacks:
+            current = latest.get(feedback.test_case_id)
+            if current is None:
+                latest[feedback.test_case_id] = feedback
+                continue
+            current_time = current.created_at or datetime.min
+            feedback_time = feedback.created_at or datetime.min
+            if feedback_time >= current_time:
+                latest[feedback.test_case_id] = feedback
+        return latest
+
+    def _build_feature_matrix(self, run: EvaluationRun) -> dict[str, list[float]]:
+        feature_map: dict[str, list[float]] = {}
+
+        for result in run.results:
+            features = [
+                self._metric_score(result, "faithfulness"),
+                self._metric_score(result, "answer_relevancy"),
+                self._metric_score(result, "context_precision"),
+                self._metric_score(result, "context_recall"),
+                self._answer_length(result.answer),
+                self._keyword_missing_rate(result.question, result.answer, result.contexts),
+                self._ttr(result.answer),
+            ]
+            feature_map[result.test_case_id] = features
+        return feature_map
+
+    def _build_labels(
+        self,
+        run: EvaluationRun,
+        feedback_index: dict[str, SatisfactionFeedback],
+    ) -> tuple[dict[str, float], dict[str, str]]:
+        labels: dict[str, float] = {}
+        sources: dict[str, str] = {}
+        for result in run.results:
+            feedback = feedback_index.get(result.test_case_id)
+            if feedback is None:
+                continue
+            if feedback.satisfaction_score is not None:
+                labels[result.test_case_id] = feedback.satisfaction_score
+                sources[result.test_case_id] = "label"
+                continue
+            mapped = self._thumb_mapping.get((feedback.thumb_feedback or "").lower())
+            if mapped is not None:
+                labels[result.test_case_id] = mapped
+                sources[result.test_case_id] = "thumb"
+        return labels, sources
+
+    def _train_models(
+        self,
+        feature_map: dict[str, list[float]],
+        labels: dict[str, float],
+        *,
+        model: str,
+        model_metrics: dict[str, dict[str, float | None]],
+    ) -> tuple[str, dict[str, Any]]:
+        from sklearn.linear_model import LinearRegression
+        from sklearn.metrics import mean_absolute_error
+        from sklearn.model_selection import train_test_split
+
+        if not labels:
+            return "linear", {}
+
+        features_matrix: list[list[float]] = []
+        labels_vector: list[float] = []
+        for test_case_id, label in labels.items():
+            features = feature_map.get(test_case_id)
+            if features is None:
+                continue
+            features_matrix.append(features)
+            labels_vector.append(label)
+
+        if not features_matrix:
+            return "linear", {}
+
+        if len(labels_vector) >= 5:
+            features_train, features_test, labels_train, labels_test = train_test_split(
+                features_matrix, labels_vector, test_size=0.2, random_state=42
+            )
+        else:
+            features_train, features_test, labels_train, labels_test = (
+                features_matrix,
+                features_matrix,
+                labels_vector,
+                labels_vector,
+            )
+
+        predictors: dict[str, Any] = {}
+
+        linear = LinearRegression()
+        linear.fit(features_train, labels_train)
+        linear_pred = linear.predict(features_test)
+        model_metrics["linear"] = self._build_metrics(labels_test, linear_pred, mean_absolute_error)
+        predictors["linear"] = linear
+
+        if model in {"xgb", "both"}:
+            try:
+                import importlib
+
+                xgb_module = importlib.import_module("xgboost")
+                xgb_regressor = xgb_module.XGBRegressor
+
+                xgb = xgb_regressor(
+                    objective="reg:squarederror",
+                    n_estimators=150,
+                    max_depth=5,
+                    learning_rate=0.1,
+                    subsample=0.8,
+                    colsample_bytree=0.8,
+                    reg_alpha=0.1,
+                    reg_lambda=1.0,
+                    n_jobs=-1,
+                    random_state=42,
+                )
+                xgb.fit(features_train, labels_train)
+                xgb_pred = xgb.predict(features_test)
+                model_metrics["xgb"] = self._build_metrics(
+                    labels_test, xgb_pred, mean_absolute_error
+                )
+                predictors["xgb"] = xgb
+            except Exception:
+                model_metrics["xgb"] = {"mae": None, "pearson": None, "spearman": None}
+
+        model_choice = "xgb" if model in {"xgb", "both"} and "xgb" in predictors else "linear"
+        return model_choice, predictors
+
+    def _build_metrics(
+        self,
+        y_true: list[float],
+        y_pred: list[float],
+        mae_func,
+    ) -> dict[str, float | None]:
+        mae = float(mae_func(y_true, y_pred)) if y_true else None
+        pearson = self._safe_corr(y_true, y_pred, method="pearson")
+        spearman = self._safe_corr(y_true, y_pred, method="spearman")
+        return {"mae": mae, "pearson": pearson, "spearman": spearman}
+
+    def _predict_or_fallback(
+        self,
+        predictor: Any | None,
+        features: list[float],
+        labels: dict[str, float],
+    ) -> float | None:
+        if predictor is not None:
+            prediction = predictor.predict([features])[0]
+            return self._clip_score(float(prediction))
+        fallback = self._fallback_mean(labels)
+        if fallback is None:
+            return None
+        return self._clip_score(fallback)
+
+    def _fallback_mean(self, labels: dict[str, float]) -> float | None:
+        if not labels:
+            return None
+        return sum(labels.values()) / len(labels)
+
+    def _build_summary(
+        self, run: EvaluationRun, feedback_index: dict[str, SatisfactionFeedback]
+    ) -> CalibrationSummary:
+        scores: list[float] = []
+        thumbs: list[str] = []
+        for result in run.results:
+            feedback = feedback_index.get(result.test_case_id)
+            if feedback is None:
+                continue
+            if feedback.satisfaction_score is not None:
+                scores.append(feedback.satisfaction_score)
+            if feedback.thumb_feedback in {"up", "down"}:
+                thumbs.append(feedback.thumb_feedback)
+        avg_score = sum(scores) / len(scores) if scores else None
+        thumb_up_rate = None
+        if thumbs:
+            thumb_up_rate = thumbs.count("up") / len(thumbs)
+        return CalibrationSummary(
+            avg_satisfaction_score=avg_score,
+            thumb_up_rate=thumb_up_rate,
+            imputed_ratio=None,
+        )
+
+    def _metric_score(self, result, name: str) -> float:
+        metric = result.get_metric(name)
+        if metric and metric.score is not None:
+            return float(metric.score)
+        return 0.0
+
+    def _answer_length(self, answer: str | None) -> float:
+        tokens = self._tokenize(answer or "")
+        return float(len(tokens))
+
+    def _keyword_missing_rate(
+        self,
+        question: str | None,
+        answer: str | None,
+        contexts: list[str] | None,
+    ) -> float:
+        question_tokens = set(self._tokenize(question or ""))
+        if not question_tokens:
+            return 0.0
+        combined = " ".join([answer or "", *(contexts or [])])
+        combined_tokens = set(self._tokenize(combined))
+        missing = [token for token in question_tokens if token not in combined_tokens]
+        return len(missing) / len(question_tokens)
+
+    def _ttr(self, answer: str | None) -> float:
+        tokens = self._tokenize(answer or "")
+        if not tokens:
+            return 0.0
+        return len(set(tokens)) / len(tokens)
+
+    def _tokenize(self, text: str) -> list[str]:
+        series = pd.Series([text])
+        tokens = series.str.findall(r"[가-힣a-zA-Z0-9]{2,}").iloc[0]
+        return [token.lower() for token in tokens]
+
+    def _clip_score(self, score: float) -> float:
+        return max(1.0, min(5.0, score))
+
+    def _safe_corr(self, y_true: list[float], y_pred: list[float], *, method: str) -> float | None:
+        if len(y_true) < 2 or len(y_pred) < 2:
+            return None
+        series_a = pd.Series(y_true)
+        series_b = pd.Series(y_pred)
+        if method == "spearman":
+            series_a = series_a.rank()
+            series_b = series_b.rank()
+        try:
+            corr = series_a.corr(series_b)
+            return float(corr) if corr is not None else None
+        except Exception:
+            return None

evalvault/ports/outbound/storage_port.py

@@ -4,9 +4,11 @@ from typing import Any, Protocol
 
 from evalvault.domain.entities import (
     EvaluationRun,
+    FeedbackSummary,
     PromptSetBundle,
     RunClusterMap,
     RunClusterMapInfo,
+    SatisfactionFeedback,
 )
 from evalvault.domain.entities.experiment import Experiment
 
@@ -76,6 +78,8 @@ class StoragePort(Protocol):
         """
         ...
 
+    def update_run_metadata(self, run_id: str, metadata: dict[str, Any]) -> None: ...
+
     def save_run_cluster_map(
         self,
         run_id: str,
@@ -99,6 +103,12 @@ class StoragePort(Protocol):
         """런별 클러스터 맵을 삭제합니다."""
         ...
 
+    def save_feedback(self, feedback: SatisfactionFeedback) -> str: ...
+
+    def list_feedback(self, run_id: str) -> list[SatisfactionFeedback]: ...
+
+    def get_feedback_summary(self, run_id: str) -> FeedbackSummary: ...
+
     # Experiment 관련 메서드
 
     def save_experiment(self, experiment: Experiment) -> str:

{evalvault-1.61.0.dist-info → evalvault-1.62.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.61.0
+Version: 1.62.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme
@@ -46,6 +46,7 @@ Requires-Dist: uvicorn>=0.40.0
 Requires-Dist: xlrd
 Provides-Extra: analysis
 Requires-Dist: scikit-learn>=1.3.0; extra == 'analysis'
+Requires-Dist: xgboost>=2.0.0; extra == 'analysis'
 Provides-Extra: anthropic
 Requires-Dist: anthropic; extra == 'anthropic'
 Requires-Dist: langchain-anthropic; extra == 'anthropic'
@@ -86,6 +87,7 @@ Requires-Dist: rank-bm25>=0.2.2; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
 Requires-Dist: scikit-learn<1.4.0,>=1.3.0; extra == 'dev'
 Requires-Dist: sentence-transformers>=5.2.0; extra == 'dev'
+Requires-Dist: xgboost>=2.0.0; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
 Requires-Dist: mkdocs>=1.5.0; extra == 'docs'

{evalvault-1.61.0.dist-info → evalvault-1.62.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ evalvault/mkdocs_helpers.py,sha256=1AKVQ1W2_VO4qclhfyefyU9Dz1Hzkh1DWDwsFMe24jc,3
 evalvault/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalvault/adapters/inbound/__init__.py,sha256=SG1svel1PwqetnqVpKFLSv612_WwGwLTbFpYgwk6FMw,166
 evalvault/adapters/inbound/api/__init__.py,sha256=LeVVttCA3tLKoHA2PO4z3y8VkfVcf3Bq8CZSzo91lf4,34
-evalvault/adapters/inbound/api/adapter.py,sha256=6L95Csns-ac_9Q1rbVjYA8G7mu0wb981G5lsbvcqzcI,59820
+evalvault/adapters/inbound/api/adapter.py,sha256=_giGdt-grmd6bkWMhRb3KdloxI_2jUMknProC76KqWY,61140
 evalvault/adapters/inbound/api/main.py,sha256=KdlAxKn0QfGI3UuoTrBDBbUs2xCvP8lnWOY1ce3svcU,2619
 evalvault/adapters/inbound/api/routers/__init__.py,sha256=q07_YF9TnBl68bqcRCvhPU4-zRTyvmPoHVehwO6W7QM,19
 evalvault/adapters/inbound/api/routers/benchmark.py,sha256=yevntbZcNtMvbVODsITUBgR1Ka4pdFQrXBJJ4K4Jyr4,4477
@@ -13,14 +13,15 @@ evalvault/adapters/inbound/api/routers/config.py,sha256=CN-FH2cn0Ive-BD3WacWY6PF
 evalvault/adapters/inbound/api/routers/domain.py,sha256=RsR7GIFMjccDN7vpG1uDyk9n1DnCTH18JDGAX7o4Qqc,3648
 evalvault/adapters/inbound/api/routers/knowledge.py,sha256=7mgyoUM1PepFb4X8_Ntn0vd7ZZYcNbM3_9nyD10g4Aw,5307
 evalvault/adapters/inbound/api/routers/pipeline.py,sha256=8UgQzNFHcuqS61s69mOrPee4OMwfxVdvRWHJ2_qYBF0,17175
-evalvault/adapters/inbound/api/routers/runs.py,sha256=Xn0Tj6sbxijdG9-x7rXFiLvKOAzdJ18QSZR0j5VEMYQ,33561
+evalvault/adapters/inbound/api/routers/runs.py,sha256=KyIar-5RJemO7i3dvRLM1IeKWVF57tZXrrixKpGOg7M,38029
 evalvault/adapters/inbound/cli/__init__.py,sha256=a42flC5NK-VfbdbBrE49IrUL5zAyKdXZYJVM6E3NTE0,675
 evalvault/adapters/inbound/cli/app.py,sha256=ytNgHRg9ZTAl33AkB1wIL8RKfQ_Cf8fsy0gSsLTs7Ew,1603
-evalvault/adapters/inbound/cli/commands/__init__.py,sha256=ciIHbHgP0gtasVi4l5cHjVojERrb-uipga_E0EwCrqM,3431
+evalvault/adapters/inbound/cli/commands/__init__.py,sha256=cNPPhsudTQWdlh_OJm9mU8LGBnJLGMswJBcIV9MAlkI,3530
 evalvault/adapters/inbound/cli/commands/agent.py,sha256=YlOYMEzzS1aSKDKD_a7UK3St18X6GXGkdTatrzyd8Zc,7555
 evalvault/adapters/inbound/cli/commands/analyze.py,sha256=aMi1BEDOX3yhN-ppBftDssPQLB5TdzIfpx9U7CZEgWo,48932
 evalvault/adapters/inbound/cli/commands/api.py,sha256=YdbJ_-QEajnFcjTa7P2heLMjFKpeQ4nWP_p-HvfYkEo,1943
 evalvault/adapters/inbound/cli/commands/benchmark.py,sha256=RZ4nRTF7d6hDZug-Pw8dGcFEyWdOKclwqkvS-gN4VWo,41097
+evalvault/adapters/inbound/cli/commands/calibrate.py,sha256=-UnT0LQH40U5lzMLqMJ7DOTLa3mt5P_fJL2XzqIkvu4,4223
 evalvault/adapters/inbound/cli/commands/config.py,sha256=Mv9IQHBFHZ3I2stUzHDgLDn-Znt_Awdy3j-sk5ruUmw,6069
 evalvault/adapters/inbound/cli/commands/debug.py,sha256=KU-hL1gLhpjV2ZybDQgGMwRfm-hCynkrqY4UzETfL9k,2234
 evalvault/adapters/inbound/cli/commands/domain.py,sha256=dL9iqBlnr5mDeS1unXW6uxE0qp6yfnxj-ls6k3EenwI,27279
@@ -74,8 +75,8 @@ evalvault/adapters/outbound/analysis/model_analyzer_module.py,sha256=28rHdXBXYIF
 evalvault/adapters/outbound/analysis/morpheme_analyzer_module.py,sha256=Hrh4mluMsOhQHPrliD2w0FVKokJpfikXOFKT6sNwk74,4158
 evalvault/adapters/outbound/analysis/morpheme_quality_checker_module.py,sha256=_uRKDXdwGbfYduf_3XT77vF8X3-_zW3stHYc3HKYQTE,2216
 evalvault/adapters/outbound/analysis/network_analyzer_module.py,sha256=ITUVnt_CI5pHy5SAESBSi004yMtiAhGFsbhC61VTezk,8475
-evalvault/adapters/outbound/analysis/nlp_adapter.py,sha256=U7verYM4XTSPLTlb2z0b7yYzTP4kkNFl5LQ91XhXu_A,27432
-evalvault/adapters/outbound/analysis/nlp_analyzer_module.py,sha256=KtoMJNa4NE-91iTEpEWIid-mciaezwmhU2xlKbYl4fg,8221
+evalvault/adapters/outbound/analysis/nlp_adapter.py,sha256=aLtF_fns-7IEtitwON2EYS_lweq_IdldFsRm47alN0Q,29561
+evalvault/adapters/outbound/analysis/nlp_analyzer_module.py,sha256=kVuG9pVMQO6OYY5zxj_w9nNQZ1-qIO0y6XcXo6lG-n0,8221
 evalvault/adapters/outbound/analysis/pattern_detector_module.py,sha256=SyCDO_VS-r-tjGh8WrW-t1GCSC9ouxirdVk4NizFPXo,1882
 evalvault/adapters/outbound/analysis/pipeline_factory.py,sha256=XvcCbKCN_otv1pGUzk0oE76RV19yFga8r6RngBvgEFo,3691
 evalvault/adapters/outbound/analysis/pipeline_helpers.py,sha256=8E8IrYI5JvRrpnjxe0DS7srbPzB0XAxxXhLLYgfwsgU,5756
@@ -164,12 +165,12 @@ evalvault/adapters/outbound/report/dashboard_generator.py,sha256=Dcu18NTK4lS8XNK
 evalvault/adapters/outbound/report/llm_report_generator.py,sha256=sp2YRCmPOhn08vb8Bq_ayo-ZjgyBBxRhzRFvzlaDhsA,24063
 evalvault/adapters/outbound/report/markdown_adapter.py,sha256=5PS72h_qe4ZtYs-umhX5TqQL2k5SuDaCUc6rRw9AKRw,16761
 evalvault/adapters/outbound/storage/__init__.py,sha256=n5R6thAPTx1leSwv6od6nBWcLWFa-UYD6cOLzN89T8I,614
-evalvault/adapters/outbound/storage/base_sql.py,sha256=pPKlT2P7fCTMxCAIi0tzYr0d5rkroPTB4dHQNDDRxsI,18167
+evalvault/adapters/outbound/storage/base_sql.py,sha256=Og-YRWHsCFQP2vnyvsgfWr4C2_ZE89ZmPXcPLiHeggU,21976
 evalvault/adapters/outbound/storage/benchmark_storage_adapter.py,sha256=Qgf9xSSIkYQRpG4uLzcUdoYO9LTQDQ4tFRkkMYer-WA,9803
-evalvault/adapters/outbound/storage/postgres_adapter.py,sha256=ro3DrE2e4l2jimoPidcmY0xBufhqi5M6_4VF-Ta0e-I,38133
-evalvault/adapters/outbound/storage/postgres_schema.sql,sha256=BLDQ7ynzS9Aw6NxN3efALpG3eN1ZfsmBcEeWugHwF98,7075
-evalvault/adapters/outbound/storage/schema.sql,sha256=R9Y3j76qR3_UbbeX1olhHs1hbsLjS3YfiSFyUgqTM28,10057
-evalvault/adapters/outbound/storage/sqlite_adapter.py,sha256=cldMzXW9_0jdAC0YkwhFznleKX6yF4RkFNpmYc8_lZQ,47446
+evalvault/adapters/outbound/storage/postgres_adapter.py,sha256=IaijoeCIRi7JO2d5yfgfmF-ejobOnU7Izlx332mSUP8,39020
+evalvault/adapters/outbound/storage/postgres_schema.sql,sha256=aAfgwxWEqCBGGpn_QRD_BbzXR2Q-9cd9GMsCbFeohNY,7632
+evalvault/adapters/outbound/storage/schema.sql,sha256=LknvBvNVLvkW7c_hHTLHrxSf4TZApzbRyAk1ctuROUc,10608
+evalvault/adapters/outbound/storage/sqlite_adapter.py,sha256=SKZ9IZjchi7w89WNkZ6aTelAzaV0MqUC7cexrkndTNY,48555
 evalvault/adapters/outbound/tracer/__init__.py,sha256=xrvQQuAvF_UI02mKLMV7GTrG3zn836n5zwCRrrmhq_U,1054
 evalvault/adapters/outbound/tracer/open_rag_log_handler.py,sha256=aq96FIWD-bBaSkq-bygWhQArC9LWghSwi-S03Mga0mI,2827
 evalvault/adapters/outbound/tracer/open_rag_trace_adapter.py,sha256=P-4PN1UweITXu5uN3LJVCEL3wRwiExzhgs3y2GN78xM,4784
@@ -190,7 +191,7 @@ evalvault/config/phoenix_support.py,sha256=e6RPWd6Qb7KU6Q8pLaYTpJGWULtvEEU6B0xHW
 evalvault/config/settings.py,sha256=T92GShlYKDaVinwbsbWX2DmNfm91Cvcvh8Te8pNOTsw,12875
 evalvault/config/playbooks/improvement_playbook.yaml,sha256=9F9WVVCydFfz6zUuGYzZ4PKdW1LLtcBKVF36T7xT764,26965
 evalvault/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalvault/domain/entities/__init__.py,sha256=C63BX4ytkh0FCEfNFIy0MDY2tcYxp9G430IdyBxqqAk,2794
+evalvault/domain/entities/__init__.py,sha256=CZU7VfTq2vart_j2pLemOX_TtSKzmpzB151pW-jSinw,3097
 evalvault/domain/entities/analysis.py,sha256=gcMtumC66g-AIqb2LgfMpm5BMzwJhJkjg-zuybNoJCM,15208
 evalvault/domain/entities/analysis_pipeline.py,sha256=hD9rFHMa4rUq0InRkSKhh6HQ9ZeNYAHKADzs-kWRP04,16845
 evalvault/domain/entities/benchmark.py,sha256=CVbz_eW7Y9eM7wG7xA_xmldTIs72csdoTmu3E0NKoMU,18475
@@ -198,6 +199,7 @@ evalvault/domain/entities/benchmark_run.py,sha256=2ZJOq5Ny_pfvRKM7E4RuIKxfxvoYK-
 evalvault/domain/entities/dataset.py,sha256=WsC_5ivGluy-o2nXxLGmoC8DYl5UafVSo2hSowb3rvs,1886
 evalvault/domain/entities/debug.py,sha256=r92lgvOpq2svw70syJIo78muRAvrSn5h1JByH_Hvz-s,1493
 evalvault/domain/entities/experiment.py,sha256=oWjbu0IJZ6oIRcnA-8ppeJDgp57Tv8ZjQ3UOZ0X9KJ8,2576
+evalvault/domain/entities/feedback.py,sha256=xiaZaUQhyuxyW_i2scXt8eKZshMC6tXe3981e-uukw8,1604
 evalvault/domain/entities/improvement.py,sha256=WHI7q1jXRxkuHhBWOrpk8UdLaH0UwjZVjRIDsqVDyZo,19322
 evalvault/domain/entities/kg.py,sha256=8awN1M4vxAGQZk_ZG8i2CXKTizQ8FA1VCLhUWHZq0U8,3001
 evalvault/domain/entities/memory.py,sha256=bfS75q8K8_jNrB7IYh4mjP8Lkyj-I0TVsmHCP0FuICw,8423
@@ -253,6 +255,7 @@ evalvault/domain/services/prompt_status.py,sha256=r1dFLGz4SfRxXaxsULQsr0-HpJkG9Y
 evalvault/domain/services/ragas_prompt_overrides.py,sha256=4BecYE2KrreUBbIM3ssP9WzHcK_wRc8jW7CE_k58QOU,1412
 evalvault/domain/services/retrieval_metrics.py,sha256=dtrQPLMrXSyWLcgF8EGcLNFwzwA59WDzEh41JRToHAY,2980
 evalvault/domain/services/retriever_context.py,sha256=ySQ-GuadiggS0LVAib4AxA_0JpasYz4S9hbjau0eyIA,6482
+evalvault/domain/services/satisfaction_calibration_service.py,sha256=H7Z8opOyPHRO5qVIw-XDsNhIwdCteAS9_a3BTlfIqHg,11906
 evalvault/domain/services/stage_event_builder.py,sha256=ScTgyeRiH7z_rnNI_2p-i9szVRIRwUxGSJvpEj3zto4,9645
 evalvault/domain/services/stage_metric_guide_service.py,sha256=_JdRsBRWirO24qYFlh6hG-dkoWlX6_XWEYKf_uUlKIQ,8807
 evalvault/domain/services/stage_metric_service.py,sha256=KukIWWhWVOtclrET6uyWJ17jG76LfkKiqrUrDIDJ3gw,15327
@@ -286,15 +289,15 @@ evalvault/ports/outbound/nlp_analysis_port.py,sha256=QDJHAsSpynTenuaKp78t1s--U03
 evalvault/ports/outbound/relation_augmenter_port.py,sha256=cMcHQnmK111WzZr50vYr7affeHhOtpFZxPARwkg9xbk,651
 evalvault/ports/outbound/report_port.py,sha256=wgReSYL4SupXIoALFh0QFWfX2kzPftXpWTvGLCMd2B8,1315
 evalvault/ports/outbound/stage_storage_port.py,sha256=Nlf9upsXxgCABQB5cJdpLQYsoZNiGRAU5zE5D-Ptp2I,1201
-evalvault/ports/outbound/storage_port.py,sha256=V1ZvV_M3ztQtAHabRn2dfcCLpgRofWB_CUp9pAuocHU,4505
+evalvault/ports/outbound/storage_port.py,sha256=d9f8bvAtPA2aytKrHvrfrWGOmaQSepLn23Bd_52QSbI,4862
 evalvault/ports/outbound/tracer_port.py,sha256=kTqJCUIJHnvvDzMxxGhHSfiz8_Q4CZ0WSPvIUVVOcyw,623
 evalvault/ports/outbound/tracker_port.py,sha256=05LA3AWnuE1XmGQC16Zle9i2sEV3q69Nt8ZUye_w1_Y,2532
 evalvault/reports/__init__.py,sha256=Bb1X4871msAN8I6PM6nKGED3psPwZt88hXZBAOdH06Y,113
 evalvault/reports/release_notes.py,sha256=pZj0PBFT-4F_Ty-Kv5P69BuoOnmTCn4kznDcORFJd0w,4011
 evalvault/scripts/__init__.py,sha256=NwEeIFQbkX4ml2R_PhtIoNtArDSX_suuoymgG_7Kwso,89
 evalvault/scripts/regression_runner.py,sha256=SxZori5BZ8jVQ057Mf5V5FPgIVDccrV5oRONmnhuk8w,8438
-evalvault-1.61.0.dist-info/METADATA,sha256=c8UQ9kRQCONSqFbgSj17CEE70xhNbFs1GAPBg1jVH9A,14058
-evalvault-1.61.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evalvault-1.61.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
-evalvault-1.61.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
-evalvault-1.61.0.dist-info/RECORD,,
+evalvault-1.62.0.dist-info/METADATA,sha256=2Nt0heOPN0il1jF3de3EAJtq9CQawjyaa27GQAGncmk,14155
+evalvault-1.62.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evalvault-1.62.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
+evalvault-1.62.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
+evalvault-1.62.0.dist-info/RECORD,,