evalvault 1.62.1__py3-none-any.whl → 1.63.1__py3-none-any.whl

evalvault/adapters/outbound/storage/postgres_adapter.py

@@ -27,6 +27,7 @@ from evalvault.domain.entities.analysis import (
 )
 from evalvault.domain.entities.experiment import Experiment
 from evalvault.domain.entities.prompt import Prompt, PromptSet, PromptSetBundle, PromptSetItem
+from evalvault.domain.entities.stage import StageEvent, StageMetric
 
 
 class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
@@ -823,6 +824,56 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
             )
             conn.commit()
 
+    def save_analysis_report(
+        self,
+        *,
+        report_id: str | None,
+        run_id: str | None,
+        experiment_id: str | None,
+        report_type: str,
+        format: str,
+        content: str | None,
+        metadata: dict[str, Any] | None = None,
+        created_at: str | None = None,
+    ) -> str:
+        report_id = report_id or str(uuid.uuid4())
+        if created_at is None:
+            created_at_value = datetime.now(UTC)
+        else:
+            created_at_value = (
+                datetime.fromisoformat(created_at) if isinstance(created_at, str) else created_at
+            )
+
+        with self._get_connection() as conn:
+            conn.execute(
+                """
+                INSERT INTO analysis_reports (
+                    report_id, run_id, experiment_id, report_type, format, content, metadata, created_at
+                ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                ON CONFLICT (report_id) DO UPDATE SET
+                    run_id = EXCLUDED.run_id,
+                    experiment_id = EXCLUDED.experiment_id,
+                    report_type = EXCLUDED.report_type,
+                    format = EXCLUDED.format,
+                    content = EXCLUDED.content,
+                    metadata = EXCLUDED.metadata,
+                    created_at = EXCLUDED.created_at
+                """,
+                (
+                    report_id,
+                    run_id,
+                    experiment_id,
+                    report_type,
+                    format,
+                    content,
+                    self._serialize_pipeline_json(metadata),
+                    created_at_value,
+                ),
+            )
+            conn.commit()
+
+        return report_id
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """
@@ -837,6 +888,164 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
             rows = conn.execute(query, (limit,)).fetchall()
         return [self._deserialize_pipeline_result(row, include_payload=False) for row in rows]
 
+    def save_stage_events(self, events: list[StageEvent]) -> int:
+        if not events:
+            return 0
+        with self._get_connection() as conn:
+            conn.executemany(
+                """
+                INSERT INTO stage_events (
+                    run_id, stage_id, parent_stage_id, stage_type, stage_name,
+                    status, attempt, started_at, finished_at, duration_ms,
+                    input_ref, output_ref, attributes, metadata, trace_id, span_id
+                ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                ON CONFLICT (run_id, stage_id) DO UPDATE SET
+                    parent_stage_id = EXCLUDED.parent_stage_id,
+                    stage_type = EXCLUDED.stage_type,
+                    stage_name = EXCLUDED.stage_name,
+                    status = EXCLUDED.status,
+                    attempt = EXCLUDED.attempt,
+                    started_at = EXCLUDED.started_at,
+                    finished_at = EXCLUDED.finished_at,
+                    duration_ms = EXCLUDED.duration_ms,
+                    input_ref = EXCLUDED.input_ref,
+                    output_ref = EXCLUDED.output_ref,
+                    attributes = EXCLUDED.attributes,
+                    metadata = EXCLUDED.metadata,
+                    trace_id = EXCLUDED.trace_id,
+                    span_id = EXCLUDED.span_id
+                """,
+                [self._serialize_stage_event(event) for event in events],
+            )
+            conn.commit()
+        return len(events)
+
+    def list_stage_events(
+        self,
+        run_id: str,
+        *,
+        stage_type: str | None = None,
+    ) -> list[StageEvent]:
+        query = (
+            "SELECT run_id, stage_id, parent_stage_id, stage_type, stage_name, status, attempt, "
+            "started_at, finished_at, duration_ms, input_ref, output_ref, attributes, metadata, "
+            "trace_id, span_id FROM stage_events WHERE run_id = %s"
+        )
+        params: list[Any] = [run_id]
+        if stage_type:
+            query += " AND stage_type = %s"
+            params.append(stage_type)
+        query += " ORDER BY id"
+        with self._get_connection() as conn:
+            rows = conn.execute(query, params).fetchall()
+        return [self._deserialize_stage_event(row) for row in rows]
+
+    def save_stage_metrics(self, metrics: list[StageMetric]) -> int:
+        if not metrics:
+            return 0
+        with self._get_connection() as conn:
+            conn.executemany(
+                """
+                INSERT INTO stage_metrics (
+                    run_id, stage_id, metric_name, score, threshold, evidence
+                ) VALUES (%s, %s, %s, %s, %s, %s)
+                """,
+                [self._serialize_stage_metric(metric) for metric in metrics],
+            )
+            conn.commit()
+        return len(metrics)
+
+    def list_stage_metrics(
+        self,
+        run_id: str,
+        *,
+        stage_id: str | None = None,
+        metric_name: str | None = None,
+    ) -> list[StageMetric]:
+        query = (
+            "SELECT run_id, stage_id, metric_name, score, threshold, evidence "
+            "FROM stage_metrics WHERE run_id = %s"
+        )
+        params: list[Any] = [run_id]
+        if stage_id:
+            query += " AND stage_id = %s"
+            params.append(stage_id)
+        if metric_name:
+            query += " AND metric_name = %s"
+            params.append(metric_name)
+        query += " ORDER BY id"
+        with self._get_connection() as conn:
+            rows = conn.execute(query, params).fetchall()
+        return [self._deserialize_stage_metric(row) for row in rows]
+
+    def _serialize_stage_event(self, event: StageEvent) -> tuple[Any, ...]:
+        return (
+            event.run_id,
+            event.stage_id,
+            event.parent_stage_id,
+            event.stage_type,
+            event.stage_name,
+            event.status,
+            event.attempt,
+            event.started_at,
+            event.finished_at,
+            event.duration_ms,
+            self._serialize_payload_ref(event.input_ref),
+            self._serialize_payload_ref(event.output_ref),
+            self._serialize_pipeline_json(event.attributes),
+            self._serialize_pipeline_json(event.metadata),
+            event.trace_id,
+            event.span_id,
+        )
+
+    def _serialize_stage_metric(self, metric: StageMetric) -> tuple[Any, ...]:
+        return (
+            metric.run_id,
+            metric.stage_id,
+            metric.metric_name,
+            metric.score,
+            metric.threshold,
+            self._serialize_pipeline_json(metric.evidence),
+        )
+
+    def _serialize_payload_ref(self, ref: Any) -> str | None:
+        if ref is None:
+            return None
+        payload = ref.to_dict() if hasattr(ref, "to_dict") else ref
+        return self._serialize_pipeline_json(payload)
+
+    def _deserialize_stage_event(self, row: dict[str, Any]) -> StageEvent:
+        payload = {
+            "run_id": row.get("run_id"),
+            "stage_id": row.get("stage_id"),
+            "parent_stage_id": row.get("parent_stage_id"),
+            "stage_type": row.get("stage_type"),
+            "stage_name": row.get("stage_name"),
+            "status": row.get("status"),
+            "attempt": row.get("attempt"),
+            "started_at": row.get("started_at"),
+            "finished_at": row.get("finished_at"),
+            "duration_ms": row.get("duration_ms"),
+            "input_ref": self._ensure_json(row.get("input_ref")),
+            "output_ref": self._ensure_json(row.get("output_ref")),
+            "attributes": self._ensure_json(row.get("attributes")) or {},
+            "metadata": self._ensure_json(row.get("metadata")) or {},
+            "trace_id": row.get("trace_id"),
+            "span_id": row.get("span_id"),
+        }
+        return StageEvent.from_dict(payload)
+
+    def _deserialize_stage_metric(self, row: dict[str, Any]) -> StageMetric:
+        payload = {
+            "run_id": row.get("run_id"),
+            "stage_id": row.get("stage_id"),
+            "metric_name": row.get("metric_name"),
+            "score": row.get("score"),
+            "threshold": row.get("threshold"),
+            "evidence": self._ensure_json(row.get("evidence")),
+        }
+        return StageMetric.from_dict(payload)
+
     def get_pipeline_result(self, result_id: str) -> dict[str, Any]:
         """저장된 파이프라인 분석 결과를 조회합니다."""
         with self._get_connection() as conn:

evalvault/adapters/outbound/storage/postgres_schema.sql

@@ -206,3 +206,41 @@ CREATE INDEX IF NOT EXISTS idx_pipeline_results_intent
     ON pipeline_results(intent);
 CREATE INDEX IF NOT EXISTS idx_pipeline_results_run_id
     ON pipeline_results(run_id);
+
+CREATE TABLE IF NOT EXISTS stage_events (
+    id BIGSERIAL PRIMARY KEY,
+    run_id UUID NOT NULL REFERENCES evaluation_runs(run_id) ON DELETE CASCADE,
+    stage_id TEXT NOT NULL,
+    parent_stage_id TEXT,
+    stage_type TEXT NOT NULL,
+    stage_name TEXT,
+    status TEXT,
+    attempt INTEGER DEFAULT 1,
+    started_at TIMESTAMP WITH TIME ZONE,
+    finished_at TIMESTAMP WITH TIME ZONE,
+    duration_ms DOUBLE PRECISION,
+    input_ref JSONB,
+    output_ref JSONB,
+    attributes JSONB,
+    metadata JSONB,
+    trace_id TEXT,
+    span_id TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_stage_events_run_stage_id
+    ON stage_events(run_id, stage_id);
+CREATE INDEX IF NOT EXISTS idx_stage_events_run_id ON stage_events(run_id);
+CREATE INDEX IF NOT EXISTS idx_stage_events_stage_type ON stage_events(stage_type);
+
+CREATE TABLE IF NOT EXISTS stage_metrics (
+    id BIGSERIAL PRIMARY KEY,
+    run_id UUID NOT NULL REFERENCES evaluation_runs(run_id) ON DELETE CASCADE,
+    stage_id TEXT NOT NULL,
+    metric_name TEXT NOT NULL,
+    score DOUBLE PRECISION NOT NULL,
+    threshold DOUBLE PRECISION,
+    evidence JSONB
+);
+
+CREATE INDEX IF NOT EXISTS idx_stage_metrics_run_id ON stage_metrics(run_id);
+CREATE INDEX IF NOT EXISTS idx_stage_metrics_stage_id ON stage_metrics(stage_id);

evalvault/adapters/outbound/storage/sqlite_adapter.py

@@ -4,10 +4,12 @@ from __future__ import annotations
 
 import json
 import sqlite3
+import uuid
+from contextlib import AbstractContextManager, closing
 from dataclasses import asdict
 from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 from evalvault.adapters.outbound.storage.base_sql import BaseSQLStorageAdapter, SQLQueries
 from evalvault.domain.entities.analysis import (
@@ -61,15 +63,20 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         conn.commit()
         conn.close()
 
-    def _connect(self) -> sqlite3.Connection:
+    def _connect(self) -> Any:
         """Create a DB-API connection with the expected options."""
         conn = sqlite3.connect(self.db_path)
         conn.row_factory = sqlite3.Row
         conn.execute("PRAGMA foreign_keys = ON")
         return conn
 
-    def _apply_migrations(self, conn: sqlite3.Connection) -> None:
+    def _get_connection(self) -> AbstractContextManager[sqlite3.Connection]:
+        conn = self._connect()
+        return closing(cast(sqlite3.Connection, conn))
+
+    def _apply_migrations(self, conn: Any) -> None:
         """Apply schema migrations for legacy databases."""
+        conn = cast(Any, conn)
         cursor = conn.execute("PRAGMA table_info(evaluation_runs)")
         columns = {row[1] for row in cursor.fetchall()}
         if "metadata" not in columns:
@@ -180,6 +187,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
     def save_prompt_set(self, bundle: PromptSetBundle) -> None:
         """Save prompt set, prompts, and join items."""
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             cursor.execute(
                 """
@@ -241,6 +249,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
     def link_prompt_set_to_run(self, run_id: str, prompt_set_id: str) -> None:
         """Attach a prompt set to a run."""
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             conn.execute(
                 """
                 INSERT OR REPLACE INTO run_prompt_sets (
@@ -258,6 +267,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
     def get_prompt_set(self, prompt_set_id: str) -> PromptSetBundle:
         """Load a prompt set bundle by ID."""
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.execute(
                 """
                 SELECT prompt_set_id, name, description, metadata, created_at
@@ -270,12 +280,17 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             if not row:
                 raise KeyError(f"Prompt set not found: {prompt_set_id}")
 
+            created_at = self._deserialize_datetime(row["created_at"])
+            if created_at is None:
+                created_at = datetime.now()
+            assert created_at is not None
+
             prompt_set = PromptSet(
                 prompt_set_id=row["prompt_set_id"],
                 name=row["name"],
                 description=row["description"] or "",
                 metadata=json.loads(row["metadata"]) if row["metadata"] else {},
-                created_at=self._deserialize_datetime(row["created_at"]),
+                created_at=created_at,
             )
 
             item_rows = conn.execute(
@@ -313,6 +328,11 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
                     tuple(prompt_ids),
                 ).fetchall()
                 for prompt_row in prompt_rows:
+                    created_at = self._deserialize_datetime(prompt_row["created_at"])
+                    if created_at is None:
+                        created_at = datetime.now()
+                    assert created_at is not None
+
                     prompts.append(
                         Prompt(
                             prompt_id=prompt_row["prompt_id"],
@@ -325,7 +345,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
                             metadata=json.loads(prompt_row["metadata"])
                             if prompt_row["metadata"]
                             else {},
-                            created_at=self._deserialize_datetime(prompt_row["created_at"]),
+                            created_at=created_at,
                         )
                     )
 
@@ -334,6 +354,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
     def get_prompt_set_for_run(self, run_id: str) -> PromptSetBundle | None:
         """Load the prompt set linked to a run."""
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             row = conn.execute(
                 """
                 SELECT prompt_set_id
@@ -360,6 +381,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             저장된 experiment의 ID
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             # Insert or replace experiment
             cursor.execute(
@@ -425,6 +447,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             KeyError: 실험을 찾을 수 없는 경우
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             # Fetch experiment
             cursor.execute(
@@ -503,6 +526,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             Experiment 객체 리스트
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             query = "SELECT experiment_id FROM experiments WHERE 1=1"
             params = []
@@ -539,6 +563,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             저장된 analysis의 ID
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             # Serialize analysis to JSON
             result_data = self._serialize_analysis(analysis)
@@ -574,6 +599,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             KeyError: 분석을 찾을 수 없는 경우
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             cursor.execute(
                 """
@@ -612,6 +638,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             StatisticalAnalysis 리스트
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             query = """
                 SELECT analysis_id, run_id, analysis_type, result_data, created_at
@@ -650,6 +677,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             삭제 성공 여부
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             cursor.execute(
                 "DELETE FROM analysis_results WHERE analysis_id = ?",
@@ -726,6 +754,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         import uuid
 
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             analysis_id = f"nlp-{analysis.run_id}-{uuid.uuid4().hex[:8]}"
             result_data = self._serialize_nlp_analysis(analysis)
@@ -761,6 +790,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             KeyError: 분석을 찾을 수 없는 경우
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             cursor.execute(
                 """
@@ -788,6 +818,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             NLPAnalysis 또는 None (분석 결과가 없는 경우)
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.cursor()
             cursor.execute(
                 """
@@ -891,6 +922,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         is_complete = 1 if record.get("is_complete", False) else 0
 
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             conn.execute(
                 """
                 INSERT OR REPLACE INTO pipeline_results (
@@ -920,6 +952,44 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             )
             conn.commit()
 
+    def save_analysis_report(
+        self,
+        *,
+        report_id: str | None,
+        run_id: str | None,
+        experiment_id: str | None,
+        report_type: str,
+        format: str,
+        content: str | None,
+        metadata: dict[str, Any] | None = None,
+        created_at: str | None = None,
+    ) -> str:
+        report_id = report_id or str(uuid.uuid4())
+        created_at = created_at or datetime.now().isoformat()
+
+        with self._get_connection() as conn:
+            conn = cast(Any, conn)
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO analysis_reports (
+                    report_id, run_id, experiment_id, report_type, format, content, metadata, created_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    report_id,
+                    run_id,
+                    experiment_id,
+                    report_type,
+                    format,
+                    content,
+                    self._serialize_json(metadata),
+                    created_at,
+                ),
+            )
+            conn.commit()
+
+        return report_id
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """
@@ -931,12 +1001,14 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             LIMIT ?
         """
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             rows = conn.execute(query, (limit,)).fetchall()
         return [self._deserialize_pipeline_result(row, include_payload=False) for row in rows]
 
     def get_pipeline_result(self, result_id: str) -> dict[str, Any]:
         """저장된 파이프라인 분석 결과를 조회합니다."""
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             row = conn.execute(
                 """
             SELECT result_id, intent, query, run_id, pipeline_id,
@@ -983,6 +1055,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
     def save_stage_event(self, event: StageEvent) -> str:
         """단계 이벤트를 저장합니다."""
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             conn.execute(
                 """
                 INSERT OR REPLACE INTO stage_events (
@@ -1001,6 +1074,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         if not events:
             return 0
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             conn.executemany(
                 """
                 INSERT OR REPLACE INTO stage_events (
@@ -1034,6 +1108,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             params.append(stage_type)
         query += " ORDER BY id"
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.execute(query, params)
             rows = cursor.fetchall()
         return [self._deserialize_stage_event(row) for row in rows]
@@ -1043,6 +1118,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         if not metrics:
             return 0
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             conn.executemany(
                 """
                 INSERT INTO stage_metrics (
@@ -1076,6 +1152,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             params.append(metric_name)
         query += " ORDER BY id"
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.execute(query, params)
             rows = cursor.fetchall()
         return [self._deserialize_stage_metric(row) for row in rows]
@@ -1155,6 +1232,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
 
     def save_benchmark_run(self, run: BenchmarkRun) -> str:
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             task_scores_json = json.dumps(
                 [
                     {
@@ -1208,6 +1286,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         )
 
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.execute(
                 """
                 SELECT run_id, benchmark_type, model_name, backend, tasks,
@@ -1288,6 +1367,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
         params.append(limit)
 
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.execute(query, params)
             run_ids = [row["run_id"] for row in cursor.fetchall()]
 
@@ -1295,6 +1375,7 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
 
     def delete_benchmark_run(self, run_id: str) -> bool:
         with self._get_connection() as conn:
+            conn = cast(Any, conn)
             cursor = conn.execute(
                 "DELETE FROM benchmark_runs WHERE run_id = ?",
                 (run_id,),

evalvault/debug_ragas.py

@@ -5,6 +5,9 @@ from unittest.mock import MagicMock
 from ragas import SingleTurnSample
 from ragas.metrics import AnswerRelevancy, Faithfulness
 
+from evalvault.adapters.outbound.llm import SettingsLLMFactory
+from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
+from evalvault.config.settings import Settings
 from evalvault.domain.services.evaluator import RagasEvaluator
 from evalvault.ports.outbound.llm_port import LLMPort
 
@@ -29,7 +32,10 @@ async def debug_ragas():
     # Actually, Ragas metrics execute validation on `score` or `ascore`.
     # Failing at LLM call (e.g. no auth) is different from failing at argument passing.
 
-    evaluator = RagasEvaluator()
+    settings = Settings()
+    llm_factory = SettingsLLMFactory(settings)
+    korean_toolkit = try_create_korean_toolkit()
+    evaluator = RagasEvaluator(korean_toolkit=korean_toolkit, llm_factory=llm_factory)
 
     # Create sample similar to what we observed
     sample = SingleTurnSample(

evalvault/debug_ragas_real.py

@@ -1,7 +1,9 @@
 import asyncio
 import logging
 
+from evalvault.adapters.outbound.llm import SettingsLLMFactory
 from evalvault.adapters.outbound.llm.openai_adapter import OpenAIAdapter
+from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
 from evalvault.config.settings import get_settings
 from evalvault.domain.entities.dataset import Dataset, TestCase
 from evalvault.domain.services.evaluator import RagasEvaluator
@@ -25,7 +27,9 @@ async def debug_ragas_real():
     print(f"Using Model: {settings.openai_model}")
 
     llm = OpenAIAdapter(settings)
-    evaluator = RagasEvaluator()
+    llm_factory = SettingsLLMFactory(settings)
+    korean_toolkit = try_create_korean_toolkit()
+    evaluator = RagasEvaluator(korean_toolkit=korean_toolkit, llm_factory=llm_factory)
 
     # Manual Dataset
     test_case = TestCase(

evalvault/domain/entities/__init__.py

@@ -37,6 +37,12 @@ from evalvault.domain.entities.improvement import (
 from evalvault.domain.entities.kg import EntityModel, RelationModel
 from evalvault.domain.entities.method import MethodInput, MethodInputDataset, MethodOutput
 from evalvault.domain.entities.prompt import Prompt, PromptSet, PromptSetBundle, PromptSetItem
+from evalvault.domain.entities.prompt_suggestion import (
+    PromptCandidate,
+    PromptCandidateSampleScore,
+    PromptCandidateScore,
+    PromptSuggestionResult,
+)
 from evalvault.domain.entities.rag_trace import (
     GenerationData,
     RAGTraceData,
@@ -110,6 +116,10 @@ __all__ = [
     "PromptSet",
     "PromptSetBundle",
     "PromptSetItem",
+    "PromptCandidate",
+    "PromptCandidateSampleScore",
+    "PromptCandidateScore",
+    "PromptSuggestionResult",
     # RAG Trace
     "GenerationData",
     "RAGTraceData",