evalvault 1.70.1__py3-none-any.whl → 1.71.0__py3-none-any.whl

evalvault/adapters/outbound/storage/base_sql.py

@@ -14,6 +14,9 @@ from evalvault.domain.entities import (
     EvaluationRun,
     FeedbackSummary,
     MetricScore,
+    MultiTurnConversationRecord,
+    MultiTurnRunRecord,
+    MultiTurnTurnResult,
     RunClusterMap,
     RunClusterMapInfo,
     SatisfactionFeedback,
@@ -72,6 +75,45 @@ class SQLQueries:
         ) VALUES ({values})
         """
 
+    def insert_multiturn_run(self) -> str:
+        values = self._values(12)
+        return f"""
+        INSERT INTO multiturn_runs (
+            run_id, dataset_name, dataset_version, model_name,
+            started_at, finished_at, conversation_count, turn_count,
+            metrics_evaluated, drift_threshold, summary, metadata
+        ) VALUES ({values})
+        """
+
+    def insert_multiturn_conversation(self) -> str:
+        values = self._values(7)
+        return f"""
+        INSERT INTO multiturn_conversations (
+            run_id, conversation_id, turn_count, drift_score, drift_threshold,
+            drift_detected, summary
+        ) VALUES ({values})
+        """
+
+    def insert_multiturn_turn(self) -> str:
+        values = self._values(8)
+        query = f"""
+        INSERT INTO multiturn_turn_results (
+            run_id, conversation_id, turn_id, turn_index, role,
+            passed, latency_ms, metadata
+        ) VALUES ({values})
+        """
+        if self._test_case_returning:
+            query = f"{query.strip()} {self._test_case_returning}"
+        return query
+
+    def insert_multiturn_metric_score(self) -> str:
+        values = self._values(4)
+        return f"""
+        INSERT INTO multiturn_metric_scores (
+            turn_result_id, metric_name, score, threshold
+        ) VALUES ({values})
+        """
+
     def insert_cluster_map(self) -> str:
         values = self._values(7)
         return f"""
@@ -127,6 +169,41 @@ class SQLQueries:
         ORDER BY id
         """
 
+    def select_multiturn_run(self) -> str:
+        return f"""
+        SELECT run_id, dataset_name, dataset_version, model_name,
+               started_at, finished_at, conversation_count, turn_count,
+               metrics_evaluated, drift_threshold, summary, metadata, created_at
+        FROM multiturn_runs
+        WHERE run_id = {self.placeholder}
+        """
+
+    def select_multiturn_conversations(self) -> str:
+        return f"""
+        SELECT run_id, conversation_id, turn_count, drift_score, drift_threshold,
+               drift_detected, summary
+        FROM multiturn_conversations
+        WHERE run_id = {self.placeholder}
+        ORDER BY id
+        """
+
+    def select_multiturn_turn_results(self) -> str:
+        return f"""
+        SELECT id, run_id, conversation_id, turn_id, turn_index, role,
+               passed, latency_ms, metadata
+        FROM multiturn_turn_results
+        WHERE run_id = {self.placeholder}
+        ORDER BY id
+        """
+
+    def select_multiturn_metric_scores(self) -> str:
+        return f"""
+        SELECT turn_result_id, metric_name, score, threshold
+        FROM multiturn_metric_scores
+        WHERE turn_result_id = {self.placeholder}
+        ORDER BY id
+        """
+
     def select_cluster_map(self) -> str:
         return f"""
         SELECT test_case_id, cluster_id, source, map_id, created_at, metadata
@@ -223,6 +300,48 @@ class BaseSQLStorageAdapter(ABC):
             conn.commit()
             return run.run_id
 
+    def save_multiturn_run(
+        self,
+        run: MultiTurnRunRecord,
+        conversations: list[MultiTurnConversationRecord],
+        turn_results: list[MultiTurnTurnResult],
+        *,
+        metric_thresholds: dict[str, float] | None = None,
+    ) -> str:
+        with self._get_connection() as conn:
+            self._execute(
+                conn, self.queries.insert_multiturn_run(), self._multiturn_run_params(run)
+            )
+
+            for conversation in conversations:
+                self._execute(
+                    conn,
+                    self.queries.insert_multiturn_conversation(),
+                    self._multiturn_conversation_params(conversation),
+                )
+
+            for turn in turn_results:
+                cursor = self._execute(
+                    conn,
+                    self.queries.insert_multiturn_turn(),
+                    self._multiturn_turn_params(run.run_id, turn),
+                )
+                turn_result_id = self._fetch_lastrowid(cursor)
+                for metric_name, score in (turn.metrics or {}).items():
+                    threshold = None
+                    if metric_thresholds and metric_name in metric_thresholds:
+                        threshold = metric_thresholds[metric_name]
+                    self._execute(
+                        conn,
+                        self.queries.insert_multiturn_metric_score(),
+                        self._multiturn_metric_params(
+                            turn_result_id, metric_name, score, threshold
+                        ),
+                    )
+
+            conn.commit()
+            return run.run_id
+
     def _insert_test_case(self, conn, run_id: str, result: TestCaseResult) -> int:
         cursor = self._execute(
             conn,
@@ -515,6 +634,56 @@ class BaseSQLStorageAdapter(ABC):
             metric.reason,
         )
 
+    def _multiturn_run_params(self, run: MultiTurnRunRecord) -> Sequence[Any]:
+        return (
+            run.run_id,
+            run.dataset_name,
+            run.dataset_version,
+            run.model_name,
+            self._serialize_datetime(run.started_at),
+            self._serialize_datetime(run.finished_at),
+            run.conversation_count,
+            run.turn_count,
+            self._serialize_json(run.metrics_evaluated),
+            run.drift_threshold,
+            self._serialize_json(run.summary),
+            self._serialize_json(run.metadata),
+        )
+
+    def _multiturn_conversation_params(
+        self, conversation: MultiTurnConversationRecord
+    ) -> Sequence[Any]:
+        return (
+            conversation.run_id,
+            conversation.conversation_id,
+            conversation.turn_count,
+            conversation.drift_score,
+            conversation.drift_threshold,
+            int(conversation.drift_detected),
+            self._serialize_json(conversation.summary),
+        )
+
+    def _multiturn_turn_params(self, run_id: str, turn: MultiTurnTurnResult) -> Sequence[Any]:
+        return (
+            run_id,
+            turn.conversation_id,
+            turn.turn_id,
+            turn.turn_index,
+            turn.role,
+            int(turn.passed),
+            turn.latency_ms,
+            self._serialize_json(turn.metadata),
+        )
+
+    def _multiturn_metric_params(
+        self,
+        turn_result_id: int,
+        metric_name: str,
+        score: float,
+        threshold: float | None,
+    ) -> Sequence[Any]:
+        return (turn_result_id, metric_name, score, threshold)
+
     def _row_to_test_case(self, conn, row) -> TestCaseResult:
         result_id = row["id"]
         metrics = self._fetch_metric_scores(conn, result_id)
@@ -1132,3 +1301,125 @@ class BaseSQLStorageAdapter(ABC):
 
         workbook.save(output)
         return output
+
+    def export_multiturn_run_to_excel(self, run_id: str, output_path) -> Path:
+        from openpyxl import Workbook
+
+        output = Path(output_path)
+        output.parent.mkdir(parents=True, exist_ok=True)
+        placeholder = self.queries.placeholder
+
+        with self._get_connection() as conn:
+            run_row = self._execute(conn, self.queries.select_multiturn_run(), (run_id,)).fetchone()
+            if not run_row:
+                raise KeyError(f"Multiturn run not found: {run_id}")
+
+            run_rows = self._normalize_rows(
+                [run_row],
+                json_columns={"metrics_evaluated", "summary", "metadata"},
+            )
+
+            conversation_rows = self._execute(
+                conn, self.queries.select_multiturn_conversations(), (run_id,)
+            ).fetchall()
+            conversation_payloads = self._normalize_rows(
+                conversation_rows,
+                json_columns={"summary"},
+            )
+
+            turn_rows = self._execute(
+                conn, self.queries.select_multiturn_turn_results(), (run_id,)
+            ).fetchall()
+            turn_payloads = self._normalize_rows(
+                turn_rows,
+                json_columns={"metadata"},
+            )
+
+            metric_rows = self._execute(
+                conn,
+                (
+                    "SELECT m.turn_result_id, t.conversation_id, t.turn_id, t.turn_index, "
+                    "m.metric_name, m.score, m.threshold "
+                    "FROM multiturn_metric_scores m "
+                    "JOIN multiturn_turn_results t ON m.turn_result_id = t.id "
+                    f"WHERE t.run_id = {placeholder} ORDER BY m.id"
+                ),
+                (run_id,),
+            ).fetchall()
+            metric_payloads = self._normalize_rows(metric_rows)
+
+        sheet_order: list[tuple[str, list[dict[str, Any]], list[str]]] = [
+            (
+                "MultiTurnRun",
+                run_rows,
+                [
+                    "run_id",
+                    "dataset_name",
+                    "dataset_version",
+                    "model_name",
+                    "started_at",
+                    "finished_at",
+                    "conversation_count",
+                    "turn_count",
+                    "metrics_evaluated",
+                    "drift_threshold",
+                    "summary",
+                    "metadata",
+                    "created_at",
+                ],
+            ),
+            (
+                "MultiTurnConversations",
+                conversation_payloads,
+                [
+                    "run_id",
+                    "conversation_id",
+                    "turn_count",
+                    "drift_score",
+                    "drift_threshold",
+                    "drift_detected",
+                    "summary",
+                ],
+            ),
+            (
+                "MultiTurnTurns",
+                turn_payloads,
+                [
+                    "id",
+                    "run_id",
+                    "conversation_id",
+                    "turn_id",
+                    "turn_index",
+                    "role",
+                    "passed",
+                    "latency_ms",
+                    "metadata",
+                ],
+            ),
+            (
+                "MultiTurnTurnMetrics",
+                metric_payloads,
+                [
+                    "turn_result_id",
+                    "conversation_id",
+                    "turn_id",
+                    "turn_index",
+                    "metric_name",
+                    "score",
+                    "threshold",
+                ],
+            ),
+        ]
+
+        workbook = Workbook()
+        default_sheet = workbook.active
+        workbook.remove(default_sheet)
+
+        for sheet_name, rows, headers in sheet_order:
+            sheet = workbook.create_sheet(title=sheet_name)
+            sheet.append(headers)
+            for row in rows:
+                sheet.append([self._row_value(row, header) for header in headers])
+
+        workbook.save(output)
+        return output

evalvault/adapters/outbound/storage/postgres_adapter.py

@@ -221,6 +221,90 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
             "CREATE INDEX IF NOT EXISTS idx_feedback_test_case_id ON satisfaction_feedback(test_case_id)"
         )
 
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS multiturn_runs (
+                run_id UUID PRIMARY KEY,
+                dataset_name VARCHAR(255) NOT NULL,
+                dataset_version VARCHAR(50),
+                model_name VARCHAR(255),
+                started_at TIMESTAMP WITH TIME ZONE NOT NULL,
+                finished_at TIMESTAMP WITH TIME ZONE,
+                conversation_count INTEGER DEFAULT 0,
+                turn_count INTEGER DEFAULT 0,
+                metrics_evaluated JSONB,
+                drift_threshold DOUBLE PRECISION,
+                summary JSONB,
+                metadata JSONB,
+                created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+            )
+            """
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_runs_dataset ON multiturn_runs(dataset_name)"
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_runs_started_at ON multiturn_runs(started_at DESC)"
+        )
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS multiturn_conversations (
+                id SERIAL PRIMARY KEY,
+                run_id UUID NOT NULL REFERENCES multiturn_runs(run_id) ON DELETE CASCADE,
+                conversation_id VARCHAR(255) NOT NULL,
+                turn_count INTEGER DEFAULT 0,
+                drift_score DOUBLE PRECISION,
+                drift_threshold DOUBLE PRECISION,
+                drift_detected BOOLEAN DEFAULT FALSE,
+                summary JSONB
+            )
+            """
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_run_id ON multiturn_conversations(run_id)"
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_conv_id ON multiturn_conversations(conversation_id)"
+        )
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS multiturn_turn_results (
+                id SERIAL PRIMARY KEY,
+                run_id UUID NOT NULL REFERENCES multiturn_runs(run_id) ON DELETE CASCADE,
+                conversation_id VARCHAR(255) NOT NULL,
+                turn_id VARCHAR(255) NOT NULL,
+                turn_index INTEGER,
+                role VARCHAR(50) NOT NULL,
+                passed BOOLEAN DEFAULT FALSE,
+                latency_ms INTEGER,
+                metadata JSONB
+            )
+            """
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_turns_run_id ON multiturn_turn_results(run_id)"
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_turns_conv_id ON multiturn_turn_results(conversation_id)"
+        )
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS multiturn_metric_scores (
+                id SERIAL PRIMARY KEY,
+                turn_result_id INTEGER NOT NULL REFERENCES multiturn_turn_results(id) ON DELETE CASCADE,
+                metric_name VARCHAR(100) NOT NULL,
+                score DECIMAL(5, 4) NOT NULL,
+                threshold DECIMAL(5, 4)
+            )
+            """
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_scores_turn_id ON multiturn_metric_scores(turn_result_id)"
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_multiturn_scores_metric_name ON multiturn_metric_scores(metric_name)"
+        )
+
     # Prompt set methods
 
     def save_prompt_set(self, bundle: PromptSetBundle) -> None:
@@ -874,6 +958,52 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
 
         return report_id
 
+    def list_analysis_reports(
+        self,
+        *,
+        run_id: str,
+        report_type: str | None = None,
+        format: str | None = None,
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        clauses = ["run_id = %s"]
+        params: list[Any] = [run_id]
+        if report_type:
+            clauses.append("report_type = %s")
+            params.append(report_type)
+        if format:
+            clauses.append("format = %s")
+            params.append(format)
+        params.append(limit)
+
+        query = (
+            "SELECT report_id, run_id, experiment_id, report_type, format, content, metadata, created_at "
+            "FROM analysis_reports WHERE "
+            + " AND ".join(clauses)
+            + " ORDER BY created_at DESC LIMIT %s"
+        )
+
+        with self._get_connection() as conn:
+            rows = conn.execute(query, tuple(params)).fetchall()
+
+        reports: list[dict[str, Any]] = []
+        for row in rows:
+            reports.append(
+                {
+                    "report_id": row["report_id"],
+                    "run_id": row["run_id"],
+                    "experiment_id": row["experiment_id"],
+                    "report_type": row["report_type"],
+                    "format": row["format"],
+                    "content": row["content"],
+                    "metadata": self._deserialize_json(row["metadata"]),
+                    "created_at": row["created_at"].isoformat()
+                    if isinstance(row["created_at"], datetime)
+                    else row["created_at"],
+                }
+            )
+        return reports
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """

evalvault/adapters/outbound/storage/postgres_schema.sql

@@ -86,6 +86,66 @@ CREATE TABLE IF NOT EXISTS metric_scores (
 CREATE INDEX IF NOT EXISTS idx_scores_result_id ON metric_scores(result_id);
 CREATE INDEX IF NOT EXISTS idx_scores_name ON metric_scores(name);
 
+-- Multiturn evaluation tables
+CREATE TABLE IF NOT EXISTS multiturn_runs (
+    run_id UUID PRIMARY KEY,
+    dataset_name VARCHAR(255) NOT NULL,
+    dataset_version VARCHAR(50),
+    model_name VARCHAR(255),
+    started_at TIMESTAMP WITH TIME ZONE NOT NULL,
+    finished_at TIMESTAMP WITH TIME ZONE,
+    conversation_count INTEGER DEFAULT 0,
+    turn_count INTEGER DEFAULT 0,
+    metrics_evaluated JSONB,
+    drift_threshold DOUBLE PRECISION,
+    summary JSONB,
+    metadata JSONB,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_runs_dataset ON multiturn_runs(dataset_name);
+CREATE INDEX IF NOT EXISTS idx_multiturn_runs_started_at ON multiturn_runs(started_at DESC);
+
+CREATE TABLE IF NOT EXISTS multiturn_conversations (
+    id SERIAL PRIMARY KEY,
+    run_id UUID NOT NULL REFERENCES multiturn_runs(run_id) ON DELETE CASCADE,
+    conversation_id VARCHAR(255) NOT NULL,
+    turn_count INTEGER DEFAULT 0,
+    drift_score DOUBLE PRECISION,
+    drift_threshold DOUBLE PRECISION,
+    drift_detected BOOLEAN DEFAULT FALSE,
+    summary JSONB
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_run_id ON multiturn_conversations(run_id);
+CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_conv_id ON multiturn_conversations(conversation_id);
+
+CREATE TABLE IF NOT EXISTS multiturn_turn_results (
+    id SERIAL PRIMARY KEY,
+    run_id UUID NOT NULL REFERENCES multiturn_runs(run_id) ON DELETE CASCADE,
+    conversation_id VARCHAR(255) NOT NULL,
+    turn_id VARCHAR(255) NOT NULL,
+    turn_index INTEGER,
+    role VARCHAR(50) NOT NULL,
+    passed BOOLEAN DEFAULT FALSE,
+    latency_ms INTEGER,
+    metadata JSONB
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_turns_run_id ON multiturn_turn_results(run_id);
+CREATE INDEX IF NOT EXISTS idx_multiturn_turns_conv_id ON multiturn_turn_results(conversation_id);
+
+CREATE TABLE IF NOT EXISTS multiturn_metric_scores (
+    id SERIAL PRIMARY KEY,
+    turn_result_id INTEGER NOT NULL REFERENCES multiturn_turn_results(id) ON DELETE CASCADE,
+    metric_name VARCHAR(100) NOT NULL,
+    score DECIMAL(5, 4) NOT NULL,
+    threshold DECIMAL(5, 4)
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_scores_turn_id ON multiturn_metric_scores(turn_result_id);
+CREATE INDEX IF NOT EXISTS idx_multiturn_scores_metric_name ON multiturn_metric_scores(metric_name);
+
 -- Prompt storage tables
 CREATE TABLE IF NOT EXISTS prompts (
     prompt_id UUID PRIMARY KEY,

evalvault/adapters/outbound/storage/schema.sql

@@ -90,6 +90,69 @@ CREATE TABLE IF NOT EXISTS metric_scores (
 CREATE INDEX IF NOT EXISTS idx_scores_result_id ON metric_scores(result_id);
 CREATE INDEX IF NOT EXISTS idx_scores_metric_name ON metric_scores(metric_name);
 
+-- Multiturn evaluation tables
+CREATE TABLE IF NOT EXISTS multiturn_runs (
+    run_id TEXT PRIMARY KEY,
+    dataset_name TEXT NOT NULL,
+    dataset_version TEXT,
+    model_name TEXT,
+    started_at TIMESTAMP NOT NULL,
+    finished_at TIMESTAMP,
+    conversation_count INTEGER DEFAULT 0,
+    turn_count INTEGER DEFAULT 0,
+    metrics_evaluated TEXT,  -- JSON array of metric names
+    drift_threshold REAL,
+    summary TEXT,  -- JSON summary
+    metadata TEXT,  -- JSON metadata
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_runs_dataset ON multiturn_runs(dataset_name);
+CREATE INDEX IF NOT EXISTS idx_multiturn_runs_started_at ON multiturn_runs(started_at DESC);
+
+CREATE TABLE IF NOT EXISTS multiturn_conversations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id TEXT NOT NULL,
+    conversation_id TEXT NOT NULL,
+    turn_count INTEGER DEFAULT 0,
+    drift_score REAL,
+    drift_threshold REAL,
+    drift_detected INTEGER DEFAULT 0,
+    summary TEXT,  -- JSON summary
+    FOREIGN KEY (run_id) REFERENCES multiturn_runs(run_id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_run_id ON multiturn_conversations(run_id);
+CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_conv_id ON multiturn_conversations(conversation_id);
+
+CREATE TABLE IF NOT EXISTS multiturn_turn_results (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id TEXT NOT NULL,
+    conversation_id TEXT NOT NULL,
+    turn_id TEXT NOT NULL,
+    turn_index INTEGER,
+    role TEXT NOT NULL,
+    passed INTEGER DEFAULT 0,
+    latency_ms INTEGER,
+    metadata TEXT,  -- JSON metadata
+    FOREIGN KEY (run_id) REFERENCES multiturn_runs(run_id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_turns_run_id ON multiturn_turn_results(run_id);
+CREATE INDEX IF NOT EXISTS idx_multiturn_turns_conv_id ON multiturn_turn_results(conversation_id);
+
+CREATE TABLE IF NOT EXISTS multiturn_metric_scores (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    turn_result_id INTEGER NOT NULL,
+    metric_name TEXT NOT NULL,
+    score REAL NOT NULL,
+    threshold REAL,
+    FOREIGN KEY (turn_result_id) REFERENCES multiturn_turn_results(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_multiturn_scores_turn_id ON multiturn_metric_scores(turn_result_id);
+CREATE INDEX IF NOT EXISTS idx_multiturn_scores_metric_name ON multiturn_metric_scores(metric_name);
+
 -- Prompt storage tables
 CREATE TABLE IF NOT EXISTS prompts (
     prompt_id TEXT PRIMARY KEY,