evalvault 1.70.1__py3-none-any.whl → 1.71.0__py3-none-any.whl

evalvault/adapters/outbound/storage/sqlite_adapter.py

@@ -182,6 +182,71 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             if "metadata" not in pipeline_columns:
                 conn.execute("ALTER TABLE pipeline_results ADD COLUMN metadata TEXT")
 
+        multiturn_cursor = conn.execute("PRAGMA table_info(multiturn_runs)")
+        multiturn_columns = {row[1] for row in multiturn_cursor.fetchall()}
+        if not multiturn_columns:
+            conn.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS multiturn_runs (
+                    run_id TEXT PRIMARY KEY,
+                    dataset_name TEXT NOT NULL,
+                    dataset_version TEXT,
+                    model_name TEXT,
+                    started_at TIMESTAMP NOT NULL,
+                    finished_at TIMESTAMP,
+                    conversation_count INTEGER DEFAULT 0,
+                    turn_count INTEGER DEFAULT 0,
+                    metrics_evaluated TEXT,
+                    drift_threshold REAL,
+                    summary TEXT,
+                    metadata TEXT,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                );
+                CREATE INDEX IF NOT EXISTS idx_multiturn_runs_dataset ON multiturn_runs(dataset_name);
+                CREATE INDEX IF NOT EXISTS idx_multiturn_runs_started_at ON multiturn_runs(started_at DESC);
+
+                CREATE TABLE IF NOT EXISTS multiturn_conversations (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    run_id TEXT NOT NULL,
+                    conversation_id TEXT NOT NULL,
+                    turn_count INTEGER DEFAULT 0,
+                    drift_score REAL,
+                    drift_threshold REAL,
+                    drift_detected INTEGER DEFAULT 0,
+                    summary TEXT,
+                    FOREIGN KEY (run_id) REFERENCES multiturn_runs(run_id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_run_id ON multiturn_conversations(run_id);
+                CREATE INDEX IF NOT EXISTS idx_multiturn_conversations_conv_id ON multiturn_conversations(conversation_id);
+
+                CREATE TABLE IF NOT EXISTS multiturn_turn_results (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    run_id TEXT NOT NULL,
+                    conversation_id TEXT NOT NULL,
+                    turn_id TEXT NOT NULL,
+                    turn_index INTEGER,
+                    role TEXT NOT NULL,
+                    passed INTEGER DEFAULT 0,
+                    latency_ms INTEGER,
+                    metadata TEXT,
+                    FOREIGN KEY (run_id) REFERENCES multiturn_runs(run_id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS idx_multiturn_turns_run_id ON multiturn_turn_results(run_id);
+                CREATE INDEX IF NOT EXISTS idx_multiturn_turns_conv_id ON multiturn_turn_results(conversation_id);
+
+                CREATE TABLE IF NOT EXISTS multiturn_metric_scores (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    turn_result_id INTEGER NOT NULL,
+                    metric_name TEXT NOT NULL,
+                    score REAL NOT NULL,
+                    threshold REAL,
+                    FOREIGN KEY (turn_result_id) REFERENCES multiturn_turn_results(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS idx_multiturn_scores_turn_id ON multiturn_metric_scores(turn_result_id);
+                CREATE INDEX IF NOT EXISTS idx_multiturn_scores_metric_name ON multiturn_metric_scores(metric_name);
+                """
+            )
+
     # Prompt set methods
 
     def save_prompt_set(self, bundle: PromptSetBundle) -> None:
@@ -990,6 +1055,48 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
 
         return report_id
 
+    def list_analysis_reports(
+        self,
+        *,
+        run_id: str,
+        report_type: str | None = None,
+        format: str | None = None,
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        query = (
+            "SELECT report_id, run_id, experiment_id, report_type, format, content, metadata, created_at "
+            "FROM analysis_reports WHERE run_id = ?"
+        )
+        params: list[Any] = [run_id]
+        if report_type:
+            query += " AND report_type = ?"
+            params.append(report_type)
+        if format:
+            query += " AND format = ?"
+            params.append(format)
+        query += " ORDER BY created_at DESC LIMIT ?"
+        params.append(limit)
+
+        with self._get_connection() as conn:
+            conn = cast(Any, conn)
+            rows = conn.execute(query, tuple(params)).fetchall()
+
+        reports: list[dict[str, Any]] = []
+        for row in rows:
+            reports.append(
+                {
+                    "report_id": row["report_id"],
+                    "run_id": row["run_id"],
+                    "experiment_id": row["experiment_id"],
+                    "report_type": row["report_type"],
+                    "format": row["format"],
+                    "content": row["content"],
+                    "metadata": self._deserialize_json(row["metadata"]),
+                    "created_at": row["created_at"],
+                }
+            )
+        return reports
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """

evalvault/domain/entities/__init__.py

@@ -21,6 +21,7 @@ from evalvault.domain.entities.feedback import (
     FeedbackSummary,
     SatisfactionFeedback,
 )
+from evalvault.domain.entities.graph_rag import EntityNode, KnowledgeSubgraph, RelationEdge
 from evalvault.domain.entities.improvement import (
     EffortLevel,
     EvidenceSource,
@@ -42,6 +43,15 @@ from evalvault.domain.entities.judge_calibration import (
 )
 from evalvault.domain.entities.kg import EntityModel, RelationModel
 from evalvault.domain.entities.method import MethodInput, MethodInputDataset, MethodOutput
+from evalvault.domain.entities.multiturn import (
+    ConversationTurn,
+    DriftAnalysis,
+    MultiTurnConversationRecord,
+    MultiTurnEvaluationResult,
+    MultiTurnRunRecord,
+    MultiTurnTestCase,
+    MultiTurnTurnResult,
+)
 from evalvault.domain.entities.prompt import Prompt, PromptSet, PromptSetBundle, PromptSetItem
 from evalvault.domain.entities.prompt_suggestion import (
     PromptCandidate,
@@ -114,6 +124,16 @@ __all__ = [
     "JudgeCalibrationMetric",
     "JudgeCalibrationResult",
     "JudgeCalibrationSummary",
+    "ConversationTurn",
+    "MultiTurnConversationRecord",
+    "MultiTurnTestCase",
+    "MultiTurnTurnResult",
+    "MultiTurnEvaluationResult",
+    "DriftAnalysis",
+    "MultiTurnRunRecord",
+    "EntityNode",
+    "KnowledgeSubgraph",
+    "RelationEdge",
     # KG
     "EntityModel",
     "RelationModel",

evalvault/domain/entities/graph_rag.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class EntityNode:
+    entity_id: str
+    name: str
+    entity_type: str
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class RelationEdge:
+    source_id: str
+    target_id: str
+    relation_type: str
+    weight: float = 1.0
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class KnowledgeSubgraph:
+    """질의에 대해 추출된 관련 서브그래프."""
+
+    nodes: list[EntityNode]
+    edges: list[RelationEdge]
+    relevance_score: float

evalvault/domain/entities/multiturn.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Literal
+
+
+@dataclass
+class ConversationTurn:
+    turn_id: str
+    role: Literal["user", "assistant"]
+    content: str
+    contexts: list[str] | None = None
+    ground_truth: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class MultiTurnTestCase:
+    conversation_id: str
+    turns: list[ConversationTurn]
+    expected_final_answer: str | None = None
+    drift_tolerance: float = 0.1
+
+
+@dataclass
+class MultiTurnTurnResult:
+    conversation_id: str
+    turn_id: str
+    turn_index: int | None
+    role: Literal["user", "assistant"]
+    metrics: dict[str, float] = field(default_factory=dict)
+    passed: bool = False
+    latency_ms: int | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class MultiTurnEvaluationResult:
+    conversation_id: str
+    turn_results: list[MultiTurnTurnResult] = field(default_factory=list)
+    summary: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class DriftAnalysis:
+    conversation_id: str
+    drift_score: float
+    drift_threshold: float
+    drift_detected: bool
+    notes: list[str] = field(default_factory=list)
+
+
+@dataclass
+class MultiTurnRunRecord:
+    run_id: str
+    dataset_name: str
+    dataset_version: str | None
+    model_name: str | None
+    started_at: datetime
+    finished_at: datetime | None
+    conversation_count: int
+    turn_count: int
+    metrics_evaluated: list[str] = field(default_factory=list)
+    drift_threshold: float | None = None
+    summary: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class MultiTurnConversationRecord:
+    run_id: str
+    conversation_id: str
+    turn_count: int
+    drift_score: float | None = None
+    drift_threshold: float | None = None
+    drift_detected: bool = False
+    summary: dict[str, Any] = field(default_factory=dict)

evalvault/domain/metrics/__init__.py

@@ -4,6 +4,12 @@ from evalvault.domain.metrics.confidence import ConfidenceScore
 from evalvault.domain.metrics.contextual_relevancy import ContextualRelevancy
 from evalvault.domain.metrics.entity_preservation import EntityPreservation
 from evalvault.domain.metrics.insurance import InsuranceTermAccuracy
+from evalvault.domain.metrics.multiturn_metrics import (
+    calculate_context_coherence,
+    calculate_drift_rate,
+    calculate_turn_faithfulness,
+    calculate_turn_latency_p95,
+)
 from evalvault.domain.metrics.no_answer import NoAnswerAccuracy, is_no_answer
 from evalvault.domain.metrics.retrieval_rank import MRR, NDCG, HitRate
 from evalvault.domain.metrics.summary_accuracy import SummaryAccuracy
@@ -28,4 +34,8 @@ __all__ = [
     "SummaryNonDefinitive",
     "SummaryRiskCoverage",
     "is_no_answer",
+    "calculate_context_coherence",
+    "calculate_drift_rate",
+    "calculate_turn_faithfulness",
+    "calculate_turn_latency_p95",
 ]

evalvault/domain/metrics/multiturn_metrics.py

@@ -0,0 +1,113 @@
+"""
+Utilities for multi-turn evaluation metrics.
+
+Metrics:
+- turn_faithfulness: average per-turn faithfulness
+- context_coherence: coherence across turn contexts
+- drift_rate: distance between initial intent and final response
+- turn_latency: p95 latency across turns
+"""
+
+from __future__ import annotations
+
+import math
+import re
+import unicodedata
+from collections.abc import Iterable
+
+from evalvault.domain.entities.multiturn import ConversationTurn, MultiTurnTurnResult
+
+
+def _normalize_text(text: str) -> str:
+    if not text:
+        return ""
+    text = unicodedata.normalize("NFC", text)
+    text = text.lower()
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def _tokenize(text: str) -> set[str]:
+    if not text:
+        return set()
+    text = _normalize_text(text)
+    tokens = re.findall(r"[\w가-힣]+", text)
+    return set(tokens)
+
+
+def _jaccard_similarity(left: str, right: str) -> float:
+    left_tokens = _tokenize(left)
+    right_tokens = _tokenize(right)
+    if not left_tokens and not right_tokens:
+        return 1.0
+    if not left_tokens or not right_tokens:
+        return 0.0
+    intersection = left_tokens.intersection(right_tokens)
+    union = left_tokens.union(right_tokens)
+    if not union:
+        return 0.0
+    return len(intersection) / len(union)
+
+
+def _turn_context_text(turn: ConversationTurn) -> str:
+    if turn.contexts:
+        return " ".join([ctx for ctx in turn.contexts if ctx])
+    return turn.content or ""
+
+
+def calculate_turn_faithfulness(turn_results: Iterable[MultiTurnTurnResult]) -> float:
+    scores: list[float] = []
+    for result in turn_results:
+        score = result.metrics.get("faithfulness") if result.metrics else None
+        if score is not None:
+            scores.append(score)
+    if not scores:
+        return 0.0
+    return sum(scores) / len(scores)
+
+
+def calculate_context_coherence(turns: Iterable[ConversationTurn]) -> float:
+    turn_list = list(turns)
+    if len(turn_list) < 2:
+        return 1.0
+    scores: list[float] = []
+    for prev, curr in zip(turn_list, turn_list[1:], strict=False):
+        left = _turn_context_text(prev)
+        right = _turn_context_text(curr)
+        scores.append(_jaccard_similarity(left, right))
+    if not scores:
+        return 0.0
+    return sum(scores) / len(scores)
+
+
+def calculate_drift_rate(turns: Iterable[ConversationTurn]) -> float:
+    turn_list = list(turns)
+    if not turn_list:
+        return 0.0
+    first_user = next((t for t in turn_list if t.role == "user"), None)
+    last_assistant = next((t for t in reversed(turn_list) if t.role == "assistant"), None)
+    if not first_user or not last_assistant:
+        return 0.0
+    similarity = _jaccard_similarity(first_user.content, last_assistant.content)
+    drift = 1.0 - similarity
+    if drift < 0.0:
+        return 0.0
+    if drift > 1.0:
+        return 1.0
+    return drift
+
+
+def calculate_turn_latency_p95(latencies_ms: Iterable[int | None]) -> float:
+    values = [float(value) for value in latencies_ms if value is not None]
+    if not values:
+        return 0.0
+    values.sort()
+    if len(values) == 1:
+        return values[0]
+    rank = 0.95 * (len(values) - 1)
+    lower = int(math.floor(rank))
+    upper = int(math.ceil(rank))
+    if lower == upper:
+        return values[lower]
+    fraction = rank - lower
+    return values[lower] + (values[upper] - values[lower]) * fraction

evalvault/domain/metrics/registry.py

@@ -139,6 +139,42 @@ _METRIC_SPECS: tuple[MetricSpec, ...] = (
         category="summary",
         signal_group="summary_fidelity",
     ),
+    MetricSpec(
+        name="turn_faithfulness",
+        description="(Multi-turn) Average faithfulness across assistant turns",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="qa",
+        signal_group="groundedness",
+    ),
+    MetricSpec(
+        name="context_coherence",
+        description="(Multi-turn) Context continuity across turns",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="qa",
+        signal_group="intent_alignment",
+    ),
+    MetricSpec(
+        name="drift_rate",
+        description="(Multi-turn) Distance between initial intent and final response",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="qa",
+        signal_group="intent_alignment",
+    ),
+    MetricSpec(
+        name="turn_latency",
+        description="(Multi-turn) P95 response latency across turns (ms)",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="qa",
+        signal_group="efficiency",
+    ),
     MetricSpec(
         name="entity_preservation",
         description="(Rule) Measures preservation of key insurance entities in summaries",

evalvault/domain/services/__init__.py

@@ -4,9 +4,14 @@ from evalvault.domain.services.analysis_service import AnalysisService
 from evalvault.domain.services.dataset_preprocessor import DatasetPreprocessor
 from evalvault.domain.services.domain_learning_hook import DomainLearningHook
 from evalvault.domain.services.evaluator import RagasEvaluator
+from evalvault.domain.services.graph_rag_experiment import (
+    GraphRAGExperiment,
+    GraphRAGExperimentResult,
+)
 from evalvault.domain.services.holdout_splitter import split_dataset_holdout
 from evalvault.domain.services.improvement_guide_service import ImprovementGuideService
 from evalvault.domain.services.method_runner import MethodRunnerService, MethodRunResult
+from evalvault.domain.services.multiturn_evaluator import MultiTurnEvaluator
 from evalvault.domain.services.prompt_scoring_service import PromptScoringService
 from evalvault.domain.services.prompt_suggestion_reporter import PromptSuggestionReporter
 
@@ -17,8 +22,11 @@ __all__ = [
     "ImprovementGuideService",
     "MethodRunnerService",
     "MethodRunResult",
+    "GraphRAGExperiment",
+    "GraphRAGExperimentResult",
     "PromptScoringService",
     "PromptSuggestionReporter",
     "RagasEvaluator",
+    "MultiTurnEvaluator",
     "split_dataset_holdout",
 ]

evalvault/domain/services/evaluator.py

@@ -63,9 +63,12 @@ _SUMMARY_FAITHFULNESS_PROMPT_EN = (
 
 def _patch_ragas_faithfulness_output() -> None:
     try:
-        from ragas.metrics import Faithfulness
+        from ragas.metrics.collections import Faithfulness
     except Exception:
-        return
+        try:
+            from ragas.metrics import Faithfulness
+        except Exception:
+            return
 
     prompt = getattr(Faithfulness, "nli_statements_prompt", None)
     if prompt is None:

evalvault/domain/services/graph_rag_experiment.py

@@ -0,0 +1,155 @@
+"""GraphRAG experiment helper for baseline vs graph comparison."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from evalvault.domain.entities import Dataset, EvaluationRun, TestCase
+from evalvault.domain.entities.analysis import ComparisonResult
+from evalvault.domain.entities.graph_rag import KnowledgeSubgraph
+from evalvault.domain.services.analysis_service import AnalysisService
+from evalvault.domain.services.evaluator import RagasEvaluator
+from evalvault.ports.outbound.graph_retriever_port import GraphRetrieverPort
+from evalvault.ports.outbound.korean_nlp_port import RetrieverPort
+from evalvault.ports.outbound.llm_port import LLMPort
+
+
+@dataclass
+class GraphRAGExperimentResult:
+    baseline_run: EvaluationRun
+    graph_run: EvaluationRun
+    comparisons: list[ComparisonResult]
+    graph_subgraphs: dict[str, KnowledgeSubgraph]
+    graph_contexts: dict[str, str]
+
+
+class GraphRAGExperiment:
+    """Compare baseline retrieval with GraphRAG context generation."""
+
+    def __init__(
+        self,
+        *,
+        evaluator: RagasEvaluator,
+        analysis_service: AnalysisService,
+    ) -> None:
+        self._evaluator = evaluator
+        self._analysis = analysis_service
+
+    async def run_comparison(
+        self,
+        *,
+        dataset: Dataset,
+        baseline_retriever: RetrieverPort,
+        graph_retriever: GraphRetrieverPort,
+        metrics: list[str],
+        llm: LLMPort,
+        thresholds: dict[str, float] | None = None,
+        retriever_top_k: int = 5,
+        graph_max_hops: int = 2,
+        graph_max_nodes: int = 20,
+        parallel: bool = False,
+        batch_size: int = 5,
+        prompt_overrides: dict[str, str] | None = None,
+        claim_level: bool = False,
+        language: str | None = None,
+    ) -> GraphRAGExperimentResult:
+        baseline_dataset = self._clone_dataset(dataset)
+        graph_dataset = self._clone_dataset(dataset)
+
+        graph_subgraphs, graph_contexts = self._apply_graph_contexts(
+            graph_dataset,
+            graph_retriever,
+            max_hops=graph_max_hops,
+            max_nodes=graph_max_nodes,
+        )
+
+        baseline_run = await self._evaluator.evaluate(
+            baseline_dataset,
+            metrics,
+            llm,
+            thresholds=thresholds,
+            parallel=parallel,
+            batch_size=batch_size,
+            retriever=baseline_retriever,
+            retriever_top_k=retriever_top_k,
+            prompt_overrides=prompt_overrides,
+            claim_level=claim_level,
+            language=language,
+        )
+
+        graph_run = await self._evaluator.evaluate(
+            graph_dataset,
+            metrics,
+            llm,
+            thresholds=thresholds,
+            parallel=parallel,
+            batch_size=batch_size,
+            retriever=None,
+            prompt_overrides=prompt_overrides,
+            claim_level=claim_level,
+            language=language,
+        )
+
+        comparisons = self._analysis.compare_runs(
+            baseline_run,
+            graph_run,
+            metrics=metrics,
+        )
+
+        return GraphRAGExperimentResult(
+            baseline_run=baseline_run,
+            graph_run=graph_run,
+            comparisons=comparisons,
+            graph_subgraphs=graph_subgraphs,
+            graph_contexts=graph_contexts,
+        )
+
+    @staticmethod
+    def _clone_dataset(dataset: Dataset) -> Dataset:
+        test_cases = [
+            TestCase(
+                id=case.id,
+                question=case.question,
+                answer=case.answer,
+                contexts=list(case.contexts),
+                ground_truth=case.ground_truth,
+                metadata=dict(case.metadata),
+            )
+            for case in dataset.test_cases
+        ]
+        return Dataset(
+            name=dataset.name,
+            version=dataset.version,
+            test_cases=test_cases,
+            metadata=dict(dataset.metadata),
+            source_file=dataset.source_file,
+            thresholds=dict(dataset.thresholds),
+        )
+
+    @staticmethod
+    def _apply_graph_contexts(
+        dataset: Dataset,
+        graph_retriever: GraphRetrieverPort,
+        *,
+        max_hops: int,
+        max_nodes: int,
+    ) -> tuple[dict[str, KnowledgeSubgraph], dict[str, str]]:
+        subgraphs: dict[str, KnowledgeSubgraph] = {}
+        contexts: dict[str, str] = {}
+        for case in dataset.test_cases:
+            if case.contexts and any(context.strip() for context in case.contexts):
+                continue
+            subgraph = graph_retriever.build_subgraph(
+                case.question,
+                max_hops=max_hops,
+                max_nodes=max_nodes,
+            )
+            context_text = graph_retriever.generate_context(subgraph)
+            if context_text:
+                case.contexts = [context_text]
+                contexts[case.id] = context_text
+            subgraphs[case.id] = subgraph
+        return subgraphs, contexts
+
+
+__all__ = ["GraphRAGExperiment", "GraphRAGExperimentResult"]