evalvault 1.62.1__py3-none-any.whl → 1.63.0__py3-none-any.whl

evalvault/domain/entities/prompt_suggestion.py

@@ -0,0 +1,50 @@
+"""Prompt suggestion entities."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class PromptCandidate:
+    """Single prompt candidate for suggestion workflow."""
+
+    candidate_id: str
+    source: str
+    content: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class PromptCandidateSampleScore:
+    sample_index: int
+    scores: dict[str, float]
+    weighted_score: float
+    responses: list[dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class PromptCandidateScore:
+    """Evaluation score for a prompt candidate."""
+
+    candidate_id: str
+    scores: dict[str, float]
+    weighted_score: float
+    sample_scores: list[PromptCandidateSampleScore] = field(default_factory=list)
+    selected_sample_index: int | None = None
+
+
+@dataclass(frozen=True)
+class PromptSuggestionResult:
+    """Aggregated prompt suggestion results."""
+
+    run_id: str
+    role: str
+    metrics: list[str]
+    weights: dict[str, float]
+    candidates: list[PromptCandidate]
+    scores: list[PromptCandidateScore]
+    ranking: list[str]
+    holdout_ratio: float
+    metadata: dict[str, Any] = field(default_factory=dict)

evalvault/domain/services/__init__.py

@@ -4,8 +4,11 @@ from evalvault.domain.services.analysis_service import AnalysisService
 from evalvault.domain.services.dataset_preprocessor import DatasetPreprocessor
 from evalvault.domain.services.domain_learning_hook import DomainLearningHook
 from evalvault.domain.services.evaluator import RagasEvaluator
+from evalvault.domain.services.holdout_splitter import split_dataset_holdout
 from evalvault.domain.services.improvement_guide_service import ImprovementGuideService
 from evalvault.domain.services.method_runner import MethodRunnerService, MethodRunResult
+from evalvault.domain.services.prompt_scoring_service import PromptScoringService
+from evalvault.domain.services.prompt_suggestion_reporter import PromptSuggestionReporter
 
 __all__ = [
     "AnalysisService",
@@ -14,5 +17,8 @@ __all__ = [
     "ImprovementGuideService",
     "MethodRunnerService",
     "MethodRunResult",
+    "PromptScoringService",
+    "PromptSuggestionReporter",
     "RagasEvaluator",
+    "split_dataset_holdout",
 ]

evalvault/domain/services/evaluator.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import importlib
 import json
 import logging
 import math
@@ -10,36 +11,10 @@ from collections.abc import Callable, Sequence
 from contextlib import suppress
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Any
+from typing import Any, Literal, cast, overload
 
 from ragas import SingleTurnSample
 
-try:  # Ragas >=0.2.0
-    from ragas.metrics.collections import (
-        AnswerRelevancy,
-        ContextPrecision,
-        ContextRecall,
-        FactualCorrectness,
-        Faithfulness,
-        SemanticSimilarity,
-    )
-except ImportError:  # pragma: no cover - fallback for older Ragas versions
-    from ragas.metrics import (
-        AnswerRelevancy,
-        ContextPrecision,
-        ContextRecall,
-        FactualCorrectness,
-        Faithfulness,
-        SemanticSimilarity,
-    )
-try:  # SummaryScore lives in different modules depending on Ragas version
-    from ragas.metrics.collections import SummaryScore as RagasSummaryScore
-except ImportError:  # pragma: no cover - fallback for older Ragas versions
-    try:
-        from ragas.metrics import SummarizationScore as RagasSummaryScore
-    except ImportError:  # pragma: no cover - no summary support available
-        RagasSummaryScore = None
-
 from evalvault.domain.entities import (
     ClaimLevelResult,
     ClaimVerdict,
@@ -59,9 +34,55 @@ from evalvault.domain.metrics.text_match import ExactMatch, F1Score
 from evalvault.domain.services.batch_executor import run_in_batches
 from evalvault.domain.services.dataset_preprocessor import DatasetPreprocessor
 from evalvault.domain.services.retriever_context import apply_retriever_to_dataset
-from evalvault.ports.outbound.korean_nlp_port import RetrieverPort
+from evalvault.ports.outbound.korean_nlp_port import KoreanNLPToolkitPort, RetrieverPort
+from evalvault.ports.outbound.llm_factory_port import LLMFactoryPort
 from evalvault.ports.outbound.llm_port import LLMPort
 
+_SUMMARY_FAITHFULNESS_PROMPT_KO = (
+    "당신은 요약 충실도 판정자입니다.\n"
+    "컨텍스트와 요약을 보고 요약의 모든 주장이 컨텍스트에 의해 뒷받침되는지 판단하세요.\n"
+    "숫자, 조건, 면책, 기간, 자격 등이 누락되거나 추가되거나 모순되면 verdict는 unsupported입니다.\n"
+    'JSON만 반환: {"verdict": "supported|unsupported", "reason": "..."}\n\n'
+    "컨텍스트:\n{context}\n\n요약:\n{summary}\n"
+)
+_SUMMARY_FAITHFULNESS_PROMPT_EN = (
+    "You are a strict summarization faithfulness judge.\n"
+    "Given the CONTEXT and SUMMARY, determine whether every claim in SUMMARY is supported by CONTEXT.\n"
+    "If any numbers, conditions, exclusions, durations, or eligibility are missing, added, or "
+    "contradicted, verdict is unsupported.\n"
+    'Return JSON only: {"verdict": "supported|unsupported", "reason": "..."}\n\n'
+    "CONTEXT:\n{context}\n\nSUMMARY:\n{summary}\n"
+)
+
+
+def _import_metric(name: str) -> type[Any]:
+    for module_name in ("ragas.metrics.collections", "ragas.metrics"):
+        try:
+            module = importlib.import_module(module_name)
+        except Exception:
+            continue
+        if hasattr(module, name):
+            return cast(type[Any], getattr(module, name))
+    raise ImportError(f"Missing ragas metric: {name}")
+
+
+def _import_optional_metric(names: list[str]) -> type[Any] | None:
+    for name in names:
+        try:
+            return _import_metric(name)
+        except Exception:
+            continue
+    return None
+
+
+AnswerRelevancy = _import_metric("AnswerRelevancy")
+ContextPrecision = _import_metric("ContextPrecision")
+ContextRecall = _import_metric("ContextRecall")
+FactualCorrectness = _import_metric("FactualCorrectness")
+Faithfulness = _import_metric("Faithfulness")
+SemanticSimilarity = _import_metric("SemanticSimilarity")
+RagasSummaryScore = _import_optional_metric(["SummaryScore", "SummarizationScore"])
+
 logger = logging.getLogger(__name__)
 
 
@@ -247,9 +268,16 @@ class RagasEvaluator:
         "openai/gpt-5-nano": (5.00, 15.00),
     }
 
-    def __init__(self, *, preprocessor: DatasetPreprocessor | None = None) -> None:
+    def __init__(
+        self,
+        *,
+        preprocessor: DatasetPreprocessor | None = None,
+        korean_toolkit: KoreanNLPToolkitPort | None = None,
+        llm_factory: LLMFactoryPort | None = None,
+    ) -> None:
         self._preprocessor = preprocessor or DatasetPreprocessor()
-        self._korean_toolkit = None
+        self._korean_toolkit = korean_toolkit
+        self._llm_factory = llm_factory
         self._faithfulness_ragas_failed = False
         self._faithfulness_fallback_llm = None
         self._faithfulness_fallback_metric = None
@@ -258,6 +286,7 @@ class RagasEvaluator:
         self._active_llm_provider = None
         self._active_llm_model = None
         self._active_llm = None
+        self._prompt_language = None
 
     async def evaluate(
         self,
@@ -273,6 +302,7 @@ class RagasEvaluator:
         on_progress: Callable[[int, int, str], None] | None = None,
         prompt_overrides: dict[str, str] | None = None,
         claim_level: bool = False,
+        language: str | None = None,
     ) -> EvaluationRun:
         """데이터셋을 Ragas로 평가.
 
@@ -299,6 +329,7 @@ class RagasEvaluator:
         self._active_llm_provider = getattr(llm, "provider_name", None)
         self._active_llm_model = llm.get_model_name()
         self._active_llm = llm
+        self._prompt_language = self._normalize_language_hint(language) if language else None
         # Resolve thresholds: CLI > dataset > default(0.7)
         resolved_thresholds = {}
         for metric in metrics:
@@ -359,7 +390,11 @@ class RagasEvaluator:
         eval_results_by_test_case = {}
         if ragas_metrics:
             run.tracker_metadata["ragas_config"] = self._build_ragas_config(llm)
-            eval_results_by_test_case, override_status = await self._evaluate_with_ragas(
+            (
+                eval_results_by_test_case,
+                override_status,
+                prompt_snapshots,
+            ) = await self._evaluate_with_ragas(
                 dataset=dataset,
                 metrics=ragas_metrics,
                 llm=llm,
@@ -370,6 +405,8 @@ class RagasEvaluator:
             )
             if override_status:
                 run.tracker_metadata["ragas_prompt_overrides"] = override_status
+            if prompt_snapshots:
+                run.tracker_metadata["ragas_prompt_snapshots"] = prompt_snapshots
         elif prompt_overrides:
             logger.warning("Ragas prompt overrides provided but no Ragas metrics requested.")
 
@@ -485,7 +522,7 @@ class RagasEvaluator:
         batch_size: int = 5,
         on_progress: Callable[[int, int, str], None] | None = None,
         prompt_overrides: dict[str, str] | None = None,
-    ) -> tuple[dict[str, TestCaseEvalResult], dict[str, str]]:
+    ) -> tuple[dict[str, TestCaseEvalResult], dict[str, str], dict[str, dict[str, Any]]]:
         """Ragas로 실제 평가 수행.
 
         Args:
@@ -496,7 +533,7 @@ class RagasEvaluator:
             batch_size: 병렬 처리 시 배치 크기
 
         Returns:
-            (테스트 케이스 ID별 평가 결과, 프롬프트 오버라이드 적용 상태)
+            (테스트 케이스 ID별 평가 결과, 프롬프트 오버라이드 적용 상태, 프롬프트 스냅샷)
             예: {"tc-001": TestCaseEvalResult(...)}
         """
 
@@ -554,6 +591,12 @@ class RagasEvaluator:
         if prompt_overrides:
             override_status = self._apply_prompt_overrides(ragas_metrics, prompt_overrides)
 
+        prompt_snapshots = self._collect_ragas_prompt_snapshots(
+            ragas_metrics,
+            prompt_overrides,
+            override_status,
+        )
+
         # 병렬 처리 vs 순차 처리
         if parallel and len(ragas_samples) > 1:
             return (
@@ -566,6 +609,7 @@ class RagasEvaluator:
                     on_progress=on_progress,
                 ),
                 override_status,
+                prompt_snapshots,
             )
         return (
             await self._evaluate_sequential(
@@ -576,6 +620,7 @@ class RagasEvaluator:
                 on_progress=on_progress,
             ),
             override_status,
+            prompt_snapshots,
         )
 
     def _apply_answer_relevancy_prompt_defaults(
@@ -619,6 +664,8 @@ class RagasEvaluator:
             self._apply_korean_factual_correctness_prompts(metric)
 
     def _resolve_dataset_language(self, dataset: Dataset) -> str | None:
+        if self._prompt_language:
+            return self._prompt_language
         metadata = dataset.metadata if isinstance(dataset.metadata, dict) else {}
         for key in ("language", "lang", "locale"):
             normalized = self._normalize_language_hint(metadata.get(key))
@@ -784,10 +831,10 @@ class RagasEvaluator:
             if isinstance(target, str):
                 metric.prompt = prompt_text
                 return True
-            if hasattr(target, "template"):
+            if target is not None and hasattr(target, "template"):
                 target.template = prompt_text
                 return True
-            if hasattr(target, "instruction"):
+            if target is not None and hasattr(target, "instruction"):
                 target.instruction = prompt_text
                 return True
 
@@ -796,10 +843,10 @@ class RagasEvaluator:
             if isinstance(target, str):
                 metric.question_generation = prompt_text
                 return True
-            if hasattr(target, "template"):
+            if target is not None and hasattr(target, "template"):
                 target.template = prompt_text
                 return True
-            if hasattr(target, "instruction"):
+            if target is not None and hasattr(target, "instruction"):
                 target.instruction = prompt_text
                 return True
 
@@ -829,6 +876,68 @@ class RagasEvaluator:
 
         return False
 
+    @staticmethod
+    def _extract_prompt_text(value: Any) -> str | None:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            return value
+        for attr in ("template", "instruction", "prompt", "text"):
+            try:
+                candidate = getattr(value, attr)
+            except Exception:
+                continue
+            if isinstance(candidate, str) and candidate.strip():
+                return candidate
+        return None
+
+    def _collect_metric_prompt_text(self, metric: Any) -> str | None:
+        for attr in ("prompt", "question_generation"):
+            if hasattr(metric, attr):
+                try:
+                    value = getattr(metric, attr)
+                except Exception:
+                    continue
+                text = self._extract_prompt_text(value)
+                if text:
+                    return text
+        for attr in dir(metric):
+            if not attr.endswith("_prompt") or attr == "prompt":
+                continue
+            try:
+                value = getattr(metric, attr)
+            except Exception:
+                continue
+            text = self._extract_prompt_text(value)
+            if text:
+                return text
+        return None
+
+    def _collect_ragas_prompt_snapshots(
+        self,
+        ragas_metrics: list[Any],
+        prompt_overrides: dict[str, str] | None,
+        override_status: dict[str, str],
+    ) -> dict[str, dict[str, Any]]:
+        snapshots: dict[str, dict[str, Any]] = {}
+        for metric in ragas_metrics:
+            metric_name = getattr(metric, "name", None)
+            if not metric_name:
+                continue
+            prompt_text = self._collect_metric_prompt_text(metric)
+            if not prompt_text:
+                continue
+            requested = bool(prompt_overrides and metric_name in prompt_overrides)
+            status = override_status.get(metric_name)
+            source = "override" if status == "applied" else "default"
+            snapshots[str(metric_name)] = {
+                "prompt": prompt_text,
+                "source": source,
+                "override_requested": requested,
+                "override_status": status,
+            }
+        return snapshots
+
     async def _evaluate_sequential(
         self,
         dataset: Dataset,
@@ -1173,6 +1282,22 @@ class RagasEvaluator:
     def default_threshold_for(cls, metric_name: str) -> float:
         return cls.DEFAULT_METRIC_THRESHOLDS.get(metric_name, cls.DEFAULT_THRESHOLD_FALLBACK)
 
+    @overload
+    def _fallback_korean_faithfulness(
+        self,
+        sample: SingleTurnSample,
+        *,
+        return_details: Literal[True],
+    ) -> ClaimLevelResult | None: ...
+
+    @overload
+    def _fallback_korean_faithfulness(
+        self,
+        sample: SingleTurnSample,
+        *,
+        return_details: Literal[False] = False,
+    ) -> float | None: ...
+
     def _fallback_korean_faithfulness(
         self, sample: SingleTurnSample, *, return_details: bool = False
     ) -> float | ClaimLevelResult | None:
@@ -1194,11 +1319,7 @@ class RagasEvaluator:
             return None
 
         if self._korean_toolkit is None:
-            try:
-                from evalvault.adapters.outbound.nlp.korean.toolkit import KoreanNLPToolkit
-            except Exception:  # pragma: no cover - optional dependency
-                return None
-            self._korean_toolkit = KoreanNLPToolkit()
+            return None
 
         try:
             result = self._korean_toolkit.check_faithfulness(
@@ -1212,6 +1333,8 @@ class RagasEvaluator:
             return self._convert_to_claim_level_result(result, test_case_id="")
 
         score = getattr(result, "score", None)
+        if score is None:
+            return None
         try:
             return float(score)
         except (TypeError, ValueError):
@@ -1291,14 +1414,11 @@ class RagasEvaluator:
             return None
 
         context = "\n\n".join(sample.retrieved_contexts)
-        prompt = (
-            "You are a strict summarization faithfulness judge.\n"
-            "Given the CONTEXT and SUMMARY, determine whether every claim in SUMMARY is supported by CONTEXT.\n"
-            "If any numbers, conditions, exclusions, durations, or eligibility are missing, added, or "
-            "contradicted, verdict is unsupported.\n"
-            'Return JSON only: {"verdict": "supported|unsupported", "reason": "..."}\n\n'
-            f"CONTEXT:\n{context}\n\nSUMMARY:\n{sample.response}\n"
+        language = self._prompt_language or "ko"
+        template = (
+            _SUMMARY_FAITHFULNESS_PROMPT_EN if language == "en" else _SUMMARY_FAITHFULNESS_PROMPT_KO
         )
+        prompt = template.format(context=context, summary=sample.response)
 
         try:
             response_text = await asyncio.to_thread(llm.generate_text, prompt, json_mode=True)
@@ -1340,7 +1460,7 @@ class RagasEvaluator:
     ) -> float | None:
         metric = self._get_faithfulness_fallback_metric()
         if metric is None:
-            return self._fallback_korean_faithfulness(sample)
+            return self._fallback_korean_faithfulness(sample, return_details=False)
 
         try:
             if hasattr(metric, "ascore"):
@@ -1368,6 +1488,8 @@ class RagasEvaluator:
             else:
                 score_value = result
 
+            if score_value is None:
+                raise ValueError("Metric returned None")
             score_value = float(score_value)
             if math.isnan(score_value):
                 raise ValueError("Metric returned NaN")
@@ -1379,7 +1501,7 @@ class RagasEvaluator:
                     self._summarize_ragas_error(exc),
                 )
                 self._faithfulness_fallback_failed = True
-            return self._fallback_korean_faithfulness(sample)
+            return self._fallback_korean_faithfulness(sample, return_details=False)
 
     def _get_faithfulness_fallback_metric(self):
         if self._faithfulness_fallback_failed:
@@ -1411,29 +1533,14 @@ class RagasEvaluator:
             return None
         if self._faithfulness_fallback_llm is not None:
             return self._faithfulness_fallback_llm
-
-        try:
-            from evalvault.adapters.outbound.llm import create_llm_adapter_for_model
-            from evalvault.config.settings import Settings
-        except Exception:
-            return None
-
-        settings = Settings()
-        provider, model = self._resolve_faithfulness_fallback_config(settings)
-        if not provider or not model:
+        if self._llm_factory is None:
             return None
 
         try:
-            llm = create_llm_adapter_for_model(provider, model, settings)
-            self._faithfulness_fallback_llm = llm
-            if not self._faithfulness_fallback_logged:
-                logger.warning(
-                    "Faithfulness fallback LLM enabled: %s/%s",
-                    provider,
-                    model,
-                )
-                self._faithfulness_fallback_logged = True
-            return llm
+            llm = self._llm_factory.create_faithfulness_fallback(
+                self._active_llm_provider,
+                self._active_llm_model,
+            )
         except Exception as exc:
             if not self._faithfulness_fallback_failed:
                 logger.warning(
@@ -1443,39 +1550,20 @@ class RagasEvaluator:
                 self._faithfulness_fallback_failed = True
             return None
 
-    def _resolve_faithfulness_fallback_config(self, settings) -> tuple[str | None, str | None]:
-        provider = (
-            settings.faithfulness_fallback_provider.strip().lower()
-            if settings.faithfulness_fallback_provider
-            else None
-        )
-        model = settings.faithfulness_fallback_model
-        active_provider = (
-            self._active_llm_provider.strip().lower()
-            if isinstance(self._active_llm_provider, str) and self._active_llm_provider.strip()
-            else None
-        )
-        default_provider = active_provider or settings.llm_provider.lower()
-
-        if not provider and model:
-            provider = default_provider
-        if provider and not model:
-            model = self._default_faithfulness_fallback_model(provider)
-        if not provider and not model:
-            provider = default_provider
-            model = self._default_faithfulness_fallback_model(default_provider)
-
-        if not provider or not model:
-            return None, None
-        return provider, model
+        if llm is None:
+            return None
 
-    @staticmethod
-    def _default_faithfulness_fallback_model(provider: str) -> str | None:
-        if provider == "ollama":
-            return "gpt-oss-safeguard:20b"
-        if provider == "vllm":
-            return "gpt-oss-120b"
-        return None
+        self._faithfulness_fallback_llm = llm
+        if not self._faithfulness_fallback_logged:
+            provider = getattr(llm, "provider_name", None)
+            model = llm.get_model_name()
+            logger.warning(
+                "Faithfulness fallback LLM enabled: %s/%s",
+                provider,
+                model,
+            )
+            self._faithfulness_fallback_logged = True
+        return llm
 
     @staticmethod
     def _contains_korean(text: str) -> bool:

evalvault/domain/services/holdout_splitter.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import random
+
+from evalvault.domain.entities import Dataset, TestCase
+
+
+def split_dataset_holdout(
+    *,
+    dataset: Dataset,
+    holdout_ratio: float,
+    seed: int | None,
+) -> tuple[Dataset, Dataset]:
+    if holdout_ratio < 0 or holdout_ratio >= 1:
+        raise ValueError("holdout_ratio must be in [0, 1).")
+
+    total = len(dataset.test_cases)
+    if total == 0:
+        return _clone_dataset(dataset, "dev", []), _clone_dataset(dataset, "holdout", [])
+
+    holdout_size = int(total * holdout_ratio)
+    if holdout_ratio > 0 and holdout_size == 0:
+        holdout_size = 1
+    if holdout_size >= total:
+        holdout_size = total - 1
+
+    rng = random.Random(seed)
+    indices = list(range(total))
+    rng.shuffle(indices)
+
+    holdout_indices = set(indices[:holdout_size])
+    dev_cases: list[TestCase] = []
+    holdout_cases: list[TestCase] = []
+
+    for idx, test_case in enumerate(dataset.test_cases):
+        if idx in holdout_indices:
+            holdout_cases.append(test_case)
+        else:
+            dev_cases.append(test_case)
+
+    return (
+        _clone_dataset(dataset, "dev", dev_cases, holdout_ratio, seed),
+        _clone_dataset(dataset, "holdout", holdout_cases, holdout_ratio, seed),
+    )
+
+
+def _clone_dataset(
+    dataset: Dataset,
+    split: str,
+    test_cases: list[TestCase],
+    holdout_ratio: float | None = None,
+    seed: int | None = None,
+) -> Dataset:
+    metadata = dict(dataset.metadata or {})
+    metadata["split"] = split
+    if holdout_ratio is not None:
+        metadata.setdefault("holdout_ratio", holdout_ratio)
+    if seed is not None:
+        metadata.setdefault("split_seed", seed)
+    return Dataset(
+        name=dataset.name,
+        version=dataset.version,
+        test_cases=list(test_cases),
+        metadata=metadata,
+        source_file=dataset.source_file,
+        thresholds=dict(dataset.thresholds),
+    )