evalvault 1.65.0__py3-none-any.whl → 1.66.0__py3-none-any.whl

evalvault/domain/services/custom_metric_snapshot.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import hashlib
+import inspect
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.metrics.registry import get_metric_spec_map
+
+SCHEMA_VERSION = 1
+
+_CUSTOM_METRIC_DETAILS: dict[str, dict[str, Any]] = {
+    "entity_preservation": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "contexts"],
+        "output": "0.0-1.0 (preserved_entities / context_entities)",
+        "evaluation_process": "Extract numeric/keyword entities from contexts and measure how many appear in the summary.",
+        "rules": {
+            "numeric_entities": ["percent", "currency", "duration", "date"],
+            "keywords_ko": [
+                "면책",
+                "제외",
+                "단서",
+                "다만",
+                "조건",
+                "자기부담",
+                "한도",
+                "감액",
+            ],
+            "keywords_en": [
+                "exclusion",
+                "deductible",
+                "limit",
+                "cap",
+                "copay",
+                "coinsurance",
+            ],
+        },
+        "notes": "Insurance-risk oriented entity coverage check.",
+    },
+    "insurance_term_accuracy": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "contexts"],
+        "output": "0.0-1.0 (verified_terms / answer_terms)",
+        "evaluation_process": "Detect insurance terms in the answer and verify their presence in contexts.",
+        "rules": {"terms_dictionary": "terms_dictionary.json"},
+        "notes": "Insurance glossary matching with canonical/variant terms.",
+    },
+    "summary_accuracy": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "contexts"],
+        "output": "0.0-1.0 (supported_summary_entities / summary_entities)",
+        "evaluation_process": "Extract numeric/keyword entities from summary and verify their presence in contexts.",
+        "rules": {
+            "numeric_entities": ["percent", "currency", "duration", "date"],
+            "keywords_ko": ["면책", "제외", "단서", "다만", "조건", "자기부담", "한도", "감액"],
+            "keywords_en": ["exclusion", "deductible", "limit", "cap", "waiting period"],
+        },
+        "notes": "Penalizes summary entities not grounded in contexts.",
+    },
+    "summary_risk_coverage": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "metadata.summary_tags"],
+        "output": "0.0-1.0 (covered_tags / expected_tags)",
+        "evaluation_process": "Check if summary mentions expected insurance risk tags.",
+        "rules": {
+            "exclusion": ["면책", "보장 제외", "지급 불가", "exclusion"],
+            "deductible": ["자기부담", "본인부담금", "deductible", "copay"],
+            "limit": ["한도", "상한", "최대", "limit", "cap"],
+            "waiting_period": ["면책기간", "대기기간", "waiting period"],
+            "condition": ["조건", "단서", "다만", "condition"],
+            "documents_required": ["서류", "진단서", "영수증", "documents"],
+            "needs_followup": ["확인 필요", "추가 확인", "담당자 확인", "재문의", "follow up"],
+        },
+        "notes": "Uses metadata summary_tags to define expected coverage.",
+    },
+    "summary_non_definitive": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer"],
+        "output": "1.0 if definitive claims absent else 0.0",
+        "evaluation_process": "Detect definitive expressions that increase liability risk.",
+        "rules": {
+            "patterns_ko": ["무조건", "반드시", "100%", "전액 지급", "확실히", "분명히", "절대"],
+            "patterns_en": [
+                "always",
+                "guaranteed",
+                "definitely",
+                "certainly",
+                "absolutely",
+                "100%",
+            ],
+        },
+        "notes": "Higher is safer; penalizes absolute guarantees.",
+    },
+    "summary_needs_followup": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "metadata.summary_tags"],
+        "output": "1.0 if follow-up guidance matches expected need",
+        "evaluation_process": "Check follow-up guidance when needs_followup tag exists.",
+        "rules": {
+            "followup_keywords": [
+                "확인 필요",
+                "추가 확인",
+                "담당자 확인",
+                "재문의",
+                "추가 문의",
+                "follow up",
+            ]
+        },
+        "notes": "Requires tags to avoid false penalties.",
+    },
+    "no_answer_accuracy": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "ground_truth"],
+        "output": "1.0 if abstention behavior matches, else 0.0",
+        "evaluation_process": "Detect abstention patterns in answer and ground_truth and compare behavior.",
+        "rules": {"patterns": "Korean/English regex patterns"},
+        "notes": "Hallucination/abstention behavior check.",
+    },
+    "exact_match": {
+        "evaluation_method": "string-match",
+        "inputs": ["answer", "ground_truth"],
+        "output": "1.0 exact match else 0.0",
+        "evaluation_process": "Normalize text and compare exact match with optional strict number matching.",
+        "rules": {"normalize": True, "number_strict": True},
+        "notes": "Token/number strict matching for factual answers.",
+    },
+    "f1_score": {
+        "evaluation_method": "token-overlap",
+        "inputs": ["answer", "ground_truth"],
+        "output": "0.0-1.0 (weighted F1)",
+        "evaluation_process": "Tokenize, compute weighted precision/recall/F1 with number emphasis.",
+        "rules": {"number_weight": 2.0},
+        "notes": "Token-level overlap with numeric weighting.",
+    },
+    "mrr": {
+        "evaluation_method": "retrieval-rank",
+        "inputs": ["ground_truth", "contexts"],
+        "output": "0.0-1.0 (1/rank of first relevant context)",
+        "evaluation_process": "Compute relevance by token overlap and take reciprocal rank of first hit.",
+        "rules": {"relevance_threshold": 0.3},
+        "notes": "Ranking quality of retrieved contexts.",
+    },
+    "ndcg": {
+        "evaluation_method": "retrieval-rank",
+        "inputs": ["ground_truth", "contexts"],
+        "output": "0.0-1.0 (NDCG@K)",
+        "evaluation_process": "Compute graded relevance per context and calculate NDCG.",
+        "rules": {"k": 10, "use_graded": True},
+        "notes": "Ranking quality across all relevant contexts.",
+    },
+    "hit_rate": {
+        "evaluation_method": "retrieval-rank",
+        "inputs": ["ground_truth", "contexts"],
+        "output": "1.0 if any relevant context in top K else 0.0",
+        "evaluation_process": "Check whether top-K contexts contain a relevant hit.",
+        "rules": {"k": 10, "relevance_threshold": 0.3},
+        "notes": "Recall@K style coverage check.",
+    },
+    "confidence_score": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "ground_truth", "contexts"],
+        "output": "0.0-1.0 (weighted confidence)",
+        "evaluation_process": "Combine context coverage, answer specificity, and consistency scores.",
+        "rules": {"coverage": 0.4, "specificity": 0.3, "consistency": 0.3},
+        "notes": "Heuristic confidence signal for human escalation.",
+    },
+    "contextual_relevancy": {
+        "evaluation_method": "token-overlap",
+        "inputs": ["question", "contexts"],
+        "output": "0.0-1.0 (avg relevancy)",
+        "evaluation_process": "Measure question-context token overlap and average across contexts.",
+        "rules": {"relevance_threshold": 0.35},
+        "notes": "Reference-free context relevance check.",
+    },
+}
+
+
+def _hash_file(path: str | Path | None) -> str | None:
+    if not path:
+        return None
+    file_path = Path(path)
+    if not file_path.exists():
+        return None
+    payload = file_path.read_bytes()
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _resolve_source_path(metric_class: type[Any]) -> str | None:
+    try:
+        source = inspect.getsourcefile(metric_class)
+    except TypeError:
+        return None
+    if not source:
+        return None
+    return str(Path(source).resolve())
+
+
+def build_custom_metric_snapshot(
+    metric_classes: dict[str, type[Any]],
+    metrics: Iterable[str],
+) -> dict[str, Any] | None:
+    custom_names = [name for name in metrics if name in metric_classes]
+    if not custom_names:
+        return None
+
+    spec_map = get_metric_spec_map()
+    rows: list[dict[str, Any]] = []
+    for metric_name in custom_names:
+        metric_class = metric_classes.get(metric_name)
+        if metric_class is None:
+            continue
+        source_path = _resolve_source_path(metric_class)
+        details = _CUSTOM_METRIC_DETAILS.get(metric_name, {})
+        spec = spec_map.get(metric_name)
+        rows.append(
+            {
+                "metric_name": metric_name,
+                "source": "custom",
+                "description": spec.description if spec else None,
+                "evaluation_method": details.get("evaluation_method"),
+                "inputs": details.get("inputs"),
+                "output": details.get("output"),
+                "evaluation_process": details.get("evaluation_process"),
+                "rules": details.get("rules"),
+                "notes": details.get("notes"),
+                "implementation_path": source_path,
+                "implementation_hash": _hash_file(source_path),
+            }
+        )
+
+    return {"schema_version": SCHEMA_VERSION, "metrics": rows}

evalvault/domain/services/evaluator.py

@@ -11,8 +11,9 @@ from collections.abc import Callable, Sequence
 from contextlib import suppress
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Any, Literal, cast, overload
+from typing import Any, Literal, overload
 
+from pydantic import BaseModel, Field, field_validator
 from ragas import SingleTurnSample
 
 from evalvault.domain.entities import (
@@ -30,8 +31,13 @@ from evalvault.domain.metrics.entity_preservation import EntityPreservation
 from evalvault.domain.metrics.insurance import InsuranceTermAccuracy
 from evalvault.domain.metrics.no_answer import NoAnswerAccuracy
 from evalvault.domain.metrics.retrieval_rank import MRR, NDCG, HitRate
+from evalvault.domain.metrics.summary_accuracy import SummaryAccuracy
+from evalvault.domain.metrics.summary_needs_followup import SummaryNeedsFollowup
+from evalvault.domain.metrics.summary_non_definitive import SummaryNonDefinitive
+from evalvault.domain.metrics.summary_risk_coverage import SummaryRiskCoverage
 from evalvault.domain.metrics.text_match import ExactMatch, F1Score
 from evalvault.domain.services.batch_executor import run_in_batches
+from evalvault.domain.services.custom_metric_snapshot import build_custom_metric_snapshot
 from evalvault.domain.services.dataset_preprocessor import DatasetPreprocessor
 from evalvault.domain.services.retriever_context import apply_retriever_to_dataset
 from evalvault.ports.outbound.korean_nlp_port import KoreanNLPToolkitPort, RetrieverPort
@@ -55,14 +61,53 @@ _SUMMARY_FAITHFULNESS_PROMPT_EN = (
 )
 
 
+def _patch_ragas_faithfulness_output() -> None:
+    try:
+        from ragas.metrics import Faithfulness
+    except Exception:
+        return
+
+    prompt = getattr(Faithfulness, "nli_statements_prompt", None)
+    if prompt is None:
+        return
+
+    output_model = getattr(prompt, "output_model", None)
+    if output_model is None:
+        return
+
+    class _StatementFaithfulnessAnswer(BaseModel):
+        statement: str = Field(..., description="the original statement, word-by-word")
+        reason: str = Field(..., description="the reason of the verdict")
+        verdict: int = Field(..., description="the verdict(0/1) of the faithfulness.")
+
+        @field_validator("verdict", mode="before")
+        @classmethod
+        def _coerce_verdict(cls, value):
+            if isinstance(value, str):
+                normalized = value.strip()
+                if normalized.isdigit():
+                    return int(normalized)
+            return value
+
+    class _NLIStatementOutput(BaseModel):
+        statements: list[_StatementFaithfulnessAnswer]
+
+    try:
+        prompt.output_model = _NLIStatementOutput
+    except Exception:
+        return
+
+
 def _import_metric(name: str) -> type[Any]:
     for module_name in ("ragas.metrics.collections", "ragas.metrics"):
         try:
             module = importlib.import_module(module_name)
-        except Exception:
+            if hasattr(module, name):
+                if name == "Faithfulness":
+                    _patch_ragas_faithfulness_output()
+                return getattr(module, name)
+        except ImportError:
             continue
-        if hasattr(module, name):
-            return cast(type[Any], getattr(module, name))
     raise ImportError(f"Missing ragas metric: {name}")
 
 
@@ -147,6 +192,10 @@ class RagasEvaluator:
     CUSTOM_METRIC_MAP = {
         "insurance_term_accuracy": InsuranceTermAccuracy,
         "entity_preservation": EntityPreservation,
+        "summary_accuracy": SummaryAccuracy,
+        "summary_risk_coverage": SummaryRiskCoverage,
+        "summary_non_definitive": SummaryNonDefinitive,
+        "summary_needs_followup": SummaryNeedsFollowup,
         "exact_match": ExactMatch,
         "f1_score": F1Score,
         "no_answer_accuracy": NoAnswerAccuracy,
@@ -198,6 +247,10 @@ class RagasEvaluator:
         "summary_faithfulness": 0.9,
         "summary_score": 0.85,
         "entity_preservation": 0.9,
+        "summary_accuracy": 0.9,
+        "summary_risk_coverage": 0.9,
+        "summary_non_definitive": 0.8,
+        "summary_needs_followup": 0.8,
         "contextual_relevancy": 0.35,
     }
     LANGUAGE_SAMPLE_LIMIT = 5
@@ -225,10 +278,28 @@ class RagasEvaluator:
         "예시의 원자성 수준을 따르세요."
     )
     FACTUAL_CORRECTNESS_NLI_INSTRUCTION = (
-        "다음 CONTEXT를 바탕으로 각 STATEMENT가 직접적으로 "
-        "추론 가능한지 판단하세요. "
-        "가능하면 verdict=1, 불가능하면 verdict=0으로 표시하고, "
-        "간단한 이유를 한국어로 적으세요."
+        "주어진 컨텍스트를 보고 각 진술이 직접적으로 도출 가능한지 판단하세요. "
+        "가능하면 verdict=1, 불가능하면 verdict=0을 JSON으로 반환하세요."
+    )
+    SUMMARY_SCORE_QUESTION_INSTRUCTION = (
+        "다음 텍스트와 핵심 키워드를 기반으로, "
+        "텍스트에 근거해 반드시 1로 답할 수 있는 폐쇄형 질문을 생성하세요. "
+        "질문은 한국어로 작성하세요."
+    )
+    SUMMARY_SCORE_ANSWER_INSTRUCTION = (
+        "다음 질문 목록에 대해, 제공된 요약이 각 질문에 답할 수 있으면 '1', "
+        "그렇지 않으면 '0'을 JSON 배열로 반환하세요."
+    )
+    SUMMARY_SCORE_KEYPHRASE_INSTRUCTION = (
+        "다음 텍스트에서 인물, 기관, 위치, 날짜/시간, 금액, 비율과 같은 핵심 키워드를 추출하세요."
+    )
+    SUMMARY_FAITHFULNESS_STATEMENT_INSTRUCTION = (
+        "질문과 답변을 보고 각 문장을 이해 가능한 주장으로 분해하세요. "
+        "각 주장은 대명사 없이 독립적으로 이해 가능해야 합니다."
+    )
+    SUMMARY_FAITHFULNESS_NLI_INSTRUCTION = (
+        "주어진 컨텍스트를 보고 각 진술이 직접적으로 도출 가능한지 판단하세요. "
+        "가능하면 verdict=1, 불가능하면 verdict=0을 JSON으로 반환하세요."
     )
     FACTUAL_CORRECTNESS_CLAIM_EXAMPLES = [
         {
@@ -390,6 +461,7 @@ class RagasEvaluator:
 
         # Evaluate with Ragas (if any Ragas metrics)
         eval_results_by_test_case = {}
+        prompt_snapshots = {}
         if ragas_metrics:
             run.tracker_metadata["ragas_config"] = self._build_ragas_config(llm)
             (
@@ -412,6 +484,13 @@ class RagasEvaluator:
         elif prompt_overrides:
             logger.warning("Ragas prompt overrides provided but no Ragas metrics requested.")
 
+        custom_snapshot = build_custom_metric_snapshot(self.CUSTOM_METRIC_MAP, metrics)
+        if custom_snapshot:
+            run.tracker_metadata["custom_metric_snapshot"] = custom_snapshot
+            custom_prompt_snapshots = self._build_custom_prompt_snapshots(custom_snapshot)
+            if custom_prompt_snapshots:
+                run.tracker_metadata["custom_prompt_snapshots"] = custom_prompt_snapshots
+
         # Evaluate with custom metrics (if any custom metrics)
         if custom_metrics:
             custom_results = await self._evaluate_with_custom_metrics(
@@ -583,6 +662,11 @@ class RagasEvaluator:
             ragas_metrics=ragas_metrics,
             prompt_overrides=prompt_overrides,
         )
+        self._apply_summary_prompt_defaults(
+            dataset=dataset,
+            ragas_metrics=ragas_metrics,
+            prompt_overrides=prompt_overrides,
+        )
         self._apply_factual_correctness_prompt_defaults(
             dataset=dataset,
             ragas_metrics=ragas_metrics,
@@ -645,6 +729,30 @@ class RagasEvaluator:
                 continue
             self._apply_korean_answer_relevancy_prompt(metric)
 
+    def _apply_summary_prompt_defaults(
+        self,
+        *,
+        dataset: Dataset,
+        ragas_metrics: list[Any],
+        prompt_overrides: dict[str, str] | None,
+    ) -> None:
+        if not ragas_metrics:
+            return
+        if prompt_overrides and any(
+            metric in prompt_overrides for metric in ("summary_score", "summary_faithfulness")
+        ):
+            return
+        resolved_language = self._resolve_dataset_language(dataset)
+        if resolved_language == "en":
+            return
+
+        for metric in ragas_metrics:
+            metric_name = getattr(metric, "name", None)
+            if metric_name == "summary_score":
+                self._apply_korean_summary_score_prompts(metric)
+            elif metric_name == "summary_faithfulness":
+                self._apply_korean_summary_faithfulness_prompts(metric)
+
     def _apply_factual_correctness_prompt_defaults(
         self,
         *,
@@ -745,6 +853,56 @@ class RagasEvaluator:
                 prompt.language = "ko"
         return True
 
+    def _apply_korean_summary_score_prompts(self, metric: Any) -> bool:
+        question_prompt = getattr(metric, "question_generation_prompt", None)
+        answer_prompt = getattr(metric, "answer_generation_prompt", None)
+        keyphrase_prompt = getattr(metric, "extract_keyphrases_prompt", None)
+        applied = False
+
+        if question_prompt and hasattr(question_prompt, "instruction"):
+            question_prompt.instruction = self.SUMMARY_SCORE_QUESTION_INSTRUCTION
+            if hasattr(question_prompt, "language"):
+                with suppress(Exception):
+                    question_prompt.language = "ko"
+            applied = True
+
+        if answer_prompt and hasattr(answer_prompt, "instruction"):
+            answer_prompt.instruction = self.SUMMARY_SCORE_ANSWER_INSTRUCTION
+            if hasattr(answer_prompt, "language"):
+                with suppress(Exception):
+                    answer_prompt.language = "ko"
+            applied = True
+
+        if keyphrase_prompt and hasattr(keyphrase_prompt, "instruction"):
+            keyphrase_prompt.instruction = self.SUMMARY_SCORE_KEYPHRASE_INSTRUCTION
+            if hasattr(keyphrase_prompt, "language"):
+                with suppress(Exception):
+                    keyphrase_prompt.language = "ko"
+            applied = True
+
+        return applied
+
+    def _apply_korean_summary_faithfulness_prompts(self, metric: Any) -> bool:
+        statement_prompt = getattr(metric, "statement_generator_prompt", None)
+        nli_prompt = getattr(metric, "nli_statements_prompt", None)
+        applied = False
+
+        if statement_prompt and hasattr(statement_prompt, "instruction"):
+            statement_prompt.instruction = self.SUMMARY_FAITHFULNESS_STATEMENT_INSTRUCTION
+            if hasattr(statement_prompt, "language"):
+                with suppress(Exception):
+                    statement_prompt.language = "ko"
+            applied = True
+
+        if nli_prompt and hasattr(nli_prompt, "instruction"):
+            nli_prompt.instruction = self.SUMMARY_FAITHFULNESS_NLI_INSTRUCTION
+            if hasattr(nli_prompt, "language"):
+                with suppress(Exception):
+                    nli_prompt.language = "ko"
+            applied = True
+
+        return applied
+
     def _apply_korean_factual_correctness_prompts(self, metric: Any) -> bool:
         claim_prompt = getattr(metric, "claim_decomposition_prompt", None)
         nli_prompt = getattr(metric, "nli_prompt", None)
@@ -819,6 +977,8 @@ class RagasEvaluator:
                 continue
             prompt_text = prompt_overrides[metric_name]
             applied = self._override_metric_prompt(metric, prompt_text)
+            if not applied and metric_name == "faithfulness":
+                applied = self._override_faithfulness_prompt(metric, prompt_text)
             statuses[metric_name] = "applied" if applied else "unsupported"
             if not applied:
                 logger.warning("Prompt override for metric '%s' could not be applied.", metric_name)
@@ -878,6 +1038,16 @@ class RagasEvaluator:
 
         return False
 
+    @staticmethod
+    def _override_faithfulness_prompt(metric: Any, prompt_text: str) -> bool:
+        target = getattr(metric, "nli_statements_prompt", None)
+        if target is None:
+            return False
+        if hasattr(target, "instruction"):
+            target.instruction = prompt_text
+            return True
+        return False
+
     @staticmethod
     def _extract_prompt_text(value: Any) -> str | None:
         if value is None:
@@ -926,18 +1096,50 @@ class RagasEvaluator:
             metric_name = getattr(metric, "name", None)
             if not metric_name:
                 continue
-            prompt_text = self._collect_metric_prompt_text(metric)
-            if not prompt_text:
-                continue
             requested = bool(prompt_overrides and metric_name in prompt_overrides)
             status = override_status.get(metric_name)
             source = "override" if status == "applied" else "default"
-            snapshots[str(metric_name)] = {
-                "prompt": prompt_text,
-                "source": source,
-                "override_requested": requested,
-                "override_status": status,
-            }
+
+            prompts: dict[str, str] = {}
+            if metric_name == "summary_score":
+                prompts["question_generation"] = (
+                    self._extract_prompt_text(getattr(metric, "question_generation_prompt", None))
+                    or ""
+                )
+                prompts["answer_generation"] = (
+                    self._extract_prompt_text(getattr(metric, "answer_generation_prompt", None))
+                    or ""
+                )
+                prompts["extract_keyphrases"] = (
+                    self._extract_prompt_text(getattr(metric, "extract_keyphrases_prompt", None))
+                    or ""
+                )
+                prompts = {k: v for k, v in prompts.items() if v}
+            elif metric_name == "summary_faithfulness":
+                prompts["statement_generation"] = (
+                    self._extract_prompt_text(getattr(metric, "statement_generator_prompt", None))
+                    or ""
+                )
+                prompts["nli_statements"] = (
+                    self._extract_prompt_text(getattr(metric, "nli_statements_prompt", None)) or ""
+                )
+                prompts = {k: v for k, v in prompts.items() if v}
+
+            prompt_text = self._collect_metric_prompt_text(metric)
+            if prompts:
+                snapshots[str(metric_name)] = {
+                    "prompts": prompts,
+                    "source": source,
+                    "override_requested": requested,
+                    "override_status": status,
+                }
+            elif prompt_text:
+                snapshots[str(metric_name)] = {
+                    "prompt": prompt_text,
+                    "source": source,
+                    "override_requested": requested,
+                    "override_status": status,
+                }
         return snapshots
 
     async def _evaluate_sequential(
@@ -1135,16 +1337,26 @@ class RagasEvaluator:
         claim_details: dict[str, ClaimLevelResult] = {}
 
         for metric in ragas_metrics:
-            if metric.name in self.FAITHFULNESS_METRICS and self._faithfulness_ragas_failed:
-                if metric.name == "summary_faithfulness":
-                    judge_score = await self._score_summary_faithfulness_judge(sample)
-                    if judge_score is not None:
-                        scores[metric.name] = judge_score
+            if metric.name in self.FAITHFULNESS_METRICS:
+                if self._active_llm_provider == "ollama":
+                    fallback_score = self._fallback_korean_faithfulness(
+                        sample, return_details=False
+                    )
+                    if fallback_score is None:
+                        fallback_score = await self._score_faithfulness_with_fallback(sample)
+                    if fallback_score is not None:
+                        scores[metric.name] = fallback_score
+                        continue
+                if self._faithfulness_ragas_failed:
+                    if metric.name == "summary_faithfulness":
+                        judge_score = await self._score_summary_faithfulness_judge(sample)
+                        if judge_score is not None:
+                            scores[metric.name] = judge_score
+                            continue
+                    fallback_score = await self._score_faithfulness_with_fallback(sample)
+                    if fallback_score is not None:
+                        scores[metric.name] = fallback_score
                         continue
-                fallback_score = await self._score_faithfulness_with_fallback(sample)
-                if fallback_score is not None:
-                    scores[metric.name] = fallback_score
-                    continue
             try:
                 # Ragas >=0.4 uses ascore() with kwargs
                 if hasattr(metric, "ascore"):
@@ -1272,6 +1484,32 @@ class RagasEvaluator:
         normalized = str(domain).strip().lower()
         return cls.SUMMARY_SCORE_COEFF_BY_DOMAIN.get(normalized, cls.SUMMARY_SCORE_COEFF)
 
+    def _build_custom_prompt_snapshots(self, snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]:
+        entries = snapshot.get("metrics") if isinstance(snapshot, dict) else None
+        if not isinstance(entries, list):
+            return {}
+        prompt_snapshot: dict[str, dict[str, Any]] = {}
+        for entry in entries:
+            if not isinstance(entry, dict):
+                continue
+            name = entry.get("metric_name")
+            if not isinstance(name, str) or not name:
+                continue
+            evaluation_process = entry.get("evaluation_process")
+            if not isinstance(evaluation_process, str) or not evaluation_process:
+                continue
+            rules = entry.get("rules") if isinstance(entry.get("rules"), dict) else None
+            prompts: dict[str, str] = {"rule": evaluation_process}
+            if rules:
+                prompts["rules"] = json.dumps(rules, ensure_ascii=False, indent=2)
+            prompt_snapshot[name] = {
+                "prompts": prompts,
+                "source": "custom_rules",
+                "rules": rules,
+                "inputs": entry.get("inputs"),
+            }
+        return prompt_snapshot
+
     def _build_summary_score_metric(self, metric_class, ragas_llm, coeff: float | None = None):
         if coeff is None:
             coeff = self.SUMMARY_SCORE_COEFF
@@ -1653,9 +1891,11 @@ class RagasEvaluator:
                             contexts=test_case.contexts,
                         )
                     else:
-                        score = metric_instance.score(
+                        score = self._score_custom_metric_with_metadata(
+                            metric_instance,
                             answer=test_case.answer,
                             contexts=test_case.contexts,
+                            metadata=test_case.metadata,
                         )
                 scores[metric_name] = score
 
@@ -1676,6 +1916,19 @@ class RagasEvaluator:
 
         return results
 
+    def _score_custom_metric_with_metadata(
+        self,
+        metric_instance: Any,
+        *,
+        answer: str,
+        contexts: list[str],
+        metadata: dict[str, Any],
+    ) -> float:
+        try:
+            return float(metric_instance.score(answer=answer, contexts=contexts, metadata=metadata))
+        except TypeError:
+            return float(metric_instance.score(answer=answer, contexts=contexts))
+
     def _calculate_cost(self, model_name: str, prompt_tokens: int, completion_tokens: int) -> float:
         """Calculate estimated cost in USD based on model pricing."""
         # Find matching model key (exact or substring match)