evalvault 1.64.0__py3-none-any.whl → 1.66.0__py3-none-any.whl

evalvault/domain/entities/judge_calibration.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class JudgeCalibrationCase:
+    test_case_id: str
+    raw_score: float
+    calibrated_score: float
+    label: float | None = None
+    label_source: str | None = None
+
+
+@dataclass
+class JudgeCalibrationMetric:
+    metric: str
+    method: str
+    sample_count: int
+    label_count: int
+    mae: float | None
+    pearson: float | None
+    spearman: float | None
+    temperature: float | None = None
+    parameters: dict[str, float | None] = field(default_factory=dict)
+    gate_passed: bool | None = None
+    warning: str | None = None
+
+
+@dataclass
+class JudgeCalibrationSummary:
+    run_id: str
+    labels_source: str
+    method: str
+    metrics: list[str]
+    holdout_ratio: float
+    seed: int
+    total_labels: int
+    total_samples: int
+    gate_passed: bool
+    gate_threshold: float | None = None
+    notes: list[str] = field(default_factory=list)
+
+
+@dataclass
+class JudgeCalibrationResult:
+    summary: JudgeCalibrationSummary
+    metrics: list[JudgeCalibrationMetric] = field(default_factory=list)
+    case_results: dict[str, list[JudgeCalibrationCase]] = field(default_factory=dict)
+    warnings: list[str] = field(default_factory=list)

evalvault/domain/entities/prompt.py

@@ -7,7 +7,7 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import uuid4
 
-PromptKind = Literal["system", "ragas"]
+PromptKind = Literal["system", "ragas", "custom"]
 
 
 @dataclass

evalvault/domain/entities/stage.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any
+from typing import Any, Literal, overload
 from uuid import uuid4
 
 REQUIRED_STAGE_TYPES: tuple[str, ...] = ("system_prompt", "input", "retrieval", "output")
@@ -82,8 +82,8 @@ class StageEvent:
             duration_ms=_optional_float(payload.get("duration_ms")),
             input_ref=input_ref,
             output_ref=output_ref,
-            attributes=_ensure_dict(payload.get("attributes")),
-            metadata=_ensure_dict(payload.get("metadata")),
+            attributes=_ensure_dict(payload.get("attributes"), allow_none=False),
+            metadata=_ensure_dict(payload.get("metadata"), allow_none=False),
             trace_id=_optional_str(payload.get("trace_id") or trace_payload.get("trace_id")),
             span_id=_optional_str(payload.get("span_id") or trace_payload.get("span_id")),
         )
@@ -187,6 +187,14 @@ def _parse_datetime(value: Any) -> datetime | None:
     raise ValueError("Invalid datetime value")
 
 
+@overload
+def _ensure_dict(value: None, *, allow_none: Literal[True]) -> None: ...
+
+
+@overload
+def _ensure_dict(value: Any, *, allow_none: Literal[False] = False) -> dict[str, Any]: ...
+
+
 def _ensure_dict(value: Any, *, allow_none: bool = False) -> dict[str, Any] | None:
     if value is None:
         return None if allow_none else {}

evalvault/domain/metrics/__init__.py

@@ -6,6 +6,10 @@ from evalvault.domain.metrics.entity_preservation import EntityPreservation
 from evalvault.domain.metrics.insurance import InsuranceTermAccuracy
 from evalvault.domain.metrics.no_answer import NoAnswerAccuracy, is_no_answer
 from evalvault.domain.metrics.retrieval_rank import MRR, NDCG, HitRate
+from evalvault.domain.metrics.summary_accuracy import SummaryAccuracy
+from evalvault.domain.metrics.summary_needs_followup import SummaryNeedsFollowup
+from evalvault.domain.metrics.summary_non_definitive import SummaryNonDefinitive
+from evalvault.domain.metrics.summary_risk_coverage import SummaryRiskCoverage
 from evalvault.domain.metrics.text_match import ExactMatch, F1Score
 
 __all__ = [
@@ -19,5 +23,9 @@ __all__ = [
     "MRR",
     "NDCG",
     "NoAnswerAccuracy",
+    "SummaryAccuracy",
+    "SummaryNeedsFollowup",
+    "SummaryNonDefinitive",
+    "SummaryRiskCoverage",
     "is_no_answer",
 ]

evalvault/domain/metrics/registry.py

@@ -123,7 +123,7 @@ _METRIC_SPECS: tuple[MetricSpec, ...] = (
     ),
     MetricSpec(
         name="summary_score",
-        description="Measures summary coverage and conciseness against contexts",
+        description="(LLM) Measures summary coverage and conciseness against contexts",
         requires_ground_truth=False,
         requires_embeddings=False,
         source="ragas",
@@ -132,7 +132,7 @@ _METRIC_SPECS: tuple[MetricSpec, ...] = (
     ),
     MetricSpec(
         name="summary_faithfulness",
-        description="Measures whether summary statements are grounded in contexts",
+        description="(LLM) Measures whether summary statements are grounded in contexts",
         requires_ground_truth=False,
         requires_embeddings=False,
         source="ragas",
@@ -141,7 +141,43 @@ _METRIC_SPECS: tuple[MetricSpec, ...] = (
     ),
     MetricSpec(
         name="entity_preservation",
-        description="Measures preservation of key insurance entities in summaries",
+        description="(Rule) Measures preservation of key insurance entities in summaries",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="summary",
+        signal_group="summary_fidelity",
+    ),
+    MetricSpec(
+        name="summary_accuracy",
+        description="(Rule) Measures whether summary entities are grounded in contexts",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="summary",
+        signal_group="summary_fidelity",
+    ),
+    MetricSpec(
+        name="summary_risk_coverage",
+        description="(Rule) Measures coverage of expected insurance risk tags in summaries",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="summary",
+        signal_group="summary_fidelity",
+    ),
+    MetricSpec(
+        name="summary_non_definitive",
+        description="(Rule) Measures avoidance of definitive claims in summaries",
+        requires_ground_truth=False,
+        requires_embeddings=False,
+        source="custom",
+        category="summary",
+        signal_group="summary_fidelity",
+    ),
+    MetricSpec(
+        name="summary_needs_followup",
+        description="(Rule) Measures follow-up guidance when required",
         requires_ground_truth=False,
         requires_embeddings=False,
         source="custom",

evalvault/domain/metrics/summary_accuracy.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import re
+from decimal import Decimal, InvalidOperation
+
+
+class SummaryAccuracy:
+    """Measure whether summary entities are supported by contexts."""
+
+    name = "summary_accuracy"
+
+    _PERCENT_RE = re.compile(r"(?P<number>\d+(?:[.,]\d+)?)\s*(?P<unit>%|퍼센트|percent)", re.I)
+    _CURRENCY_RE = re.compile(
+        r"(?P<number>\d+(?:[.,]\d+)?)\s*(?P<unit>원|만원|억원|달러|usd|krw|won)",
+        re.I,
+    )
+    _CURRENCY_PREFIX_RE = re.compile(r"(?P<unit>[$₩])\s*(?P<number>\d+(?:[.,]\d+)?)")
+    _DURATION_RE = re.compile(
+        r"(?P<number>\d+(?:[.,]\d+)?)\s*(?P<unit>년|개월|월|일|years?|months?|days?)",
+        re.I,
+    )
+    _DATE_RE = re.compile(r"\b\d{4}[./-]\d{1,2}[./-]\d{1,2}\b")
+
+    _CURRENCY_MULTIPLIERS = {"만원": Decimal("10000"), "억원": Decimal("100000000")}
+    _KRW_UNITS = {"원", "krw", "won", "₩", "만원", "억원"}
+    _USD_UNITS = {"달러", "usd", "$"}
+    _DURATION_UNITS = {
+        "년": "year",
+        "year": "year",
+        "years": "year",
+        "개월": "month",
+        "월": "month",
+        "month": "month",
+        "months": "month",
+        "일": "day",
+        "day": "day",
+        "days": "day",
+    }
+
+    _KEYWORDS_KO = (
+        "면책",
+        "제외",
+        "단서",
+        "다만",
+        "조건",
+        "자기부담",
+        "한도",
+        "감액",
+    )
+    _KEYWORDS_EN = (
+        "exclusion",
+        "excluded",
+        "exception",
+        "except",
+        "condition",
+        "deductible",
+        "limit",
+        "cap",
+        "waiting period",
+        "co-pay",
+        "copay",
+        "co-insurance",
+        "coinsurance",
+    )
+
+    def score(self, answer: str, contexts: list[str]) -> float:
+        if not contexts:
+            return 0.0
+
+        context_text = " ".join([ctx for ctx in contexts if ctx])
+        context_entities = self._extract_entities(context_text)
+        summary_entities = self._extract_entities(answer or "")
+
+        if not summary_entities:
+            return 0.5 if context_entities else 0.0
+        if not context_entities:
+            return 0.0
+
+        supported = summary_entities.intersection(context_entities)
+        return len(supported) / len(summary_entities)
+
+    def _extract_entities(self, text: str) -> set[str]:
+        entities = set()
+        entities.update(self._extract_numeric_entities(text))
+        entities.update(self._extract_keyword_entities(text))
+        return entities
+
+    def _extract_numeric_entities(self, text: str) -> set[str]:
+        entities: set[str] = set()
+
+        for match in self._PERCENT_RE.finditer(text):
+            number = self._normalize_number(match.group("number"))
+            if number:
+                entities.add(f"percent:{number}")
+
+        for match in self._CURRENCY_RE.finditer(text):
+            number = self._normalize_number(match.group("number"))
+            unit = match.group("unit").lower()
+            normalized = self._normalize_currency(number, unit)
+            if normalized:
+                entities.add(f"currency:{normalized}")
+
+        for match in self._CURRENCY_PREFIX_RE.finditer(text):
+            number = self._normalize_number(match.group("number"))
+            unit = match.group("unit")
+            normalized = self._normalize_currency(number, unit)
+            if normalized:
+                entities.add(f"currency:{normalized}")
+
+        for match in self._DURATION_RE.finditer(text):
+            number = self._normalize_number(match.group("number"))
+            unit = match.group("unit").lower()
+            normalized = self._normalize_duration(number, unit)
+            if normalized:
+                entities.add(f"duration:{normalized}")
+
+        for match in self._DATE_RE.finditer(text):
+            entities.add(f"date:{self._normalize_date(match.group(0))}")
+
+        return entities
+
+    def _extract_keyword_entities(self, text: str) -> set[str]:
+        entities: set[str] = set()
+        lower = text.lower()
+
+        for keyword in self._KEYWORDS_KO:
+            if keyword in text:
+                entities.add(f"kw:{keyword}")
+
+        for keyword in self._KEYWORDS_EN:
+            if keyword in lower:
+                entities.add(f"kw:{keyword}")
+
+        return entities
+
+    def _normalize_currency(self, number: str | None, unit: str) -> str | None:
+        if number is None:
+            return None
+        try:
+            value = Decimal(number)
+        except InvalidOperation:
+            return None
+
+        unit_key = unit.lower()
+        multiplier = self._CURRENCY_MULTIPLIERS.get(unit_key)
+        if multiplier:
+            value *= multiplier
+
+        if unit_key in self._KRW_UNITS:
+            currency = "krw"
+        elif unit_key in self._USD_UNITS:
+            currency = "usd"
+        else:
+            currency = unit_key
+
+        return f"{currency}:{self._format_decimal(value)}"
+
+    def _normalize_duration(self, number: str | None, unit: str) -> str | None:
+        if number is None:
+            return None
+        try:
+            value = Decimal(number)
+        except InvalidOperation:
+            return None
+        base_unit = self._DURATION_UNITS.get(unit, unit)
+        return f"{self._format_decimal(value)}{base_unit}"
+
+    @staticmethod
+    def _normalize_date(raw: str) -> str:
+        return re.sub(r"[./-]", "", raw)
+
+    @staticmethod
+    def _normalize_number(raw: str | None) -> str | None:
+        if raw is None:
+            return None
+        cleaned = raw.replace(",", "").strip()
+        if not cleaned:
+            return None
+        try:
+            value = Decimal(cleaned)
+        except InvalidOperation:
+            return None
+        return SummaryAccuracy._format_decimal(value)
+
+    @staticmethod
+    def _format_decimal(value: Decimal) -> str:
+        if value == value.to_integral_value():
+            return str(value.to_integral_value())
+        return format(value.normalize(), "f").rstrip("0").rstrip(".")

evalvault/domain/metrics/summary_needs_followup.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+
+class SummaryNeedsFollowup:
+    """Check if follow-up guidance appears when required."""
+
+    name = "summary_needs_followup"
+
+    _FOLLOWUP_KEYWORDS = [
+        "확인 필요",
+        "추가 확인",
+        "담당자 확인",
+        "재문의",
+        "추가 문의",
+        "서류 확인",
+        "follow up",
+        "follow-up",
+    ]
+
+    def score(self, answer: str, contexts: list[str], metadata: dict | None = None) -> float:
+        text = answer or ""
+        has_followup = self._has_followup(text)
+        expected = self._expects_followup(metadata)
+
+        if expected:
+            return 1.0 if has_followup else 0.0
+        return 1.0 if not has_followup else 0.0
+
+    def _expects_followup(self, metadata: dict | None) -> bool:
+        if not metadata:
+            return False
+        raw = metadata.get("summary_tags")
+        if not raw:
+            return False
+        if isinstance(raw, list):
+            tags = [str(item).strip().lower() for item in raw if str(item).strip()]
+        else:
+            tags = [str(raw).strip().lower()]
+        return "needs_followup" in tags
+
+    def _has_followup(self, text: str) -> bool:
+        lowered = text.lower()
+        return any(
+            keyword in text or keyword.lower() in lowered for keyword in self._FOLLOWUP_KEYWORDS
+        )

evalvault/domain/metrics/summary_non_definitive.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import re
+
+
+class SummaryNonDefinitive:
+    """Penalize definitive statements in summaries."""
+
+    name = "summary_non_definitive"
+
+    _DEFINITIVE_PATTERNS_KO = [
+        r"무조건",
+        r"반드시",
+        r"100%",
+        r"전액\s*지급",
+        r"확실히",
+        r"분명히",
+        r"절대",
+        r"항상",
+    ]
+    _DEFINITIVE_PATTERNS_EN = [
+        r"always",
+        r"guaranteed",
+        r"definitely",
+        r"certainly",
+        r"absolutely",
+        r"100%",
+    ]
+
+    def score(self, answer: str, contexts: list[str]) -> float:
+        text = answer or ""
+        if self._has_definitive_pattern(text):
+            return 0.0
+        return 1.0
+
+    def _has_definitive_pattern(self, text: str) -> bool:
+        for pattern in self._DEFINITIVE_PATTERNS_KO:
+            if re.search(pattern, text):
+                return True
+        lowered = text.lower()
+        return any(re.search(pattern, lowered) for pattern in self._DEFINITIVE_PATTERNS_EN)

evalvault/domain/metrics/summary_risk_coverage.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+
+class SummaryRiskCoverage:
+    """Measure coverage of expected insurance risk tags in summary."""
+
+    name = "summary_risk_coverage"
+
+    _TAG_KEYWORDS = {
+        "exclusion": ["면책", "보장 제외", "지급 불가", "exclusion"],
+        "deductible": ["자기부담", "본인부담금", "deductible", "copay"],
+        "limit": ["한도", "상한", "최대", "limit", "cap"],
+        "waiting_period": ["면책기간", "대기기간", "waiting period"],
+        "condition": ["조건", "단서", "다만", "condition"],
+        "documents_required": ["서류", "진단서", "영수증", "documents"],
+        "needs_followup": ["확인 필요", "추가 확인", "담당자 확인", "재문의", "follow up"],
+    }
+
+    def score(self, answer: str, contexts: list[str], metadata: dict | None = None) -> float:
+        expected_tags = self._extract_expected_tags(metadata)
+        if not expected_tags:
+            return 1.0
+
+        text = answer or ""
+        covered = 0
+        for tag in expected_tags:
+            if self._has_tag_keyword(text, tag):
+                covered += 1
+
+        return covered / len(expected_tags)
+
+    def _extract_expected_tags(self, metadata: dict | None) -> list[str]:
+        if not metadata:
+            return []
+        raw = metadata.get("summary_tags")
+        if not raw:
+            return []
+        if isinstance(raw, list):
+            return [str(item).strip().lower() for item in raw if str(item).strip()]
+        return [str(raw).strip().lower()]
+
+    def _has_tag_keyword(self, text: str, tag: str) -> bool:
+        keywords = self._TAG_KEYWORDS.get(tag, [])
+        lowered = text.lower()
+        return any(keyword in text or keyword.lower() in lowered for keyword in keywords)