evalvault 1.64.0__py3-none-any.whl → 1.66.0__py3-none-any.whl

evalvault/domain/services/artifact_lint_service.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from evalvault.ports.outbound.artifact_fs_port import ArtifactFileSystemPort
+
+logger = logging.getLogger(__name__)
+
+
+LintLevel = Literal["error", "warning"]
+LintStatus = Literal["ok", "warning", "error"]
+
+
+@dataclass(frozen=True)
+class ArtifactLintIssue:
+    level: LintLevel
+    code: str
+    message: str
+    path: str | None = None
+
+
+@dataclass(frozen=True)
+class ArtifactLintSummary:
+    status: LintStatus
+    issues: list[ArtifactLintIssue]
+    artifacts_dir: Path
+    index_path: Path
+    started_at: datetime
+    finished_at: datetime
+    duration_ms: int
+    strict: bool
+
+
+class ArtifactLintService:
+    def __init__(self, fs: ArtifactFileSystemPort) -> None:
+        self._fs = fs
+
+    def lint(self, artifacts_dir: Path, *, strict: bool = False) -> ArtifactLintSummary:
+        started_at = datetime.now(UTC)
+        issues: list[ArtifactLintIssue] = []
+        index_path = artifacts_dir / "index.json"
+        logger.info("Artifact lint started: %s", artifacts_dir)
+
+        try:
+            self._validate_dir(artifacts_dir, issues)
+            if not self._fs.exists(index_path):
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.missing",
+                        "index.json is missing.",
+                        path=str(index_path),
+                    )
+                )
+            elif self._fs.exists(artifacts_dir) and self._fs.is_dir(artifacts_dir):
+                index_payload = self._load_index(index_path, issues)
+                if index_payload is not None:
+                    self._validate_index(
+                        index_payload,
+                        artifacts_dir,
+                        issues,
+                        strict=strict,
+                    )
+        except Exception as exc:
+            logger.exception("Artifact lint failed: %s", artifacts_dir)
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.lint.exception",
+                    f"Unexpected error: {exc}",
+                )
+            )
+
+        finished_at = datetime.now(UTC)
+        duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+        status = _resolve_status(issues)
+        logger.info("Artifact lint finished: %s (%s)", artifacts_dir, status)
+        return ArtifactLintSummary(
+            status=status,
+            issues=issues,
+            artifacts_dir=artifacts_dir,
+            index_path=index_path,
+            started_at=started_at,
+            finished_at=finished_at,
+            duration_ms=duration_ms,
+            strict=strict,
+        )
+
+    def _validate_dir(self, artifacts_dir: Path, issues: list[ArtifactLintIssue]) -> None:
+        if not self._fs.exists(artifacts_dir):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.dir.missing",
+                    "Artifacts directory is missing.",
+                    path=str(artifacts_dir),
+                )
+            )
+            return
+        if not self._fs.is_dir(artifacts_dir):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.dir.not_directory",
+                    "Artifacts path is not a directory.",
+                    path=str(artifacts_dir),
+                )
+            )
+
+    def _load_index(
+        self,
+        index_path: Path,
+        issues: list[ArtifactLintIssue],
+    ) -> dict[str, object] | None:
+        try:
+            payload = json.loads(self._fs.read_text(index_path))
+        except json.JSONDecodeError as exc:
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.invalid_json",
+                    f"index.json parse failed: {exc}",
+                    path=str(index_path),
+                )
+            )
+            return None
+        except OSError as exc:
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.read_failed",
+                    f"index.json read failed: {exc}",
+                    path=str(index_path),
+                )
+            )
+            return None
+
+        if not isinstance(payload, dict):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.invalid_schema",
+                    "index.json root must be an object.",
+                    path=str(index_path),
+                )
+            )
+            return None
+        return payload
+
+    def _validate_index(
+        self,
+        payload: dict[str, object],
+        artifacts_dir: Path,
+        issues: list[ArtifactLintIssue],
+        *,
+        strict: bool,
+    ) -> None:
+        pipeline_id = payload.get("pipeline_id")
+        if not isinstance(pipeline_id, str) or not pipeline_id.strip():
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.pipeline_id.missing",
+                    "pipeline_id is missing.",
+                )
+            )
+
+        nodes = payload.get("nodes")
+        if not isinstance(nodes, list):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.nodes.invalid",
+                    "nodes list is missing or invalid.",
+                )
+            )
+            return
+
+        for idx, node in enumerate(nodes, start=1):
+            if not isinstance(node, dict):
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.node.invalid",
+                        f"nodes[{idx}] entry must be an object.",
+                    )
+                )
+                continue
+            node_id = node.get("node_id")
+            if not isinstance(node_id, str) or not node_id.strip():
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.node_id.missing",
+                        f"nodes[{idx}] node_id is missing.",
+                    )
+                )
+            path_value = node.get("path")
+            self._validate_path(
+                path_value,
+                artifacts_dir,
+                issues,
+                strict=strict,
+                code="artifacts.index.node.path.missing",
+                message=f"nodes[{idx}] path is missing.",
+            )
+
+        final_output = payload.get("final_output_path")
+        if final_output:
+            self._validate_path(
+                final_output,
+                artifacts_dir,
+                issues,
+                strict=strict,
+                code="artifacts.index.final_output.missing",
+                message="final_output_path is missing.",
+            )
+
+    def _validate_path(
+        self,
+        path_value: object,
+        artifacts_dir: Path,
+        issues: list[ArtifactLintIssue],
+        *,
+        strict: bool,
+        code: str,
+        message: str,
+    ) -> None:
+        if not isinstance(path_value, str) or not path_value.strip():
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    code,
+                    message,
+                )
+            )
+            return
+
+        resolved = _resolve_artifact_path(artifacts_dir, Path(path_value))
+        if self._fs.exists(resolved):
+            return
+        issues.append(
+            ArtifactLintIssue(
+                "error" if strict else "warning",
+                code,
+                "Artifact file is missing.",
+                path=str(resolved),
+            )
+        )
+
+
+def _resolve_artifact_path(base_dir: Path, candidate: Path) -> Path:
+    if candidate.is_absolute():
+        return candidate
+    return base_dir / candidate
+
+
+def _resolve_status(issues: list[ArtifactLintIssue]) -> LintStatus:
+    if any(issue.level == "error" for issue in issues):
+        return "error"
+    if any(issue.level == "warning" for issue in issues):
+        return "warning"
+    return "ok"

evalvault/domain/services/benchmark_runner.py

@@ -414,12 +414,7 @@ class KoreanRAGBenchmarkRunner:
             try:
                 # 형태소 분석 기반 검색
                 if retriever:
-                    if self.use_hybrid_search and hasattr(retriever, "has_embeddings"):
-                        results = retriever.search(
-                            query, top_k=recall_k, use_dense=retriever.has_embeddings
-                        )
-                    else:
-                        results = retriever.search(query, top_k=recall_k)
+                    results = retriever.search(query, top_k=recall_k)
                     retrieved_doc_ids = [
                         resolve_doc_id(getattr(res, "doc_id", None), doc_ids, idx)
                         for idx, res in enumerate(results, start=1)

evalvault/domain/services/custom_metric_snapshot.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import hashlib
+import inspect
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.metrics.registry import get_metric_spec_map
+
+SCHEMA_VERSION = 1
+
+_CUSTOM_METRIC_DETAILS: dict[str, dict[str, Any]] = {
+    "entity_preservation": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "contexts"],
+        "output": "0.0-1.0 (preserved_entities / context_entities)",
+        "evaluation_process": "Extract numeric/keyword entities from contexts and measure how many appear in the summary.",
+        "rules": {
+            "numeric_entities": ["percent", "currency", "duration", "date"],
+            "keywords_ko": [
+                "면책",
+                "제외",
+                "단서",
+                "다만",
+                "조건",
+                "자기부담",
+                "한도",
+                "감액",
+            ],
+            "keywords_en": [
+                "exclusion",
+                "deductible",
+                "limit",
+                "cap",
+                "copay",
+                "coinsurance",
+            ],
+        },
+        "notes": "Insurance-risk oriented entity coverage check.",
+    },
+    "insurance_term_accuracy": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "contexts"],
+        "output": "0.0-1.0 (verified_terms / answer_terms)",
+        "evaluation_process": "Detect insurance terms in the answer and verify their presence in contexts.",
+        "rules": {"terms_dictionary": "terms_dictionary.json"},
+        "notes": "Insurance glossary matching with canonical/variant terms.",
+    },
+    "summary_accuracy": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "contexts"],
+        "output": "0.0-1.0 (supported_summary_entities / summary_entities)",
+        "evaluation_process": "Extract numeric/keyword entities from summary and verify their presence in contexts.",
+        "rules": {
+            "numeric_entities": ["percent", "currency", "duration", "date"],
+            "keywords_ko": ["면책", "제외", "단서", "다만", "조건", "자기부담", "한도", "감액"],
+            "keywords_en": ["exclusion", "deductible", "limit", "cap", "waiting period"],
+        },
+        "notes": "Penalizes summary entities not grounded in contexts.",
+    },
+    "summary_risk_coverage": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "metadata.summary_tags"],
+        "output": "0.0-1.0 (covered_tags / expected_tags)",
+        "evaluation_process": "Check if summary mentions expected insurance risk tags.",
+        "rules": {
+            "exclusion": ["면책", "보장 제외", "지급 불가", "exclusion"],
+            "deductible": ["자기부담", "본인부담금", "deductible", "copay"],
+            "limit": ["한도", "상한", "최대", "limit", "cap"],
+            "waiting_period": ["면책기간", "대기기간", "waiting period"],
+            "condition": ["조건", "단서", "다만", "condition"],
+            "documents_required": ["서류", "진단서", "영수증", "documents"],
+            "needs_followup": ["확인 필요", "추가 확인", "담당자 확인", "재문의", "follow up"],
+        },
+        "notes": "Uses metadata summary_tags to define expected coverage.",
+    },
+    "summary_non_definitive": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer"],
+        "output": "1.0 if definitive claims absent else 0.0",
+        "evaluation_process": "Detect definitive expressions that increase liability risk.",
+        "rules": {
+            "patterns_ko": ["무조건", "반드시", "100%", "전액 지급", "확실히", "분명히", "절대"],
+            "patterns_en": [
+                "always",
+                "guaranteed",
+                "definitely",
+                "certainly",
+                "absolutely",
+                "100%",
+            ],
+        },
+        "notes": "Higher is safer; penalizes absolute guarantees.",
+    },
+    "summary_needs_followup": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "metadata.summary_tags"],
+        "output": "1.0 if follow-up guidance matches expected need",
+        "evaluation_process": "Check follow-up guidance when needs_followup tag exists.",
+        "rules": {
+            "followup_keywords": [
+                "확인 필요",
+                "추가 확인",
+                "담당자 확인",
+                "재문의",
+                "추가 문의",
+                "follow up",
+            ]
+        },
+        "notes": "Requires tags to avoid false penalties.",
+    },
+    "no_answer_accuracy": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "ground_truth"],
+        "output": "1.0 if abstention behavior matches, else 0.0",
+        "evaluation_process": "Detect abstention patterns in answer and ground_truth and compare behavior.",
+        "rules": {"patterns": "Korean/English regex patterns"},
+        "notes": "Hallucination/abstention behavior check.",
+    },
+    "exact_match": {
+        "evaluation_method": "string-match",
+        "inputs": ["answer", "ground_truth"],
+        "output": "1.0 exact match else 0.0",
+        "evaluation_process": "Normalize text and compare exact match with optional strict number matching.",
+        "rules": {"normalize": True, "number_strict": True},
+        "notes": "Token/number strict matching for factual answers.",
+    },
+    "f1_score": {
+        "evaluation_method": "token-overlap",
+        "inputs": ["answer", "ground_truth"],
+        "output": "0.0-1.0 (weighted F1)",
+        "evaluation_process": "Tokenize, compute weighted precision/recall/F1 with number emphasis.",
+        "rules": {"number_weight": 2.0},
+        "notes": "Token-level overlap with numeric weighting.",
+    },
+    "mrr": {
+        "evaluation_method": "retrieval-rank",
+        "inputs": ["ground_truth", "contexts"],
+        "output": "0.0-1.0 (1/rank of first relevant context)",
+        "evaluation_process": "Compute relevance by token overlap and take reciprocal rank of first hit.",
+        "rules": {"relevance_threshold": 0.3},
+        "notes": "Ranking quality of retrieved contexts.",
+    },
+    "ndcg": {
+        "evaluation_method": "retrieval-rank",
+        "inputs": ["ground_truth", "contexts"],
+        "output": "0.0-1.0 (NDCG@K)",
+        "evaluation_process": "Compute graded relevance per context and calculate NDCG.",
+        "rules": {"k": 10, "use_graded": True},
+        "notes": "Ranking quality across all relevant contexts.",
+    },
+    "hit_rate": {
+        "evaluation_method": "retrieval-rank",
+        "inputs": ["ground_truth", "contexts"],
+        "output": "1.0 if any relevant context in top K else 0.0",
+        "evaluation_process": "Check whether top-K contexts contain a relevant hit.",
+        "rules": {"k": 10, "relevance_threshold": 0.3},
+        "notes": "Recall@K style coverage check.",
+    },
+    "confidence_score": {
+        "evaluation_method": "rule-based",
+        "inputs": ["answer", "ground_truth", "contexts"],
+        "output": "0.0-1.0 (weighted confidence)",
+        "evaluation_process": "Combine context coverage, answer specificity, and consistency scores.",
+        "rules": {"coverage": 0.4, "specificity": 0.3, "consistency": 0.3},
+        "notes": "Heuristic confidence signal for human escalation.",
+    },
+    "contextual_relevancy": {
+        "evaluation_method": "token-overlap",
+        "inputs": ["question", "contexts"],
+        "output": "0.0-1.0 (avg relevancy)",
+        "evaluation_process": "Measure question-context token overlap and average across contexts.",
+        "rules": {"relevance_threshold": 0.35},
+        "notes": "Reference-free context relevance check.",
+    },
+}
+
+
+def _hash_file(path: str | Path | None) -> str | None:
+    if not path:
+        return None
+    file_path = Path(path)
+    if not file_path.exists():
+        return None
+    payload = file_path.read_bytes()
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _resolve_source_path(metric_class: type[Any]) -> str | None:
+    try:
+        source = inspect.getsourcefile(metric_class)
+    except TypeError:
+        return None
+    if not source:
+        return None
+    return str(Path(source).resolve())
+
+
+def build_custom_metric_snapshot(
+    metric_classes: dict[str, type[Any]],
+    metrics: Iterable[str],
+) -> dict[str, Any] | None:
+    custom_names = [name for name in metrics if name in metric_classes]
+    if not custom_names:
+        return None
+
+    spec_map = get_metric_spec_map()
+    rows: list[dict[str, Any]] = []
+    for metric_name in custom_names:
+        metric_class = metric_classes.get(metric_name)
+        if metric_class is None:
+            continue
+        source_path = _resolve_source_path(metric_class)
+        details = _CUSTOM_METRIC_DETAILS.get(metric_name, {})
+        spec = spec_map.get(metric_name)
+        rows.append(
+            {
+                "metric_name": metric_name,
+                "source": "custom",
+                "description": spec.description if spec else None,
+                "evaluation_method": details.get("evaluation_method"),
+                "inputs": details.get("inputs"),
+                "output": details.get("output"),
+                "evaluation_process": details.get("evaluation_process"),
+                "rules": details.get("rules"),
+                "notes": details.get("notes"),
+                "implementation_path": source_path,
+                "implementation_hash": _hash_file(source_path),
+            }
+        )
+
+    return {"schema_version": SCHEMA_VERSION, "metrics": rows}

evalvault/domain/services/dataset_preprocessor.py

@@ -17,9 +17,22 @@ REFERENCE_REQUIRED_METRICS = {
 }
 
 _WHITESPACE_RE = re.compile(r"\s+")
+_PUNCT_ONLY_RE = re.compile(r"^[\W_]+$")
 _HANGUL_RE = re.compile(r"[\uac00-\ud7a3]")
 _LATIN_RE = re.compile(r"[A-Za-z]")
 
+_PLACEHOLDER_TEXT = {
+    "n/a",
+    "na",
+    "none",
+    "null",
+    "nil",
+    "unknown",
+    "tbd",
+    "todo",
+    "undefined",
+}
+
 
 @dataclass(frozen=True)
 class DatasetPreprocessConfig:
@@ -205,8 +218,18 @@ class DatasetPreprocessor:
         if self._config.trim_whitespace:
             text = text.replace("\u00a0", " ")
             text = _WHITESPACE_RE.sub(" ", text).strip()
+        if self._is_noise_text(text):
+            return ""
         return text
 
+    def _is_noise_text(self, text: str) -> bool:
+        if not text:
+            return True
+        if _PUNCT_ONLY_RE.fullmatch(text):
+            return True
+        lower_text = text.casefold()
+        return lower_text in _PLACEHOLDER_TEXT
+
     def _normalize_contexts(self, contexts: Any) -> tuple[list[str], dict[str, int]]:
         removed = 0
         deduped = 0
@@ -292,6 +315,9 @@ class DatasetPreprocessor:
                 elif source == "context":
                     filled_from_context = 1
 
+        if reference:
+            reference = self._normalize_text(reference)
+
         if reference and self._config.max_reference_chars > 0:
             reference, did_truncate = self._truncate_text(
                 reference, self._config.max_reference_chars

evalvault/domain/services/difficulty_profile_reporter.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.ports.outbound.difficulty_profile_port import DifficultyProfileWriterPort
+
+
+class DifficultyProfileReporter:
+    def __init__(self, writer: DifficultyProfileWriterPort) -> None:
+        self._writer = writer
+
+    def write(
+        self,
+        *,
+        output_path: Path,
+        artifacts_dir: Path,
+        envelope: dict[str, object],
+        artifacts: dict[str, object],
+    ) -> dict[str, object]:
+        return self._writer.write_profile(
+            output_path=output_path,
+            artifacts_dir=artifacts_dir,
+            envelope=envelope,
+            artifacts=artifacts,
+        )