evalvault 1.63.1__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/domain/services/artifact_lint_service.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from evalvault.ports.outbound.artifact_fs_port import ArtifactFileSystemPort
+
+logger = logging.getLogger(__name__)
+
+
+LintLevel = Literal["error", "warning"]
+LintStatus = Literal["ok", "warning", "error"]
+
+
+@dataclass(frozen=True)
+class ArtifactLintIssue:
+    level: LintLevel
+    code: str
+    message: str
+    path: str | None = None
+
+
+@dataclass(frozen=True)
+class ArtifactLintSummary:
+    status: LintStatus
+    issues: list[ArtifactLintIssue]
+    artifacts_dir: Path
+    index_path: Path
+    started_at: datetime
+    finished_at: datetime
+    duration_ms: int
+    strict: bool
+
+
+class ArtifactLintService:
+    def __init__(self, fs: ArtifactFileSystemPort) -> None:
+        self._fs = fs
+
+    def lint(self, artifacts_dir: Path, *, strict: bool = False) -> ArtifactLintSummary:
+        started_at = datetime.now(UTC)
+        issues: list[ArtifactLintIssue] = []
+        index_path = artifacts_dir / "index.json"
+        logger.info("Artifact lint started: %s", artifacts_dir)
+
+        try:
+            self._validate_dir(artifacts_dir, issues)
+            if not self._fs.exists(index_path):
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.missing",
+                        "index.json is missing.",
+                        path=str(index_path),
+                    )
+                )
+            elif self._fs.exists(artifacts_dir) and self._fs.is_dir(artifacts_dir):
+                index_payload = self._load_index(index_path, issues)
+                if index_payload is not None:
+                    self._validate_index(
+                        index_payload,
+                        artifacts_dir,
+                        issues,
+                        strict=strict,
+                    )
+        except Exception as exc:
+            logger.exception("Artifact lint failed: %s", artifacts_dir)
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.lint.exception",
+                    f"Unexpected error: {exc}",
+                )
+            )
+
+        finished_at = datetime.now(UTC)
+        duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+        status = _resolve_status(issues)
+        logger.info("Artifact lint finished: %s (%s)", artifacts_dir, status)
+        return ArtifactLintSummary(
+            status=status,
+            issues=issues,
+            artifacts_dir=artifacts_dir,
+            index_path=index_path,
+            started_at=started_at,
+            finished_at=finished_at,
+            duration_ms=duration_ms,
+            strict=strict,
+        )
+
+    def _validate_dir(self, artifacts_dir: Path, issues: list[ArtifactLintIssue]) -> None:
+        if not self._fs.exists(artifacts_dir):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.dir.missing",
+                    "Artifacts directory is missing.",
+                    path=str(artifacts_dir),
+                )
+            )
+            return
+        if not self._fs.is_dir(artifacts_dir):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.dir.not_directory",
+                    "Artifacts path is not a directory.",
+                    path=str(artifacts_dir),
+                )
+            )
+
+    def _load_index(
+        self,
+        index_path: Path,
+        issues: list[ArtifactLintIssue],
+    ) -> dict[str, object] | None:
+        try:
+            payload = json.loads(self._fs.read_text(index_path))
+        except json.JSONDecodeError as exc:
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.invalid_json",
+                    f"index.json parse failed: {exc}",
+                    path=str(index_path),
+                )
+            )
+            return None
+        except OSError as exc:
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.read_failed",
+                    f"index.json read failed: {exc}",
+                    path=str(index_path),
+                )
+            )
+            return None
+
+        if not isinstance(payload, dict):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.invalid_schema",
+                    "index.json root must be an object.",
+                    path=str(index_path),
+                )
+            )
+            return None
+        return payload
+
+    def _validate_index(
+        self,
+        payload: dict[str, object],
+        artifacts_dir: Path,
+        issues: list[ArtifactLintIssue],
+        *,
+        strict: bool,
+    ) -> None:
+        pipeline_id = payload.get("pipeline_id")
+        if not isinstance(pipeline_id, str) or not pipeline_id.strip():
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.pipeline_id.missing",
+                    "pipeline_id is missing.",
+                )
+            )
+
+        nodes = payload.get("nodes")
+        if not isinstance(nodes, list):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.nodes.invalid",
+                    "nodes list is missing or invalid.",
+                )
+            )
+            return
+
+        for idx, node in enumerate(nodes, start=1):
+            if not isinstance(node, dict):
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.node.invalid",
+                        f"nodes[{idx}] entry must be an object.",
+                    )
+                )
+                continue
+            node_id = node.get("node_id")
+            if not isinstance(node_id, str) or not node_id.strip():
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.node_id.missing",
+                        f"nodes[{idx}] node_id is missing.",
+                    )
+                )
+            path_value = node.get("path")
+            self._validate_path(
+                path_value,
+                artifacts_dir,
+                issues,
+                strict=strict,
+                code="artifacts.index.node.path.missing",
+                message=f"nodes[{idx}] path is missing.",
+            )
+
+        final_output = payload.get("final_output_path")
+        if final_output:
+            self._validate_path(
+                final_output,
+                artifacts_dir,
+                issues,
+                strict=strict,
+                code="artifacts.index.final_output.missing",
+                message="final_output_path is missing.",
+            )
+
+    def _validate_path(
+        self,
+        path_value: object,
+        artifacts_dir: Path,
+        issues: list[ArtifactLintIssue],
+        *,
+        strict: bool,
+        code: str,
+        message: str,
+    ) -> None:
+        if not isinstance(path_value, str) or not path_value.strip():
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    code,
+                    message,
+                )
+            )
+            return
+
+        resolved = _resolve_artifact_path(artifacts_dir, Path(path_value))
+        if self._fs.exists(resolved):
+            return
+        issues.append(
+            ArtifactLintIssue(
+                "error" if strict else "warning",
+                code,
+                "Artifact file is missing.",
+                path=str(resolved),
+            )
+        )
+
+
+def _resolve_artifact_path(base_dir: Path, candidate: Path) -> Path:
+    if candidate.is_absolute():
+        return candidate
+    return base_dir / candidate
+
+
+def _resolve_status(issues: list[ArtifactLintIssue]) -> LintStatus:
+    if any(issue.level == "error" for issue in issues):
+        return "error"
+    if any(issue.level == "warning" for issue in issues):
+        return "warning"
+    return "ok"

evalvault/domain/services/benchmark_runner.py

@@ -414,12 +414,7 @@ class KoreanRAGBenchmarkRunner:
             try:
                 # 형태소 분석 기반 검색
                 if retriever:
-                    if self.use_hybrid_search and hasattr(retriever, "has_embeddings"):
-                        results = retriever.search(
-                            query, top_k=recall_k, use_dense=retriever.has_embeddings
-                        )
-                    else:
-                        results = retriever.search(query, top_k=recall_k)
+                    results = retriever.search(query, top_k=recall_k)
                     retrieved_doc_ids = [
                         resolve_doc_id(getattr(res, "doc_id", None), doc_ids, idx)
                         for idx, res in enumerate(results, start=1)

evalvault/domain/services/dataset_preprocessor.py

@@ -17,9 +17,22 @@ REFERENCE_REQUIRED_METRICS = {
 }
 
 _WHITESPACE_RE = re.compile(r"\s+")
+_PUNCT_ONLY_RE = re.compile(r"^[\W_]+$")
 _HANGUL_RE = re.compile(r"[\uac00-\ud7a3]")
 _LATIN_RE = re.compile(r"[A-Za-z]")
 
+_PLACEHOLDER_TEXT = {
+    "n/a",
+    "na",
+    "none",
+    "null",
+    "nil",
+    "unknown",
+    "tbd",
+    "todo",
+    "undefined",
+}
+
 
 @dataclass(frozen=True)
 class DatasetPreprocessConfig:
@@ -205,8 +218,18 @@ class DatasetPreprocessor:
         if self._config.trim_whitespace:
             text = text.replace("\u00a0", " ")
             text = _WHITESPACE_RE.sub(" ", text).strip()
+        if self._is_noise_text(text):
+            return ""
         return text
 
+    def _is_noise_text(self, text: str) -> bool:
+        if not text:
+            return True
+        if _PUNCT_ONLY_RE.fullmatch(text):
+            return True
+        lower_text = text.casefold()
+        return lower_text in _PLACEHOLDER_TEXT
+
     def _normalize_contexts(self, contexts: Any) -> tuple[list[str], dict[str, int]]:
         removed = 0
         deduped = 0
@@ -292,6 +315,9 @@ class DatasetPreprocessor:
                 elif source == "context":
                     filled_from_context = 1
 
+        if reference:
+            reference = self._normalize_text(reference)
+
         if reference and self._config.max_reference_chars > 0:
             reference, did_truncate = self._truncate_text(
                 reference, self._config.max_reference_chars

evalvault/domain/services/difficulty_profile_reporter.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.ports.outbound.difficulty_profile_port import DifficultyProfileWriterPort
+
+
+class DifficultyProfileReporter:
+    def __init__(self, writer: DifficultyProfileWriterPort) -> None:
+        self._writer = writer
+
+    def write(
+        self,
+        *,
+        output_path: Path,
+        artifacts_dir: Path,
+        envelope: dict[str, object],
+        artifacts: dict[str, object],
+    ) -> dict[str, object]:
+        return self._writer.write_profile(
+            output_path=output_path,
+            artifacts_dir=artifacts_dir,
+            envelope=envelope,
+            artifacts=artifacts,
+        )

evalvault/domain/services/difficulty_profiling_service.py

@@ -0,0 +1,304 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.entities import EvaluationRun, MetricScore, TestCaseResult
+from evalvault.domain.services.difficulty_profile_reporter import DifficultyProfileReporter
+from evalvault.ports.outbound.storage_port import StoragePort
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class DifficultyProfileRequest:
+    dataset_name: str | None
+    run_id: str | None
+    limit_runs: int | None
+    metrics: tuple[str, ...] | None
+    bucket_count: int
+    min_samples: int
+    output_path: Path
+    artifacts_dir: Path
+    parallel: bool
+    concurrency: int | None
+
+
+@dataclass(frozen=True)
+class DifficultyCaseProfile:
+    run_id: str
+    test_case_id: str
+    metric_scores: dict[str, float]
+    avg_score: float
+    difficulty_score: float
+    bucket: str
+    passed: bool
+
+
+@dataclass(frozen=True)
+class DifficultyBucketSummary:
+    label: str
+    count: int
+    ratio: float
+    avg_score: float | None
+    pass_rate: float | None
+
+
+class DifficultyProfilingService:
+    def __init__(self, *, storage: StoragePort, reporter: DifficultyProfileReporter) -> None:
+        self._storage = storage
+        self._reporter = reporter
+
+    def profile(self, request: DifficultyProfileRequest) -> dict[str, Any]:
+        started_at = datetime.now(UTC)
+        logger.info(
+            "difficulty profiling started",
+            extra={
+                "dataset_name": request.dataset_name,
+                "run_id": request.run_id,
+                "bucket_count": request.bucket_count,
+                "parallel": request.parallel,
+            },
+        )
+
+        runs = self._load_runs(request)
+        metrics = self._resolve_metrics(request, runs)
+        cases = self._collect_cases(runs, metrics)
+
+        if len(cases) < request.min_samples:
+            logger.warning(
+                "difficulty profiling aborted: insufficient samples",
+                extra={"sample_count": len(cases), "min_samples": request.min_samples},
+            )
+            raise ValueError("insufficient history to build difficulty profile")
+
+        case_profiles, bucket_summaries = self._assign_buckets(
+            cases, bucket_count=request.bucket_count
+        )
+        breakdown = _build_breakdown(bucket_summaries)
+        failure_concentration = _build_failure_concentration(bucket_summaries)
+
+        data = {
+            "run_id": request.run_id,
+            "dataset_name": request.dataset_name,
+            "run_ids": sorted({case.run_id for case in case_profiles}),
+            "metrics": list(metrics),
+            "bucket_count": request.bucket_count,
+            "min_samples": request.min_samples,
+            "total_cases": len(case_profiles),
+            "dataset_difficulty_distribution": breakdown,
+            "accuracy_by_difficulty": _build_accuracy(bucket_summaries),
+            "failure_concentration": failure_concentration,
+            "buckets": [
+                {
+                    "label": bucket.label,
+                    "count": bucket.count,
+                    "ratio": bucket.ratio,
+                    "avg_score": bucket.avg_score,
+                    "pass_rate": bucket.pass_rate,
+                }
+                for bucket in bucket_summaries
+            ],
+        }
+
+        finished_at = datetime.now(UTC)
+        duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+        envelope = {
+            "command": "profile-difficulty",
+            "version": 1,
+            "status": "ok",
+            "started_at": started_at.isoformat(),
+            "finished_at": finished_at.isoformat(),
+            "duration_ms": duration_ms,
+            "artifacts": {},
+            "data": data,
+        }
+
+        artifacts_payload = {
+            "breakdown": {
+                "run_id": request.run_id,
+                "dataset_difficulty_distribution": breakdown,
+                "accuracy_by_difficulty": _build_accuracy(bucket_summaries),
+                "failure_concentration": failure_concentration,
+            },
+            "cases": [
+                {
+                    "run_id": case.run_id,
+                    "test_case_id": case.test_case_id,
+                    "metric_scores": case.metric_scores,
+                    "avg_score": case.avg_score,
+                    "difficulty_score": case.difficulty_score,
+                    "bucket": case.bucket,
+                    "passed": case.passed,
+                }
+                for case in case_profiles
+            ],
+        }
+
+        artifacts_index = self._reporter.write(
+            output_path=request.output_path,
+            artifacts_dir=request.artifacts_dir,
+            envelope=envelope,
+            artifacts=artifacts_payload,
+        )
+        envelope["artifacts"] = artifacts_index
+        logger.info(
+            "difficulty profiling completed",
+            extra={"artifact_dir": artifacts_index.get("dir"), "case_count": len(case_profiles)},
+        )
+        return envelope
+
+    def _load_runs(self, request: DifficultyProfileRequest) -> list[EvaluationRun]:
+        if request.run_id:
+            run = self._storage.get_run(request.run_id)
+            return [run]
+        if not request.dataset_name:
+            raise ValueError("dataset_name or run_id is required")
+        limit = request.limit_runs or 50
+        runs = self._storage.list_runs(limit=limit, dataset_name=request.dataset_name)
+        if not runs:
+            raise ValueError("no runs found for dataset")
+        return runs
+
+    def _resolve_metrics(
+        self, request: DifficultyProfileRequest, runs: list[EvaluationRun]
+    ) -> tuple[str, ...]:
+        if request.metrics:
+            return request.metrics
+        metrics: set[str] = set()
+        for run in runs:
+            metrics.update(run.metrics_evaluated)
+        if not metrics:
+            raise ValueError("no metrics available for difficulty profiling")
+        return tuple(sorted(metrics))
+
+    def _collect_cases(
+        self, runs: list[EvaluationRun], metrics: tuple[str, ...]
+    ) -> list[tuple[str, TestCaseResult, dict[str, float], bool]]:
+        cases = []
+        for run in runs:
+            for result in run.results:
+                metric_scores, passed = _extract_metric_scores(result, metrics)
+                if not metric_scores:
+                    continue
+                cases.append((run.run_id, result, metric_scores, passed))
+        return cases
+
+    def _assign_buckets(
+        self, cases: list[tuple[str, TestCaseResult, dict[str, float], bool]], *, bucket_count: int
+    ) -> tuple[list[DifficultyCaseProfile], list[DifficultyBucketSummary]]:
+        sorted_cases = sorted(cases, key=lambda item: _difficulty_score(item[2]))
+        total_cases = len(sorted_cases)
+        if total_cases == 0:
+            return [], []
+        labels = _bucket_labels(bucket_count)
+
+        bucket_map: dict[str, list[DifficultyCaseProfile]] = {label: [] for label in labels}
+
+        for index, (run_id, result, metric_scores, passed) in enumerate(sorted_cases):
+            bucket_index = min(int(index / total_cases * bucket_count), bucket_count - 1)
+            label = labels[bucket_index]
+            avg_score = sum(metric_scores.values()) / len(metric_scores)
+            difficulty_score = _difficulty_score(metric_scores)
+            profile = DifficultyCaseProfile(
+                run_id=run_id,
+                test_case_id=result.test_case_id,
+                metric_scores=metric_scores,
+                avg_score=avg_score,
+                difficulty_score=difficulty_score,
+                bucket=label,
+                passed=passed,
+            )
+            bucket_map[label].append(profile)
+
+        bucket_summaries: list[DifficultyBucketSummary] = []
+        case_profiles: list[DifficultyCaseProfile] = []
+        for label in labels:
+            bucket_cases = bucket_map[label]
+            case_profiles.extend(bucket_cases)
+            count = len(bucket_cases)
+            ratio = count / total_cases if total_cases else 0.0
+            avg_score = _safe_average([case.avg_score for case in bucket_cases])
+            pass_rate = _safe_average([1.0 if case.passed else 0.0 for case in bucket_cases])
+            bucket_summaries.append(
+                DifficultyBucketSummary(
+                    label=label,
+                    count=count,
+                    ratio=ratio,
+                    avg_score=avg_score,
+                    pass_rate=pass_rate,
+                )
+            )
+        return case_profiles, bucket_summaries
+
+
+def _extract_metric_scores(
+    result: TestCaseResult, metrics: tuple[str, ...]
+) -> tuple[dict[str, float], bool]:
+    scores: dict[str, float] = {}
+    passed_all = True
+    for metric_name in metrics:
+        metric: MetricScore | None = result.get_metric(metric_name)
+        if metric is None:
+            continue
+        scores[metric_name] = float(metric.score)
+        passed_all = passed_all and metric.score >= metric.threshold
+    if not scores:
+        passed_all = False
+    return scores, passed_all
+
+
+def _difficulty_score(metric_scores: dict[str, float]) -> float:
+    if not metric_scores:
+        return 1.0
+    avg_score = sum(metric_scores.values()) / len(metric_scores)
+    score = 1.0 - avg_score
+    return min(max(score, 0.0), 1.0)
+
+
+def _safe_average(values: list[float]) -> float | None:
+    if not values:
+        return None
+    return sum(values) / len(values)
+
+
+def _bucket_labels(bucket_count: int) -> list[str]:
+    if bucket_count == 3:
+        return ["easy", "medium", "hard"]
+    return [f"bucket_{index}" for index in range(1, bucket_count + 1)]
+
+
+def _build_breakdown(buckets: list[DifficultyBucketSummary]) -> dict[str, float]:
+    return {bucket.label: bucket.ratio for bucket in buckets}
+
+
+def _build_accuracy(buckets: list[DifficultyBucketSummary]) -> dict[str, float | None]:
+    return {bucket.label: bucket.pass_rate for bucket in buckets}
+
+
+def _build_failure_concentration(buckets: list[DifficultyBucketSummary]) -> dict[str, Any]:
+    if not buckets:
+        return {
+            "primary_difficulty": None,
+            "primary_flags": [],
+            "actionable_insight": "난이도 분포 데이터가 없습니다.",
+        }
+    primary = max(buckets, key=lambda bucket: (1 - (bucket.pass_rate or 0.0), bucket.count))
+    flags: list[str] = []
+    if primary.pass_rate is not None and primary.pass_rate < 0.5:
+        flags.append("low_pass_rate")
+    if primary.avg_score is not None and primary.avg_score < 0.5:
+        flags.append("low_avg_score")
+    insight = "난이도 분포에 큰 편차가 없습니다."
+    if "low_pass_rate" in flags:
+        insight = "해당 난이도 구간에서 정답률이 낮습니다."
+    elif "low_avg_score" in flags:
+        insight = "메트릭 평균 점수가 낮습니다."
+    return {
+        "primary_difficulty": primary.label,
+        "primary_flags": flags,
+        "actionable_insight": insight,
+    }

evalvault/domain/services/evaluator.py

@@ -330,6 +330,8 @@ class RagasEvaluator:
         self._active_llm_model = llm.get_model_name()
         self._active_llm = llm
         self._prompt_language = self._normalize_language_hint(language) if language else None
+        if self._prompt_language is None:
+            self._prompt_language = self._resolve_dataset_language(dataset)
         # Resolve thresholds: CLI > dataset > default(0.7)
         resolved_thresholds = {}
         for metric in metrics: