evalvault 1.64.0__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/adapters/outbound/tracker/phoenix_adapter.py

@@ -26,8 +26,7 @@ from evalvault.domain.entities import (
 from evalvault.ports.outbound.tracker_port import TrackerPort
 
 if TYPE_CHECKING:
-    from opentelemetry.sdk.trace import Span, TracerProvider
-    from opentelemetry.trace import Tracer
+    from opentelemetry.sdk.trace import TracerProvider
 
 
 class PhoenixAdapter(TrackerPort):
@@ -62,9 +61,10 @@ class PhoenixAdapter(TrackerPort):
         """
         self._endpoint = endpoint
         self._service_name = service_name
-        self._tracer: Tracer | None = None
+        self._tracer: Any | None = None
         self._tracer_provider: TracerProvider | None = None
-        self._active_spans: dict[str, Span] = {}
+        self._active_spans: dict[str, Any] = {}
+        self._tracer_any: Any | None = None
         self._initialized = False
 
     def _ensure_initialized(self) -> None:
@@ -90,7 +90,8 @@ class PhoenixAdapter(TrackerPort):
                 provider = get_tracer_provider()
                 if provider:
                     self._tracer_provider = provider
-                    self._tracer = trace.get_tracer(__name__)
+                    self._tracer_any = trace.get_tracer(__name__)
+                    self._tracer = self._tracer_any
                     self._initialized = True
                     return
 
@@ -109,7 +110,8 @@ class PhoenixAdapter(TrackerPort):
             trace.set_tracer_provider(self._tracer_provider)
 
             # Get tracer
-            self._tracer = trace.get_tracer(__name__)
+            self._tracer_any = trace.get_tracer(__name__)
+            self._tracer = self._tracer_any
             self._initialized = True
 
         except ImportError as e:
@@ -134,7 +136,12 @@ class PhoenixAdapter(TrackerPort):
         self._ensure_initialized()
 
         # Start a new span as root
-        span = self._tracer.start_span(name)
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
+        span = tracer.start_span(name)
         trace_id = str(uuid.uuid4())
 
         # Set metadata as span attributes
@@ -173,10 +180,15 @@ class PhoenixAdapter(TrackerPort):
 
         from opentelemetry import trace
 
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span(name, context=context) as span:
+        with tracer.start_span(name, context=context) as span:
             if input_data is not None:
                 safe_input = sanitize_payload(input_data, max_chars=MAX_LOG_CHARS)
                 span.set_attribute("input", json.dumps(safe_input, default=str))
@@ -279,7 +291,7 @@ class PhoenixAdapter(TrackerPort):
             passed_count = sum(
                 1
                 for r in run.results
-                if r.get_metric(metric_name) and r.get_metric(metric_name).passed
+                if (metric := r.get_metric(metric_name)) and metric.passed is True
             )
             avg_score = run.get_avg_score(metric_name)
             threshold = run.thresholds.get(metric_name, 0.7)
@@ -369,10 +381,15 @@ class PhoenixAdapter(TrackerPort):
         """
         from opentelemetry import trace
 
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span(
+        with tracer.start_span(
             f"test-case-{result.test_case_id}",
             context=context,
         ) as span:
@@ -478,7 +495,12 @@ class PhoenixAdapter(TrackerPort):
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span("retrieval", context=context) as span:
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
+        with tracer.start_span("retrieval", context=context) as span:
             # Set retrieval attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)
@@ -560,7 +582,12 @@ class PhoenixAdapter(TrackerPort):
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span("generation", context=context) as span:
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
+        with tracer.start_span("generation", context=context) as span:
             # Set generation attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)

evalvault/domain/entities/__init__.py

@@ -34,6 +34,12 @@ from evalvault.domain.entities.improvement import (
     RAGComponent,
     RAGImprovementGuide,
 )
+from evalvault.domain.entities.judge_calibration import (
+    JudgeCalibrationCase,
+    JudgeCalibrationMetric,
+    JudgeCalibrationResult,
+    JudgeCalibrationSummary,
+)
 from evalvault.domain.entities.kg import EntityModel, RelationModel
 from evalvault.domain.entities.method import MethodInput, MethodInputDataset, MethodOutput
 from evalvault.domain.entities.prompt import Prompt, PromptSet, PromptSetBundle, PromptSetItem
@@ -104,6 +110,10 @@ __all__ = [
     "PatternType",
     "RAGComponent",
     "RAGImprovementGuide",
+    "JudgeCalibrationCase",
+    "JudgeCalibrationMetric",
+    "JudgeCalibrationResult",
+    "JudgeCalibrationSummary",
     # KG
     "EntityModel",
     "RelationModel",

evalvault/domain/entities/judge_calibration.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class JudgeCalibrationCase:
+    test_case_id: str
+    raw_score: float
+    calibrated_score: float
+    label: float | None = None
+    label_source: str | None = None
+
+
+@dataclass
+class JudgeCalibrationMetric:
+    metric: str
+    method: str
+    sample_count: int
+    label_count: int
+    mae: float | None
+    pearson: float | None
+    spearman: float | None
+    temperature: float | None = None
+    parameters: dict[str, float | None] = field(default_factory=dict)
+    gate_passed: bool | None = None
+    warning: str | None = None
+
+
+@dataclass
+class JudgeCalibrationSummary:
+    run_id: str
+    labels_source: str
+    method: str
+    metrics: list[str]
+    holdout_ratio: float
+    seed: int
+    total_labels: int
+    total_samples: int
+    gate_passed: bool
+    gate_threshold: float | None = None
+    notes: list[str] = field(default_factory=list)
+
+
+@dataclass
+class JudgeCalibrationResult:
+    summary: JudgeCalibrationSummary
+    metrics: list[JudgeCalibrationMetric] = field(default_factory=list)
+    case_results: dict[str, list[JudgeCalibrationCase]] = field(default_factory=dict)
+    warnings: list[str] = field(default_factory=list)

evalvault/domain/entities/stage.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any
+from typing import Any, Literal, overload
 from uuid import uuid4
 
 REQUIRED_STAGE_TYPES: tuple[str, ...] = ("system_prompt", "input", "retrieval", "output")
@@ -82,8 +82,8 @@ class StageEvent:
             duration_ms=_optional_float(payload.get("duration_ms")),
             input_ref=input_ref,
             output_ref=output_ref,
-            attributes=_ensure_dict(payload.get("attributes")),
-            metadata=_ensure_dict(payload.get("metadata")),
+            attributes=_ensure_dict(payload.get("attributes"), allow_none=False),
+            metadata=_ensure_dict(payload.get("metadata"), allow_none=False),
             trace_id=_optional_str(payload.get("trace_id") or trace_payload.get("trace_id")),
             span_id=_optional_str(payload.get("span_id") or trace_payload.get("span_id")),
         )
@@ -187,6 +187,14 @@ def _parse_datetime(value: Any) -> datetime | None:
     raise ValueError("Invalid datetime value")
 
 
+@overload
+def _ensure_dict(value: None, *, allow_none: Literal[True]) -> None: ...
+
+
+@overload
+def _ensure_dict(value: Any, *, allow_none: Literal[False] = False) -> dict[str, Any]: ...
+
+
 def _ensure_dict(value: Any, *, allow_none: bool = False) -> dict[str, Any] | None:
     if value is None:
         return None if allow_none else {}

evalvault/domain/services/artifact_lint_service.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from evalvault.ports.outbound.artifact_fs_port import ArtifactFileSystemPort
+
+logger = logging.getLogger(__name__)
+
+
+LintLevel = Literal["error", "warning"]
+LintStatus = Literal["ok", "warning", "error"]
+
+
+@dataclass(frozen=True)
+class ArtifactLintIssue:
+    level: LintLevel
+    code: str
+    message: str
+    path: str | None = None
+
+
+@dataclass(frozen=True)
+class ArtifactLintSummary:
+    status: LintStatus
+    issues: list[ArtifactLintIssue]
+    artifacts_dir: Path
+    index_path: Path
+    started_at: datetime
+    finished_at: datetime
+    duration_ms: int
+    strict: bool
+
+
+class ArtifactLintService:
+    def __init__(self, fs: ArtifactFileSystemPort) -> None:
+        self._fs = fs
+
+    def lint(self, artifacts_dir: Path, *, strict: bool = False) -> ArtifactLintSummary:
+        started_at = datetime.now(UTC)
+        issues: list[ArtifactLintIssue] = []
+        index_path = artifacts_dir / "index.json"
+        logger.info("Artifact lint started: %s", artifacts_dir)
+
+        try:
+            self._validate_dir(artifacts_dir, issues)
+            if not self._fs.exists(index_path):
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.missing",
+                        "index.json is missing.",
+                        path=str(index_path),
+                    )
+                )
+            elif self._fs.exists(artifacts_dir) and self._fs.is_dir(artifacts_dir):
+                index_payload = self._load_index(index_path, issues)
+                if index_payload is not None:
+                    self._validate_index(
+                        index_payload,
+                        artifacts_dir,
+                        issues,
+                        strict=strict,
+                    )
+        except Exception as exc:
+            logger.exception("Artifact lint failed: %s", artifacts_dir)
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.lint.exception",
+                    f"Unexpected error: {exc}",
+                )
+            )
+
+        finished_at = datetime.now(UTC)
+        duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+        status = _resolve_status(issues)
+        logger.info("Artifact lint finished: %s (%s)", artifacts_dir, status)
+        return ArtifactLintSummary(
+            status=status,
+            issues=issues,
+            artifacts_dir=artifacts_dir,
+            index_path=index_path,
+            started_at=started_at,
+            finished_at=finished_at,
+            duration_ms=duration_ms,
+            strict=strict,
+        )
+
+    def _validate_dir(self, artifacts_dir: Path, issues: list[ArtifactLintIssue]) -> None:
+        if not self._fs.exists(artifacts_dir):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.dir.missing",
+                    "Artifacts directory is missing.",
+                    path=str(artifacts_dir),
+                )
+            )
+            return
+        if not self._fs.is_dir(artifacts_dir):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.dir.not_directory",
+                    "Artifacts path is not a directory.",
+                    path=str(artifacts_dir),
+                )
+            )
+
+    def _load_index(
+        self,
+        index_path: Path,
+        issues: list[ArtifactLintIssue],
+    ) -> dict[str, object] | None:
+        try:
+            payload = json.loads(self._fs.read_text(index_path))
+        except json.JSONDecodeError as exc:
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.invalid_json",
+                    f"index.json parse failed: {exc}",
+                    path=str(index_path),
+                )
+            )
+            return None
+        except OSError as exc:
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.read_failed",
+                    f"index.json read failed: {exc}",
+                    path=str(index_path),
+                )
+            )
+            return None
+
+        if not isinstance(payload, dict):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.invalid_schema",
+                    "index.json root must be an object.",
+                    path=str(index_path),
+                )
+            )
+            return None
+        return payload
+
+    def _validate_index(
+        self,
+        payload: dict[str, object],
+        artifacts_dir: Path,
+        issues: list[ArtifactLintIssue],
+        *,
+        strict: bool,
+    ) -> None:
+        pipeline_id = payload.get("pipeline_id")
+        if not isinstance(pipeline_id, str) or not pipeline_id.strip():
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.pipeline_id.missing",
+                    "pipeline_id is missing.",
+                )
+            )
+
+        nodes = payload.get("nodes")
+        if not isinstance(nodes, list):
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    "artifacts.index.nodes.invalid",
+                    "nodes list is missing or invalid.",
+                )
+            )
+            return
+
+        for idx, node in enumerate(nodes, start=1):
+            if not isinstance(node, dict):
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.node.invalid",
+                        f"nodes[{idx}] entry must be an object.",
+                    )
+                )
+                continue
+            node_id = node.get("node_id")
+            if not isinstance(node_id, str) or not node_id.strip():
+                issues.append(
+                    ArtifactLintIssue(
+                        "error",
+                        "artifacts.index.node_id.missing",
+                        f"nodes[{idx}] node_id is missing.",
+                    )
+                )
+            path_value = node.get("path")
+            self._validate_path(
+                path_value,
+                artifacts_dir,
+                issues,
+                strict=strict,
+                code="artifacts.index.node.path.missing",
+                message=f"nodes[{idx}] path is missing.",
+            )
+
+        final_output = payload.get("final_output_path")
+        if final_output:
+            self._validate_path(
+                final_output,
+                artifacts_dir,
+                issues,
+                strict=strict,
+                code="artifacts.index.final_output.missing",
+                message="final_output_path is missing.",
+            )
+
+    def _validate_path(
+        self,
+        path_value: object,
+        artifacts_dir: Path,
+        issues: list[ArtifactLintIssue],
+        *,
+        strict: bool,
+        code: str,
+        message: str,
+    ) -> None:
+        if not isinstance(path_value, str) or not path_value.strip():
+            issues.append(
+                ArtifactLintIssue(
+                    "error",
+                    code,
+                    message,
+                )
+            )
+            return
+
+        resolved = _resolve_artifact_path(artifacts_dir, Path(path_value))
+        if self._fs.exists(resolved):
+            return
+        issues.append(
+            ArtifactLintIssue(
+                "error" if strict else "warning",
+                code,
+                "Artifact file is missing.",
+                path=str(resolved),
+            )
+        )
+
+
+def _resolve_artifact_path(base_dir: Path, candidate: Path) -> Path:
+    if candidate.is_absolute():
+        return candidate
+    return base_dir / candidate
+
+
+def _resolve_status(issues: list[ArtifactLintIssue]) -> LintStatus:
+    if any(issue.level == "error" for issue in issues):
+        return "error"
+    if any(issue.level == "warning" for issue in issues):
+        return "warning"
+    return "ok"

evalvault/domain/services/benchmark_runner.py

@@ -414,12 +414,7 @@ class KoreanRAGBenchmarkRunner:
             try:
                 # 형태소 분석 기반 검색
                 if retriever:
-                    if self.use_hybrid_search and hasattr(retriever, "has_embeddings"):
-                        results = retriever.search(
-                            query, top_k=recall_k, use_dense=retriever.has_embeddings
-                        )
-                    else:
-                        results = retriever.search(query, top_k=recall_k)
+                    results = retriever.search(query, top_k=recall_k)
                     retrieved_doc_ids = [
                         resolve_doc_id(getattr(res, "doc_id", None), doc_ids, idx)
                         for idx, res in enumerate(results, start=1)

evalvault/domain/services/dataset_preprocessor.py

@@ -17,9 +17,22 @@ REFERENCE_REQUIRED_METRICS = {
 }
 
 _WHITESPACE_RE = re.compile(r"\s+")
+_PUNCT_ONLY_RE = re.compile(r"^[\W_]+$")
 _HANGUL_RE = re.compile(r"[\uac00-\ud7a3]")
 _LATIN_RE = re.compile(r"[A-Za-z]")
 
+_PLACEHOLDER_TEXT = {
+    "n/a",
+    "na",
+    "none",
+    "null",
+    "nil",
+    "unknown",
+    "tbd",
+    "todo",
+    "undefined",
+}
+
 
 @dataclass(frozen=True)
 class DatasetPreprocessConfig:
@@ -205,8 +218,18 @@ class DatasetPreprocessor:
         if self._config.trim_whitespace:
             text = text.replace("\u00a0", " ")
             text = _WHITESPACE_RE.sub(" ", text).strip()
+        if self._is_noise_text(text):
+            return ""
         return text
 
+    def _is_noise_text(self, text: str) -> bool:
+        if not text:
+            return True
+        if _PUNCT_ONLY_RE.fullmatch(text):
+            return True
+        lower_text = text.casefold()
+        return lower_text in _PLACEHOLDER_TEXT
+
     def _normalize_contexts(self, contexts: Any) -> tuple[list[str], dict[str, int]]:
         removed = 0
         deduped = 0
@@ -292,6 +315,9 @@ class DatasetPreprocessor:
                 elif source == "context":
                     filled_from_context = 1
 
+        if reference:
+            reference = self._normalize_text(reference)
+
         if reference and self._config.max_reference_chars > 0:
             reference, did_truncate = self._truncate_text(
                 reference, self._config.max_reference_chars

evalvault/domain/services/difficulty_profile_reporter.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.ports.outbound.difficulty_profile_port import DifficultyProfileWriterPort
+
+
+class DifficultyProfileReporter:
+    def __init__(self, writer: DifficultyProfileWriterPort) -> None:
+        self._writer = writer
+
+    def write(
+        self,
+        *,
+        output_path: Path,
+        artifacts_dir: Path,
+        envelope: dict[str, object],
+        artifacts: dict[str, object],
+    ) -> dict[str, object]:
+        return self._writer.write_profile(
+            output_path=output_path,
+            artifacts_dir=artifacts_dir,
+            envelope=envelope,
+            artifacts=artifacts,
+        )