evalvault 1.64.0__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/domain/services/regression_gate_service.py

@@ -0,0 +1,199 @@
+"""Regression gate service for CLI automation."""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+
+from evalvault.domain.entities.analysis import ComparisonResult, EffectSizeLevel
+from evalvault.ports.outbound.analysis_port import AnalysisPort
+from evalvault.ports.outbound.storage_port import StoragePort
+
+logger = logging.getLogger(__name__)
+
+TestType = str
+
+
+@dataclass(frozen=True)
+class RegressionMetricResult:
+    metric: str
+
+    baseline_score: float
+    candidate_score: float
+    diff: float
+    diff_percent: float
+    p_value: float
+    effect_size: float
+    effect_level: EffectSizeLevel
+    is_significant: bool
+    regression: bool
+
+    @classmethod
+    def from_comparison(
+        cls,
+        comparison: ComparisonResult,
+        *,
+        fail_on_regression: float,
+    ) -> RegressionMetricResult:
+        regression = comparison.diff < -fail_on_regression
+        return cls(
+            metric=comparison.metric,
+            baseline_score=comparison.mean_a,
+            candidate_score=comparison.mean_b,
+            diff=comparison.diff,
+            diff_percent=comparison.diff_percent,
+            p_value=comparison.p_value,
+            effect_size=comparison.effect_size,
+            effect_level=comparison.effect_level,
+            is_significant=comparison.is_significant,
+            regression=regression,
+        )
+
+    def to_dict(self) -> dict[str, float | str | bool]:
+        return {
+            "metric": self.metric,
+            "baseline_score": self.baseline_score,
+            "candidate_score": self.candidate_score,
+            "diff": self.diff,
+            "diff_percent": self.diff_percent,
+            "p_value": self.p_value,
+            "effect_size": self.effect_size,
+            "effect_level": self.effect_level.value,
+            "is_significant": self.is_significant,
+            "regression": self.regression,
+        }
+
+
+@dataclass(frozen=True)
+class RegressionGateReport:
+    candidate_run_id: str
+    baseline_run_id: str
+    results: list[RegressionMetricResult]
+    regression_detected: bool
+    fail_on_regression: float
+    test_type: TestType
+    metrics: list[str]
+    started_at: datetime
+    finished_at: datetime
+    duration_ms: int
+    parallel: bool
+    concurrency: int | None
+
+    @property
+    def status(self) -> str:
+        return "failed" if self.regression_detected else "passed"
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "candidate_run_id": self.candidate_run_id,
+            "baseline_run_id": self.baseline_run_id,
+            "status": self.status,
+            "regression_detected": self.regression_detected,
+            "fail_on_regression": self.fail_on_regression,
+            "test": self.test_type,
+            "metrics": list(self.metrics),
+            "results": [result.to_dict() for result in self.results],
+            "parallel": self.parallel,
+            "concurrency": self.concurrency,
+        }
+
+
+class RegressionGateService:
+    def __init__(self, storage: StoragePort, analysis_adapter: AnalysisPort) -> None:
+        self._storage = storage
+        self._analysis = analysis_adapter
+
+    def run_gate(
+        self,
+        candidate_run_id: str,
+        baseline_run_id: str,
+        *,
+        metrics: list[str] | None = None,
+        test_type: TestType = "t-test",
+        fail_on_regression: float = 0.05,
+        parallel: bool = True,
+        concurrency: int | None = None,
+    ) -> RegressionGateReport:
+        start_time = time.monotonic()
+        started_at = datetime.now(UTC)
+        logger.info(
+            "Regression gate start: candidate=%s baseline=%s",
+            candidate_run_id,
+            baseline_run_id,
+        )
+        try:
+            candidate = self._storage.get_run(candidate_run_id)
+            baseline = self._storage.get_run(baseline_run_id)
+
+            requested_metrics = [m for m in (metrics or []) if m]
+            if requested_metrics:
+                metric_list = requested_metrics
+            else:
+                metric_list = sorted(
+                    set(candidate.metrics_evaluated) & set(baseline.metrics_evaluated)
+                )
+
+            if not metric_list:
+                raise ValueError("No shared metrics available for regression gate.")
+
+            comparisons = self._analysis.compare_runs(
+                baseline,
+                candidate,
+                metrics=metric_list,
+                test_type=test_type,
+            )
+            if not comparisons:
+                raise ValueError("No comparable metrics found for regression gate.")
+
+            comparison_map = {result.metric: result for result in comparisons}
+            missing = [metric for metric in metric_list if metric not in comparison_map]
+            if missing:
+                raise ValueError("Missing comparison results for metrics: " + ", ".join(missing))
+
+            ordered = [comparison_map[metric] for metric in metric_list]
+            results = [
+                RegressionMetricResult.from_comparison(
+                    comparison,
+                    fail_on_regression=fail_on_regression,
+                )
+                for comparison in ordered
+            ]
+            regression_detected = any(result.regression for result in results)
+            finished_at = datetime.now(UTC)
+            duration_ms = int((time.monotonic() - start_time) * 1000)
+            logger.info(
+                "Regression gate complete: candidate=%s baseline=%s regressions=%s",
+                candidate_run_id,
+                baseline_run_id,
+                regression_detected,
+            )
+            return RegressionGateReport(
+                candidate_run_id=candidate_run_id,
+                baseline_run_id=baseline_run_id,
+                results=results,
+                regression_detected=regression_detected,
+                fail_on_regression=fail_on_regression,
+                test_type=test_type,
+                metrics=metric_list,
+                started_at=started_at,
+                finished_at=finished_at,
+                duration_ms=duration_ms,
+                parallel=parallel,
+                concurrency=concurrency,
+            )
+        except Exception:
+            logger.exception(
+                "Regression gate failed: candidate=%s baseline=%s",
+                candidate_run_id,
+                baseline_run_id,
+            )
+            raise
+
+
+__all__ = [
+    "RegressionGateReport",
+    "RegressionGateService",
+    "RegressionMetricResult",
+]

evalvault/domain/services/run_comparison_service.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from evalvault.domain.entities.analysis import ComparisonResult
+from evalvault.domain.entities.analysis_pipeline import AnalysisIntent, PipelineResult
+from evalvault.ports.outbound.analysis_port import AnalysisPort
+from evalvault.ports.outbound.comparison_pipeline_port import ComparisonPipelinePort
+from evalvault.ports.outbound.storage_port import StoragePort
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class RunComparisonRequest:
+    run_id_a: str
+    run_id_b: str
+    metrics: list[str] | None = None
+    test_type: str = "t-test"
+    parallel: bool = False
+    concurrency: int | None = None
+    report_type: str = "comparison"
+    use_llm_report: bool = True
+
+
+@dataclass
+class RunComparisonOutcome:
+    run_ids: tuple[str, str]
+    comparisons: list[ComparisonResult]
+    pipeline_result: PipelineResult
+    report_text: str
+    status: str
+    started_at: datetime
+    finished_at: datetime
+    duration_ms: int
+    degraded_reasons: list[str] = field(default_factory=list)
+
+    @property
+    def is_degraded(self) -> bool:
+        return self.status != "ok"
+
+
+class RunComparisonError(Exception):
+    def __init__(self, message: str, *, exit_code: int = 1):
+        super().__init__(message)
+        self.exit_code = exit_code
+
+
+class RunComparisonService:
+    def __init__(
+        self,
+        *,
+        storage: StoragePort,
+        analysis_port: AnalysisPort,
+        pipeline_port: ComparisonPipelinePort,
+    ) -> None:
+        self._storage = storage
+        self._analysis = analysis_port
+        self._pipeline = pipeline_port
+
+    def compare_runs(self, request: RunComparisonRequest) -> RunComparisonOutcome:
+        started_at = datetime.now(UTC)
+        logger.info("Starting run comparison: %s vs %s", request.run_id_a, request.run_id_b)
+
+        try:
+            run_a = self._storage.get_run(request.run_id_a)
+            run_b = self._storage.get_run(request.run_id_b)
+        except KeyError as exc:
+            logger.error("Run not found during comparison: %s", exc)
+            raise RunComparisonError("Run을 찾을 수 없습니다.", exit_code=1) from exc
+
+        comparisons = self._analysis.compare_runs(
+            run_a,
+            run_b,
+            metrics=request.metrics,
+            test_type=request.test_type,
+        )
+        if not comparisons:
+            logger.warning("No common metrics to compare for %s vs %s", run_a.run_id, run_b.run_id)
+            raise RunComparisonError("공통 메트릭이 없습니다.", exit_code=1)
+
+        pipeline_error: Exception | None = None
+        try:
+            pipeline_result = self._pipeline.run_comparison(
+                run_ids=[run_a.run_id, run_b.run_id],
+                compare_metrics=request.metrics,
+                test_type=request.test_type,
+                parallel=request.parallel,
+                concurrency=request.concurrency,
+                report_type=request.report_type,
+                use_llm_report=request.use_llm_report,
+            )
+        except Exception as exc:
+            pipeline_error = exc
+            logger.exception("Comparison pipeline failed: %s", exc)
+            pipeline_result = PipelineResult(
+                pipeline_id=f"compare-{run_a.run_id[:8]}-{run_b.run_id[:8]}",
+                intent=AnalysisIntent.GENERATE_COMPARISON,
+            )
+            pipeline_result.mark_complete()
+
+        report_text, report_found = self._extract_markdown_report(pipeline_result)
+        degraded_reasons: list[str] = []
+        if pipeline_error is not None:
+            degraded_reasons.append("pipeline_error")
+        if not report_found:
+            degraded_reasons.append("report_missing")
+        if not pipeline_result.all_succeeded:
+            degraded_reasons.append("pipeline_failed")
+
+        status = "degraded" if degraded_reasons else "ok"
+        if status == "degraded":
+            logger.warning("Comparison report degraded: %s", degraded_reasons)
+        finished_at = datetime.now(UTC)
+        duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+
+        logger.info("Completed run comparison: status=%s duration_ms=%s", status, duration_ms)
+
+        return RunComparisonOutcome(
+            run_ids=(run_a.run_id, run_b.run_id),
+            comparisons=comparisons,
+            pipeline_result=pipeline_result,
+            report_text=report_text,
+            status=status,
+            started_at=started_at,
+            finished_at=finished_at,
+            duration_ms=duration_ms,
+            degraded_reasons=degraded_reasons,
+        )
+
+    @staticmethod
+    def _extract_markdown_report(pipeline_result: PipelineResult) -> tuple[str, bool]:
+        final_output = pipeline_result.final_output
+        if isinstance(final_output, dict):
+            report = RunComparisonService._find_report(final_output)
+            if report:
+                return report, True
+        return "# 비교 분석 보고서\n\n보고서 본문을 찾지 못했습니다.\n", False
+
+    @staticmethod
+    def _find_report(output: dict) -> str | None:
+        if "report" in output and isinstance(output["report"], str):
+            return output["report"]
+        for value in output.values():
+            if isinstance(value, dict):
+                nested = RunComparisonService._find_report(value)
+                if nested:
+                    return nested
+        return None
+
+
+__all__ = [
+    "RunComparisonService",
+    "RunComparisonRequest",
+    "RunComparisonOutcome",
+    "RunComparisonError",
+]

evalvault/domain/services/stage_event_builder.py

@@ -255,7 +255,12 @@ def _normalize_scores(value: Any) -> list[float]:
     if value is None:
         return []
     if isinstance(value, list | tuple | set):
-        return [_coerce_float(item) for item in value if _coerce_float(item) is not None]
+        scores: list[float] = []
+        for item in value:
+            score = _coerce_float(item)
+            if score is not None:
+                scores.append(score)
+        return scores
     coerced = _coerce_float(value)
     return [coerced] if coerced is not None else []
 

evalvault/domain/services/stage_metric_service.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import math
 from collections.abc import Iterable, Mapping, Sequence
 from typing import Any
 
@@ -83,8 +84,16 @@ class StageMetricService:
         relevance_map: Mapping[str, set[str]],
     ) -> list[StageMetric]:
         metrics: list[StageMetric] = []
-        doc_ids = _to_str_list(event.attributes.get("doc_ids"))
-        scores = _to_float_list(event.attributes.get("scores"))
+        raw_doc_ids = event.attributes.get("doc_ids")
+        raw_scores = event.attributes.get("scores")
+        unordered_doc_ids = isinstance(raw_doc_ids, set | frozenset)
+        unordered_scores = isinstance(raw_scores, set | frozenset)
+        doc_ids = _to_str_list(raw_doc_ids)
+        scores = _to_float_list(raw_scores)
+        order_reconstructed = None
+        if unordered_doc_ids:
+            doc_ids = sorted(doc_ids)
+            order_reconstructed = "doc_id_asc"
 
         metrics.append(
             StageMetric(
@@ -92,19 +101,36 @@ class StageMetricService:
                 stage_id=event.stage_id,
                 metric_name="retrieval.result_count",
                 score=float(len(doc_ids)),
-                evidence={"count": len(doc_ids)},
+                evidence=_with_order_evidence({"count": len(doc_ids)}, unordered_doc_ids, None),
             )
         )
+        if unordered_doc_ids or unordered_scores:
+            metrics.append(
+                StageMetric(
+                    run_id=event.run_id,
+                    stage_id=event.stage_id,
+                    metric_name="retrieval.ordering_warning",
+                    score=1.0,
+                    evidence=_with_order_evidence(
+                        {
+                            "doc_ids_unordered": unordered_doc_ids,
+                            "scores_unordered": unordered_scores,
+                        },
+                        True,
+                        order_reconstructed,
+                    ),
+                )
+            )
 
         if scores:
-            avg_score = sum(scores) / len(scores)
+            avg_score = _safe_avg(scores)
             metrics.append(
                 StageMetric(
                     run_id=event.run_id,
                     stage_id=event.stage_id,
                     metric_name="retrieval.avg_score",
                     score=avg_score,
-                    evidence={"count": len(scores)},
+                    evidence=_with_order_evidence({"count": len(scores)}, unordered_scores, None),
                 )
             )
             if len(scores) > 1:
@@ -115,14 +141,22 @@ class StageMetricService:
                         stage_id=event.stage_id,
                         metric_name="retrieval.score_gap",
                         score=score_gap,
-                        evidence={"max": max(scores), "min": min(scores)},
+                        evidence=_with_order_evidence(
+                            {"max": max(scores), "min": min(scores)}, unordered_scores, None
+                        ),
                     )
                 )
 
         relevant_docs = _get_relevant_docs(event, relevance_map)
         if doc_ids and relevant_docs:
             top_k = _coerce_int(event.attributes.get("top_k"), default=len(doc_ids))
-            k = min(top_k, len(doc_ids)) if top_k > 0 else len(doc_ids)
+            k = len(doc_ids) if top_k is None or top_k <= 0 else min(top_k, len(doc_ids))
+            if unordered_scores and scores:
+                score_pairs = list(zip(doc_ids, scores, strict=False))
+                score_pairs.sort(key=lambda item: (-item[1], item[0]))
+                doc_ids = [doc_id for doc_id, _score in score_pairs]
+                scores = [score for _doc_id, score in score_pairs]
+                order_reconstructed = "score_desc_then_id"
             retrieved_top_k = doc_ids[:k]
             relevant_found = len(set(retrieved_top_k) & relevant_docs)
 
@@ -135,11 +169,15 @@ class StageMetricService:
                     stage_id=event.stage_id,
                     metric_name="retrieval.precision_at_k",
                     score=precision,
-                    evidence={
-                        "k": k,
-                        "relevant_found": relevant_found,
-                        "retrieved_count": k,
-                    },
+                    evidence=_with_order_evidence(
+                        {
+                            "k": k,
+                            "relevant_found": relevant_found,
+                            "retrieved_count": k,
+                        },
+                        unordered_doc_ids or unordered_scores,
+                        order_reconstructed,
+                    ),
                 )
             )
             metrics.append(
@@ -148,11 +186,15 @@ class StageMetricService:
                     stage_id=event.stage_id,
                     metric_name="retrieval.recall_at_k",
                     score=recall,
-                    evidence={
-                        "k": k,
-                        "relevant_found": relevant_found,
-                        "relevant_total": len(relevant_docs),
-                    },
+                    evidence=_with_order_evidence(
+                        {
+                            "k": k,
+                            "relevant_found": relevant_found,
+                            "relevant_total": len(relevant_docs),
+                        },
+                        unordered_doc_ids or unordered_scores,
+                        order_reconstructed,
+                    ),
                 )
             )
 
@@ -180,7 +222,7 @@ class StageMetricService:
 
         scores = _to_float_list(event.attributes.get("scores"))
         if scores:
-            avg_score = sum(scores) / len(scores)
+            avg_score = _safe_avg(scores)
             metrics.append(
                 StageMetric(
                     run_id=event.run_id,
@@ -358,6 +400,8 @@ def _to_str_list(value: Any) -> list[str]:
         return []
     if isinstance(value, str):
         return [value]
+    if isinstance(value, set | frozenset):
+        return [str(item) for item in value if not isinstance(item, bytes | bytearray)]
     if isinstance(value, Sequence):
         return [str(item) for item in value if not isinstance(item, bytes | bytearray)]
     return [str(value)]
@@ -370,6 +414,8 @@ def _to_str_set(value: Any) -> set[str]:
 def _to_float_list(value: Any) -> list[float]:
     if value is None:
         return []
+    if isinstance(value, set | frozenset):
+        return [float(item) for item in value]
     if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
         return [float(item) for item in value]
     return [float(value)]
@@ -390,6 +436,25 @@ def _coerce_float(value: Any) -> float | None:
         return None
 
 
+def _safe_avg(values: Sequence[float]) -> float:
+    if not values:
+        return 0.0
+    total = math.fsum(values)
+    return total / len(values)
+
+
+def _with_order_evidence(
+    evidence: dict[str, Any], unordered: bool, order_reconstructed: str | None
+) -> dict[str, Any]:
+    if not unordered:
+        return evidence
+    enriched = dict(evidence)
+    enriched["unordered_input"] = True
+    if order_reconstructed:
+        enriched["order_reconstructed"] = order_reconstructed
+    return enriched
+
+
 def _extract_violation_count(attributes: Mapping[str, Any]) -> int | None:
     violations = attributes.get("violations")
     if isinstance(violations, list | tuple | set):

evalvault/ports/outbound/__init__.py

@@ -11,6 +11,7 @@ from evalvault.ports.outbound.benchmark_port import (
     BenchmarkTaskResult,
 )
 from evalvault.ports.outbound.causal_analysis_port import CausalAnalysisPort
+from evalvault.ports.outbound.comparison_pipeline_port import ComparisonPipelinePort
 from evalvault.ports.outbound.dataset_port import DatasetPort
 from evalvault.ports.outbound.domain_memory_port import (
     BehaviorMemoryPort,
@@ -38,6 +39,7 @@ from evalvault.ports.outbound.improvement_port import (
     PlaybookPort,
 )
 from evalvault.ports.outbound.intent_classifier_port import IntentClassifierPort
+from evalvault.ports.outbound.judge_calibration_port import JudgeCalibrationPort
 from evalvault.ports.outbound.korean_nlp_port import (
     FaithfulnessResultProtocol,
     KoreanNLPToolkitPort,
@@ -58,6 +60,7 @@ from evalvault.ports.outbound.tracker_port import TrackerPort
 __all__ = [
     "AnalysisCachePort",
     "AnalysisPort",
+    "ComparisonPipelinePort",
     "CausalAnalysisPort",
     "DatasetPort",
     "DomainMemoryPort",
@@ -83,6 +86,7 @@ __all__ = [
     "PatternDefinitionProtocol",
     "MetricPlaybookProtocol",
     "ClaimImprovementProtocol",
+    "JudgeCalibrationPort",
     "LLMFactoryPort",
     "LLMPort",
     "MethodRuntime",

evalvault/ports/outbound/artifact_fs_port.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+
+class ArtifactFileSystemPort(Protocol):
+    def exists(self, path: Path) -> bool: ...
+
+    def is_dir(self, path: Path) -> bool: ...
+
+    def read_text(self, path: Path) -> str: ...

evalvault/ports/outbound/comparison_pipeline_port.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+from evalvault.domain.entities.analysis_pipeline import PipelineResult
+
+
+class ComparisonPipelinePort(Protocol):
+    def run_comparison(
+        self,
+        *,
+        run_ids: list[str],
+        compare_metrics: list[str] | None,
+        test_type: str,
+        parallel: bool,
+        concurrency: int | None,
+        report_type: str,
+        use_llm_report: bool,
+    ) -> PipelineResult: ...
+
+
+__all__ = ["ComparisonPipelinePort"]

evalvault/ports/outbound/difficulty_profile_port.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+
+class DifficultyProfileWriterPort(Protocol):
+    def write_profile(
+        self,
+        *,
+        output_path: Path,
+        artifacts_dir: Path,
+        envelope: dict[str, object],
+        artifacts: dict[str, object],
+    ) -> dict[str, object]: ...

evalvault/ports/outbound/judge_calibration_port.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+from evalvault.domain.entities import EvaluationRun, SatisfactionFeedback
+from evalvault.domain.entities.judge_calibration import JudgeCalibrationResult
+
+
+class JudgeCalibrationPort(Protocol):
+    def calibrate(
+        self,
+        run: EvaluationRun,
+        feedbacks: list[SatisfactionFeedback],
+        *,
+        labels_source: str,
+        method: str,
+        metrics: list[str],
+        holdout_ratio: float,
+        seed: int,
+        parallel: bool = False,
+        concurrency: int = 8,
+    ) -> JudgeCalibrationResult: ...

evalvault/ports/outbound/ops_snapshot_port.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Protocol
+
+
+class OpsSnapshotWriterPort(Protocol):
+    def write_snapshot(self, path: Path, payload: dict[str, Any]) -> None: ...

{evalvault-1.64.0.dist-info → evalvault-1.65.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.64.0
+Version: 1.65.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme