evalvault 1.64.0__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/profile_difficulty.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from evalvault.adapters.outbound.filesystem.difficulty_profile_writer import DifficultyProfileWriter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings
+from evalvault.domain.services.difficulty_profile_reporter import DifficultyProfileReporter
+from evalvault.domain.services.difficulty_profiling_service import (
+    DifficultyProfileRequest,
+    DifficultyProfilingService,
+)
+
+from ..utils.console import print_cli_error, progress_spinner
+from ..utils.options import db_option
+from ..utils.validators import parse_csv_option, validate_choices
+
+logger = logging.getLogger(__name__)
+
+
+def register_profile_difficulty_commands(
+    app: typer.Typer,
+    console: Console,
+    available_metrics: Sequence[str],
+) -> None:
+    @app.command("profile-difficulty")
+    def profile_difficulty(
+        dataset_name: str | None = typer.Option(
+            None, "--dataset-name", help="Dataset name to profile."
+        ),
+        run_id: str | None = typer.Option(None, "--run-id", help="Run ID to profile."),
+        limit_runs: int | None = typer.Option(
+            None, "--limit-runs", help="Limit number of runs to analyze."
+        ),
+        metrics: str | None = typer.Option(
+            None, "--metrics", "-m", help="Comma-separated metric allowlist."
+        ),
+        bucket_count: int = typer.Option(
+            3, "--bucket-count", help="Number of difficulty buckets.", min=2
+        ),
+        min_samples: int = typer.Option(
+            10, "--min-samples", help="Minimum samples required for profiling.", min=1
+        ),
+        output_path: Path | None = typer.Option(None, "--output", "-o", help="Output JSON path."),
+        artifacts_dir: Path | None = typer.Option(
+            None, "--artifacts-dir", help="Artifacts directory path."
+        ),
+        parallel: bool = typer.Option(
+            False, "--parallel/--no-parallel", help="Enable parallel execution."
+        ),
+        concurrency: int | None = typer.Option(
+            None, "--concurrency", help="Max concurrency when parallel is enabled.", min=1
+        ),
+        db_path: Path | None = db_option(help_text="SQLite DB path."),
+    ) -> None:
+        if not dataset_name and not run_id:
+            print_cli_error(
+                console,
+                "--dataset-name 또는 --run-id 중 하나는 필수입니다.",
+                fixes=["예: --dataset-name insurance-qa", "또는 --run-id run_123"],
+            )
+            raise typer.Exit(1)
+        if dataset_name and run_id:
+            print_cli_error(
+                console,
+                "--dataset-name과 --run-id는 동시에 사용할 수 없습니다.",
+                fixes=["둘 중 하나만 지정하세요."],
+            )
+            raise typer.Exit(1)
+
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            print_cli_error(
+                console,
+                "DB 경로가 필요합니다.",
+                fixes=["--db 옵션으로 SQLite DB 경로를 지정하세요."],
+            )
+            raise typer.Exit(1)
+
+        metric_list = parse_csv_option(metrics)
+        if metric_list:
+            validate_choices(metric_list, available_metrics, console, value_label="metric")
+        resolved_metrics = tuple(metric_list) if metric_list else None
+
+        identifier = _safe_identifier(run_id or dataset_name or "difficulty")
+        prefix = f"difficulty_{identifier}"
+        resolved_output = output_path or Path("reports") / "difficulty" / f"{prefix}.json"
+        resolved_artifacts_dir = artifacts_dir or resolved_output.parent / "artifacts" / prefix
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        writer = DifficultyProfileWriter()
+        reporter = DifficultyProfileReporter(writer)
+        service = DifficultyProfilingService(storage=storage, reporter=reporter)
+        request = DifficultyProfileRequest(
+            dataset_name=dataset_name,
+            run_id=run_id,
+            limit_runs=limit_runs,
+            metrics=resolved_metrics,
+            bucket_count=bucket_count,
+            min_samples=min_samples,
+            output_path=resolved_output,
+            artifacts_dir=resolved_artifacts_dir,
+            parallel=parallel,
+            concurrency=concurrency,
+        )
+
+        with progress_spinner(console, "난이도 프로파일링 실행 중..."):
+            started_at = datetime.now(UTC)
+            logger.info("profile-difficulty started", extra={"dataset_name": dataset_name})
+            try:
+                envelope = service.profile(request)
+            except KeyError as exc:
+                logger.exception("profile-difficulty run missing")
+                print_cli_error(
+                    console,
+                    "Run을 찾지 못했습니다.",
+                    details=str(exc),
+                    fixes=["--run-id 값과 --db 경로를 확인하세요."],
+                )
+                raise typer.Exit(1) from exc
+            except ValueError as exc:
+                logger.exception("profile-difficulty validation failed")
+                print_cli_error(
+                    console,
+                    "난이도 프로파일링 조건을 만족하지 못했습니다.",
+                    details=str(exc),
+                    fixes=["--min-samples 값을 낮추거나 충분한 실행 이력을 준비하세요."],
+                )
+                raise typer.Exit(1) from exc
+            except Exception as exc:
+                logger.exception("profile-difficulty failed")
+                print_cli_error(
+                    console,
+                    "난이도 프로파일링 중 오류가 발생했습니다.",
+                    details=str(exc),
+                )
+                raise typer.Exit(1) from exc
+
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+
+        console.print("[green]난이도 프로파일링 완료[/green]")
+        console.print(f"- output: {resolved_output}")
+        console.print(f"- artifacts: {envelope.get('artifacts', {}).get('dir')}")
+        console.print(f"- duration_ms: {duration_ms}")
+
+
+def _safe_identifier(value: str) -> str:
+    sanitized = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("_")
+    return sanitized or "difficulty"
+
+
+__all__ = ["register_profile_difficulty_commands"]

evalvault/adapters/inbound/cli/commands/regress.py

@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.adapters.outbound.analysis.statistical_adapter import (
+    StatisticalAnalysisAdapter,
+)
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.domain.services.regression_gate_service import (
+    RegressionGateReport,
+    RegressionGateService,
+    TestType,
+)
+
+from ..utils.formatters import format_diff, format_score, format_status
+from ..utils.options import db_option
+from ..utils.validators import parse_csv_option, validate_choice
+
+
+def _coerce_test_type(value: Literal["t-test", "mann-whitney"]) -> TestType:
+    if value == "t-test":
+        return "t-test"
+    return "mann-whitney"
+
+
+OutputFormat = Literal["table", "json", "github-actions"]
+
+
+def _format_timestamp(value: datetime) -> str:
+    return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _build_envelope(
+    *,
+    report: RegressionGateReport | None,
+    status: str,
+    started_at: datetime,
+    finished_at: datetime,
+    duration_ms: int,
+    message: str | None = None,
+    error_type: str | None = None,
+) -> dict[str, object]:
+    payload: dict[str, object] = {
+        "command": "regress",
+        "version": 1,
+        "status": status,
+        "started_at": _format_timestamp(started_at),
+        "finished_at": _format_timestamp(finished_at),
+        "duration_ms": duration_ms,
+        "artifacts": None,
+        "data": report.to_dict() if report else None,
+    }
+    if message:
+        payload["message"] = message
+    if error_type:
+        payload["error_type"] = error_type
+    return payload
+
+
+def register_regress_commands(app: typer.Typer, console: Console) -> None:
+    @app.command()
+    def regress(
+        run_id: str = typer.Argument(..., help="Candidate run ID to check."),
+        baseline: str = typer.Option(
+            ...,
+            "--baseline",
+            "-b",
+            help="Baseline run ID for regression detection.",
+        ),
+        fail_on_regression: float = typer.Option(
+            0.05,
+            "--fail-on-regression",
+            "-r",
+            help="Fail if metric drops by more than this amount (default: 0.05).",
+        ),
+        test: TestType = typer.Option(
+            "t-test",
+            "--test",
+            "-t",
+            help="Statistical test (t-test, mann-whitney).",
+        ),
+        metrics: str | None = typer.Option(
+            None,
+            "--metrics",
+            "-m",
+            help="Comma-separated list of metrics to check.",
+        ),
+        output_format: OutputFormat = typer.Option(
+            "table",
+            "--format",
+            "-f",
+            help="Output format: table, json, or github-actions.",
+        ),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="Write JSON summary to a file.",
+        ),
+        parallel: bool = typer.Option(
+            True,
+            "--parallel/--no-parallel",
+            help="Enable parallel execution for metric checks.",
+        ),
+        concurrency: int = typer.Option(
+            8,
+            "--concurrency",
+            help="Concurrency level when running in parallel.",
+        ),
+        db_path: Path | None = db_option(help_text="Database path"),
+    ) -> None:
+        started_at = datetime.now(UTC)
+        if db_path is None:
+            console.print("[red]Error:[/red] Database path is not configured.")
+            raise typer.Exit(1)
+
+        validate_choice(test, ["t-test", "mann-whitney"], console, value_label="test")
+        metric_list = parse_csv_option(metrics)
+
+        storage = SQLiteStorageAdapter(db_path=db_path)
+        analysis_adapter = StatisticalAnalysisAdapter()
+        service = RegressionGateService(storage=storage, analysis_adapter=analysis_adapter)
+
+        try:
+            report = service.run_gate(
+                run_id,
+                baseline,
+                metrics=metric_list or None,
+                test_type=_coerce_test_type(test),
+                fail_on_regression=fail_on_regression,
+                parallel=parallel,
+                concurrency=concurrency,
+            )
+        except (KeyError, ValueError) as exc:
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+            payload = _build_envelope(
+                report=None,
+                status="error",
+                started_at=started_at,
+                finished_at=finished_at,
+                duration_ms=duration_ms,
+                message=str(exc),
+                error_type=type(exc).__name__,
+            )
+            if output:
+                write_json(output, payload)
+            if output_format == "json":
+                console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+            else:
+                console.print(f"[red]Error:[/red] {exc}")
+            raise typer.Exit(1) from exc
+        except Exception as exc:
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+            payload = _build_envelope(
+                report=None,
+                status="error",
+                started_at=started_at,
+                finished_at=finished_at,
+                duration_ms=duration_ms,
+                message=str(exc),
+                error_type=type(exc).__name__,
+            )
+            if output:
+                write_json(output, payload)
+            if output_format == "json":
+                console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+            else:
+                console.print(f"[red]Error:[/red] {exc}")
+            raise typer.Exit(3) from exc
+
+        finished_at = report.finished_at
+        duration_ms = report.duration_ms
+        payload = _build_envelope(
+            report=report,
+            status="ok",
+            started_at=report.started_at,
+            finished_at=finished_at,
+            duration_ms=duration_ms,
+        )
+        if output:
+            write_json(output, payload)
+
+        if output_format == "json":
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+        elif output_format == "github-actions":
+            _render_github_actions(report, console)
+        else:
+            _render_table(report, console)
+
+        if report.regression_detected:
+            raise typer.Exit(2)
+
+
+def _render_table(report: RegressionGateReport, console: Console) -> None:
+    console.print(f"\n[bold]Regression Gate Check: {report.candidate_run_id}[/bold]\n")
+    console.print(f"Baseline: {report.baseline_run_id}")
+    console.print(f"Test: {report.test_type}\n")
+    table = Table(show_header=True, header_style="bold cyan")
+    table.add_column("Metric")
+    table.add_column("Baseline", justify="right")
+    table.add_column("Candidate", justify="right")
+    table.add_column("Diff", justify="right")
+    table.add_column("p-value", justify="right")
+    table.add_column("Regression", justify="center")
+
+    for result in report.results:
+        table.add_row(
+            result.metric,
+            format_score(result.baseline_score),
+            format_score(result.candidate_score),
+            format_diff(result.diff),
+            f"{result.p_value:.4f}",
+            format_status(not result.regression, success_text="NO", failure_text="YES"),
+        )
+
+    console.print(table)
+    if report.regression_detected:
+        regressed = [r.metric for r in report.results if r.regression]
+        console.print("\n[bold red]Regression detected[/bold red]")
+        console.print(f"[red]Regressed metrics: {', '.join(regressed)}[/red]")
+    else:
+        console.print("\n[bold green]Regression gate PASSED[/bold green]")
+    console.print()
+
+
+def _render_github_actions(report: RegressionGateReport, console: Console) -> None:
+    for result in report.results:
+        status = "✅" if not result.regression else "❌"
+        reg_status = " (REGRESSION)" if result.regression else ""
+        console.print(
+            f"{status} {result.metric}: {result.candidate_score:.3f} "
+            f"(baseline: {result.baseline_score:.3f}, diff: {result.diff:+.3f}){reg_status}"
+        )
+
+    console.print(f"::set-output name=passed::{str(not report.regression_detected).lower()}")
+    if report.regression_detected:
+        regressed = [r.metric for r in report.results if r.regression]
+        console.print(f"::error::Regression detected in: {', '.join(regressed)}")
+
+
+__all__ = ["register_regress_commands"]

evalvault/adapters/outbound/analysis/comparison_pipeline_adapter.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import asyncio
+
+from evalvault.domain.entities.analysis_pipeline import AnalysisIntent, PipelineResult
+from evalvault.domain.services.pipeline_orchestrator import AnalysisPipelineService
+from evalvault.ports.outbound.comparison_pipeline_port import ComparisonPipelinePort
+
+
+class ComparisonPipelineAdapter(ComparisonPipelinePort):
+    def __init__(self, service: AnalysisPipelineService) -> None:
+        self._service = service
+
+    def run_comparison(
+        self,
+        *,
+        run_ids: list[str],
+        compare_metrics: list[str] | None,
+        test_type: str,
+        parallel: bool,
+        concurrency: int | None,
+        report_type: str,
+        use_llm_report: bool,
+    ) -> PipelineResult:
+        params = {
+            "run_ids": run_ids,
+            "compare_metrics": compare_metrics,
+            "test_type": test_type,
+            "report_type": report_type,
+            "use_llm_report": use_llm_report,
+        }
+        if parallel:
+            if concurrency is not None:
+                params["max_concurrency"] = concurrency
+            return asyncio.run(
+                self._service.analyze_intent_async(
+                    AnalysisIntent.GENERATE_COMPARISON,
+                    run_id=run_ids[0] if run_ids else None,
+                    **params,
+                )
+            )
+        return self._service.analyze_intent(
+            AnalysisIntent.GENERATE_COMPARISON,
+            run_id=run_ids[0] if run_ids else None,
+            **params,
+        )
+
+
+__all__ = ["ComparisonPipelineAdapter"]

evalvault/adapters/outbound/artifact_fs.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.ports.outbound.artifact_fs_port import ArtifactFileSystemPort
+
+
+class LocalArtifactFileSystemAdapter(ArtifactFileSystemPort):
+    def exists(self, path: Path) -> bool:
+        return path.exists()
+
+    def is_dir(self, path: Path) -> bool:
+        return path.is_dir()
+
+    def read_text(self, path: Path) -> str:
+        return path.read_text(encoding="utf-8")

evalvault/adapters/outbound/filesystem/__init__.py

@@ -0,0 +1,3 @@
+from evalvault.adapters.outbound.filesystem.ops_snapshot_writer import OpsSnapshotWriter
+
+__all__ = ["OpsSnapshotWriter"]

evalvault/adapters/outbound/filesystem/difficulty_profile_writer.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.ports.outbound.difficulty_profile_port import DifficultyProfileWriterPort
+
+
+class DifficultyProfileWriter(DifficultyProfileWriterPort):
+    def write_profile(
+        self,
+        *,
+        output_path: Path,
+        artifacts_dir: Path,
+        envelope: dict[str, object],
+        artifacts: dict[str, object],
+    ) -> dict[str, object]:
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+        breakdown_path = artifacts_dir / "difficulty_breakdown.json"
+        cases_path = artifacts_dir / "difficulty_cases.json"
+        breakdown_payload = artifacts.get("breakdown")
+        cases_payload = artifacts.get("cases")
+        write_json(
+            breakdown_path,
+            breakdown_payload if isinstance(breakdown_payload, dict) else {},
+        )
+        write_json(
+            cases_path,
+            {"cases": cases_payload} if isinstance(cases_payload, list) else {"cases": []},
+        )
+
+        index_payload = {
+            "files": {
+                "breakdown": str(breakdown_path),
+                "cases": str(cases_path),
+            }
+        }
+        index_path = artifacts_dir / "index.json"
+        write_json(index_path, index_payload)
+
+        artifacts_index = {
+            "dir": str(artifacts_dir),
+            "index": str(index_path),
+        }
+        envelope["artifacts"] = artifacts_index
+        write_json(output_path, envelope)
+
+        return artifacts_index

evalvault/adapters/outbound/filesystem/ops_snapshot_writer.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.ports.outbound.ops_snapshot_port import OpsSnapshotWriterPort
+
+
+class OpsSnapshotWriter(OpsSnapshotWriterPort):
+    def write_snapshot(self, path: Path, payload: dict[str, Any]) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        write_json(path, payload)

evalvault/adapters/outbound/judge_calibration_adapter.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from evalvault.domain.entities import EvaluationRun, SatisfactionFeedback
+from evalvault.domain.entities.judge_calibration import JudgeCalibrationResult
+from evalvault.domain.services.judge_calibration_service import JudgeCalibrationService
+from evalvault.ports.outbound.judge_calibration_port import JudgeCalibrationPort
+
+
+class JudgeCalibrationAdapter(JudgeCalibrationPort):
+    def __init__(self) -> None:
+        self._service = JudgeCalibrationService()
+
+    def calibrate(
+        self,
+        run: EvaluationRun,
+        feedbacks: list[SatisfactionFeedback],
+        *,
+        labels_source: str,
+        method: str,
+        metrics: list[str],
+        holdout_ratio: float,
+        seed: int,
+        parallel: bool = False,
+        concurrency: int = 8,
+    ) -> JudgeCalibrationResult:
+        return self._service.calibrate(
+            run,
+            feedbacks,
+            labels_source=labels_source,
+            method=method,
+            metrics=metrics,
+            holdout_ratio=holdout_ratio,
+            seed=seed,
+            parallel=parallel,
+            concurrency=concurrency,
+        )

evalvault/adapters/outbound/judge_calibration_reporter.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.entities.judge_calibration import JudgeCalibrationResult
+
+
+class JudgeCalibrationReporter:
+    def render_json(self, result: JudgeCalibrationResult) -> dict[str, Any]:
+        return {
+            "summary": asdict(result.summary),
+            "metrics": [asdict(metric) for metric in result.metrics],
+            "case_results": {
+                metric: [asdict(entry) for entry in entries]
+                for metric, entries in result.case_results.items()
+            },
+            "warnings": list(result.warnings),
+        }
+
+    def write_artifacts(
+        self,
+        *,
+        result: JudgeCalibrationResult,
+        artifacts_dir: Path,
+    ) -> dict[str, str]:
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+        index_path = artifacts_dir / "index.json"
+        payload = {
+            "run_id": result.summary.run_id,
+            "metrics": [metric.metric for metric in result.metrics],
+            "cases": {},
+        }
+        for metric, cases in result.case_results.items():
+            case_path = artifacts_dir / f"{metric}.json"
+            case_payload = [
+                {
+                    "test_case_id": case.test_case_id,
+                    "raw_score": case.raw_score,
+                    "calibrated_score": case.calibrated_score,
+                    "label": case.label,
+                    "label_source": case.label_source,
+                }
+                for case in cases
+            ]
+            case_path.write_text(
+                json.dumps(case_payload, ensure_ascii=False, indent=2),
+                encoding="utf-8",
+            )
+            payload["cases"][metric] = str(case_path)
+        index_path.write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        return {"dir": str(artifacts_dir), "index": str(index_path)}

evalvault/adapters/outbound/tracker/langfuse_adapter.py

@@ -63,13 +63,15 @@ class LangfuseAdapter(TrackerPort):
                 span.update_trace(name=name, metadata=metadata)
             self._traces[trace_id] = span
         else:
-            # Langfuse 2.x: use trace method
-            trace = self._client.trace(
+            trace_fn: Any = getattr(self._client, "trace", None)
+            if trace_fn is None:
+                raise RuntimeError("Langfuse client does not expose trace API")
+            trace_obj = trace_fn(
                 name=name,
                 metadata=metadata,
             )
-            trace_id = trace.id
-            self._traces[trace_id] = trace
+            trace_id = trace_obj.id
+            self._traces[trace_id] = trace_obj
         return trace_id
 
     def add_span(
@@ -240,7 +242,7 @@ class LangfuseAdapter(TrackerPort):
             passed_count = sum(
                 1
                 for r in run.results
-                if r.get_metric(metric_name) and r.get_metric(metric_name).passed
+                if (metric := r.get_metric(metric_name)) and metric.passed is True
             )
             avg_score = run.get_avg_score(metric_name)
             threshold = run.thresholds.get(metric_name, 0.7)
@@ -421,12 +423,15 @@ class LangfuseAdapter(TrackerPort):
             }
 
             # Span metadata: additional info
-            span_metadata = {
+            span_metadata: dict[str, float | int] = {
                 "tokens_used": result.tokens_used,
                 "latency_ms": result.latency_ms,
             }
             if result.cost_usd:
-                span_metadata["cost_usd"] = result.cost_usd
+                span_metadata = {
+                    **span_metadata,
+                    "cost_usd": float(result.cost_usd),
+                }
 
             if hasattr(root_span, "start_span"):
                 child_span = root_span.start_span(