evalvault 1.64.0__py3-none-any.whl → 1.66.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/artifacts.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from evalvault.adapters.inbound.cli.utils.console import print_cli_error
+from evalvault.adapters.inbound.cli.utils.validators import validate_choice
+from evalvault.adapters.outbound.artifact_fs import LocalArtifactFileSystemAdapter
+from evalvault.domain.services.artifact_lint_service import ArtifactLintService
+
+logger = logging.getLogger(__name__)
+
+
+def create_artifacts_app(console: Console) -> typer.Typer:
+    artifacts_app = typer.Typer(name="artifacts", help="Artifact utilities.")
+
+    @artifacts_app.command("lint")
+    def lint(
+        artifacts_dir: Path = typer.Argument(..., help="Artifacts directory."),
+        strict: bool = typer.Option(False, "--strict", help="Fail on missing files."),
+        output_format: str = typer.Option(
+            "json",
+            "--format",
+            "-f",
+            help="Output format (json).",
+        ),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="Output file path for lint result.",
+        ),
+        parallel: bool = typer.Option(
+            True,
+            "--parallel/--no-parallel",
+            help="Enable parallel validation (placeholder).",
+        ),
+        concurrency: int = typer.Option(
+            8,
+            "--concurrency",
+            min=1,
+            help="Parallel validation concurrency (placeholder).",
+        ),
+    ) -> None:
+        validate_choice(output_format, ["json"], console, value_label="format")
+
+        logger.info("Artifacts lint command started: %s", artifacts_dir)
+        fs_adapter = LocalArtifactFileSystemAdapter()
+        service = ArtifactLintService(fs_adapter)
+        summary = service.lint(artifacts_dir, strict=strict)
+
+        payload = _build_payload(summary, parallel=parallel, concurrency=concurrency)
+        if output:
+            output.parent.mkdir(parents=True, exist_ok=True)
+            output.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+            console.print(f"[green]Lint report saved:[/green] {output}")
+        else:
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+        if summary.status == "error":
+            logger.error("Artifacts lint command failed: %s", artifacts_dir)
+            print_cli_error(console, "Artifact lint failed", details=str(artifacts_dir))
+            raise typer.Exit(1)
+
+        logger.info("Artifacts lint command finished: %s", artifacts_dir)
+
+    return artifacts_app
+
+
+def _build_payload(summary, *, parallel: bool, concurrency: int) -> dict[str, object]:
+    issues = [
+        {
+            "level": issue.level,
+            "code": issue.code,
+            "message": issue.message,
+            "path": issue.path,
+        }
+        for issue in summary.issues
+    ]
+    error_count = sum(1 for issue in summary.issues if issue.level == "error")
+    warning_count = sum(1 for issue in summary.issues if issue.level == "warning")
+    return {
+        "command": "artifacts.lint",
+        "version": 1,
+        "status": summary.status,
+        "started_at": summary.started_at.isoformat(),
+        "finished_at": summary.finished_at.isoformat(),
+        "duration_ms": summary.duration_ms,
+        "artifacts": {
+            "dir": str(summary.artifacts_dir),
+            "index": str(summary.index_path),
+        },
+        "data": {
+            "strict": summary.strict,
+            "parallel": parallel,
+            "concurrency": concurrency,
+            "issue_counts": {
+                "error": error_count,
+                "warning": warning_count,
+            },
+            "issues": issues,
+        },
+    }

evalvault/adapters/inbound/cli/commands/calibrate_judge.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import resolve_artifact_dir, write_json
+from evalvault.adapters.inbound.cli.utils.console import print_cli_error, progress_spinner
+from evalvault.adapters.inbound.cli.utils.options import db_option
+from evalvault.adapters.inbound.cli.utils.validators import parse_csv_option, validate_choice
+from evalvault.adapters.outbound.judge_calibration_reporter import JudgeCalibrationReporter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings
+from evalvault.domain.services.judge_calibration_service import JudgeCalibrationService
+
+_console = Console()
+
+_ALLOWED_LABELS = ["feedback", "gold", "hybrid"]
+_ALLOWED_METHODS = ["platt", "isotonic", "temperature", "none"]
+
+
+def register_calibrate_judge_commands(app: typer.Typer, console: Console) -> None:
+    global _console
+    _console = console
+
+    @app.command(name="calibrate-judge")
+    def calibrate_judge(
+        run_id: str = typer.Argument(..., help="보정 대상 Run ID"),
+        labels_source: str = typer.Option(
+            "feedback",
+            "--labels-source",
+            help="라벨 소스 (feedback|gold|hybrid)",
+        ),
+        method: str = typer.Option(
+            "isotonic",
+            "--method",
+            help="보정 방법 (platt|isotonic|temperature|none)",
+        ),
+        metrics: str | None = typer.Option(
+            None,
+            "--metric",
+            "-m",
+            help="보정 대상 메트릭 (쉼표로 구분, 미지정 시 run 메트릭 전체)",
+        ),
+        holdout_ratio: float = typer.Option(
+            0.2,
+            "--holdout-ratio",
+            help="검증용 holdout 비율",
+        ),
+        seed: int = typer.Option(42, "--seed", help="샘플 분할 랜덤 시드"),
+        write_back: bool = typer.Option(
+            False,
+            "--write-back",
+            help="보정 결과를 Run 메타데이터에 저장",
+        ),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="JSON 결과 파일 경로",
+        ),
+        artifacts_dir: Path | None = typer.Option(
+            None,
+            "--artifacts-dir",
+            help="아티팩트 저장 디렉터리",
+        ),
+        parallel: bool = typer.Option(
+            False,
+            "--parallel/--no-parallel",
+            help="병렬 실행 활성화",
+        ),
+        concurrency: int = typer.Option(8, "--concurrency", help="동시성 수준"),
+        db_path: Path | None = db_option(help_text="DB 경로"),
+    ) -> None:
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            print_cli_error(_console, "DB 경로가 설정되지 않았습니다.")
+            raise typer.Exit(1)
+
+        labels_source = labels_source.strip().lower()
+        method = method.strip().lower()
+        validate_choice(labels_source, _ALLOWED_LABELS, _console, value_label="labels-source")
+        validate_choice(method, _ALLOWED_METHODS, _console, value_label="method")
+
+        metric_list = parse_csv_option(metrics)
+        if holdout_ratio <= 0 or holdout_ratio >= 1:
+            print_cli_error(_console, "--holdout-ratio 값은 0과 1 사이여야 합니다.")
+            raise typer.Exit(1)
+        if seed < 0:
+            print_cli_error(_console, "--seed 값은 0 이상이어야 합니다.")
+            raise typer.Exit(1)
+        if concurrency <= 0:
+            print_cli_error(_console, "--concurrency 값은 1 이상이어야 합니다.")
+            raise typer.Exit(1)
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        try:
+            run = storage.get_run(run_id)
+        except KeyError:
+            print_cli_error(_console, f"Run을 찾을 수 없습니다: {run_id}")
+            raise typer.Exit(1)
+
+        feedbacks = storage.list_feedback(run_id)
+        if labels_source in {"feedback", "hybrid"} and not feedbacks:
+            print_cli_error(_console, "피드백 라벨이 없습니다.")
+            raise typer.Exit(1)
+
+        resolved_metrics = metric_list or list(run.metrics_evaluated)
+        if not resolved_metrics:
+            print_cli_error(_console, "보정 대상 메트릭이 없습니다.")
+            raise typer.Exit(1)
+
+        prefix = f"judge_calibration_{run_id}"
+        output_path = (output or Path("reports/calibration") / f"{prefix}.json").expanduser()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        resolved_artifacts_dir = resolve_artifact_dir(
+            base_dir=artifacts_dir,
+            output_path=output_path,
+            report_path=output_path,
+            prefix=prefix,
+        )
+
+        service = JudgeCalibrationService()
+        reporter = JudgeCalibrationReporter()
+        started_at = datetime.now(UTC)
+
+        with progress_spinner(_console, "Judge 보정 실행 중..."):
+            result = service.calibrate(
+                run,
+                feedbacks,
+                labels_source=labels_source,
+                method=method,
+                metrics=resolved_metrics,
+                holdout_ratio=holdout_ratio,
+                seed=seed,
+                parallel=parallel,
+                concurrency=concurrency,
+            )
+
+        artifacts_index = reporter.write_artifacts(
+            result=result,
+            artifacts_dir=resolved_artifacts_dir,
+        )
+        finished_at = datetime.now(UTC)
+        payload = _build_envelope(
+            result,
+            artifacts_index,
+            started_at=started_at,
+            finished_at=finished_at,
+        )
+        write_json(output_path, payload)
+
+        _display_summary(result)
+        _console.print(f"[green]JSON 저장:[/green] {output_path}")
+        _console.print(
+            f"[green]아티팩트 저장:[/green] {artifacts_index['dir']} (index: {artifacts_index['index']})"
+        )
+
+        if write_back:
+            metadata = run.tracker_metadata or {}
+            metadata["judge_calibration"] = reporter.render_json(result)
+            metadata["judge_calibration"]["artifacts"] = artifacts_index
+            metadata["judge_calibration"]["output"] = str(output_path)
+            storage.update_run_metadata(run_id, metadata)
+            _console.print("[green]보정 결과를 메타데이터에 저장했습니다.[/green]")
+
+        if result.summary.gate_passed is False:
+            raise typer.Exit(2)
+
+    return None
+
+
+def _build_envelope(
+    result,
+    artifacts_index: dict[str, str],
+    *,
+    started_at: datetime,
+    finished_at: datetime,
+) -> dict[str, object]:
+    duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+    status = "ok" if result.summary.gate_passed else "degraded"
+    return {
+        "command": "calibrate-judge",
+        "version": 1,
+        "status": status,
+        "started_at": started_at.astimezone(UTC).isoformat(),
+        "finished_at": finished_at.astimezone(UTC).isoformat(),
+        "duration_ms": duration_ms,
+        "artifacts": artifacts_index,
+        "data": {
+            "summary": _serialize_summary(result.summary),
+            "metrics": [_serialize_metric(metric) for metric in result.metrics],
+            "case_results": {
+                metric: [_serialize_case(case) for case in cases]
+                for metric, cases in result.case_results.items()
+            },
+            "warnings": list(result.warnings),
+        },
+    }
+
+
+def _serialize_summary(summary) -> dict[str, object]:
+    return {
+        "run_id": summary.run_id,
+        "labels_source": summary.labels_source,
+        "method": summary.method,
+        "metrics": list(summary.metrics),
+        "holdout_ratio": summary.holdout_ratio,
+        "seed": summary.seed,
+        "total_labels": summary.total_labels,
+        "total_samples": summary.total_samples,
+        "gate_passed": summary.gate_passed,
+        "gate_threshold": summary.gate_threshold,
+        "notes": list(summary.notes),
+    }
+
+
+def _serialize_metric(metric) -> dict[str, object]:
+    return {
+        "metric": metric.metric,
+        "method": metric.method,
+        "sample_count": metric.sample_count,
+        "label_count": metric.label_count,
+        "mae": metric.mae,
+        "pearson": metric.pearson,
+        "spearman": metric.spearman,
+        "temperature": metric.temperature,
+        "parameters": dict(metric.parameters),
+        "gate_passed": metric.gate_passed,
+        "warning": metric.warning,
+    }
+
+
+def _serialize_case(case) -> dict[str, object]:
+    return {
+        "test_case_id": case.test_case_id,
+        "raw_score": case.raw_score,
+        "calibrated_score": case.calibrated_score,
+        "label": case.label,
+        "label_source": case.label_source,
+    }
+
+
+def _display_summary(result) -> None:
+    summary_table = Table(title="Judge 보정 요약", show_header=True, header_style="bold cyan")
+    summary_table.add_column("메트릭")
+    summary_table.add_column("표본", justify="right")
+    summary_table.add_column("라벨", justify="right")
+    summary_table.add_column("MAE", justify="right")
+    summary_table.add_column("Pearson", justify="right")
+    summary_table.add_column("Spearman", justify="right")
+    summary_table.add_column("Gate", justify="right")
+
+    for metric in result.metrics:
+        summary_table.add_row(
+            metric.metric,
+            str(metric.sample_count),
+            str(metric.label_count),
+            _format_metric(metric.mae),
+            _format_metric(metric.pearson),
+            _format_metric(metric.spearman),
+            "PASS" if metric.gate_passed else "FAIL",
+        )
+
+    _console.print(summary_table)
+    _console.print(
+        f"라벨 소스: {result.summary.labels_source} | "
+        f"방법: {result.summary.method} | "
+        f"Gate: {'PASS' if result.summary.gate_passed else 'FAIL'}"
+    )
+
+    if result.warnings:
+        for warning in result.warnings:
+            _console.print(f"[yellow]경고:[/yellow] {warning}")
+
+
+def _format_metric(value: float | None) -> str:
+    if value is None:
+        return "-"
+    return f"{value:.3f}"

evalvault/adapters/inbound/cli/commands/compare.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.outbound.analysis.comparison_pipeline_adapter import (
+    ComparisonPipelineAdapter,
+)
+from evalvault.adapters.outbound.analysis.pipeline_factory import build_analysis_pipeline_service
+from evalvault.adapters.outbound.analysis.statistical_adapter import StatisticalAnalysisAdapter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings, apply_profile
+from evalvault.domain.services.run_comparison_service import (
+    RunComparisonError,
+    RunComparisonRequest,
+    RunComparisonService,
+)
+
+from ..utils.analysis_io import (
+    build_comparison_scorecard,
+    resolve_artifact_dir,
+    resolve_output_paths,
+    serialize_pipeline_result,
+    write_json,
+    write_pipeline_artifacts,
+)
+from ..utils.console import print_cli_error
+from ..utils.options import db_option, profile_option
+from ..utils.validators import parse_csv_option, validate_choice
+
+
+def _coerce_test_type(value: str) -> str:
+    if value == "t-test":
+        return "t-test"
+    return "mann-whitney"
+
+
+def register_compare_commands(app: typer.Typer, console: Console) -> None:
+    @app.command(name="compare")
+    def compare(
+        run_id_a: str = typer.Argument(..., help="기준 Run ID"),
+        run_id_b: str = typer.Argument(..., help="비교 Run ID"),
+        metrics: str | None = typer.Option(
+            None,
+            "--metrics",
+            "-m",
+            help="비교할 메트릭 목록 (쉼표 구분)",
+        ),
+        test: str = typer.Option(
+            "t-test",
+            "--test",
+            "-t",
+            help="통계 검정 (t-test, mann-whitney)",
+        ),
+        output_format: str = typer.Option(
+            "table",
+            "--format",
+            "-f",
+            help="출력 형식 (table, json)",
+        ),
+        output: Path | None = typer.Option(None, "--output", "-o", help="JSON 출력 파일"),
+        report: Path | None = typer.Option(None, "--report", help="리포트 출력 파일"),
+        output_dir: Path | None = typer.Option(
+            None,
+            "--output-dir",
+            help="출력 기본 디렉터리",
+        ),
+        artifacts_dir: Path | None = typer.Option(
+            None,
+            "--artifacts-dir",
+            help="아티팩트 저장 디렉터리",
+        ),
+        parallel: bool = typer.Option(
+            False,
+            "--parallel/--no-parallel",
+            help="병렬 파이프라인 실행",
+        ),
+        concurrency: int | None = typer.Option(
+            None,
+            "--concurrency",
+            min=1,
+            help="병렬 실행 동시성 제한",
+        ),
+        db_path: Path | None = db_option(help_text="DB 경로"),
+        profile: str | None = profile_option(help_text="LLM 프로필"),
+    ) -> None:
+        validate_choice(test, ["t-test", "mann-whitney"], console, value_label="test")
+        validate_choice(output_format, ["table", "json"], console, value_label="format")
+
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            print_cli_error(console, "DB 경로가 설정되지 않았습니다.")
+            raise typer.Exit(1)
+
+        metric_list = parse_csv_option(metrics)
+        metric_list = metric_list or None
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        analysis_adapter = StatisticalAnalysisAdapter()
+
+        settings = Settings()
+        profile_name = profile or settings.evalvault_profile
+        if profile_name:
+            settings = apply_profile(settings, profile_name)
+
+        llm_adapter = None
+        try:
+            from evalvault.adapters.outbound.llm import get_llm_adapter
+
+            llm_adapter = get_llm_adapter(settings)
+        except Exception as exc:
+            console.print(f"[yellow]경고: LLM 어댑터 초기화 실패 ({exc})[/yellow]")
+
+        pipeline_service = build_analysis_pipeline_service(
+            storage=storage,
+            llm_adapter=llm_adapter,
+        )
+        pipeline_adapter = ComparisonPipelineAdapter(pipeline_service)
+
+        service = RunComparisonService(
+            storage=storage,
+            analysis_port=analysis_adapter,
+            pipeline_port=pipeline_adapter,
+        )
+
+        request = RunComparisonRequest(
+            run_id_a=run_id_a,
+            run_id_b=run_id_b,
+            metrics=metric_list,
+            test_type=_coerce_test_type(test),
+            parallel=parallel,
+            concurrency=concurrency,
+        )
+
+        try:
+            outcome = service.compare_runs(request)
+        except RunComparisonError as exc:
+            print_cli_error(console, str(exc))
+            raise typer.Exit(exc.exit_code) from exc
+
+        comparison_prefix = f"comparison_{run_id_a[:8]}_{run_id_b[:8]}"
+        resolved_base_dir = output_dir or Path("reports/comparison")
+        output_path, report_path = resolve_output_paths(
+            base_dir=resolved_base_dir,
+            output_path=output,
+            report_path=report,
+            prefix=comparison_prefix,
+        )
+        if artifacts_dir is not None:
+            resolved_artifacts_dir = artifacts_dir
+            resolved_artifacts_dir.mkdir(parents=True, exist_ok=True)
+        else:
+            resolved_artifacts_dir = resolve_artifact_dir(
+                base_dir=output_dir,
+                output_path=output_path,
+                report_path=report_path,
+                prefix=comparison_prefix,
+            )
+
+        artifact_index = write_pipeline_artifacts(
+            outcome.pipeline_result,
+            artifacts_dir=resolved_artifacts_dir,
+        )
+
+        payload = _build_envelope(outcome, artifact_index)
+        payload["run_ids"] = list(outcome.run_ids)
+        payload["data"] = serialize_pipeline_result(outcome.pipeline_result)
+        payload["data"]["run_ids"] = list(outcome.run_ids)
+        payload["data"]["artifacts"] = artifact_index
+        write_json(output_path, payload)
+        report_path.write_text(outcome.report_text, encoding="utf-8")
+
+        if output_format == "table":
+            _render_table(console, outcome)
+        else:
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+        if outcome.is_degraded:
+            console.print("[yellow]리포트가 일부 누락되었을 수 있습니다.[/yellow]")
+
+        console.print(f"[green]비교 결과 저장:[/green] {output_path}")
+        console.print(f"[green]비교 리포트 저장:[/green] {report_path}")
+        console.print(
+            "[green]비교 아티팩트 저장:[/green] "
+            f"{artifact_index['dir']} (index: {artifact_index['index']})"
+        )
+
+        if outcome.is_degraded:
+            raise typer.Exit(2)
+
+
+def _build_envelope(outcome, artifact_index: dict[str, str]) -> dict[str, object]:
+    return {
+        "command": "compare",
+        "version": 1,
+        "status": outcome.status,
+        "started_at": outcome.started_at.isoformat(),
+        "finished_at": outcome.finished_at.isoformat(),
+        "duration_ms": outcome.duration_ms,
+        "artifacts": {
+            "dir": artifact_index.get("dir"),
+            "index": artifact_index.get("index"),
+        },
+    }
+
+
+def _render_table(console: Console, outcome) -> None:
+    table = Table(title="통계 비교", show_header=True, header_style="bold cyan")
+    table.add_column("메트릭")
+    table.add_column("실행 A (평균)", justify="right")
+    table.add_column("실행 B (평균)", justify="right")
+    table.add_column("변화 (%)", justify="right")
+    table.add_column("p-값", justify="right")
+    table.add_column("효과 크기", justify="right")
+    table.add_column("유의")
+    table.add_column("승자")
+
+    for comparison in outcome.comparisons:
+        sig_style = "green" if comparison.is_significant else "dim"
+        winner = comparison.winner[:8] if comparison.winner else "-"
+        table.add_row(
+            comparison.metric,
+            f"{comparison.mean_a:.3f}",
+            f"{comparison.mean_b:.3f}",
+            f"{comparison.diff_percent:+.1f}%",
+            f"{comparison.p_value:.4f}",
+            f"{comparison.effect_size:.2f} ({comparison.effect_level.value})",
+            f"[{sig_style}]{'예' if comparison.is_significant else '아니오'}[/{sig_style}]",
+            winner,
+        )
+
+    console.print("\n[bold]실행 비교 결과[/bold]")
+    console.print(table)
+    console.print()
+
+    scorecard = build_comparison_scorecard(
+        outcome.pipeline_result.get_node_result("run_metric_comparison").output
+        if outcome.pipeline_result.get_node_result("run_metric_comparison")
+        else {}
+    )
+    if not scorecard:
+        return
+
+    summary_table = Table(title="비교 스코어카드", show_header=True, header_style="bold cyan")
+    summary_table.add_column("메트릭")
+    summary_table.add_column("A", justify="right")
+    summary_table.add_column("B", justify="right")
+    summary_table.add_column("차이", justify="right")
+    summary_table.add_column("p-값", justify="right")
+    summary_table.add_column("효과 크기", justify="right")
+    summary_table.add_column("유의 여부")
+
+    for row in scorecard:
+        effect_size = row.get("effect_size")
+        effect_level = row.get("effect_level")
+        effect_text = (
+            f"{effect_size:.2f} ({effect_level})"
+            if isinstance(effect_size, (float, int)) and effect_level
+            else "-"
+        )
+        summary_table.add_row(
+            str(row.get("metric") or "-"),
+            _format_float(row.get("mean_a")),
+            _format_float(row.get("mean_b")),
+            _format_float(row.get("diff"), signed=True),
+            _format_float(row.get("p_value")),
+            effect_text,
+            "예" if row.get("is_significant") else "아니오",
+        )
+
+    console.print(summary_table)
+    console.print()
+
+
+def _format_float(value: float | None, *, signed: bool = False) -> str:
+    if value is None:
+        return "-"
+    try:
+        if signed:
+            return f"{float(value):+.3f}"
+        return f"{float(value):.3f}"
+    except (TypeError, ValueError):
+        return "-"
+
+
+__all__ = ["register_compare_commands"]