evalvault 1.64.0__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/compare.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.outbound.analysis.comparison_pipeline_adapter import (
+    ComparisonPipelineAdapter,
+)
+from evalvault.adapters.outbound.analysis.pipeline_factory import build_analysis_pipeline_service
+from evalvault.adapters.outbound.analysis.statistical_adapter import StatisticalAnalysisAdapter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings, apply_profile
+from evalvault.domain.services.run_comparison_service import (
+    RunComparisonError,
+    RunComparisonRequest,
+    RunComparisonService,
+)
+
+from ..utils.analysis_io import (
+    build_comparison_scorecard,
+    resolve_artifact_dir,
+    resolve_output_paths,
+    serialize_pipeline_result,
+    write_json,
+    write_pipeline_artifacts,
+)
+from ..utils.console import print_cli_error
+from ..utils.options import db_option, profile_option
+from ..utils.validators import parse_csv_option, validate_choice
+
+
+def _coerce_test_type(value: str) -> str:
+    if value == "t-test":
+        return "t-test"
+    return "mann-whitney"
+
+
+def register_compare_commands(app: typer.Typer, console: Console) -> None:
+    @app.command(name="compare")
+    def compare(
+        run_id_a: str = typer.Argument(..., help="기준 Run ID"),
+        run_id_b: str = typer.Argument(..., help="비교 Run ID"),
+        metrics: str | None = typer.Option(
+            None,
+            "--metrics",
+            "-m",
+            help="비교할 메트릭 목록 (쉼표 구분)",
+        ),
+        test: str = typer.Option(
+            "t-test",
+            "--test",
+            "-t",
+            help="통계 검정 (t-test, mann-whitney)",
+        ),
+        output_format: str = typer.Option(
+            "table",
+            "--format",
+            "-f",
+            help="출력 형식 (table, json)",
+        ),
+        output: Path | None = typer.Option(None, "--output", "-o", help="JSON 출력 파일"),
+        report: Path | None = typer.Option(None, "--report", help="리포트 출력 파일"),
+        output_dir: Path | None = typer.Option(
+            None,
+            "--output-dir",
+            help="출력 기본 디렉터리",
+        ),
+        artifacts_dir: Path | None = typer.Option(
+            None,
+            "--artifacts-dir",
+            help="아티팩트 저장 디렉터리",
+        ),
+        parallel: bool = typer.Option(
+            False,
+            "--parallel/--no-parallel",
+            help="병렬 파이프라인 실행",
+        ),
+        concurrency: int | None = typer.Option(
+            None,
+            "--concurrency",
+            min=1,
+            help="병렬 실행 동시성 제한",
+        ),
+        db_path: Path | None = db_option(help_text="DB 경로"),
+        profile: str | None = profile_option(help_text="LLM 프로필"),
+    ) -> None:
+        validate_choice(test, ["t-test", "mann-whitney"], console, value_label="test")
+        validate_choice(output_format, ["table", "json"], console, value_label="format")
+
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            print_cli_error(console, "DB 경로가 설정되지 않았습니다.")
+            raise typer.Exit(1)
+
+        metric_list = parse_csv_option(metrics)
+        metric_list = metric_list or None
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        analysis_adapter = StatisticalAnalysisAdapter()
+
+        settings = Settings()
+        profile_name = profile or settings.evalvault_profile
+        if profile_name:
+            settings = apply_profile(settings, profile_name)
+
+        llm_adapter = None
+        try:
+            from evalvault.adapters.outbound.llm import get_llm_adapter
+
+            llm_adapter = get_llm_adapter(settings)
+        except Exception as exc:
+            console.print(f"[yellow]경고: LLM 어댑터 초기화 실패 ({exc})[/yellow]")
+
+        pipeline_service = build_analysis_pipeline_service(
+            storage=storage,
+            llm_adapter=llm_adapter,
+        )
+        pipeline_adapter = ComparisonPipelineAdapter(pipeline_service)
+
+        service = RunComparisonService(
+            storage=storage,
+            analysis_port=analysis_adapter,
+            pipeline_port=pipeline_adapter,
+        )
+
+        request = RunComparisonRequest(
+            run_id_a=run_id_a,
+            run_id_b=run_id_b,
+            metrics=metric_list,
+            test_type=_coerce_test_type(test),
+            parallel=parallel,
+            concurrency=concurrency,
+        )
+
+        try:
+            outcome = service.compare_runs(request)
+        except RunComparisonError as exc:
+            print_cli_error(console, str(exc))
+            raise typer.Exit(exc.exit_code) from exc
+
+        comparison_prefix = f"comparison_{run_id_a[:8]}_{run_id_b[:8]}"
+        resolved_base_dir = output_dir or Path("reports/comparison")
+        output_path, report_path = resolve_output_paths(
+            base_dir=resolved_base_dir,
+            output_path=output,
+            report_path=report,
+            prefix=comparison_prefix,
+        )
+        if artifacts_dir is not None:
+            resolved_artifacts_dir = artifacts_dir
+            resolved_artifacts_dir.mkdir(parents=True, exist_ok=True)
+        else:
+            resolved_artifacts_dir = resolve_artifact_dir(
+                base_dir=output_dir,
+                output_path=output_path,
+                report_path=report_path,
+                prefix=comparison_prefix,
+            )
+
+        artifact_index = write_pipeline_artifacts(
+            outcome.pipeline_result,
+            artifacts_dir=resolved_artifacts_dir,
+        )
+
+        payload = _build_envelope(outcome, artifact_index)
+        payload["run_ids"] = list(outcome.run_ids)
+        payload["data"] = serialize_pipeline_result(outcome.pipeline_result)
+        payload["data"]["run_ids"] = list(outcome.run_ids)
+        payload["data"]["artifacts"] = artifact_index
+        write_json(output_path, payload)
+        report_path.write_text(outcome.report_text, encoding="utf-8")
+
+        if output_format == "table":
+            _render_table(console, outcome)
+        else:
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+        if outcome.is_degraded:
+            console.print("[yellow]리포트가 일부 누락되었을 수 있습니다.[/yellow]")
+
+        console.print(f"[green]비교 결과 저장:[/green] {output_path}")
+        console.print(f"[green]비교 리포트 저장:[/green] {report_path}")
+        console.print(
+            "[green]비교 아티팩트 저장:[/green] "
+            f"{artifact_index['dir']} (index: {artifact_index['index']})"
+        )
+
+        if outcome.is_degraded:
+            raise typer.Exit(2)
+
+
+def _build_envelope(outcome, artifact_index: dict[str, str]) -> dict[str, object]:
+    return {
+        "command": "compare",
+        "version": 1,
+        "status": outcome.status,
+        "started_at": outcome.started_at.isoformat(),
+        "finished_at": outcome.finished_at.isoformat(),
+        "duration_ms": outcome.duration_ms,
+        "artifacts": {
+            "dir": artifact_index.get("dir"),
+            "index": artifact_index.get("index"),
+        },
+    }
+
+
+def _render_table(console: Console, outcome) -> None:
+    table = Table(title="통계 비교", show_header=True, header_style="bold cyan")
+    table.add_column("메트릭")
+    table.add_column("실행 A (평균)", justify="right")
+    table.add_column("실행 B (평균)", justify="right")
+    table.add_column("변화 (%)", justify="right")
+    table.add_column("p-값", justify="right")
+    table.add_column("효과 크기", justify="right")
+    table.add_column("유의")
+    table.add_column("승자")
+
+    for comparison in outcome.comparisons:
+        sig_style = "green" if comparison.is_significant else "dim"
+        winner = comparison.winner[:8] if comparison.winner else "-"
+        table.add_row(
+            comparison.metric,
+            f"{comparison.mean_a:.3f}",
+            f"{comparison.mean_b:.3f}",
+            f"{comparison.diff_percent:+.1f}%",
+            f"{comparison.p_value:.4f}",
+            f"{comparison.effect_size:.2f} ({comparison.effect_level.value})",
+            f"[{sig_style}]{'예' if comparison.is_significant else '아니오'}[/{sig_style}]",
+            winner,
+        )
+
+    console.print("\n[bold]실행 비교 결과[/bold]")
+    console.print(table)
+    console.print()
+
+    scorecard = build_comparison_scorecard(
+        outcome.pipeline_result.get_node_result("run_metric_comparison").output
+        if outcome.pipeline_result.get_node_result("run_metric_comparison")
+        else {}
+    )
+    if not scorecard:
+        return
+
+    summary_table = Table(title="비교 스코어카드", show_header=True, header_style="bold cyan")
+    summary_table.add_column("메트릭")
+    summary_table.add_column("A", justify="right")
+    summary_table.add_column("B", justify="right")
+    summary_table.add_column("차이", justify="right")
+    summary_table.add_column("p-값", justify="right")
+    summary_table.add_column("효과 크기", justify="right")
+    summary_table.add_column("유의 여부")
+
+    for row in scorecard:
+        effect_size = row.get("effect_size")
+        effect_level = row.get("effect_level")
+        effect_text = (
+            f"{effect_size:.2f} ({effect_level})"
+            if isinstance(effect_size, (float, int)) and effect_level
+            else "-"
+        )
+        summary_table.add_row(
+            str(row.get("metric") or "-"),
+            _format_float(row.get("mean_a")),
+            _format_float(row.get("mean_b")),
+            _format_float(row.get("diff"), signed=True),
+            _format_float(row.get("p_value")),
+            effect_text,
+            "예" if row.get("is_significant") else "아니오",
+        )
+
+    console.print(summary_table)
+    console.print()
+
+
+def _format_float(value: float | None, *, signed: bool = False) -> str:
+    if value is None:
+        return "-"
+    try:
+        if signed:
+            return f"{float(value):+.3f}"
+        return f"{float(value):.3f}"
+    except (TypeError, ValueError):
+        return "-"
+
+
+__all__ = ["register_compare_commands"]

evalvault/adapters/inbound/cli/commands/history.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
+from typing import cast
 
 import typer
 from rich.console import Console
@@ -46,7 +47,7 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
             "--mode",
             help="Filter by run mode: 'simple' or 'full'.",
         ),
-        db_path: Path = db_option(help_text="Path to database file."),
+        db_path: Path | None = db_option(help_text="Path to database file."),
     ) -> None:
         """Show evaluation run history.
 
@@ -75,7 +76,6 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
 
         \b
         See also:
-          evalvault compare  — Compare two runs side by side
           evalvault export   — Export run details to JSON
           evalvault run      — Create new evaluation runs
         """
@@ -88,7 +88,11 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
                     "[red]Error:[/red] --mode must be one of: " + ", ".join(RUN_MODE_CHOICES)
                 )
                 raise typer.Exit(2)
-        storage = SQLiteStorageAdapter(db_path=db_path)
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            console.print("[red]Error:[/red] Database path is not configured.")
+            raise typer.Exit(1)
+        storage = SQLiteStorageAdapter(db_path=cast(Path, resolved_db_path))
         runs = storage.list_runs(limit=limit, dataset_name=dataset, model_name=model)
         if normalized_mode:
             runs = [
@@ -157,86 +161,6 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
         console.print(table)
         console.print(f"\n[dim]Showing {len(runs)} of {limit} runs[/dim]\n")
 
-    @app.command()
-    def compare(
-        run_id1: str = typer.Argument(..., help="First run ID to compare."),
-        run_id2: str = typer.Argument(..., help="Second run ID to compare."),
-        db_path: Path = db_option(help_text="Path to database file."),
-    ) -> None:
-        """Compare two evaluation runs.
-
-        Show a side-by-side comparison of metrics, pass rates, and scores
-        between two evaluation runs.
-
-        \b
-        Examples:
-          # Compare two runs by ID
-          evalvault compare abc12345 def67890
-
-          # Compare runs from a custom database
-          evalvault compare abc12345 def67890 --db custom.db
-
-        \b
-        See also:
-          evalvault history  — List runs to find IDs
-          evalvault export   — Export run details to JSON
-          evalvault analyze  — Deep analysis of a single run
-        """
-        console.print("\n[bold]Comparing Evaluation Runs[/bold]\n")
-
-        storage = SQLiteStorageAdapter(db_path=db_path)
-
-        try:
-            run1 = storage.get_run(run_id1)
-            run2 = storage.get_run(run_id2)
-        except KeyError as exc:
-            console.print(f"[red]Error:[/red] {exc}")
-            raise typer.Exit(1) from exc
-
-        table = Table(show_header=True, header_style="bold cyan")
-        table.add_column("Metric")
-        table.add_column(f"Run 1\n{run_id1[:12]}...", justify="right")
-        table.add_column(f"Run 2\n{run_id2[:12]}...", justify="right")
-        table.add_column("Difference", justify="right")
-
-        table.add_row("Dataset", run1.dataset_name, run2.dataset_name, "-")
-        table.add_row("Model", run1.model_name, run2.model_name, "-")
-        table.add_row(
-            "Test Cases",
-            str(run1.total_test_cases),
-            str(run2.total_test_cases),
-            str(run2.total_test_cases - run1.total_test_cases),
-        )
-
-        pass_rate_diff = run2.pass_rate - run1.pass_rate
-        diff_color = "green" if pass_rate_diff > 0 else "red" if pass_rate_diff < 0 else "dim"
-        table.add_row(
-            "Pass Rate",
-            f"{run1.pass_rate:.1%}",
-            f"{run2.pass_rate:.1%}",
-            f"[{diff_color}]{pass_rate_diff:+.1%}[/{diff_color}]",
-        )
-
-        for metric in run1.metrics_evaluated:
-            if metric in run2.metrics_evaluated:
-                score1 = run1.get_avg_score(metric)
-                score2 = run2.get_avg_score(metric)
-                diff = score2 - score1 if score1 is not None and score2 is not None else None
-                diff_str = (
-                    f"[{'green' if diff and diff > 0 else 'red' if diff and diff < 0 else 'dim'}]{diff:+.3f}[/{'green' if diff and diff > 0 else 'red' if diff and diff < 0 else 'dim'}]"
-                    if diff is not None
-                    else "-"
-                )
-                table.add_row(
-                    f"Avg {metric}",
-                    f"{score1:.3f}" if score1 is not None else "-",
-                    f"{score2:.3f}" if score2 is not None else "-",
-                    diff_str,
-                )
-
-        console.print(table)
-        console.print()
-
     @app.command(name="export")
     def export_cmd(
         run_id: str = typer.Argument(..., help="Run ID to export."),
@@ -246,7 +170,7 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
             "-o",
             help="Output file path (JSON format).",
         ),
-        db_path: Path = db_option(help_text="Path to database file."),
+        db_path: Path | None = db_option(help_text="Path to database file."),
     ) -> None:
         """Export evaluation run to JSON file.
 
@@ -275,7 +199,11 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
         """
         console.print(f"\n[bold]Exporting Run {run_id}[/bold]\n")
 
-        storage = SQLiteStorageAdapter(db_path=db_path)
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            console.print("[red]Error:[/red] Database path is not configured.")
+            raise typer.Exit(1)
+        storage = SQLiteStorageAdapter(db_path=cast(Path, resolved_db_path))
 
         try:
             run = storage.get_run(run_id)

evalvault/adapters/inbound/cli/commands/ops.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from evalvault.adapters.outbound.filesystem.ops_snapshot_writer import OpsSnapshotWriter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings, apply_profile
+from evalvault.domain.services.ops_snapshot_service import (
+    OpsSnapshotRequest,
+    OpsSnapshotService,
+)
+
+from ..utils.console import print_cli_error, progress_spinner
+from ..utils.options import db_option, profile_option
+
+
+def _resolve_storage_path(db_path: Path | None) -> Path:
+    if db_path is None:
+        return Path(Settings().evalvault_db_path)
+    return db_path
+
+
+def create_ops_app(console: Console) -> typer.Typer:
+    app = typer.Typer(name="ops", help="Ops utilities.")
+
+    @app.command("snapshot")
+    def snapshot(
+        run_id: str = typer.Option(..., "--run-id", help="Run ID to snapshot."),
+        profile: str | None = profile_option(help_text="Profile name to snapshot."),
+        db_path: Path | None = db_option(help_text="Path to SQLite database file."),
+        include_model_config: bool = typer.Option(
+            False,
+            "--include-model-config",
+            help="Include model profile configuration.",
+        ),
+        include_env: bool = typer.Option(
+            False,
+            "--include-env",
+            help="Include resolved settings snapshot.",
+        ),
+        redact: list[str] = typer.Option(
+            [],
+            "--redact",
+            help="Environment keys to redact (repeatable).",
+        ),
+        output_path: Path = typer.Option(
+            ..., "--output", "-o", help="Output JSON path for snapshot."
+        ),
+    ) -> None:
+        settings = Settings()
+        resolved_profile = profile or settings.evalvault_profile
+        if resolved_profile:
+            settings = apply_profile(settings, resolved_profile)
+
+        resolved_db_path = _resolve_storage_path(db_path)
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        writer = OpsSnapshotWriter()
+        service = OpsSnapshotService(
+            storage=storage,
+            writer=writer,
+            settings=settings,
+            output_path=output_path,
+        )
+        request = OpsSnapshotRequest(
+            run_id=run_id,
+            profile=resolved_profile,
+            db_path=resolved_db_path,
+            include_model_config=include_model_config,
+            include_env=include_env,
+            redact_keys=tuple(redact),
+        )
+
+        with progress_spinner(console, "Ops snapshot 생성 중..."):
+            started_at = datetime.now(UTC)
+            try:
+                envelope = service.collect(request)
+            except KeyError as exc:
+                print_cli_error(
+                    console,
+                    "Run을 찾지 못했습니다.",
+                    details=str(exc),
+                    fixes=["--run-id 값과 --db 경로를 확인하세요."],
+                )
+                raise typer.Exit(1) from exc
+            except Exception as exc:
+                print_cli_error(
+                    console,
+                    "Ops snapshot 생성 중 오류가 발생했습니다.",
+                    details=str(exc),
+                    fixes=["--output 경로와 파일 권한을 확인하세요."],
+                )
+                raise typer.Exit(1) from exc
+
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+        console.print("[green]Ops snapshot 완료[/green]")
+        console.print(f"- output: {output_path}")
+        console.print(f"- duration_ms: {duration_ms}")
+        console.print(f"- status: {envelope.status}")
+        if envelope.data.get("model_config") is None and include_model_config:
+            console.print("[yellow]model_config을 찾지 못했습니다.[/yellow]")
+
+    return app
+
+
+__all__ = ["create_ops_app"]