evalvault 1.63.1__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/history.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
+from typing import cast
 
 import typer
 from rich.console import Console
@@ -46,7 +47,7 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
             "--mode",
             help="Filter by run mode: 'simple' or 'full'.",
         ),
-        db_path: Path = db_option(help_text="Path to database file."),
+        db_path: Path | None = db_option(help_text="Path to database file."),
     ) -> None:
         """Show evaluation run history.
 
@@ -75,7 +76,6 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
 
         \b
         See also:
-          evalvault compare  — Compare two runs side by side
           evalvault export   — Export run details to JSON
           evalvault run      — Create new evaluation runs
         """
@@ -88,7 +88,11 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
                     "[red]Error:[/red] --mode must be one of: " + ", ".join(RUN_MODE_CHOICES)
                 )
                 raise typer.Exit(2)
-        storage = SQLiteStorageAdapter(db_path=db_path)
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            console.print("[red]Error:[/red] Database path is not configured.")
+            raise typer.Exit(1)
+        storage = SQLiteStorageAdapter(db_path=cast(Path, resolved_db_path))
         runs = storage.list_runs(limit=limit, dataset_name=dataset, model_name=model)
         if normalized_mode:
             runs = [
@@ -157,86 +161,6 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
         console.print(table)
         console.print(f"\n[dim]Showing {len(runs)} of {limit} runs[/dim]\n")
 
-    @app.command()
-    def compare(
-        run_id1: str = typer.Argument(..., help="First run ID to compare."),
-        run_id2: str = typer.Argument(..., help="Second run ID to compare."),
-        db_path: Path = db_option(help_text="Path to database file."),
-    ) -> None:
-        """Compare two evaluation runs.
-
-        Show a side-by-side comparison of metrics, pass rates, and scores
-        between two evaluation runs.
-
-        \b
-        Examples:
-          # Compare two runs by ID
-          evalvault compare abc12345 def67890
-
-          # Compare runs from a custom database
-          evalvault compare abc12345 def67890 --db custom.db
-
-        \b
-        See also:
-          evalvault history  — List runs to find IDs
-          evalvault export   — Export run details to JSON
-          evalvault analyze  — Deep analysis of a single run
-        """
-        console.print("\n[bold]Comparing Evaluation Runs[/bold]\n")
-
-        storage = SQLiteStorageAdapter(db_path=db_path)
-
-        try:
-            run1 = storage.get_run(run_id1)
-            run2 = storage.get_run(run_id2)
-        except KeyError as exc:
-            console.print(f"[red]Error:[/red] {exc}")
-            raise typer.Exit(1) from exc
-
-        table = Table(show_header=True, header_style="bold cyan")
-        table.add_column("Metric")
-        table.add_column(f"Run 1\n{run_id1[:12]}...", justify="right")
-        table.add_column(f"Run 2\n{run_id2[:12]}...", justify="right")
-        table.add_column("Difference", justify="right")
-
-        table.add_row("Dataset", run1.dataset_name, run2.dataset_name, "-")
-        table.add_row("Model", run1.model_name, run2.model_name, "-")
-        table.add_row(
-            "Test Cases",
-            str(run1.total_test_cases),
-            str(run2.total_test_cases),
-            str(run2.total_test_cases - run1.total_test_cases),
-        )
-
-        pass_rate_diff = run2.pass_rate - run1.pass_rate
-        diff_color = "green" if pass_rate_diff > 0 else "red" if pass_rate_diff < 0 else "dim"
-        table.add_row(
-            "Pass Rate",
-            f"{run1.pass_rate:.1%}",
-            f"{run2.pass_rate:.1%}",
-            f"[{diff_color}]{pass_rate_diff:+.1%}[/{diff_color}]",
-        )
-
-        for metric in run1.metrics_evaluated:
-            if metric in run2.metrics_evaluated:
-                score1 = run1.get_avg_score(metric)
-                score2 = run2.get_avg_score(metric)
-                diff = score2 - score1 if score1 is not None and score2 is not None else None
-                diff_str = (
-                    f"[{'green' if diff and diff > 0 else 'red' if diff and diff < 0 else 'dim'}]{diff:+.3f}[/{'green' if diff and diff > 0 else 'red' if diff and diff < 0 else 'dim'}]"
-                    if diff is not None
-                    else "-"
-                )
-                table.add_row(
-                    f"Avg {metric}",
-                    f"{score1:.3f}" if score1 is not None else "-",
-                    f"{score2:.3f}" if score2 is not None else "-",
-                    diff_str,
-                )
-
-        console.print(table)
-        console.print()
-
     @app.command(name="export")
     def export_cmd(
         run_id: str = typer.Argument(..., help="Run ID to export."),
@@ -246,7 +170,7 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
             "-o",
             help="Output file path (JSON format).",
         ),
-        db_path: Path = db_option(help_text="Path to database file."),
+        db_path: Path | None = db_option(help_text="Path to database file."),
     ) -> None:
         """Export evaluation run to JSON file.
 
@@ -275,7 +199,11 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
         """
         console.print(f"\n[bold]Exporting Run {run_id}[/bold]\n")
 
-        storage = SQLiteStorageAdapter(db_path=db_path)
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            console.print("[red]Error:[/red] Database path is not configured.")
+            raise typer.Exit(1)
+        storage = SQLiteStorageAdapter(db_path=cast(Path, resolved_db_path))
 
         try:
             run = storage.get_run(run_id)

evalvault/adapters/inbound/cli/commands/ops.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from evalvault.adapters.outbound.filesystem.ops_snapshot_writer import OpsSnapshotWriter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings, apply_profile
+from evalvault.domain.services.ops_snapshot_service import (
+    OpsSnapshotRequest,
+    OpsSnapshotService,
+)
+
+from ..utils.console import print_cli_error, progress_spinner
+from ..utils.options import db_option, profile_option
+
+
+def _resolve_storage_path(db_path: Path | None) -> Path:
+    if db_path is None:
+        return Path(Settings().evalvault_db_path)
+    return db_path
+
+
+def create_ops_app(console: Console) -> typer.Typer:
+    app = typer.Typer(name="ops", help="Ops utilities.")
+
+    @app.command("snapshot")
+    def snapshot(
+        run_id: str = typer.Option(..., "--run-id", help="Run ID to snapshot."),
+        profile: str | None = profile_option(help_text="Profile name to snapshot."),
+        db_path: Path | None = db_option(help_text="Path to SQLite database file."),
+        include_model_config: bool = typer.Option(
+            False,
+            "--include-model-config",
+            help="Include model profile configuration.",
+        ),
+        include_env: bool = typer.Option(
+            False,
+            "--include-env",
+            help="Include resolved settings snapshot.",
+        ),
+        redact: list[str] = typer.Option(
+            [],
+            "--redact",
+            help="Environment keys to redact (repeatable).",
+        ),
+        output_path: Path = typer.Option(
+            ..., "--output", "-o", help="Output JSON path for snapshot."
+        ),
+    ) -> None:
+        settings = Settings()
+        resolved_profile = profile or settings.evalvault_profile
+        if resolved_profile:
+            settings = apply_profile(settings, resolved_profile)
+
+        resolved_db_path = _resolve_storage_path(db_path)
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        writer = OpsSnapshotWriter()
+        service = OpsSnapshotService(
+            storage=storage,
+            writer=writer,
+            settings=settings,
+            output_path=output_path,
+        )
+        request = OpsSnapshotRequest(
+            run_id=run_id,
+            profile=resolved_profile,
+            db_path=resolved_db_path,
+            include_model_config=include_model_config,
+            include_env=include_env,
+            redact_keys=tuple(redact),
+        )
+
+        with progress_spinner(console, "Ops snapshot 생성 중..."):
+            started_at = datetime.now(UTC)
+            try:
+                envelope = service.collect(request)
+            except KeyError as exc:
+                print_cli_error(
+                    console,
+                    "Run을 찾지 못했습니다.",
+                    details=str(exc),
+                    fixes=["--run-id 값과 --db 경로를 확인하세요."],
+                )
+                raise typer.Exit(1) from exc
+            except Exception as exc:
+                print_cli_error(
+                    console,
+                    "Ops snapshot 생성 중 오류가 발생했습니다.",
+                    details=str(exc),
+                    fixes=["--output 경로와 파일 권한을 확인하세요."],
+                )
+                raise typer.Exit(1) from exc
+
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+        console.print("[green]Ops snapshot 완료[/green]")
+        console.print(f"- output: {output_path}")
+        console.print(f"- duration_ms: {duration_ms}")
+        console.print(f"- status: {envelope.status}")
+        if envelope.data.get("model_config") is None and include_model_config:
+            console.print("[yellow]model_config을 찾지 못했습니다.[/yellow]")
+
+    return app
+
+
+__all__ = ["create_ops_app"]

evalvault/adapters/inbound/cli/commands/profile_difficulty.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from evalvault.adapters.outbound.filesystem.difficulty_profile_writer import DifficultyProfileWriter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings
+from evalvault.domain.services.difficulty_profile_reporter import DifficultyProfileReporter
+from evalvault.domain.services.difficulty_profiling_service import (
+    DifficultyProfileRequest,
+    DifficultyProfilingService,
+)
+
+from ..utils.console import print_cli_error, progress_spinner
+from ..utils.options import db_option
+from ..utils.validators import parse_csv_option, validate_choices
+
+logger = logging.getLogger(__name__)
+
+
+def register_profile_difficulty_commands(
+    app: typer.Typer,
+    console: Console,
+    available_metrics: Sequence[str],
+) -> None:
+    @app.command("profile-difficulty")
+    def profile_difficulty(
+        dataset_name: str | None = typer.Option(
+            None, "--dataset-name", help="Dataset name to profile."
+        ),
+        run_id: str | None = typer.Option(None, "--run-id", help="Run ID to profile."),
+        limit_runs: int | None = typer.Option(
+            None, "--limit-runs", help="Limit number of runs to analyze."
+        ),
+        metrics: str | None = typer.Option(
+            None, "--metrics", "-m", help="Comma-separated metric allowlist."
+        ),
+        bucket_count: int = typer.Option(
+            3, "--bucket-count", help="Number of difficulty buckets.", min=2
+        ),
+        min_samples: int = typer.Option(
+            10, "--min-samples", help="Minimum samples required for profiling.", min=1
+        ),
+        output_path: Path | None = typer.Option(None, "--output", "-o", help="Output JSON path."),
+        artifacts_dir: Path | None = typer.Option(
+            None, "--artifacts-dir", help="Artifacts directory path."
+        ),
+        parallel: bool = typer.Option(
+            False, "--parallel/--no-parallel", help="Enable parallel execution."
+        ),
+        concurrency: int | None = typer.Option(
+            None, "--concurrency", help="Max concurrency when parallel is enabled.", min=1
+        ),
+        db_path: Path | None = db_option(help_text="SQLite DB path."),
+    ) -> None:
+        if not dataset_name and not run_id:
+            print_cli_error(
+                console,
+                "--dataset-name 또는 --run-id 중 하나는 필수입니다.",
+                fixes=["예: --dataset-name insurance-qa", "또는 --run-id run_123"],
+            )
+            raise typer.Exit(1)
+        if dataset_name and run_id:
+            print_cli_error(
+                console,
+                "--dataset-name과 --run-id는 동시에 사용할 수 없습니다.",
+                fixes=["둘 중 하나만 지정하세요."],
+            )
+            raise typer.Exit(1)
+
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            print_cli_error(
+                console,
+                "DB 경로가 필요합니다.",
+                fixes=["--db 옵션으로 SQLite DB 경로를 지정하세요."],
+            )
+            raise typer.Exit(1)
+
+        metric_list = parse_csv_option(metrics)
+        if metric_list:
+            validate_choices(metric_list, available_metrics, console, value_label="metric")
+        resolved_metrics = tuple(metric_list) if metric_list else None
+
+        identifier = _safe_identifier(run_id or dataset_name or "difficulty")
+        prefix = f"difficulty_{identifier}"
+        resolved_output = output_path or Path("reports") / "difficulty" / f"{prefix}.json"
+        resolved_artifacts_dir = artifacts_dir or resolved_output.parent / "artifacts" / prefix
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        writer = DifficultyProfileWriter()
+        reporter = DifficultyProfileReporter(writer)
+        service = DifficultyProfilingService(storage=storage, reporter=reporter)
+        request = DifficultyProfileRequest(
+            dataset_name=dataset_name,
+            run_id=run_id,
+            limit_runs=limit_runs,
+            metrics=resolved_metrics,
+            bucket_count=bucket_count,
+            min_samples=min_samples,
+            output_path=resolved_output,
+            artifacts_dir=resolved_artifacts_dir,
+            parallel=parallel,
+            concurrency=concurrency,
+        )
+
+        with progress_spinner(console, "난이도 프로파일링 실행 중..."):
+            started_at = datetime.now(UTC)
+            logger.info("profile-difficulty started", extra={"dataset_name": dataset_name})
+            try:
+                envelope = service.profile(request)
+            except KeyError as exc:
+                logger.exception("profile-difficulty run missing")
+                print_cli_error(
+                    console,
+                    "Run을 찾지 못했습니다.",
+                    details=str(exc),
+                    fixes=["--run-id 값과 --db 경로를 확인하세요."],
+                )
+                raise typer.Exit(1) from exc
+            except ValueError as exc:
+                logger.exception("profile-difficulty validation failed")
+                print_cli_error(
+                    console,
+                    "난이도 프로파일링 조건을 만족하지 못했습니다.",
+                    details=str(exc),
+                    fixes=["--min-samples 값을 낮추거나 충분한 실행 이력을 준비하세요."],
+                )
+                raise typer.Exit(1) from exc
+            except Exception as exc:
+                logger.exception("profile-difficulty failed")
+                print_cli_error(
+                    console,
+                    "난이도 프로파일링 중 오류가 발생했습니다.",
+                    details=str(exc),
+                )
+                raise typer.Exit(1) from exc
+
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+
+        console.print("[green]난이도 프로파일링 완료[/green]")
+        console.print(f"- output: {resolved_output}")
+        console.print(f"- artifacts: {envelope.get('artifacts', {}).get('dir')}")
+        console.print(f"- duration_ms: {duration_ms}")
+
+
+def _safe_identifier(value: str) -> str:
+    sanitized = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("_")
+    return sanitized or "difficulty"
+
+
+__all__ = ["register_profile_difficulty_commands"]

evalvault/adapters/inbound/cli/commands/regress.py

@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.adapters.outbound.analysis.statistical_adapter import (
+    StatisticalAnalysisAdapter,
+)
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.domain.services.regression_gate_service import (
+    RegressionGateReport,
+    RegressionGateService,
+    TestType,
+)
+
+from ..utils.formatters import format_diff, format_score, format_status
+from ..utils.options import db_option
+from ..utils.validators import parse_csv_option, validate_choice
+
+
+def _coerce_test_type(value: Literal["t-test", "mann-whitney"]) -> TestType:
+    if value == "t-test":
+        return "t-test"
+    return "mann-whitney"
+
+
+OutputFormat = Literal["table", "json", "github-actions"]
+
+
+def _format_timestamp(value: datetime) -> str:
+    return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _build_envelope(
+    *,
+    report: RegressionGateReport | None,
+    status: str,
+    started_at: datetime,
+    finished_at: datetime,
+    duration_ms: int,
+    message: str | None = None,
+    error_type: str | None = None,
+) -> dict[str, object]:
+    payload: dict[str, object] = {
+        "command": "regress",
+        "version": 1,
+        "status": status,
+        "started_at": _format_timestamp(started_at),
+        "finished_at": _format_timestamp(finished_at),
+        "duration_ms": duration_ms,
+        "artifacts": None,
+        "data": report.to_dict() if report else None,
+    }
+    if message:
+        payload["message"] = message
+    if error_type:
+        payload["error_type"] = error_type
+    return payload
+
+
+def register_regress_commands(app: typer.Typer, console: Console) -> None:
+    @app.command()
+    def regress(
+        run_id: str = typer.Argument(..., help="Candidate run ID to check."),
+        baseline: str = typer.Option(
+            ...,
+            "--baseline",
+            "-b",
+            help="Baseline run ID for regression detection.",
+        ),
+        fail_on_regression: float = typer.Option(
+            0.05,
+            "--fail-on-regression",
+            "-r",
+            help="Fail if metric drops by more than this amount (default: 0.05).",
+        ),
+        test: TestType = typer.Option(
+            "t-test",
+            "--test",
+            "-t",
+            help="Statistical test (t-test, mann-whitney).",
+        ),
+        metrics: str | None = typer.Option(
+            None,
+            "--metrics",
+            "-m",
+            help="Comma-separated list of metrics to check.",
+        ),
+        output_format: OutputFormat = typer.Option(
+            "table",
+            "--format",
+            "-f",
+            help="Output format: table, json, or github-actions.",
+        ),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="Write JSON summary to a file.",
+        ),
+        parallel: bool = typer.Option(
+            True,
+            "--parallel/--no-parallel",
+            help="Enable parallel execution for metric checks.",
+        ),
+        concurrency: int = typer.Option(
+            8,
+            "--concurrency",
+            help="Concurrency level when running in parallel.",
+        ),
+        db_path: Path | None = db_option(help_text="Database path"),
+    ) -> None:
+        started_at = datetime.now(UTC)
+        if db_path is None:
+            console.print("[red]Error:[/red] Database path is not configured.")
+            raise typer.Exit(1)
+
+        validate_choice(test, ["t-test", "mann-whitney"], console, value_label="test")
+        metric_list = parse_csv_option(metrics)
+
+        storage = SQLiteStorageAdapter(db_path=db_path)
+        analysis_adapter = StatisticalAnalysisAdapter()
+        service = RegressionGateService(storage=storage, analysis_adapter=analysis_adapter)
+
+        try:
+            report = service.run_gate(
+                run_id,
+                baseline,
+                metrics=metric_list or None,
+                test_type=_coerce_test_type(test),
+                fail_on_regression=fail_on_regression,
+                parallel=parallel,
+                concurrency=concurrency,
+            )
+        except (KeyError, ValueError) as exc:
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+            payload = _build_envelope(
+                report=None,
+                status="error",
+                started_at=started_at,
+                finished_at=finished_at,
+                duration_ms=duration_ms,
+                message=str(exc),
+                error_type=type(exc).__name__,
+            )
+            if output:
+                write_json(output, payload)
+            if output_format == "json":
+                console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+            else:
+                console.print(f"[red]Error:[/red] {exc}")
+            raise typer.Exit(1) from exc
+        except Exception as exc:
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+            payload = _build_envelope(
+                report=None,
+                status="error",
+                started_at=started_at,
+                finished_at=finished_at,
+                duration_ms=duration_ms,
+                message=str(exc),
+                error_type=type(exc).__name__,
+            )
+            if output:
+                write_json(output, payload)
+            if output_format == "json":
+                console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+            else:
+                console.print(f"[red]Error:[/red] {exc}")
+            raise typer.Exit(3) from exc
+
+        finished_at = report.finished_at
+        duration_ms = report.duration_ms
+        payload = _build_envelope(
+            report=report,
+            status="ok",
+            started_at=report.started_at,
+            finished_at=finished_at,
+            duration_ms=duration_ms,
+        )
+        if output:
+            write_json(output, payload)
+
+        if output_format == "json":
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+        elif output_format == "github-actions":
+            _render_github_actions(report, console)
+        else:
+            _render_table(report, console)
+
+        if report.regression_detected:
+            raise typer.Exit(2)
+
+
+def _render_table(report: RegressionGateReport, console: Console) -> None:
+    console.print(f"\n[bold]Regression Gate Check: {report.candidate_run_id}[/bold]\n")
+    console.print(f"Baseline: {report.baseline_run_id}")
+    console.print(f"Test: {report.test_type}\n")
+    table = Table(show_header=True, header_style="bold cyan")
+    table.add_column("Metric")
+    table.add_column("Baseline", justify="right")
+    table.add_column("Candidate", justify="right")
+    table.add_column("Diff", justify="right")
+    table.add_column("p-value", justify="right")
+    table.add_column("Regression", justify="center")
+
+    for result in report.results:
+        table.add_row(
+            result.metric,
+            format_score(result.baseline_score),
+            format_score(result.candidate_score),
+            format_diff(result.diff),
+            f"{result.p_value:.4f}",
+            format_status(not result.regression, success_text="NO", failure_text="YES"),
+        )
+
+    console.print(table)
+    if report.regression_detected:
+        regressed = [r.metric for r in report.results if r.regression]
+        console.print("\n[bold red]Regression detected[/bold red]")
+        console.print(f"[red]Regressed metrics: {', '.join(regressed)}[/red]")
+    else:
+        console.print("\n[bold green]Regression gate PASSED[/bold green]")
+    console.print()
+
+
+def _render_github_actions(report: RegressionGateReport, console: Console) -> None:
+    for result in report.results:
+        status = "✅" if not result.regression else "❌"
+        reg_status = " (REGRESSION)" if result.regression else ""
+        console.print(
+            f"{status} {result.metric}: {result.candidate_score:.3f} "
+            f"(baseline: {result.baseline_score:.3f}, diff: {result.diff:+.3f}){reg_status}"
+        )
+
+    console.print(f"::set-output name=passed::{str(not report.regression_detected).lower()}")
+    if report.regression_detected:
+        regressed = [r.metric for r in report.results if r.regression]
+        console.print(f"::error::Regression detected in: {', '.join(regressed)}")
+
+
+__all__ = ["register_regress_commands"]