evalvault 1.64.0__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/__init__.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from collections.abc import Callable
 from dataclasses import dataclass
-from typing import Any, Protocol
+from typing import Any
 
 import typer
 from rich.console import Console
@@ -12,8 +12,11 @@ from rich.console import Console
 from .agent import register_agent_commands
 from .analyze import register_analyze_commands
 from .api import register_api_command
+from .artifacts import create_artifacts_app
 from .benchmark import create_benchmark_app
 from .calibrate import register_calibrate_commands
+from .calibrate_judge import register_calibrate_judge_commands
+from .compare import register_compare_commands
 from .config import register_config_commands
 from .debug import create_debug_app
 from .domain import create_domain_app
@@ -25,19 +28,17 @@ from .init import register_init_command
 from .kg import create_kg_app
 from .langfuse import register_langfuse_commands
 from .method import create_method_app
+from .ops import create_ops_app
 from .phoenix import create_phoenix_app
 from .pipeline import register_pipeline_commands
+from .profile_difficulty import register_profile_difficulty_commands
 from .prompts import create_prompts_app
+from .regress import register_regress_commands
 from .run import register_run_commands
 from .stage import create_stage_app
 
 CommandFactory = Callable[[Console], typer.Typer]
-
-
-class CommandRegistrar(Protocol):
-    """Callable protocol for Typer command registrars."""
-
-    def __call__(self, app: typer.Typer, console: Console, **kwargs: Any) -> None: ...
+CommandRegistrar = Callable[..., Any]
 
 
 @dataclass(frozen=True)
@@ -61,10 +62,14 @@ COMMAND_MODULES: tuple[CommandModule, ...] = (
     CommandModule(register_run_commands, needs_metrics=True),
     CommandModule(register_pipeline_commands),
     CommandModule(register_history_commands),
+    CommandModule(register_compare_commands),
     CommandModule(register_analyze_commands),
     CommandModule(register_calibrate_commands),
+    CommandModule(register_calibrate_judge_commands),
     CommandModule(register_generate_commands),
     CommandModule(register_gate_commands),
+    CommandModule(register_profile_difficulty_commands, needs_metrics=True),
+    CommandModule(register_regress_commands),
     CommandModule(register_agent_commands),
     CommandModule(register_experiment_commands),
     CommandModule(register_config_commands),
@@ -78,9 +83,11 @@ SUB_APPLICATIONS: tuple[SubAppModule, ...] = (
     SubAppModule("domain", create_domain_app),
     SubAppModule("benchmark", create_benchmark_app),
     SubAppModule("method", create_method_app),
+    SubAppModule("ops", create_ops_app),
     SubAppModule("phoenix", create_phoenix_app),
     SubAppModule("prompts", create_prompts_app),
     SubAppModule("stage", create_stage_app),
+    SubAppModule("artifacts", create_artifacts_app),
     SubAppModule("debug", create_debug_app),
 )
 

evalvault/adapters/inbound/cli/commands/artifacts.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from evalvault.adapters.inbound.cli.utils.console import print_cli_error
+from evalvault.adapters.inbound.cli.utils.validators import validate_choice
+from evalvault.adapters.outbound.artifact_fs import LocalArtifactFileSystemAdapter
+from evalvault.domain.services.artifact_lint_service import ArtifactLintService
+
+logger = logging.getLogger(__name__)
+
+
+def create_artifacts_app(console: Console) -> typer.Typer:
+    artifacts_app = typer.Typer(name="artifacts", help="Artifact utilities.")
+
+    @artifacts_app.command("lint")
+    def lint(
+        artifacts_dir: Path = typer.Argument(..., help="Artifacts directory."),
+        strict: bool = typer.Option(False, "--strict", help="Fail on missing files."),
+        output_format: str = typer.Option(
+            "json",
+            "--format",
+            "-f",
+            help="Output format (json).",
+        ),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="Output file path for lint result.",
+        ),
+        parallel: bool = typer.Option(
+            True,
+            "--parallel/--no-parallel",
+            help="Enable parallel validation (placeholder).",
+        ),
+        concurrency: int = typer.Option(
+            8,
+            "--concurrency",
+            min=1,
+            help="Parallel validation concurrency (placeholder).",
+        ),
+    ) -> None:
+        validate_choice(output_format, ["json"], console, value_label="format")
+
+        logger.info("Artifacts lint command started: %s", artifacts_dir)
+        fs_adapter = LocalArtifactFileSystemAdapter()
+        service = ArtifactLintService(fs_adapter)
+        summary = service.lint(artifacts_dir, strict=strict)
+
+        payload = _build_payload(summary, parallel=parallel, concurrency=concurrency)
+        if output:
+            output.parent.mkdir(parents=True, exist_ok=True)
+            output.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+            console.print(f"[green]Lint report saved:[/green] {output}")
+        else:
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+        if summary.status == "error":
+            logger.error("Artifacts lint command failed: %s", artifacts_dir)
+            print_cli_error(console, "Artifact lint failed", details=str(artifacts_dir))
+            raise typer.Exit(1)
+
+        logger.info("Artifacts lint command finished: %s", artifacts_dir)
+
+    return artifacts_app
+
+
+def _build_payload(summary, *, parallel: bool, concurrency: int) -> dict[str, object]:
+    issues = [
+        {
+            "level": issue.level,
+            "code": issue.code,
+            "message": issue.message,
+            "path": issue.path,
+        }
+        for issue in summary.issues
+    ]
+    error_count = sum(1 for issue in summary.issues if issue.level == "error")
+    warning_count = sum(1 for issue in summary.issues if issue.level == "warning")
+    return {
+        "command": "artifacts.lint",
+        "version": 1,
+        "status": summary.status,
+        "started_at": summary.started_at.isoformat(),
+        "finished_at": summary.finished_at.isoformat(),
+        "duration_ms": summary.duration_ms,
+        "artifacts": {
+            "dir": str(summary.artifacts_dir),
+            "index": str(summary.index_path),
+        },
+        "data": {
+            "strict": summary.strict,
+            "parallel": parallel,
+            "concurrency": concurrency,
+            "issue_counts": {
+                "error": error_count,
+                "warning": warning_count,
+            },
+            "issues": issues,
+        },
+    }

evalvault/adapters/inbound/cli/commands/calibrate_judge.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import resolve_artifact_dir, write_json
+from evalvault.adapters.inbound.cli.utils.console import print_cli_error, progress_spinner
+from evalvault.adapters.inbound.cli.utils.options import db_option
+from evalvault.adapters.inbound.cli.utils.validators import parse_csv_option, validate_choice
+from evalvault.adapters.outbound.judge_calibration_reporter import JudgeCalibrationReporter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings
+from evalvault.domain.services.judge_calibration_service import JudgeCalibrationService
+
+_console = Console()
+
+_ALLOWED_LABELS = ["feedback", "gold", "hybrid"]
+_ALLOWED_METHODS = ["platt", "isotonic", "temperature", "none"]
+
+
+def register_calibrate_judge_commands(app: typer.Typer, console: Console) -> None:
+    global _console
+    _console = console
+
+    @app.command(name="calibrate-judge")
+    def calibrate_judge(
+        run_id: str = typer.Argument(..., help="보정 대상 Run ID"),
+        labels_source: str = typer.Option(
+            "feedback",
+            "--labels-source",
+            help="라벨 소스 (feedback|gold|hybrid)",
+        ),
+        method: str = typer.Option(
+            "isotonic",
+            "--method",
+            help="보정 방법 (platt|isotonic|temperature|none)",
+        ),
+        metrics: str | None = typer.Option(
+            None,
+            "--metric",
+            "-m",
+            help="보정 대상 메트릭 (쉼표로 구분, 미지정 시 run 메트릭 전체)",
+        ),
+        holdout_ratio: float = typer.Option(
+            0.2,
+            "--holdout-ratio",
+            help="검증용 holdout 비율",
+        ),
+        seed: int = typer.Option(42, "--seed", help="샘플 분할 랜덤 시드"),
+        write_back: bool = typer.Option(
+            False,
+            "--write-back",
+            help="보정 결과를 Run 메타데이터에 저장",
+        ),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="JSON 결과 파일 경로",
+        ),
+        artifacts_dir: Path | None = typer.Option(
+            None,
+            "--artifacts-dir",
+            help="아티팩트 저장 디렉터리",
+        ),
+        parallel: bool = typer.Option(
+            False,
+            "--parallel/--no-parallel",
+            help="병렬 실행 활성화",
+        ),
+        concurrency: int = typer.Option(8, "--concurrency", help="동시성 수준"),
+        db_path: Path | None = db_option(help_text="DB 경로"),
+    ) -> None:
+        resolved_db_path = db_path or Settings().evalvault_db_path
+        if resolved_db_path is None:
+            print_cli_error(_console, "DB 경로가 설정되지 않았습니다.")
+            raise typer.Exit(1)
+
+        labels_source = labels_source.strip().lower()
+        method = method.strip().lower()
+        validate_choice(labels_source, _ALLOWED_LABELS, _console, value_label="labels-source")
+        validate_choice(method, _ALLOWED_METHODS, _console, value_label="method")
+
+        metric_list = parse_csv_option(metrics)
+        if holdout_ratio <= 0 or holdout_ratio >= 1:
+            print_cli_error(_console, "--holdout-ratio 값은 0과 1 사이여야 합니다.")
+            raise typer.Exit(1)
+        if seed < 0:
+            print_cli_error(_console, "--seed 값은 0 이상이어야 합니다.")
+            raise typer.Exit(1)
+        if concurrency <= 0:
+            print_cli_error(_console, "--concurrency 값은 1 이상이어야 합니다.")
+            raise typer.Exit(1)
+
+        storage = SQLiteStorageAdapter(db_path=resolved_db_path)
+        try:
+            run = storage.get_run(run_id)
+        except KeyError:
+            print_cli_error(_console, f"Run을 찾을 수 없습니다: {run_id}")
+            raise typer.Exit(1)
+
+        feedbacks = storage.list_feedback(run_id)
+        if labels_source in {"feedback", "hybrid"} and not feedbacks:
+            print_cli_error(_console, "피드백 라벨이 없습니다.")
+            raise typer.Exit(1)
+
+        resolved_metrics = metric_list or list(run.metrics_evaluated)
+        if not resolved_metrics:
+            print_cli_error(_console, "보정 대상 메트릭이 없습니다.")
+            raise typer.Exit(1)
+
+        prefix = f"judge_calibration_{run_id}"
+        output_path = (output or Path("reports/calibration") / f"{prefix}.json").expanduser()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        resolved_artifacts_dir = resolve_artifact_dir(
+            base_dir=artifacts_dir,
+            output_path=output_path,
+            report_path=output_path,
+            prefix=prefix,
+        )
+
+        service = JudgeCalibrationService()
+        reporter = JudgeCalibrationReporter()
+        started_at = datetime.now(UTC)
+
+        with progress_spinner(_console, "Judge 보정 실행 중..."):
+            result = service.calibrate(
+                run,
+                feedbacks,
+                labels_source=labels_source,
+                method=method,
+                metrics=resolved_metrics,
+                holdout_ratio=holdout_ratio,
+                seed=seed,
+                parallel=parallel,
+                concurrency=concurrency,
+            )
+
+        artifacts_index = reporter.write_artifacts(
+            result=result,
+            artifacts_dir=resolved_artifacts_dir,
+        )
+        finished_at = datetime.now(UTC)
+        payload = _build_envelope(
+            result,
+            artifacts_index,
+            started_at=started_at,
+            finished_at=finished_at,
+        )
+        write_json(output_path, payload)
+
+        _display_summary(result)
+        _console.print(f"[green]JSON 저장:[/green] {output_path}")
+        _console.print(
+            f"[green]아티팩트 저장:[/green] {artifacts_index['dir']} (index: {artifacts_index['index']})"
+        )
+
+        if write_back:
+            metadata = run.tracker_metadata or {}
+            metadata["judge_calibration"] = reporter.render_json(result)
+            metadata["judge_calibration"]["artifacts"] = artifacts_index
+            metadata["judge_calibration"]["output"] = str(output_path)
+            storage.update_run_metadata(run_id, metadata)
+            _console.print("[green]보정 결과를 메타데이터에 저장했습니다.[/green]")
+
+        if result.summary.gate_passed is False:
+            raise typer.Exit(2)
+
+    return None
+
+
+def _build_envelope(
+    result,
+    artifacts_index: dict[str, str],
+    *,
+    started_at: datetime,
+    finished_at: datetime,
+) -> dict[str, object]:
+    duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+    status = "ok" if result.summary.gate_passed else "degraded"
+    return {
+        "command": "calibrate-judge",
+        "version": 1,
+        "status": status,
+        "started_at": started_at.astimezone(UTC).isoformat(),
+        "finished_at": finished_at.astimezone(UTC).isoformat(),
+        "duration_ms": duration_ms,
+        "artifacts": artifacts_index,
+        "data": {
+            "summary": _serialize_summary(result.summary),
+            "metrics": [_serialize_metric(metric) for metric in result.metrics],
+            "case_results": {
+                metric: [_serialize_case(case) for case in cases]
+                for metric, cases in result.case_results.items()
+            },
+            "warnings": list(result.warnings),
+        },
+    }
+
+
+def _serialize_summary(summary) -> dict[str, object]:
+    return {
+        "run_id": summary.run_id,
+        "labels_source": summary.labels_source,
+        "method": summary.method,
+        "metrics": list(summary.metrics),
+        "holdout_ratio": summary.holdout_ratio,
+        "seed": summary.seed,
+        "total_labels": summary.total_labels,
+        "total_samples": summary.total_samples,
+        "gate_passed": summary.gate_passed,
+        "gate_threshold": summary.gate_threshold,
+        "notes": list(summary.notes),
+    }
+
+
+def _serialize_metric(metric) -> dict[str, object]:
+    return {
+        "metric": metric.metric,
+        "method": metric.method,
+        "sample_count": metric.sample_count,
+        "label_count": metric.label_count,
+        "mae": metric.mae,
+        "pearson": metric.pearson,
+        "spearman": metric.spearman,
+        "temperature": metric.temperature,
+        "parameters": dict(metric.parameters),
+        "gate_passed": metric.gate_passed,
+        "warning": metric.warning,
+    }
+
+
+def _serialize_case(case) -> dict[str, object]:
+    return {
+        "test_case_id": case.test_case_id,
+        "raw_score": case.raw_score,
+        "calibrated_score": case.calibrated_score,
+        "label": case.label,
+        "label_source": case.label_source,
+    }
+
+
+def _display_summary(result) -> None:
+    summary_table = Table(title="Judge 보정 요약", show_header=True, header_style="bold cyan")
+    summary_table.add_column("메트릭")
+    summary_table.add_column("표본", justify="right")
+    summary_table.add_column("라벨", justify="right")
+    summary_table.add_column("MAE", justify="right")
+    summary_table.add_column("Pearson", justify="right")
+    summary_table.add_column("Spearman", justify="right")
+    summary_table.add_column("Gate", justify="right")
+
+    for metric in result.metrics:
+        summary_table.add_row(
+            metric.metric,
+            str(metric.sample_count),
+            str(metric.label_count),
+            _format_metric(metric.mae),
+            _format_metric(metric.pearson),
+            _format_metric(metric.spearman),
+            "PASS" if metric.gate_passed else "FAIL",
+        )
+
+    _console.print(summary_table)
+    _console.print(
+        f"라벨 소스: {result.summary.labels_source} | "
+        f"방법: {result.summary.method} | "
+        f"Gate: {'PASS' if result.summary.gate_passed else 'FAIL'}"
+    )
+
+    if result.warnings:
+        for warning in result.warnings:
+            _console.print(f"[yellow]경고:[/yellow] {warning}")
+
+
+def _format_metric(value: float | None) -> str:
+    if value is None:
+        return "-"
+    return f"{value:.3f}"