evalvault 1.70.1__py3-none-any.whl → 1.72.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/__init__.py

@@ -23,6 +23,7 @@ from .domain import create_domain_app
 from .experiment import register_experiment_commands
 from .gate import register_gate_commands
 from .generate import register_generate_commands
+from .graph_rag import create_graph_rag_app
 from .history import register_history_commands
 from .init import register_init_command
 from .kg import create_kg_app
@@ -82,6 +83,7 @@ SUB_APPLICATIONS: tuple[SubAppModule, ...] = (
     SubAppModule("kg", create_kg_app),
     SubAppModule("domain", create_domain_app),
     SubAppModule("benchmark", create_benchmark_app),
+    SubAppModule("graphrag", create_graph_rag_app),
     SubAppModule("method", create_method_app),
     SubAppModule("ops", create_ops_app),
     SubAppModule("phoenix", create_phoenix_app),

evalvault/adapters/inbound/cli/commands/analyze.py

@@ -298,6 +298,7 @@ def register_analyze_commands(app: typer.Typer, console: Console) -> None:
             _console.print(f"\n[green]리포트 생성: {report}[/green]")
 
     @app.command(name="analyze-compare")
+    @app.command(name="compare-analysis")
     def analyze_compare(
         run_id1: str = typer.Argument(..., help="첫 번째 Run ID"),
         run_id2: str = typer.Argument(..., help="두 번째 Run ID"),

evalvault/adapters/inbound/cli/commands/compare.py

@@ -54,7 +54,7 @@ def register_compare_commands(app: typer.Typer, console: Console) -> None:
             "t-test",
             "--test",
             "-t",
-            help="통계 검정 (t-test, mann-whitney)",
+            help="통계 검정 (t-test | mann-whitney)",
         ),
         output_format: str = typer.Option(
             "table",

evalvault/adapters/inbound/cli/commands/experiment.py

@@ -12,7 +12,7 @@ from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdap
 from evalvault.domain.services.experiment_manager import ExperimentManager
 
 from ..utils.options import db_option
-from ..utils.validators import parse_csv_option
+from ..utils.validators import parse_csv_option, validate_choice
 
 
 def register_experiment_commands(app: typer.Typer, console: Console) -> None:
@@ -29,10 +29,24 @@ def register_experiment_commands(app: typer.Typer, console: Console) -> None:
             "-m",
             help="Comma-separated list of metrics to compare.",
         ),
+        control_retriever: str | None = typer.Option(
+            None,
+            "--control-retriever",
+            help="Control retriever (bm25, dense, hybrid, graphrag).",
+        ),
+        variant_retriever: str | None = typer.Option(
+            None,
+            "--variant-retriever",
+            help="Variant retriever (bm25, dense, hybrid, graphrag).",
+        ),
         db_path: Path = db_option(help_text="Path to database file."),
     ) -> None:
         """Create a new experiment for A/B testing."""
 
+        for retriever_name in (control_retriever, variant_retriever):
+            if retriever_name:
+                validate_choice(retriever_name, ["bm25", "dense", "hybrid", "graphrag"], console)
+
         console.print("\n[bold]Creating Experiment[/bold]\n")
         storage = SQLiteStorageAdapter(db_path=db_path)
         manager = ExperimentManager(storage)
@@ -44,6 +58,18 @@ def register_experiment_commands(app: typer.Typer, console: Console) -> None:
             hypothesis=hypothesis,
             metrics=metric_list,
         )
+        if control_retriever:
+            manager.add_group_to_experiment(
+                experiment.experiment_id,
+                "control",
+                f"retriever={control_retriever}",
+            )
+        if variant_retriever:
+            manager.add_group_to_experiment(
+                experiment.experiment_id,
+                "variant",
+                f"retriever={variant_retriever}",
+            )
         console.print(f"[green]Created experiment:[/green] {experiment.experiment_id}")
         console.print(f"  Name: {experiment.name}")
         console.print(f"  Status: {experiment.status}")

evalvault/adapters/inbound/cli/commands/graph_rag.py

@@ -0,0 +1,303 @@
+"""GraphRAG experiment commands for the EvalVault CLI."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from evalvault.adapters.outbound.analysis.statistical_adapter import StatisticalAnalysisAdapter
+from evalvault.adapters.outbound.dataset import get_loader
+from evalvault.adapters.outbound.llm import SettingsLLMFactory, get_llm_adapter
+from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
+from evalvault.adapters.outbound.retriever.graph_rag_adapter import GraphRAGAdapter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings, apply_profile
+from evalvault.domain.services.analysis_service import AnalysisService
+from evalvault.domain.services.evaluator import RagasEvaluator
+from evalvault.domain.services.graph_rag_experiment import GraphRAGExperiment
+from evalvault.ports.outbound.korean_nlp_port import RetrieverPort
+
+from ..utils.console import print_cli_error
+from ..utils.options import db_option, profile_option
+from ..utils.validators import parse_csv_option, validate_choice
+from .run import _build_dense_retriever
+from .run_helpers import _is_oss_open_model, load_knowledge_graph, load_retriever_documents
+
+
+def create_graph_rag_app(console: Console) -> typer.Typer:
+    app = typer.Typer(name="graphrag", help="GraphRAG experiment utilities.")
+
+    @app.command("compare")
+    def graphrag_compare(
+        dataset: Path = typer.Argument(
+            ...,
+            help="Path to dataset file (CSV, Excel, or JSON).",
+            exists=True,
+            readable=True,
+        ),
+        metrics: str = typer.Option(
+            "faithfulness,answer_relevancy",
+            "--metrics",
+            "-m",
+            help="Comma-separated list of metrics to evaluate.",
+        ),
+        baseline_retriever: str = typer.Option(
+            "bm25",
+            "--baseline-retriever",
+            help="Baseline retriever (bm25, dense, hybrid).",
+        ),
+        retriever_docs: Path = typer.Option(
+            ...,
+            "--retriever-docs",
+            help="Retriever documents file (.json/.jsonl/.txt).",
+            exists=True,
+            readable=True,
+        ),
+        kg_path: Path = typer.Option(
+            ...,
+            "--kg",
+            "-k",
+            help="Knowledge graph JSON file for GraphRAG.",
+            exists=True,
+            readable=True,
+        ),
+        retriever_top_k: int = typer.Option(
+            5,
+            "--retriever-top-k",
+            help="Retriever top-k to fill contexts.",
+        ),
+        graph_max_hops: int = typer.Option(
+            2,
+            "--graph-max-hops",
+            help="GraphRAG max hop depth.",
+        ),
+        graph_max_nodes: int = typer.Option(
+            20,
+            "--graph-max-nodes",
+            help="GraphRAG max nodes in subgraph.",
+        ),
+        model: str | None = typer.Option(
+            None,
+            "--model",
+            help="Model to use for evaluation (overrides profile).",
+        ),
+        db_path: Path | None = db_option(help_text="DB 경로 (저장 시 사용)."),
+        profile: str | None = profile_option(help_text="LLM 프로필"),
+        output: Path | None = typer.Option(
+            None,
+            "--output",
+            "-o",
+            help="JSON 출력 파일 경로",
+        ),
+        artifact_dir: Path = typer.Option(
+            Path("reports/analysis/artifacts"),
+            "--artifact-dir",
+            help="GraphRAG 아티팩트 저장 경로",
+        ),
+    ) -> None:
+        validate_choice(baseline_retriever, ["bm25", "dense", "hybrid"], console)
+
+        settings = Settings()
+        profile_name = profile or settings.evalvault_profile
+        if profile_name:
+            settings = apply_profile(settings, profile_name)
+
+        if model:
+            if _is_oss_open_model(model) and settings.llm_provider != "vllm":
+                settings.llm_provider = "ollama"
+                settings.ollama_model = model
+            elif settings.llm_provider == "ollama":
+                settings.ollama_model = model
+            elif settings.llm_provider == "vllm":
+                settings.vllm_model = model
+            else:
+                settings.openai_model = model
+
+        if settings.llm_provider == "openai" and not settings.openai_api_key:
+            print_cli_error(console, "OPENAI_API_KEY가 설정되지 않았습니다.")
+            raise typer.Exit(1)
+
+        try:
+            llm_adapter = get_llm_adapter(settings)
+        except Exception as exc:
+            print_cli_error(console, "LLM 어댑터 초기화에 실패했습니다.", details=str(exc))
+            raise typer.Exit(1) from exc
+
+        loader = get_loader(dataset)
+        ds = loader.load(dataset)
+
+        documents, doc_ids = load_retriever_documents(retriever_docs)
+        baseline = _build_baseline_retriever(
+            baseline_retriever,
+            documents=documents,
+            settings=settings,
+            profile_name=profile_name,
+        )
+        if baseline is None:
+            print_cli_error(console, "Baseline retriever 초기화에 실패했습니다.")
+            raise typer.Exit(1)
+
+        kg_graph = load_knowledge_graph(kg_path)
+        graph_adapter = GraphRAGAdapter(kg_graph)
+
+        korean_toolkit = try_create_korean_toolkit()
+        evaluator = RagasEvaluator(
+            korean_toolkit=korean_toolkit,
+            llm_factory=SettingsLLMFactory(settings),
+        )
+        analysis_service = AnalysisService(analysis_adapter=StatisticalAnalysisAdapter())
+        experiment = GraphRAGExperiment(
+            evaluator=evaluator,
+            analysis_service=analysis_service,
+        )
+
+        metric_list = parse_csv_option(metrics)
+        if not metric_list:
+            print_cli_error(console, "평가 메트릭을 지정하세요.")
+            raise typer.Exit(1)
+
+        result = asyncio.run(
+            experiment.run_comparison(
+                dataset=ds,
+                baseline_retriever=baseline,
+                graph_retriever=graph_adapter,
+                metrics=metric_list,
+                llm=llm_adapter,
+                retriever_top_k=retriever_top_k,
+                graph_max_hops=graph_max_hops,
+                graph_max_nodes=graph_max_nodes,
+            )
+        )
+
+        artifacts_path = _write_graph_rag_artifacts(
+            result=result,
+            dataset=ds,
+            graph_retriever=graph_adapter,
+            artifact_root=artifact_dir,
+        )
+        console.print(f"[green]Saved GraphRAG artifacts:[/green] {artifacts_path}")
+
+        if db_path is not None:
+            storage = SQLiteStorageAdapter(db_path=db_path)
+            storage.save_run(result.baseline_run)
+            storage.save_run(result.graph_run)
+            console.print(f"[green]Saved baseline run:[/green] {result.baseline_run.run_id}")
+            console.print(f"[green]Saved graph run:[/green] {result.graph_run.run_id}")
+
+        _render_comparison_table(console, result)
+
+        if output:
+            payload = _build_output_payload(result, doc_ids)
+            output.parent.mkdir(parents=True, exist_ok=True)
+            with output.open("w", encoding="utf-8") as f:
+                json.dump(payload, f, ensure_ascii=False, indent=2, default=str)
+            console.print(f"[green]Saved output:[/green] {output}")
+
+    return app
+
+
+def _build_baseline_retriever(
+    mode: str,
+    *,
+    documents: list[str],
+    settings: Settings,
+    profile_name: str | None,
+) -> RetrieverPort | None:
+    if mode in {"bm25", "hybrid"}:
+        toolkit = try_create_korean_toolkit()
+        if toolkit is None:
+            return None
+        return toolkit.build_retriever(documents, use_hybrid=mode == "hybrid", verbose=False)
+    return _build_dense_retriever(
+        documents=documents,
+        settings=settings,
+        profile_name=profile_name,
+    )
+
+
+def _render_comparison_table(console: Console, result: Any) -> None:
+    table = Table(show_header=True, header_style="bold cyan")
+    table.add_column("Metric")
+    table.add_column("Baseline", justify="right")
+    table.add_column("Graph", justify="right")
+    table.add_column("Diff%", justify="right")
+    table.add_column("P-Value", justify="right")
+
+    for comp in result.comparisons:
+        table.add_row(
+            comp.metric,
+            f"{comp.mean_a:.3f}",
+            f"{comp.mean_b:.3f}",
+            f"{comp.diff_percent:+.1f}%",
+            f"{comp.p_value:.4f}",
+        )
+
+    console.print("\n[bold]GraphRAG Comparison[/bold]\n")
+    console.print(table)
+    console.print()
+
+
+def _build_output_payload(result: Any, doc_ids: list[str]) -> dict[str, Any]:
+    return {
+        "baseline": result.baseline_run.to_summary_dict(),
+        "graph": result.graph_run.to_summary_dict(),
+        "comparisons": [asdict(comp) for comp in result.comparisons],
+        "graph_contexts": result.graph_contexts,
+        "graph_subgraphs": {key: asdict(value) for key, value in result.graph_subgraphs.items()},
+        "retriever_doc_ids": doc_ids,
+    }
+
+
+def _write_graph_rag_artifacts(
+    *,
+    result: Any,
+    dataset: Any,
+    graph_retriever: GraphRAGAdapter,
+    artifact_root: Path,
+) -> Path:
+    run_id = result.graph_run.run_id
+    base_dir = artifact_root / f"analysis_{run_id}"
+    graph_dir = base_dir / "graph_subgraphs"
+    entity_dir = base_dir / "entity_extraction"
+    graph_dir.mkdir(parents=True, exist_ok=True)
+    entity_dir.mkdir(parents=True, exist_ok=True)
+
+    graph_index: dict[str, str] = {}
+    for case_id, subgraph in result.graph_subgraphs.items():
+        safe_id = _safe_filename(case_id)
+        file_name = f"{safe_id}_subgraph.json"
+        file_path = graph_dir / file_name
+        with file_path.open("w", encoding="utf-8") as handle:
+            json.dump(asdict(subgraph), handle, ensure_ascii=False, indent=2)
+        graph_index[case_id] = str(Path("graph_subgraphs") / file_name)
+
+    entities_payload: dict[str, list[dict[str, object]]] = {}
+    for case in dataset.test_cases:
+        entities = graph_retriever.extract_entities(case.question)
+        entities_payload[case.id] = [asdict(entity) for entity in entities]
+    entities_path = entity_dir / "entities.json"
+    with entities_path.open("w", encoding="utf-8") as handle:
+        json.dump(entities_payload, handle, ensure_ascii=False, indent=2)
+
+    index_payload = {
+        "graph_subgraphs": graph_index,
+        "entity_extraction": str(Path("entity_extraction") / "entities.json"),
+    }
+    with (base_dir / "index.json").open("w", encoding="utf-8") as handle:
+        json.dump(index_payload, handle, ensure_ascii=False, indent=2)
+
+    return base_dir
+
+
+def _safe_filename(value: str) -> str:
+    return value.replace("/", "_").replace("\\", "_").replace(" ", "_")
+
+
+__all__ = ["create_graph_rag_app"]

evalvault/adapters/inbound/cli/commands/history.py

@@ -28,7 +28,7 @@ def register_history_commands(app: typer.Typer, console: Console) -> None:
             10,
             "--limit",
             "-n",
-            help="Maximum number of runs to show.",
+            help="Maximum number of runs to show (default: 10).",
         ),
         dataset: str | None = typer.Option(
             None,

evalvault/adapters/inbound/cli/commands/regress.py

@@ -13,6 +13,13 @@ from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
 from evalvault.adapters.outbound.analysis.statistical_adapter import (
     StatisticalAnalysisAdapter,
 )
+from evalvault.adapters.outbound.report.ci_report_formatter import (
+    CIGateMetricRow,
+    format_ci_regression_report,
+)
+from evalvault.adapters.outbound.report.pr_comment_formatter import (
+    format_ci_gate_pr_comment,
+)
 from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
 from evalvault.domain.services.regression_gate_service import (
     RegressionGateReport,
@@ -25,13 +32,14 @@ from ..utils.options import db_option
 from ..utils.validators import parse_csv_option, validate_choice
 
 
-def _coerce_test_type(value: Literal["t-test", "mann-whitney"]) -> TestType:
+def _coerce_test_type(value: str) -> TestType:
     if value == "t-test":
         return "t-test"
     return "mann-whitney"
 
 
 OutputFormat = Literal["table", "json", "github-actions"]
+CIGateOutputFormat = Literal["github", "gitlab", "json", "pr-comment"]
 
 
 def _format_timestamp(value: datetime) -> str:
@@ -200,6 +208,166 @@ def register_regress_commands(app: typer.Typer, console: Console) -> None:
         if report.regression_detected:
             raise typer.Exit(2)
 
+    @app.command(name="ci-gate")
+    def ci_gate(
+        baseline_run_id: str = typer.Argument(..., help="Baseline run ID."),
+        current_run_id: str = typer.Argument(..., help="Current run ID."),
+        regression_threshold: float = typer.Option(
+            0.05,
+            "--regression-threshold",
+            help="Fail if regression rate exceeds this threshold (default: 0.05).",
+        ),
+        output_format: str = typer.Option(
+            "github",
+            "--format",
+            "-f",
+            help="Output format: github, gitlab, json, or pr-comment.",
+        ),
+        fail_on_regression: bool = typer.Option(
+            True,
+            "--fail-on-regression/--no-fail-on-regression",
+            help="Fail the command when regression rate exceeds threshold.",
+        ),
+        db_path: Path | None = db_option(default=None, help_text="Database path"),
+    ) -> None:
+        """CI/CD 파이프라인용 회귀 게이트 체크."""
+        started_at = datetime.now(UTC)
+        if db_path is None:
+            console.print("[red]Error:[/red] Database path is not configured.")
+            raise typer.Exit(1)
+
+        validate_choice(
+            output_format,
+            ["github", "gitlab", "json", "pr-comment"],
+            console,
+            value_label="format",
+        )
+
+        storage = SQLiteStorageAdapter(db_path=db_path)
+        analysis_adapter = StatisticalAnalysisAdapter()
+        service = RegressionGateService(storage=storage, analysis_adapter=analysis_adapter)
+
+        try:
+            current_run = storage.get_run(current_run_id)
+            storage.get_run(baseline_run_id)
+            report = service.run_gate(
+                current_run_id,
+                baseline_run_id,
+            )
+        except KeyError as exc:
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+            payload = _build_envelope(
+                report=None,
+                status="error",
+                started_at=started_at,
+                finished_at=finished_at,
+                duration_ms=duration_ms,
+                message=str(exc),
+                error_type=type(exc).__name__,
+            )
+            if output_format == "json":
+                console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+            else:
+                console.print(f"[red]Error:[/red] {exc}")
+            raise typer.Exit(3) from exc
+        except (ValueError, RuntimeError) as exc:
+            finished_at = datetime.now(UTC)
+            duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+            payload = _build_envelope(
+                report=None,
+                status="error",
+                started_at=started_at,
+                finished_at=finished_at,
+                duration_ms=duration_ms,
+                message=str(exc),
+                error_type=type(exc).__name__,
+            )
+            if output_format == "json":
+                console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+            else:
+                console.print(f"[red]Error:[/red] {exc}")
+            raise typer.Exit(1) from exc
+
+        thresholds = dict.fromkeys(current_run.metrics_evaluated, 0.7)
+        thresholds.update(current_run.thresholds or {})
+
+        rows: list[CIGateMetricRow] = []
+        threshold_failures = []
+        regressed_metrics = []
+        for result in report.results:
+            avg_score = current_run.get_avg_score(result.metric)
+            threshold = thresholds.get(result.metric, 0.7)
+            threshold_passed = avg_score is not None and avg_score >= threshold
+            if not threshold_passed:
+                threshold_failures.append(result.metric)
+            if result.regression:
+                regressed_metrics.append(result.metric)
+            if result.regression:
+                status = "⚠️"
+            elif threshold_passed:
+                status = "✅"
+            else:
+                status = "❌"
+            rows.append(
+                CIGateMetricRow(
+                    metric=result.metric,
+                    baseline_score=result.baseline_score,
+                    current_score=result.candidate_score,
+                    change_percent=result.diff_percent,
+                    status=status,
+                )
+            )
+
+        regression_rate = len(regressed_metrics) / len(report.results) if report.results else 0.0
+        all_thresholds_passed = not threshold_failures
+        gate_passed = all_thresholds_passed and regression_rate < regression_threshold
+
+        finished_at = datetime.now(UTC)
+        duration_ms = int((finished_at - started_at).total_seconds() * 1000)
+        payload = {
+            "baseline_run_id": baseline_run_id,
+            "current_run_id": current_run_id,
+            "gate_passed": gate_passed,
+            "all_thresholds_passed": all_thresholds_passed,
+            "regression_rate": regression_rate,
+            "regression_threshold": regression_threshold,
+            "regressed_metrics": regressed_metrics,
+            "threshold_failures": threshold_failures,
+            "started_at": _format_timestamp(started_at),
+            "finished_at": _format_timestamp(finished_at),
+            "duration_ms": duration_ms,
+            "report": report.to_dict(),
+        }
+
+        if output_format == "json":
+            console.print(json.dumps(payload, ensure_ascii=False, indent=2))
+        elif output_format == "pr-comment":
+            markdown = format_ci_gate_pr_comment(
+                rows,
+                baseline_run_id=baseline_run_id,
+                current_run_id=current_run_id,
+                regression_rate=regression_rate,
+                regression_threshold=regression_threshold,
+                gate_passed=gate_passed,
+                threshold_failures=threshold_failures,
+                regressed_metrics=regressed_metrics,
+            )
+            console.print(markdown)
+        else:
+            markdown = format_ci_regression_report(
+                rows,
+                regression_rate=regression_rate,
+                regression_threshold=regression_threshold,
+                gate_passed=gate_passed,
+            )
+            console.print(markdown)
+
+        if not all_thresholds_passed:
+            raise typer.Exit(1)
+        if not gate_passed and fail_on_regression:
+            raise typer.Exit(2)
+
 
 def _render_table(report: RegressionGateReport, console: Console) -> None:
     console.print(f"\n[bold]Regression Gate Check: {report.candidate_run_id}[/bold]\n")