evalvault 1.70.1__py3-none-any.whl → 1.71.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/run.py

@@ -5,9 +5,11 @@ from __future__ import annotations
 import asyncio
 import os
 from collections.abc import Callable, Sequence
+from dataclasses import asdict
 from datetime import date, datetime
 from pathlib import Path
 from typing import Any, cast
+from uuid import uuid4
 
 import click
 import typer
@@ -15,7 +17,7 @@ from rich.console import Console
 from rich.table import Table
 
 from evalvault.adapters.outbound.analysis.pipeline_factory import build_analysis_pipeline_service
-from evalvault.adapters.outbound.dataset import get_loader
+from evalvault.adapters.outbound.dataset import get_loader, load_multiturn_dataset
 from evalvault.adapters.outbound.documents.versioned_loader import (
     load_versioned_chunks_from_pdf_dir,
 )
@@ -33,10 +35,16 @@ from evalvault.adapters.outbound.tracer.phoenix_tracer_adapter import PhoenixTra
 from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
 from evalvault.config.settings import Settings, apply_profile
 from evalvault.domain.entities.analysis_pipeline import AnalysisIntent
+from evalvault.domain.entities.multiturn import (
+    MultiTurnConversationRecord,
+    MultiTurnRunRecord,
+    MultiTurnTurnResult,
+)
 from evalvault.domain.services.document_versioning import parse_contract_date
 from evalvault.domain.services.evaluator import RagasEvaluator
 from evalvault.domain.services.memory_aware_evaluator import MemoryAwareEvaluator
 from evalvault.domain.services.memory_based_analysis import MemoryBasedAnalysis
+from evalvault.domain.services.multiturn_evaluator import MultiTurnEvaluator
 from evalvault.domain.services.prompt_registry import (
     PromptInput,
     build_prompt_bundle,
@@ -81,6 +89,7 @@ from .run_helpers import (
     _option_was_provided,
     _print_run_mode_banner,
     _resolve_thresholds,
+    _save_multiturn_to_db,
     _save_results,
     _save_to_db,
     _write_stage_events_jsonl,
@@ -221,21 +230,26 @@ def register_run_commands(
             False,
             "--auto-analyze",
             help="평가 완료 후 통합 분석을 자동 실행하고 보고서를 저장합니다.",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_output: Path | None = typer.Option(
             None,
             "--analysis-json",
             help="자동 분석 JSON 결과 파일 경로 (기본값: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_report: Path | None = typer.Option(
             None,
             "--analysis-report",
+            "--report",
             help="자동 분석 Markdown 보고서 경로 (기본값: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_dir: Path | None = typer.Option(
             None,
             "--analysis-dir",
             help="자동 분석 결과 저장 디렉터리 (기본: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         retriever: str | None = typer.Option(
             None,
@@ -428,6 +442,18 @@ def register_run_commands(
             help="실행 모드 선택: 'simple'은 간편 실행, 'full'은 모든 옵션 노출.",
             rich_help_panel="Run modes",
         ),
+        max_turns: int | None = typer.Option(
+            None,
+            "--max-turns",
+            help="멀티턴 모드에서 사용할 최대 턴 수 (지정 시 앞에서부터 절단).",
+            rich_help_panel="Multiturn options",
+        ),
+        drift_threshold: float = typer.Option(
+            0.1,
+            "--drift-threshold",
+            help="멀티턴 모드에서 드리프트 경고 임계값.",
+            rich_help_panel="Multiturn options",
+        ),
         db_path: Path | None = db_option(
             help_text="Path to SQLite database file for storing results.",
         ),
@@ -462,6 +488,7 @@ def register_run_commands(
             False,
             "--verbose",
             "-v",
+            "-V",
             help="Show detailed output.",
         ),
         parallel: bool = typer.Option(
@@ -983,6 +1010,191 @@ def register_run_commands(
         if threshold_profile:
             phoenix_trace_metadata["threshold.profile"] = str(threshold_profile).strip().lower()
 
+        if preset.name == "multiturn":
+            llm_factory = SettingsLLMFactory(settings)
+            korean_toolkit = try_create_korean_toolkit()
+            evaluator = RagasEvaluator(korean_toolkit=korean_toolkit, llm_factory=llm_factory)
+            try:
+                llm_adapter = get_llm_adapter(settings)
+            except Exception as exc:
+                provider = str(getattr(settings, "llm_provider", "")).strip().lower()
+                fixes: list[str]
+                if provider == "ollama":
+                    fixes = [
+                        "Ollama 서버가 실행 중인지 확인하세요 (기본: http://localhost:11434).",
+                        "필요 모델을 받아두세요: `ollama pull gpt-oss-safeguard:20b` 및 `ollama pull qwen3-embedding:0.6b`.",
+                        "URL을 바꿨다면 .env의 `OLLAMA_BASE_URL`을 확인하세요.",
+                    ]
+                elif provider == "openai":
+                    fixes = [
+                        "`.env`에 `OPENAI_API_KEY`를 설정하세요.",
+                        "프록시/네트워크가 필요한 환경이면 연결 가능 여부를 확인하세요.",
+                    ]
+                elif provider == "vllm":
+                    fixes = [
+                        "`.env`의 `VLLM_BASE_URL`/`VLLM_MODEL` 설정을 확인하세요.",
+                        "vLLM 서버가 OpenAI 호환 API로 실행 중인지 확인하세요.",
+                    ]
+                else:
+                    fixes = ["--profile 또는 환경변수 설정을 확인하세요."]
+                print_cli_error(
+                    console,
+                    "LLM/임베딩 어댑터를 초기화하지 못했습니다.",
+                    details=str(exc),
+                    fixes=fixes,
+                )
+                raise typer.Exit(1) from exc
+
+            multiturn_started_at = datetime.now()
+            _log_timestamp(console, verbose, "멀티턴 데이터셋 로딩 시작")
+            try:
+                multiturn_dataset = load_multiturn_dataset(dataset)
+            except Exception as exc:
+                _log_duration(console, verbose, "멀티턴 데이터셋 로딩 실패", multiturn_started_at)
+                print_cli_error(
+                    console,
+                    "멀티턴 데이터셋을 불러오지 못했습니다.",
+                    details=str(exc),
+                    fixes=[
+                        "파일 경로/형식을 확인하세요.",
+                        "멀티턴 스키마(turns, conversation_id)가 문서와 동일한지 확인하세요.",
+                    ],
+                )
+                raise typer.Exit(1) from exc
+            _log_duration(console, verbose, "멀티턴 데이터셋 로딩 완료", multiturn_started_at)
+
+            if stream:
+                print_cli_warning(
+                    console,
+                    "멀티턴 모드에서는 streaming 옵션을 무시합니다.",
+                    tips=["--stream을 제거하거나 일반 모드로 실행하세요."],
+                )
+            if retriever:
+                print_cli_warning(
+                    console,
+                    "멀티턴 모드에서는 retriever 적용을 지원하지 않습니다.",
+                    tips=["단일 턴 모드에서 retriever를 사용하세요."],
+                )
+            if use_domain_memory:
+                print_cli_warning(
+                    console,
+                    "멀티턴 모드에서는 Domain Memory를 지원하지 않습니다.",
+                    tips=["--use-domain-memory 옵션을 제거하세요."],
+                )
+
+            if max_turns and max_turns > 0:
+                trimmed = 0
+                for case in multiturn_dataset.test_cases:
+                    if len(case.turns) > max_turns:
+                        case.turns = case.turns[:max_turns]
+                        trimmed += 1
+                if trimmed:
+                    console.print(f"[dim]Trimmed turns in {trimmed} conversation(s).[/dim]")
+
+            evaluation_started_at = datetime.now()
+            multiturn_evaluator = MultiTurnEvaluator(evaluator=evaluator, llm=llm_adapter)
+            results = []
+            drift_flags = 0
+            turn_count = 0
+            for case in multiturn_dataset.test_cases:
+                result = multiturn_evaluator.evaluate_conversation(case, metric_list)
+                drift = multiturn_evaluator.detect_drift(case, threshold=drift_threshold)
+                result.summary["drift_detected"] = drift.drift_detected
+                result.summary["drift_threshold"] = drift.drift_threshold
+                result.summary["drift_score"] = drift.drift_score
+                results.append(result)
+                turn_count += len(result.turn_results)
+                if drift.drift_detected:
+                    drift_flags += 1
+
+            multiturn_summary: dict[str, object] = {
+                "conversation_count": len(results),
+                "turn_count": turn_count,
+                "drift_detected_count": drift_flags,
+                "drift_threshold": drift_threshold,
+            }
+            for metric in metric_list:
+                scores = [
+                    result.summary.get(metric)
+                    for result in results
+                    if isinstance(result.summary.get(metric), (int, float))
+                ]
+                if scores:
+                    multiturn_summary[metric] = sum(scores) / len(scores)
+
+            payload = {
+                "dataset": {
+                    "name": multiturn_dataset.name,
+                    "version": multiturn_dataset.version,
+                    "metadata": multiturn_dataset.metadata,
+                    "source_file": multiturn_dataset.source_file,
+                },
+                "metrics": metric_list,
+                "summary": multiturn_summary,
+                "conversations": [asdict(item) for item in results],
+            }
+
+            table = Table(title="Multi-turn Summary", show_header=True, header_style="bold cyan")
+            table.add_column("Metric", style="bold")
+            table.add_column("Value", justify="right")
+            for metric in metric_list:
+                value = multiturn_summary.get(metric)
+                if isinstance(value, float):
+                    display = f"{value:.3f}"
+                else:
+                    display = str(value) if value is not None else "-"
+                table.add_row(metric, display)
+            table.add_row("conversation_count", str(multiturn_summary.get("conversation_count")))
+            table.add_row("turn_count", str(multiturn_summary.get("turn_count")))
+            table.add_row("drift_detected", str(multiturn_summary.get("drift_detected_count")))
+            console.print(table)
+
+            if output:
+                write_json(output, payload)
+                console.print(f"[green]멀티턴 결과 저장:[/green] {output}")
+            if db_path:
+                run_id = str(uuid4())
+                run_record = MultiTurnRunRecord(
+                    run_id=run_id,
+                    dataset_name=multiturn_dataset.name,
+                    dataset_version=multiturn_dataset.version,
+                    model_name=llm_adapter.get_model_name(),
+                    started_at=evaluation_started_at,
+                    finished_at=datetime.now(),
+                    conversation_count=len(results),
+                    turn_count=turn_count,
+                    metrics_evaluated=list(metric_list),
+                    drift_threshold=drift_threshold,
+                    summary=multiturn_summary,
+                    metadata={"dataset": multiturn_dataset.metadata},
+                )
+                conversation_records = [
+                    MultiTurnConversationRecord(
+                        run_id=run_id,
+                        conversation_id=conversation.conversation_id,
+                        turn_count=len(conversation.turn_results),
+                        drift_score=conversation.summary.get("drift_score"),
+                        drift_threshold=conversation.summary.get("drift_threshold"),
+                        drift_detected=bool(conversation.summary.get("drift_detected")),
+                        summary=dict(conversation.summary),
+                    )
+                    for conversation in results
+                ]
+                turn_results: list[MultiTurnTurnResult] = []
+                for conversation in results:
+                    for turn in conversation.turn_results:
+                        turn_results.append(turn)
+                _save_multiturn_to_db(
+                    db_path,
+                    run_record,
+                    conversation_records,
+                    turn_results,
+                    console,
+                    export_excel=True,
+                    excel_output_path=excel_output,
+                )
+            return
+
         # Load dataset or configure streaming metadata
         if stream:
             stream_started_at = datetime.now()
@@ -2120,21 +2332,26 @@ def register_run_commands(
             False,
             "--auto-analyze",
             help="평가 완료 후 통합 분석을 자동 실행하고 보고서를 저장합니다.",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_output: Path | None = typer.Option(
             None,
             "--analysis-json",
             help="자동 분석 JSON 결과 파일 경로 (기본값: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_report: Path | None = typer.Option(
             None,
             "--analysis-report",
+            "--report",
             help="자동 분석 Markdown 보고서 경로 (기본값: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_dir: Path | None = typer.Option(
             None,
             "--analysis-dir",
             help="자동 분석 결과 저장 디렉터리 (기본: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         retriever: str | None = typer.Option(
             None,
@@ -2273,6 +2490,7 @@ def register_run_commands(
         verbose: bool = typer.Option(
             False,
             "--verbose",
+            "-V",
             help="Show detailed output.",
         ),
         parallel: bool = typer.Option(
@@ -2406,21 +2624,26 @@ def register_run_commands(
             False,
             "--auto-analyze",
             help="평가 완료 후 통합 분석을 자동 실행하고 보고서를 저장합니다.",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_output: Path | None = typer.Option(
             None,
             "--analysis-json",
             help="자동 분석 JSON 결과 파일 경로 (기본값: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_report: Path | None = typer.Option(
             None,
             "--analysis-report",
+            "--report",
             help="자동 분석 Markdown 보고서 경로 (기본값: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         analysis_dir: Path | None = typer.Option(
             None,
             "--analysis-dir",
             help="자동 분석 결과 저장 디렉터리 (기본: reports/analysis).",
+            rich_help_panel="Auto Analysis",
         ),
         retriever: str | None = typer.Option(
             None,
@@ -2559,6 +2782,7 @@ def register_run_commands(
         verbose: bool = typer.Option(
             False,
             "--verbose",
+            "-V",
             help="Show detailed output.",
         ),
         parallel: bool = typer.Option(

evalvault/adapters/inbound/cli/commands/run_helpers.py

@@ -29,6 +29,9 @@ from evalvault.domain.entities import (
     Dataset,
     EvaluationRun,
     GenerationData,
+    MultiTurnConversationRecord,
+    MultiTurnRunRecord,
+    MultiTurnTurnResult,
     PromptSetBundle,
     RAGTraceData,
     RetrievalData,
@@ -86,6 +89,14 @@ RUN_MODE_PRESETS: dict[str, RunModePreset] = {
         label="Full",
         description="모든 CLI 옵션과 Domain Memory, Prompt manifest를 활용하는 전체 모드.",
     ),
+    "multiturn": RunModePreset(
+        name="multiturn",
+        label="Multiturn",
+        description="멀티턴 대화 평가 전용 모드 (멀티턴 메트릭만 지원).",
+        default_metrics=("turn_faithfulness", "context_coherence", "drift_rate"),
+        allow_domain_memory=False,
+        allow_prompt_metadata=False,
+    ),
 }
 
 SUMMARY_METRIC_ORDER = (
@@ -490,6 +501,52 @@ def _save_to_db(
             )
 
 
+def _save_multiturn_to_db(
+    db_path: Path,
+    run_record: MultiTurnRunRecord,
+    conversations: list[MultiTurnConversationRecord],
+    turn_results: list[MultiTurnTurnResult],
+    console: Console,
+    *,
+    storage_cls: type[SQLiteStorageAdapter] = SQLiteStorageAdapter,
+    export_excel: bool = True,
+    excel_output_path: Path | None = None,
+    metric_thresholds: dict[str, float] | None = None,
+) -> None:
+    """Persist multiturn evaluation run to SQLite database."""
+    with console.status(f"[bold green]Saving multiturn run to {db_path}..."):
+        try:
+            storage = storage_cls(db_path=db_path)
+            storage.save_multiturn_run(
+                run_record,
+                conversations,
+                turn_results,
+                metric_thresholds=metric_thresholds,
+            )
+            if export_excel:
+                excel_path = excel_output_path or (
+                    db_path.parent / f"evalvault_multiturn_{run_record.run_id}.xlsx"
+                )
+                try:
+                    storage.export_multiturn_run_to_excel(run_record.run_id, excel_path)
+                    console.print(f"[green]Multiturn Excel export saved: {excel_path}[/green]")
+                except Exception as exc:
+                    print_cli_warning(
+                        console,
+                        "멀티턴 엑셀 내보내기에 실패했습니다.",
+                        tips=[str(exc)],
+                    )
+            console.print(f"[green]Multiturn results saved to database: {db_path}[/green]")
+            console.print(f"[dim]Run ID: {run_record.run_id}[/dim]")
+        except Exception as exc:  # pragma: no cover - persistence errors
+            print_cli_error(
+                console,
+                "멀티턴 결과를 데이터베이스에 저장하지 못했습니다.",
+                details=str(exc),
+                fixes=["경로 권한과 DB 파일 잠금 상태를 확인하세요."],
+            )
+
+
 def _save_results(output: Path, result, console: Console) -> None:
     """Write evaluation summary to disk."""
     with console.status(f"[bold green]Saving to {output}..."):

evalvault/adapters/outbound/analysis/network_analyzer_module.py

@@ -1,7 +1,11 @@
 from __future__ import annotations
 
+import contextlib
+import os
+import sys
 from dataclasses import dataclass, field
 from datetime import datetime
+from importlib import import_module
 from typing import Any
 
 import networkx as nx
@@ -12,10 +16,18 @@ from evalvault.adapters.outbound.analysis.pipeline_helpers import (
     to_serializable,
 )
 
-try:
-    import matplotlib.pyplot as plt
-except ImportError:
-    plt = None
+
+def _get_matplotlib_pyplot() -> Any | None:
+    try:
+        if "matplotlib.pyplot" in sys.modules:
+            return import_module("matplotlib.pyplot")
+        os.environ.setdefault("MPLBACKEND", "Agg")
+        matplotlib = import_module("matplotlib")
+        with contextlib.suppress(Exception):
+            matplotlib.use("Agg", force=True)
+        return import_module("matplotlib.pyplot")
+    except ModuleNotFoundError:
+        return None
 
 
 @dataclass
@@ -173,6 +185,7 @@ class NetworkAnalyzerModule(BaseAnalysisModule):
         output_path: str | None = None,
         figsize: tuple[int, int] = (12, 8),
     ) -> Any | None:
+        plt = _get_matplotlib_pyplot()
         if plt is None:
             return None
 

evalvault/adapters/outbound/dataset/__init__.py

@@ -6,6 +6,10 @@ from evalvault.adapters.outbound.dataset.excel_loader import ExcelDatasetLoader
 from evalvault.adapters.outbound.dataset.json_loader import JSONDatasetLoader
 from evalvault.adapters.outbound.dataset.loader_factory import get_loader, register_loader
 from evalvault.adapters.outbound.dataset.method_input_loader import MethodInputDatasetLoader
+from evalvault.adapters.outbound.dataset.multiturn_json_loader import (
+    MultiTurnDataset,
+    load_multiturn_dataset,
+)
 from evalvault.adapters.outbound.dataset.streaming_loader import (
     StreamingConfig,
     StreamingCSVLoader,
@@ -23,6 +27,7 @@ __all__ = [
     "ExcelDatasetLoader",
     "JSONDatasetLoader",
     "MethodInputDatasetLoader",
+    "MultiTurnDataset",
     "StreamingCSVLoader",
     "StreamingConfig",
     "StreamingDatasetLoader",
@@ -31,6 +36,7 @@ __all__ = [
     "StreamingTestCaseIterator",
     "get_loader",
     "load_in_chunks",
+    "load_multiturn_dataset",
     "register_loader",
     "stream_file",
 ]

evalvault/adapters/outbound/dataset/multiturn_json_loader.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.entities.multiturn import ConversationTurn, MultiTurnTestCase
+
+
+@dataclass(frozen=True)
+class MultiTurnDataset:
+    name: str
+    version: str
+    test_cases: list[MultiTurnTestCase]
+    metadata: dict[str, Any]
+    source_file: str | None = None
+
+
+def load_multiturn_dataset(file_path: str | Path) -> MultiTurnDataset:
+    path = Path(file_path)
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+    if not path.is_file():
+        raise ValueError(f"Path is not a file: {file_path}")
+
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON file: {exc}") from exc
+
+    name = path.stem
+    version = "1.0.0"
+    metadata: dict[str, Any] = {}
+    raw_cases: list[dict[str, Any]]
+
+    if isinstance(payload, list):
+        raw_cases = payload
+    elif isinstance(payload, dict):
+        name = str(payload.get("name") or name)
+        version = str(payload.get("version") or version)
+        metadata = payload.get("metadata") or {}
+        if not isinstance(metadata, dict):
+            raise ValueError("metadata must be a JSON object")
+        raw_cases = payload.get("test_cases") or payload.get("conversations") or []
+    else:
+        raise ValueError("JSON must be an array or object with 'test_cases' key")
+
+    if not isinstance(raw_cases, list):
+        raise ValueError("test_cases must be a list")
+
+    test_cases: list[MultiTurnTestCase] = []
+    for idx, raw_case in enumerate(raw_cases, start=1):
+        if not isinstance(raw_case, dict):
+            raise ValueError(f"test_cases[{idx}] must be an object")
+        conversation_id = raw_case.get("conversation_id") or raw_case.get("id")
+        if not conversation_id:
+            raise ValueError(f"test_cases[{idx}] missing conversation_id")
+        raw_turns = raw_case.get("turns")
+        if not isinstance(raw_turns, list) or not raw_turns:
+            raise ValueError(f"test_cases[{idx}] missing turns list")
+
+        turns: list[ConversationTurn] = []
+        for t_idx, raw_turn in enumerate(raw_turns, start=1):
+            if not isinstance(raw_turn, dict):
+                raise ValueError(f"turns[{t_idx}] must be an object")
+            role = raw_turn.get("role")
+            if role not in {"user", "assistant"}:
+                raise ValueError(f"turns[{t_idx}] role must be 'user' or 'assistant'")
+            content = raw_turn.get("content")
+            if content is None:
+                raise ValueError(f"turns[{t_idx}] missing content")
+            turn_id = raw_turn.get("turn_id") or f"t{t_idx:02d}"
+            contexts = raw_turn.get("contexts")
+            if contexts is None:
+                contexts = []
+            if isinstance(contexts, str):
+                contexts = [contexts]
+            if not isinstance(contexts, list):
+                raise ValueError(f"turns[{t_idx}] contexts must be a list")
+            ground_truth = raw_turn.get("ground_truth")
+            metadata_value = raw_turn.get("metadata") or {}
+            if not isinstance(metadata_value, dict):
+                raise ValueError(f"turns[{t_idx}] metadata must be an object")
+            turns.append(
+                ConversationTurn(
+                    turn_id=str(turn_id),
+                    role=role,
+                    content=str(content),
+                    contexts=[str(ctx) for ctx in contexts],
+                    ground_truth=ground_truth,
+                    metadata=metadata_value,
+                )
+            )
+
+        test_cases.append(
+            MultiTurnTestCase(
+                conversation_id=str(conversation_id),
+                turns=turns,
+                expected_final_answer=raw_case.get("expected_final_answer"),
+                drift_tolerance=float(raw_case.get("drift_tolerance", 0.1)),
+            )
+        )
+
+    return MultiTurnDataset(
+        name=name,
+        version=version,
+        test_cases=test_cases,
+        metadata=metadata,
+        source_file=str(path),
+    )

evalvault/adapters/outbound/report/__init__.py

@@ -1,5 +1,9 @@
 """Report generation adapters."""
 
+from evalvault.adapters.outbound.report.ci_report_formatter import (
+    CIGateMetricRow,
+    format_ci_regression_report,
+)
 from evalvault.adapters.outbound.report.dashboard_generator import DashboardGenerator
 from evalvault.adapters.outbound.report.llm_report_generator import (
     LLMReport,
@@ -9,7 +13,9 @@ from evalvault.adapters.outbound.report.llm_report_generator import (
 from evalvault.adapters.outbound.report.markdown_adapter import MarkdownReportAdapter
 
 __all__ = [
+    "CIGateMetricRow",
     "DashboardGenerator",
+    "format_ci_regression_report",
     "LLMReport",
     "LLMReportGenerator",
     "LLMReportSection",

evalvault/adapters/outbound/report/ci_report_formatter.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class CIGateMetricRow:
+    metric: str
+    baseline_score: float
+    current_score: float
+    change_percent: float
+    status: str
+
+
+def format_ci_regression_report(
+    rows: list[CIGateMetricRow],
+    *,
+    regression_rate: float,
+    regression_threshold: float,
+    gate_passed: bool,
+) -> str:
+    lines: list[str] = ["## RAG Regression Gate Results", ""]
+    lines.append("| Metric | Baseline | Current | Change | Status |")
+    lines.append("|--------|----------|---------|--------|--------|")
+    for row in rows:
+        change = f"{row.change_percent:+.1f}%"
+        lines.append(
+            f"| {row.metric} | {row.baseline_score:.3f} | {row.current_score:.3f} | {change} | {row.status} |"
+        )
+    lines.append("")
+    if gate_passed:
+        status_line = "✅ PASSED"
+        comparison = "<"
+    else:
+        status_line = "❌ FAILED"
+        comparison = ">="
+    lines.append(
+        f"**Gate Status**: {status_line} (regression: {regression_rate:.1%} {comparison} {regression_threshold:.1%} threshold)"
+    )
+    return "\n".join(lines).strip()
+
+
+__all__ = ["CIGateMetricRow", "format_ci_regression_report"]