evalvault 1.62.1__py3-none-any.whl → 1.63.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -3,13 +3,14 @@
 from __future__ import annotations
 
 import asyncio
+import difflib
 import json
 import logging
 import time
 from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any, Literal, cast
 from urllib.request import urlopen
 
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
@@ -19,6 +20,7 @@ from evalvault.domain.entities import (
     FeedbackSummary,
     SatisfactionFeedback,
 )
+from evalvault.domain.entities.debug import DebugReport
 from evalvault.domain.entities.prompt import PromptSetBundle
 from evalvault.domain.metrics.registry import (
     get_metric_descriptions as registry_metric_descriptions,
@@ -28,9 +30,11 @@ from evalvault.domain.metrics.registry import (
     list_metric_specs,
 )
 from evalvault.domain.services.cluster_map_builder import build_cluster_map
+from evalvault.domain.services.debug_report_service import DebugReportService
 from evalvault.domain.services.prompt_registry import (
     PromptInput,
     build_prompt_bundle,
+    build_prompt_inputs_from_snapshots,
     build_prompt_summary,
 )
 from evalvault.domain.services.prompt_status import extract_prompt_entries
@@ -47,12 +51,16 @@ from evalvault.ports.inbound.web_port import (
     RunFilters,
     RunSummary,
 )
+from evalvault.ports.outbound.stage_storage_port import StageStoragePort
 
 if TYPE_CHECKING:
     from evalvault.domain.entities import EvaluationRun, RunClusterMap, RunClusterMapInfo
     from evalvault.domain.entities.improvement import ImprovementReport
     from evalvault.domain.entities.stage import StageEvent, StageMetric
+    from evalvault.domain.services.evaluator import RagasEvaluator
+    from evalvault.ports.outbound.dataset_port import DatasetPort
     from evalvault.ports.outbound.llm_port import LLMPort
+    from evalvault.ports.outbound.report_port import ReportPort
     from evalvault.ports.outbound.storage_port import StoragePort
 
 logger = logging.getLogger(__name__)
@@ -90,10 +98,10 @@ class WebUIAdapter:
     def __init__(
         self,
         storage: StoragePort | None = None,
-        evaluator: object | None = None,
-        report_generator: object | None = None,
+        evaluator: RagasEvaluator | None = None,
+        report_generator: ReportPort | None = None,
         llm_adapter: LLMPort | None = None,
-        data_loader: object | None = None,
+        data_loader: DatasetPort | None = None,
         settings: Settings | None = None,
     ):
         """어댑터 초기화.
@@ -105,12 +113,21 @@ class WebUIAdapter:
             llm_adapter: LLM 어댑터 (선택적)
             data_loader: 데이터 로더 (선택적)
         """
+        resolved_settings = settings
+        if storage is None:
+            resolved_settings = settings or Settings()
+            db_path = getattr(resolved_settings, "evalvault_db_path", None)
+            if db_path:
+                from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+
+                storage = SQLiteStorageAdapter(db_path=db_path)
+
         self._storage = storage
         self._evaluator = evaluator
         self._report_generator = report_generator
         self._llm_adapter = llm_adapter
         self._data_loader = data_loader
-        self._settings = settings
+        self._settings = resolved_settings
         self._phoenix_resolver: PhoenixExperimentResolver | None = None
         self._phoenix_resolver_checked = False
 
@@ -362,13 +379,14 @@ class WebUIAdapter:
         """
         if self._evaluator is None:
             raise RuntimeError("Evaluator not configured")
+        evaluator = self._evaluator
 
         # LLM Adapter Resolution
-        llm_adapter = self._get_llm_for_model(request.model_name)
-        if llm_adapter is None:
+        resolved_llm = self._get_llm_for_model(request.model_name)
+        if resolved_llm is None:
             if self._llm_adapter is None:
                 raise RuntimeError("LLM adapter not configured")
-            llm_adapter = self._llm_adapter
+            resolved_llm = self._llm_adapter
             logger.warning(f"Using default LLM adapter instead of requested {request.model_name}")
 
         # 1. 데이터셋 로드 (비동기 처리)
@@ -540,7 +558,7 @@ class WebUIAdapter:
                 result = await memory_evaluator.evaluate_with_memory(
                     dataset=dataset,
                     metrics=request.metrics,
-                    llm=llm_adapter,
+                    llm=resolved_llm,
                     thresholds=resolved_thresholds,
                     parallel=request.parallel,
                     batch_size=request.batch_size,
@@ -553,10 +571,10 @@ class WebUIAdapter:
                     on_progress=adaptor_progress,
                 )
             else:
-                result = await self._evaluator.evaluate(
+                result = await evaluator.evaluate(
                     dataset=dataset,
                     metrics=request.metrics,
-                    llm=llm_adapter,
+                    llm=resolved_llm,
                     thresholds=resolved_thresholds,
                     parallel=request.parallel,
                     batch_size=request.batch_size,
@@ -573,6 +591,34 @@ class WebUIAdapter:
                 on_progress(EvalProgress(0, 0, "", 0.0, "failed", str(e)))
             raise e
 
+        tracker_meta = result.tracker_metadata or {}
+        result.tracker_metadata = tracker_meta
+        ragas_snapshots = tracker_meta.get("ragas_prompt_snapshots")
+        ragas_snapshot_inputs = build_prompt_inputs_from_snapshots(
+            ragas_snapshots if isinstance(ragas_snapshots, dict) else None,
+        )
+        override_status: dict[str, str] = {}
+        raw_override = tracker_meta.get("ragas_prompt_overrides")
+        if isinstance(raw_override, dict):
+            override_status = cast(dict[str, str], raw_override)
+        if override_status:
+            prompt_inputs = [
+                entry
+                for entry in prompt_inputs
+                if not (
+                    entry.kind == "ragas"
+                    and override_status.get(entry.role) is not None
+                    and override_status.get(entry.role) != "applied"
+                )
+            ]
+
+        if ragas_snapshot_inputs:
+            existing_roles = {entry.role for entry in prompt_inputs if entry.kind == "ragas"}
+            for entry in ragas_snapshot_inputs:
+                if entry.role in existing_roles and override_status.get(entry.role) == "applied":
+                    continue
+                prompt_inputs.append(entry)
+
         prompt_bundle = None
         if prompt_inputs:
             prompt_bundle = build_prompt_bundle(
@@ -684,7 +730,15 @@ class WebUIAdapter:
                     prompt_bundle.prompt_set.prompt_set_id,
                 )
             try:
-                self._auto_generate_cluster_map(result, llm_adapter)
+                export_settings = self._settings or Settings()
+                export_base = Path(export_settings.evalvault_db_path)
+                excel_path = export_base.parent / f"evalvault_run_{result.run_id}.xlsx"
+                if hasattr(self._storage, "export_run_to_excel"):
+                    self._storage.export_run_to_excel(result.run_id, excel_path)
+            except Exception as exc:
+                logger.warning("Excel export failed for run %s: %s", result.run_id, exc)
+            try:
+                self._auto_generate_cluster_map(result, resolved_llm)
             except Exception as exc:
                 logger.warning("Cluster map auto-generation failed: %s", exc)
 
@@ -957,6 +1011,116 @@ class WebUIAdapter:
             ]
         return metrics
 
+    def compare_prompt_sets(
+        self,
+        base_run_id: str,
+        target_run_id: str,
+        *,
+        max_lines: int = 40,
+        include_diff: bool = True,
+    ) -> dict[str, Any]:
+        if self._storage is None or not hasattr(self._storage, "get_prompt_set_for_run"):
+            raise RuntimeError("Storage not configured")
+
+        base_bundle = self._storage.get_prompt_set_for_run(base_run_id)
+        target_bundle = self._storage.get_prompt_set_for_run(target_run_id)
+        if not base_bundle or not target_bundle:
+            raise KeyError("Prompt set not found")
+
+        base_roles = self._prompt_bundle_role_map(base_bundle)
+        target_roles = self._prompt_bundle_role_map(target_bundle)
+        all_roles = sorted(set(base_roles) | set(target_roles))
+
+        summary: list[dict[str, Any]] = []
+        diffs: list[dict[str, Any]] = []
+
+        for role in all_roles:
+            base = base_roles.get(role)
+            target = target_roles.get(role)
+            if not base or not target:
+                summary.append(
+                    {
+                        "role": role,
+                        "base_checksum": base["checksum"] if base else None,
+                        "target_checksum": target["checksum"] if target else None,
+                        "status": "missing",
+                        "base_name": base["name"] if base else None,
+                        "target_name": target["name"] if target else None,
+                        "base_kind": base["kind"] if base else None,
+                        "target_kind": target["kind"] if target else None,
+                    }
+                )
+                continue
+
+            status = "same" if base["checksum"] == target["checksum"] else "diff"
+            summary.append(
+                {
+                    "role": role,
+                    "base_checksum": base["checksum"],
+                    "target_checksum": target["checksum"],
+                    "status": status,
+                    "base_name": base["name"],
+                    "target_name": target["name"],
+                    "base_kind": base["kind"],
+                    "target_kind": target["kind"],
+                }
+            )
+
+            if include_diff and status == "diff":
+                diff_lines = list(
+                    difflib.unified_diff(
+                        base["content"].splitlines(),
+                        target["content"].splitlines(),
+                        fromfile=f"{base_run_id[:8]}:{role}",
+                        tofile=f"{target_run_id[:8]}:{role}",
+                        lineterm="",
+                    )
+                )
+                truncated = len(diff_lines) > max_lines
+                diffs.append(
+                    {
+                        "role": role,
+                        "lines": diff_lines[:max_lines],
+                        "truncated": truncated,
+                    }
+                )
+
+        return {
+            "base_run_id": base_run_id,
+            "target_run_id": target_run_id,
+            "summary": summary,
+            "diffs": diffs,
+        }
+
+    def _prompt_bundle_role_map(self, bundle: PromptSetBundle) -> dict[str, dict[str, str]]:
+        prompt_map = {prompt.prompt_id: prompt for prompt in bundle.prompts}
+        roles: dict[str, dict[str, str]] = {}
+        for item in bundle.items:
+            prompt = prompt_map.get(item.prompt_id)
+            if not prompt:
+                continue
+            roles[item.role] = {
+                "checksum": prompt.checksum,
+                "content": prompt.content,
+                "name": prompt.name,
+                "kind": prompt.kind,
+            }
+        return roles
+
+    def build_debug_report(self, run_id: str) -> DebugReport:
+        if self._storage is None:
+            raise RuntimeError("Storage not configured")
+        if not hasattr(self._storage, "list_stage_events"):
+            raise RuntimeError("Stage storage not configured")
+
+        service = DebugReportService()
+        stage_storage = cast(StageStoragePort, self._storage)
+        return service.build_report(
+            run_id,
+            storage=self._storage,
+            stage_storage=stage_storage,
+        )
+
     def delete_run(self, run_id: str) -> bool:
         """평가 삭제.
 
@@ -1138,6 +1302,8 @@ class WebUIAdapter:
             raise RuntimeError("Evaluator not configured")
         if self._llm_adapter is None:
             raise RuntimeError("LLM adapter not configured. .env에 OPENAI_API_KEY를 설정하세요.")
+        evaluator = self._evaluator
+        llm_adapter = self._llm_adapter
 
         # 진행률 초기화
         if on_progress:
@@ -1156,10 +1322,10 @@ class WebUIAdapter:
         logger.info(f"Starting evaluation ({mode}) with metrics: {metrics}")
 
         async def run_async_evaluation():
-            return await self._evaluator.evaluate(
+            return await evaluator.evaluate(
                 dataset=dataset,
                 metrics=metrics,
-                llm=self._llm_adapter,
+                llm=llm_adapter,
                 thresholds=thresholds or {},
                 parallel=parallel,
                 batch_size=batch_size,
@@ -1356,6 +1522,7 @@ class WebUIAdapter:
         metrics_to_analyze: list[str] | None = None,
         thresholds: dict[str, float] | None = None,
         model_id: str | None = None,
+        language: str | None = None,
     ):
         """LLM 기반 지능형 보고서 생성.
 
@@ -1395,6 +1562,7 @@ class WebUIAdapter:
             llm_adapter=llm_adapter,
             include_research_insights=True,
             include_action_items=True,
+            language=language or "ko",
         )
 
         # 동기 방식으로 보고서 생성
@@ -1462,7 +1630,7 @@ class WebUIAdapter:
 
         return str(file_path.absolute())
 
-    def list_models(self, provider: str | None = None) -> list[dict[str, str]]:
+    def list_models(self, provider: str | None = None) -> list[dict[str, str | bool]]:
         """사용 가능한 모델 목록 조회."""
         settings = self._settings or Settings()
         provider_key = provider.lower() if provider else None
@@ -1476,7 +1644,7 @@ class WebUIAdapter:
         if provider_key:
             return self._list_other_models(provider_key)
 
-        models: list[dict[str, str]] = []
+        models: list[dict[str, str | bool]] = []
         models.extend(self._list_ollama_models(settings))
         models.extend(self._list_openai_models())
         models.extend(self._list_vllm_models(settings))
@@ -1571,7 +1739,7 @@ class WebUIAdapter:
         lowered = model_name.lower()
         return any(lowered == entry or lowered.startswith(f"{entry}:") for entry in allowlist)
 
-    def _list_other_models(self, provider: str | None = None) -> list[dict[str, str]]:
+    def _list_other_models(self, provider: str | None = None) -> list[dict[str, str | bool]]:
         if provider and provider not in {"anthropic", "azure"}:
             return []
         return [
@@ -1612,7 +1780,8 @@ def create_adapter() -> WebUIAdapter:
 
     설정에 따라 적절한 저장소와 서비스를 주입합니다.
     """
-    from evalvault.adapters.outbound.llm import get_llm_adapter
+    from evalvault.adapters.outbound.llm import SettingsLLMFactory, get_llm_adapter
+    from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
     from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
     from evalvault.config.settings import get_settings
     from evalvault.domain.services.evaluator import RagasEvaluator
@@ -1633,7 +1802,9 @@ def create_adapter() -> WebUIAdapter:
         logger.warning(f"LLM adapter initialization failed: {e}")
 
     # Evaluator 생성
-    evaluator = RagasEvaluator()
+    llm_factory = SettingsLLMFactory(settings)
+    korean_toolkit = try_create_korean_toolkit()
+    evaluator = RagasEvaluator(korean_toolkit=korean_toolkit, llm_factory=llm_factory)
 
     return WebUIAdapter(
         storage=storage,

evalvault/adapters/inbound/api/routers/runs.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from typing import Any, Literal
 
 from fastapi import APIRouter, File, HTTPException, Query, UploadFile
-from fastapi.responses import Response, StreamingResponse
+from fastapi.responses import PlainTextResponse, Response, StreamingResponse
 from pydantic import BaseModel
 
 from evalvault.adapters.inbound.api.main import AdapterDep
@@ -19,6 +19,7 @@ from evalvault.adapters.outbound.dataset.templates import (
     render_dataset_template_json,
     render_dataset_template_xlsx,
 )
+from evalvault.adapters.outbound.debug.report_renderer import render_markdown
 from evalvault.adapters.outbound.domain_memory.sqlite_adapter import SQLiteDomainMemoryAdapter
 from evalvault.config.settings import get_settings
 from evalvault.domain.entities import (
@@ -81,6 +82,30 @@ class QualityGateReportResponse(BaseModel):
     regression_amount: float | None = None
 
 
+class PromptDiffSummaryItem(BaseModel):
+    role: str
+    base_checksum: str | None = None
+    target_checksum: str | None = None
+    status: Literal["same", "diff", "missing"]
+    base_name: str | None = None
+    target_name: str | None = None
+    base_kind: str | None = None
+    target_kind: str | None = None
+
+
+class PromptDiffEntry(BaseModel):
+    role: str
+    lines: list[str]
+    truncated: bool
+
+
+class PromptDiffResponse(BaseModel):
+    base_run_id: str
+    target_run_id: str
+    summary: list[PromptDiffSummaryItem]
+    diffs: list[PromptDiffEntry]
+
+
 class StartEvaluationRequest(BaseModel):
     dataset_path: str
     metrics: list[str]
@@ -1067,6 +1092,27 @@ def list_stage_metrics(
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.get("/prompt-diff", response_model=PromptDiffResponse)
+def prompt_diff(
+    adapter: AdapterDep,
+    base_run_id: str = Query(..., description="Base run id"),
+    target_run_id: str = Query(..., description="Target run id"),
+    max_lines: int = Query(40, ge=1, le=200),
+    include_diff: bool = Query(True),
+):
+    try:
+        return adapter.compare_prompt_sets(
+            base_run_id,
+            target_run_id,
+            max_lines=max_lines,
+            include_diff=include_diff,
+        )
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Prompt set not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.get("/{run_id}/quality-gate", response_model=QualityGateReportResponse)
 def check_quality_gate(run_id: str, adapter: AdapterDep):
     """Check quality gate status for a run."""
@@ -1079,6 +1125,23 @@ def check_quality_gate(run_id: str, adapter: AdapterDep):
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.get("/{run_id}/debug-report", response_model=None)
+def get_debug_report(
+    run_id: str,
+    adapter: AdapterDep,
+    format: Literal["json", "markdown"] = Query("json", description="Report format"),
+):
+    try:
+        report = adapter.build_debug_report(run_id)
+        if format == "markdown":
+            return PlainTextResponse(render_markdown(report))
+        return report.to_dict()
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Run not found")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.get("/{run_id}/improvement")
 def get_improvement_guide(
     run_id: str,
@@ -1101,10 +1164,11 @@ def generate_llm_report(
     run_id: str,
     adapter: AdapterDep,
     model_id: str | None = None,
+    language: str | None = Query(None, description="Report language (ko/en)"),
 ):
     """Generate LLM-based detailed report."""
     try:
-        report = adapter.generate_llm_report(run_id, model_id=model_id)
+        report = adapter.generate_llm_report(run_id, model_id=model_id, language=language)
         return report
     except KeyError:
         raise HTTPException(status_code=404, detail="Run not found")

evalvault/adapters/inbound/cli/commands/method.py

@@ -15,8 +15,9 @@ from rich.console import Console
 from rich.table import Table
 
 from evalvault.adapters.outbound.dataset.method_input_loader import MethodInputDatasetLoader
-from evalvault.adapters.outbound.llm import get_llm_adapter
+from evalvault.adapters.outbound.llm import SettingsLLMFactory, get_llm_adapter
 from evalvault.adapters.outbound.methods import ExternalCommandMethod, MethodRegistry
+from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
 from evalvault.config.settings import Settings, apply_profile
 from evalvault.domain.entities import Dataset
 from evalvault.domain.entities.method import MethodOutput
@@ -376,7 +377,9 @@ def create_method_app(console: Console) -> typer.Typer:
             raise typer.Exit(1)
 
         llm_adapter = get_llm_adapter(settings)
-        evaluator = RagasEvaluator()
+        llm_factory = SettingsLLMFactory(settings)
+        korean_toolkit = try_create_korean_toolkit()
+        evaluator = RagasEvaluator(korean_toolkit=korean_toolkit, llm_factory=llm_factory)
         resolved_thresholds = _resolve_thresholds(metric_list, method_result.dataset)
 
         with progress_spinner(console, "🤖 Evaluation in progress") as update_progress: