evalvault 1.73.2__py3-none-any.whl → 1.75.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -21,6 +21,7 @@ from evalvault.adapters.outbound.analysis import (
 )
 from evalvault.adapters.outbound.cache import MemoryCacheAdapter
 from evalvault.adapters.outbound.judge_calibration_reporter import JudgeCalibrationReporter
+from evalvault.adapters.outbound.ops.report_renderer import render_json, render_markdown
 from evalvault.adapters.outbound.report import MarkdownReportAdapter
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
 from evalvault.config.settings import Settings
@@ -43,6 +44,7 @@ from evalvault.domain.services.analysis_service import AnalysisService
 from evalvault.domain.services.cluster_map_builder import build_cluster_map
 from evalvault.domain.services.debug_report_service import DebugReportService
 from evalvault.domain.services.judge_calibration_service import JudgeCalibrationService
+from evalvault.domain.services.ops_report_service import OpsReportService
 from evalvault.domain.services.prompt_registry import (
     PromptInput,
     build_prompt_bundle,
@@ -125,14 +127,15 @@ class WebUIAdapter:
             llm_adapter: LLM 어댑터 (선택적)
             data_loader: 데이터 로더 (선택적)
         """
-        resolved_settings = settings
+        resolved_settings = settings or Settings()
         if storage is None:
-            resolved_settings = settings or Settings()
-            db_path = getattr(resolved_settings, "evalvault_db_path", None)
-            if db_path:
-                from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+            from evalvault.adapters.outbound.storage.factory import build_storage_adapter
 
-                storage = SQLiteStorageAdapter(db_path=db_path)
+            try:
+                storage = build_storage_adapter(settings=resolved_settings)
+            except Exception as exc:
+                logger.warning("Storage initialization failed: %s", exc)
+                storage = None
 
         self._storage = storage
         self._evaluator = evaluator
@@ -448,7 +451,12 @@ class WebUIAdapter:
         memory_domain = memory_config.get("domain") or dataset.metadata.get("domain") or "default"
         memory_language = memory_config.get("language") or "ko"
         memory_augment = bool(memory_config.get("augment_context"))
-        memory_db_path = memory_config.get("db_path") or settings.evalvault_memory_db_path
+        if memory_config.get("db_path"):
+            memory_db_path = memory_config.get("db_path")
+        elif settings.db_backend == "sqlite":
+            memory_db_path = settings.evalvault_memory_db_path
+        else:
+            memory_db_path = None
         memory_evaluator = None
         requested_thresholds = request.thresholds or {}
         if request.threshold_profile or requested_thresholds:
@@ -470,16 +478,17 @@ class WebUIAdapter:
         memory_active = False
         if memory_enabled:
             try:
-                from evalvault.adapters.outbound.domain_memory.sqlite_adapter import (
-                    SQLiteDomainMemoryAdapter,
-                )
+                from evalvault.adapters.outbound.domain_memory import build_domain_memory_adapter
                 from evalvault.adapters.outbound.tracer.phoenix_tracer_adapter import (
                     PhoenixTracerAdapter,
                 )
                 from evalvault.domain.services.memory_aware_evaluator import MemoryAwareEvaluator
 
                 tracer = PhoenixTracerAdapter() if tracker_provider == "phoenix" else None
-                memory_adapter = SQLiteDomainMemoryAdapter(memory_db_path)
+                memory_adapter = build_domain_memory_adapter(
+                    settings=self._settings,
+                    db_path=Path(memory_db_path) if memory_db_path else None,
+                )
                 memory_evaluator = MemoryAwareEvaluator(
                     evaluator=self._evaluator,
                     memory_port=memory_adapter,
@@ -812,6 +821,7 @@ class WebUIAdapter:
     def list_runs(
         self,
         limit: int = 50,
+        offset: int = 0,
         filters: RunFilters | None = None,
     ) -> list[RunSummary]:
         """평가 목록 조회.
@@ -831,7 +841,7 @@ class WebUIAdapter:
 
         try:
             # 저장소에서 평가 목록 조회
-            runs = self._storage.list_runs(limit=limit)
+            runs = self._storage.list_runs(limit=limit, offset=offset)
 
             # RunSummary로 변환
             summaries = []
@@ -1027,7 +1037,11 @@ class WebUIAdapter:
         run = self.get_run_details(run_id)
         feedbacks = storage.list_feedback(run_id)
         if labels_source in {"feedback", "hybrid"} and not feedbacks:
-            raise ValueError("Feedback labels are required for this labels_source")
+            raise ValueError(
+                f"No feedback labels found for run '{run_id}'. "
+                f"Calibration with labels_source='{labels_source}' requires at least one feedback label. "
+                "Please add feedback labels via the UI or API, or use labels_source='gold' if gold labels are available."
+            )
         resolved_metrics = metrics or list(run.metrics_evaluated)
         if not resolved_metrics:
             raise ValueError("No metrics available for calibration")
@@ -1329,6 +1343,42 @@ class WebUIAdapter:
             stage_storage=stage_storage,
         )
 
+    def generate_ops_report(
+        self,
+        run_id: str,
+        *,
+        output_format: str,
+        save: bool,
+    ) -> dict[str, Any] | str:
+        if self._storage is None:
+            raise RuntimeError("Storage not configured")
+        if not hasattr(self._storage, "list_stage_events"):
+            raise RuntimeError("Stage storage not configured")
+
+        service = OpsReportService()
+        stage_storage = cast(StageStoragePort, self._storage)
+        report = service.build_report(
+            run_id,
+            storage=self._storage,
+            stage_storage=stage_storage,
+        )
+
+        content = render_markdown(report) if output_format == "markdown" else render_json(report)
+
+        if save:
+            self._storage.save_ops_report(
+                report_id=None,
+                run_id=run_id,
+                report_type="ops_report",
+                format=output_format,
+                content=content,
+                metadata={"source": "api"},
+            )
+
+        if output_format == "markdown":
+            return content
+        return report.to_dict()
+
     def delete_run(self, run_id: str) -> bool:
         """평가 삭제.
 
@@ -2160,16 +2210,15 @@ def create_adapter() -> WebUIAdapter:
     """
     from evalvault.adapters.outbound.llm import SettingsLLMFactory, get_llm_adapter
     from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
-    from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+    from evalvault.adapters.outbound.storage.factory import build_storage_adapter
     from evalvault.config.settings import get_settings
     from evalvault.domain.services.evaluator import RagasEvaluator
 
     # 설정 로드
     settings = get_settings()
 
-    # Storage 생성 (기본 SQLite)
-    db_path = Path(settings.evalvault_db_path)
-    storage = SQLiteStorageAdapter(db_path=db_path)
+    # Storage 생성
+    storage = build_storage_adapter(settings=settings)
 
     # LLM adapter 생성 (API 키 없으면 None)
     llm_adapter = None

evalvault/adapters/inbound/api/routers/calibration.py

@@ -113,6 +113,15 @@ def run_judge_calibration(
     return JudgeCalibrationResponse.model_validate(payload)
 
 
+@router.get("/judge/history", response_model=list[JudgeCalibrationHistoryItem])
+def list_calibrations(
+    adapter: AdapterDep,
+    limit: int = Query(20, ge=1, le=200),
+) -> list[JudgeCalibrationHistoryItem]:
+    entries = adapter.list_judge_calibrations(limit=limit)
+    return [JudgeCalibrationHistoryItem.model_validate(entry) for entry in entries]
+
+
 @router.get("/judge/{calibration_id}", response_model=JudgeCalibrationResponse)
 def get_calibration_result(calibration_id: str, adapter: AdapterDep) -> JudgeCalibrationResponse:
     try:
@@ -122,12 +131,3 @@ def get_calibration_result(calibration_id: str, adapter: AdapterDep) -> JudgeCal
     except RuntimeError as exc:
         raise HTTPException(status_code=500, detail=str(exc)) from exc
     return JudgeCalibrationResponse.model_validate(payload)
-
-
-@router.get("/judge/history", response_model=list[JudgeCalibrationHistoryItem])
-def list_calibrations(
-    adapter: AdapterDep,
-    limit: int = Query(20, ge=1, le=200),
-) -> list[JudgeCalibrationHistoryItem]:
-    entries = adapter.list_judge_calibrations(limit=limit)
-    return [JudgeCalibrationHistoryItem.model_validate(entry) for entry in entries]