evalvault 1.74.0__py3-none-any.whl → 1.76.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -24,7 +24,7 @@ from evalvault.adapters.outbound.judge_calibration_reporter import JudgeCalibrat
 from evalvault.adapters.outbound.ops.report_renderer import render_json, render_markdown
 from evalvault.adapters.outbound.report import MarkdownReportAdapter
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
-from evalvault.config.settings import Settings
+from evalvault.config.settings import Settings, resolve_tracker_providers
 from evalvault.domain.entities import (
     CalibrationResult,
     FeedbackSummary,
@@ -127,14 +127,15 @@ class WebUIAdapter:
             llm_adapter: LLM 어댑터 (선택적)
             data_loader: 데이터 로더 (선택적)
         """
-        resolved_settings = settings
+        resolved_settings = settings or Settings()
         if storage is None:
-            resolved_settings = settings or Settings()
-            db_path = getattr(resolved_settings, "evalvault_db_path", None)
-            if db_path:
-                from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+            from evalvault.adapters.outbound.storage.factory import build_storage_adapter
 
-                storage = SQLiteStorageAdapter(db_path=db_path)
+            try:
+                storage = build_storage_adapter(settings=resolved_settings)
+            except Exception as exc:
+                logger.warning("Storage initialization failed: %s", exc)
+                storage = None
 
         self._storage = storage
         self._evaluator = evaluator
@@ -216,56 +217,83 @@ class WebUIAdapter:
             logger.warning(f"Failed to create LLM adapter for {model_id}: {e}, using default")
             return self._llm_adapter
 
-    def _get_tracker(
+    def _get_trackers(
         self,
         settings: Settings,
         tracker_config: dict[str, Any] | None,
-    ) -> tuple[str | None, Any | None]:
-        provider = (tracker_config or {}).get("provider") or "none"
-        provider = provider.lower()
-
-        if provider in {"none", ""}:
-            return None, None
-
-        if provider == "langfuse":
-            if not settings.langfuse_public_key or not settings.langfuse_secret_key:
-                logger.warning("Langfuse credentials missing; skipping tracker logging.")
-                return None, None
-            from evalvault.adapters.outbound.tracker.langfuse_adapter import LangfuseAdapter
-
-            return provider, LangfuseAdapter(
-                public_key=settings.langfuse_public_key,
-                secret_key=settings.langfuse_secret_key,
-                host=settings.langfuse_host,
-            )
+    ) -> list[tuple[str, Any]]:
+        provider = (tracker_config or {}).get("provider") or settings.tracker_provider or "none"
+        providers = resolve_tracker_providers(provider)
+        if not providers or providers == ["none"]:
+            return []
+        required = {"mlflow", "phoenix"}
+        if not required.issubset(set(providers)):
+            raise RuntimeError("Tracker must include both mlflow and phoenix")
+
+        trackers: list[tuple[str, Any]] = []
+        for entry in providers:
+            if entry == "langfuse":
+                if not settings.langfuse_public_key or not settings.langfuse_secret_key:
+                    raise RuntimeError("Langfuse credentials missing")
+                from evalvault.adapters.outbound.tracker.langfuse_adapter import LangfuseAdapter
+
+                trackers.append(
+                    (
+                        entry,
+                        LangfuseAdapter(
+                            public_key=settings.langfuse_public_key,
+                            secret_key=settings.langfuse_secret_key,
+                            host=settings.langfuse_host,
+                        ),
+                    )
+                )
+                continue
 
-        if provider == "phoenix":
-            from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
+            if entry == "phoenix":
+                from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
 
-            ensure_phoenix_instrumentation(settings, force=True)
-            try:
-                from evalvault.adapters.outbound.tracker.phoenix_adapter import PhoenixAdapter
-            except ImportError as exc:
-                logger.warning("Phoenix extras not installed: %s", exc)
-                return None, None
-            return provider, PhoenixAdapter(endpoint=settings.phoenix_endpoint)
-
-        if provider == "mlflow":
-            if not settings.mlflow_tracking_uri:
-                logger.warning("MLflow tracking URI missing; skipping tracker logging.")
-                return None, None
-            try:
-                from evalvault.adapters.outbound.tracker.mlflow_adapter import MLflowAdapter
-            except ImportError as exc:
-                logger.warning("MLflow adapter unavailable: %s", exc)
-                return None, None
-            return provider, MLflowAdapter(
-                tracking_uri=settings.mlflow_tracking_uri,
-                experiment_name=settings.mlflow_experiment_name,
-            )
+                ensure_phoenix_instrumentation(settings, force=True)
+                try:
+                    from evalvault.adapters.outbound.tracker.phoenix_adapter import PhoenixAdapter
+                except ImportError as exc:
+                    raise RuntimeError("Phoenix extras not installed") from exc
+                trackers.append(
+                    (
+                        entry,
+                        PhoenixAdapter(
+                            endpoint=settings.phoenix_endpoint,
+                            project_name=getattr(settings, "phoenix_project_name", None),
+                            annotations_enabled=getattr(
+                                settings,
+                                "phoenix_annotations_enabled",
+                                True,
+                            ),
+                        ),
+                    )
+                )
+                continue
+
+            if entry == "mlflow":
+                if not settings.mlflow_tracking_uri:
+                    raise RuntimeError("MLflow tracking URI missing")
+                try:
+                    from evalvault.adapters.outbound.tracker.mlflow_adapter import MLflowAdapter
+                except ImportError as exc:
+                    raise RuntimeError("MLflow adapter unavailable") from exc
+                trackers.append(
+                    (
+                        entry,
+                        MLflowAdapter(
+                            tracking_uri=settings.mlflow_tracking_uri,
+                            experiment_name=settings.mlflow_experiment_name,
+                        ),
+                    )
+                )
+                continue
+
+            raise RuntimeError(f"Unknown tracker provider: {entry}")
 
-        logger.warning("Unknown tracker provider: %s", provider)
-        return None, None
+        return trackers
 
     @staticmethod
     def _build_phoenix_trace_url(endpoint: str, trace_id: str) -> str:
@@ -424,7 +452,11 @@ class WebUIAdapter:
             dataset.metadata["domain"] = requested_domain
 
         settings = self._settings or Settings()
-        tracker_provider, tracker = self._get_tracker(settings, request.tracker_config)
+        try:
+            trackers = self._get_trackers(settings, request.tracker_config)
+        except RuntimeError as exc:
+            raise RuntimeError(f"Tracker configuration error: {exc}") from exc
+        tracker_providers = [provider for provider, _ in trackers]
         stage_store = bool(request.stage_store)
 
         retriever_instance = None
@@ -450,7 +482,12 @@ class WebUIAdapter:
         memory_domain = memory_config.get("domain") or dataset.metadata.get("domain") or "default"
         memory_language = memory_config.get("language") or "ko"
         memory_augment = bool(memory_config.get("augment_context"))
-        memory_db_path = memory_config.get("db_path") or settings.evalvault_memory_db_path
+        if memory_config.get("db_path"):
+            memory_db_path = memory_config.get("db_path")
+        elif settings.db_backend == "sqlite":
+            memory_db_path = settings.evalvault_memory_db_path
+        else:
+            memory_db_path = None
         memory_evaluator = None
         requested_thresholds = request.thresholds or {}
         if request.threshold_profile or requested_thresholds:
@@ -472,16 +509,17 @@ class WebUIAdapter:
         memory_active = False
         if memory_enabled:
             try:
-                from evalvault.adapters.outbound.domain_memory.sqlite_adapter import (
-                    SQLiteDomainMemoryAdapter,
-                )
+                from evalvault.adapters.outbound.domain_memory import build_domain_memory_adapter
                 from evalvault.adapters.outbound.tracer.phoenix_tracer_adapter import (
                     PhoenixTracerAdapter,
                 )
                 from evalvault.domain.services.memory_aware_evaluator import MemoryAwareEvaluator
 
-                tracer = PhoenixTracerAdapter() if tracker_provider == "phoenix" else None
-                memory_adapter = SQLiteDomainMemoryAdapter(memory_db_path)
+                tracer = PhoenixTracerAdapter() if "phoenix" in tracker_providers else None
+                memory_adapter = build_domain_memory_adapter(
+                    settings=self._settings,
+                    db_path=Path(memory_db_path) if memory_db_path else None,
+                )
                 memory_evaluator = MemoryAwareEvaluator(
                     evaluator=self._evaluator,
                     memory_port=memory_adapter,
@@ -689,22 +727,27 @@ class WebUIAdapter:
                 str(request.threshold_profile).strip().lower()
             )
 
-        if tracker and tracker_provider:
-            try:
-                trace_id = tracker.log_evaluation_run(result)
-                if tracker_provider == "phoenix":
-                    endpoint = settings.phoenix_endpoint or "http://localhost:6006/v1/traces"
-                    phoenix_meta = result.tracker_metadata.setdefault("phoenix", {})
-                    phoenix_meta.update(
-                        {
-                            "trace_id": trace_id,
-                            "endpoint": endpoint,
-                            "trace_url": self._build_phoenix_trace_url(endpoint, trace_id),
-                            "schema_version": 2,
-                        }
-                    )
-            except Exception as exc:
-                logger.warning("Tracker logging failed: %s", exc)
+        if trackers:
+            result.tracker_metadata.setdefault("tracker_providers", tracker_providers)
+            for provider, tracker in trackers:
+                try:
+                    trace_id = tracker.log_evaluation_run(result)
+                    provider_meta = result.tracker_metadata.setdefault(provider, {})
+                    if isinstance(provider_meta, dict):
+                        provider_meta.setdefault("trace_id", trace_id)
+                    if provider == "phoenix":
+                        endpoint = settings.phoenix_endpoint or "http://localhost:6006/v1/traces"
+                        phoenix_meta = result.tracker_metadata.setdefault("phoenix", {})
+                        phoenix_meta.update(
+                            {
+                                "trace_id": trace_id,
+                                "endpoint": endpoint,
+                                "trace_url": self._build_phoenix_trace_url(endpoint, trace_id),
+                                "schema_version": 2,
+                            }
+                        )
+                except Exception as exc:
+                    raise RuntimeError(f"Tracker logging failed for {provider}: {exc}") from exc
 
         if stage_store and self._storage and hasattr(self._storage, "save_stage_events"):
             try:
@@ -814,6 +857,7 @@ class WebUIAdapter:
     def list_runs(
         self,
         limit: int = 50,
+        offset: int = 0,
         filters: RunFilters | None = None,
     ) -> list[RunSummary]:
         """평가 목록 조회.
@@ -833,7 +877,7 @@ class WebUIAdapter:
 
         try:
             # 저장소에서 평가 목록 조회
-            runs = self._storage.list_runs(limit=limit)
+            runs = self._storage.list_runs(limit=limit, offset=offset)
 
             # RunSummary로 변환
             summaries = []
@@ -1029,7 +1073,11 @@ class WebUIAdapter:
         run = self.get_run_details(run_id)
         feedbacks = storage.list_feedback(run_id)
         if labels_source in {"feedback", "hybrid"} and not feedbacks:
-            raise ValueError("Feedback labels are required for this labels_source")
+            raise ValueError(
+                f"No feedback labels found for run '{run_id}'. "
+                f"Calibration with labels_source='{labels_source}' requires at least one feedback label. "
+                "Please add feedback labels via the UI or API, or use labels_source='gold' if gold labels are available."
+            )
         resolved_metrics = metrics or list(run.metrics_evaluated)
         if not resolved_metrics:
             raise ValueError("No metrics available for calibration")
@@ -2198,16 +2246,15 @@ def create_adapter() -> WebUIAdapter:
     """
     from evalvault.adapters.outbound.llm import SettingsLLMFactory, get_llm_adapter
     from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
-    from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+    from evalvault.adapters.outbound.storage.factory import build_storage_adapter
     from evalvault.config.settings import get_settings
     from evalvault.domain.services.evaluator import RagasEvaluator
 
     # 설정 로드
     settings = get_settings()
 
-    # Storage 생성 (기본 SQLite)
-    db_path = Path(settings.evalvault_db_path)
-    storage = SQLiteStorageAdapter(db_path=db_path)
+    # Storage 생성
+    storage = build_storage_adapter(settings=settings)
 
     # LLM adapter 생성 (API 키 없으면 None)
     llm_adapter = None

evalvault/adapters/inbound/api/routers/calibration.py

@@ -113,6 +113,15 @@ def run_judge_calibration(
     return JudgeCalibrationResponse.model_validate(payload)
 
 
+@router.get("/judge/history", response_model=list[JudgeCalibrationHistoryItem])
+def list_calibrations(
+    adapter: AdapterDep,
+    limit: int = Query(20, ge=1, le=200),
+) -> list[JudgeCalibrationHistoryItem]:
+    entries = adapter.list_judge_calibrations(limit=limit)
+    return [JudgeCalibrationHistoryItem.model_validate(entry) for entry in entries]
+
+
 @router.get("/judge/{calibration_id}", response_model=JudgeCalibrationResponse)
 def get_calibration_result(calibration_id: str, adapter: AdapterDep) -> JudgeCalibrationResponse:
     try:
@@ -122,12 +131,3 @@ def get_calibration_result(calibration_id: str, adapter: AdapterDep) -> JudgeCal
     except RuntimeError as exc:
         raise HTTPException(status_code=500, detail=str(exc)) from exc
     return JudgeCalibrationResponse.model_validate(payload)
-
-
-@router.get("/judge/history", response_model=list[JudgeCalibrationHistoryItem])
-def list_calibrations(
-    adapter: AdapterDep,
-    limit: int = Query(20, ge=1, le=200),
-) -> list[JudgeCalibrationHistoryItem]:
-    entries = adapter.list_judge_calibrations(limit=limit)
-    return [JudgeCalibrationHistoryItem.model_validate(entry) for entry in entries]