evalvault 1.75.0__py3-none-any.whl → 1.77.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -24,7 +24,7 @@ from evalvault.adapters.outbound.judge_calibration_reporter import JudgeCalibrat
 from evalvault.adapters.outbound.ops.report_renderer import render_json, render_markdown
 from evalvault.adapters.outbound.report import MarkdownReportAdapter
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
-from evalvault.config.settings import Settings
+from evalvault.config.settings import Settings, resolve_tracker_providers
 from evalvault.domain.entities import (
     CalibrationResult,
     FeedbackSummary,
@@ -217,56 +217,83 @@ class WebUIAdapter:
             logger.warning(f"Failed to create LLM adapter for {model_id}: {e}, using default")
             return self._llm_adapter
 
-    def _get_tracker(
+    def _get_trackers(
         self,
         settings: Settings,
         tracker_config: dict[str, Any] | None,
-    ) -> tuple[str | None, Any | None]:
-        provider = (tracker_config or {}).get("provider") or "none"
-        provider = provider.lower()
-
-        if provider in {"none", ""}:
-            return None, None
-
-        if provider == "langfuse":
-            if not settings.langfuse_public_key or not settings.langfuse_secret_key:
-                logger.warning("Langfuse credentials missing; skipping tracker logging.")
-                return None, None
-            from evalvault.adapters.outbound.tracker.langfuse_adapter import LangfuseAdapter
-
-            return provider, LangfuseAdapter(
-                public_key=settings.langfuse_public_key,
-                secret_key=settings.langfuse_secret_key,
-                host=settings.langfuse_host,
-            )
+    ) -> list[tuple[str, Any]]:
+        provider = (tracker_config or {}).get("provider") or settings.tracker_provider or "none"
+        providers = resolve_tracker_providers(provider)
+        if not providers or providers == ["none"]:
+            return []
+        required = {"mlflow", "phoenix"}
+        if not required.issubset(set(providers)):
+            raise RuntimeError("Tracker must include both mlflow and phoenix")
+
+        trackers: list[tuple[str, Any]] = []
+        for entry in providers:
+            if entry == "langfuse":
+                if not settings.langfuse_public_key or not settings.langfuse_secret_key:
+                    raise RuntimeError("Langfuse credentials missing")
+                from evalvault.adapters.outbound.tracker.langfuse_adapter import LangfuseAdapter
+
+                trackers.append(
+                    (
+                        entry,
+                        LangfuseAdapter(
+                            public_key=settings.langfuse_public_key,
+                            secret_key=settings.langfuse_secret_key,
+                            host=settings.langfuse_host,
+                        ),
+                    )
+                )
+                continue
 
-        if provider == "phoenix":
-            from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
+            if entry == "phoenix":
+                from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
 
-            ensure_phoenix_instrumentation(settings, force=True)
-            try:
-                from evalvault.adapters.outbound.tracker.phoenix_adapter import PhoenixAdapter
-            except ImportError as exc:
-                logger.warning("Phoenix extras not installed: %s", exc)
-                return None, None
-            return provider, PhoenixAdapter(endpoint=settings.phoenix_endpoint)
-
-        if provider == "mlflow":
-            if not settings.mlflow_tracking_uri:
-                logger.warning("MLflow tracking URI missing; skipping tracker logging.")
-                return None, None
-            try:
-                from evalvault.adapters.outbound.tracker.mlflow_adapter import MLflowAdapter
-            except ImportError as exc:
-                logger.warning("MLflow adapter unavailable: %s", exc)
-                return None, None
-            return provider, MLflowAdapter(
-                tracking_uri=settings.mlflow_tracking_uri,
-                experiment_name=settings.mlflow_experiment_name,
-            )
+                ensure_phoenix_instrumentation(settings, force=True)
+                try:
+                    from evalvault.adapters.outbound.tracker.phoenix_adapter import PhoenixAdapter
+                except ImportError as exc:
+                    raise RuntimeError("Phoenix extras not installed") from exc
+                trackers.append(
+                    (
+                        entry,
+                        PhoenixAdapter(
+                            endpoint=settings.phoenix_endpoint,
+                            project_name=getattr(settings, "phoenix_project_name", None),
+                            annotations_enabled=getattr(
+                                settings,
+                                "phoenix_annotations_enabled",
+                                True,
+                            ),
+                        ),
+                    )
+                )
+                continue
 
-        logger.warning("Unknown tracker provider: %s", provider)
-        return None, None
+            if entry == "mlflow":
+                if not settings.mlflow_tracking_uri:
+                    raise RuntimeError("MLflow tracking URI missing")
+                try:
+                    from evalvault.adapters.outbound.tracker.mlflow_adapter import MLflowAdapter
+                except ImportError as exc:
+                    raise RuntimeError("MLflow adapter unavailable") from exc
+                trackers.append(
+                    (
+                        entry,
+                        MLflowAdapter(
+                            tracking_uri=settings.mlflow_tracking_uri,
+                            experiment_name=settings.mlflow_experiment_name,
+                        ),
+                    )
+                )
+                continue
+
+            raise RuntimeError(f"Unknown tracker provider: {entry}")
+
+        return trackers
 
     @staticmethod
     def _build_phoenix_trace_url(endpoint: str, trace_id: str) -> str:
@@ -425,7 +452,11 @@ class WebUIAdapter:
             dataset.metadata["domain"] = requested_domain
 
         settings = self._settings or Settings()
-        tracker_provider, tracker = self._get_tracker(settings, request.tracker_config)
+        try:
+            trackers = self._get_trackers(settings, request.tracker_config)
+        except RuntimeError as exc:
+            raise RuntimeError(f"Tracker configuration error: {exc}") from exc
+        tracker_providers = [provider for provider, _ in trackers]
         stage_store = bool(request.stage_store)
 
         retriever_instance = None
@@ -484,7 +515,7 @@ class WebUIAdapter:
                 )
                 from evalvault.domain.services.memory_aware_evaluator import MemoryAwareEvaluator
 
-                tracer = PhoenixTracerAdapter() if tracker_provider == "phoenix" else None
+                tracer = PhoenixTracerAdapter() if "phoenix" in tracker_providers else None
                 memory_adapter = build_domain_memory_adapter(
                     settings=self._settings,
                     db_path=Path(memory_db_path) if memory_db_path else None,
@@ -536,11 +567,31 @@ class WebUIAdapter:
 
         # 2. 진행률 초기화
         start_time = time.monotonic()
+        total_cases = len(dataset.test_cases)
+
+        def emit_progress(message: str, *, status: str = "running") -> None:
+            if not on_progress:
+                return
+            elapsed = time.monotonic() - start_time
+            rate = (total_cases / elapsed) if total_cases > 0 and elapsed > 0 else None
+            on_progress(
+                EvalProgress(
+                    current=total_cases,
+                    total=total_cases,
+                    current_metric=message,
+                    percent=100.0 if total_cases > 0 else 0.0,
+                    status=status,
+                    elapsed_seconds=elapsed,
+                    eta_seconds=0.0,
+                    rate=rate,
+                )
+            )
+
         if on_progress:
             on_progress(
                 EvalProgress(
                     current=0,
-                    total=len(dataset.test_cases),
+                    total=total_cases,
                     current_metric="",
                     percent=0.0,
                     status="running",
@@ -696,24 +747,31 @@ class WebUIAdapter:
                 str(request.threshold_profile).strip().lower()
             )
 
-        if tracker and tracker_provider:
-            try:
-                trace_id = tracker.log_evaluation_run(result)
-                if tracker_provider == "phoenix":
-                    endpoint = settings.phoenix_endpoint or "http://localhost:6006/v1/traces"
-                    phoenix_meta = result.tracker_metadata.setdefault("phoenix", {})
-                    phoenix_meta.update(
-                        {
-                            "trace_id": trace_id,
-                            "endpoint": endpoint,
-                            "trace_url": self._build_phoenix_trace_url(endpoint, trace_id),
-                            "schema_version": 2,
-                        }
-                    )
-            except Exception as exc:
-                logger.warning("Tracker logging failed: %s", exc)
+        if trackers:
+            emit_progress("Logging trackers...", status="finalizing")
+            result.tracker_metadata.setdefault("tracker_providers", tracker_providers)
+            for provider, tracker in trackers:
+                try:
+                    trace_id = tracker.log_evaluation_run(result)
+                    provider_meta = result.tracker_metadata.setdefault(provider, {})
+                    if isinstance(provider_meta, dict):
+                        provider_meta.setdefault("trace_id", trace_id)
+                    if provider == "phoenix":
+                        endpoint = settings.phoenix_endpoint or "http://localhost:6006/v1/traces"
+                        phoenix_meta = result.tracker_metadata.setdefault("phoenix", {})
+                        phoenix_meta.update(
+                            {
+                                "trace_id": trace_id,
+                                "endpoint": endpoint,
+                                "trace_url": self._build_phoenix_trace_url(endpoint, trace_id),
+                                "schema_version": 2,
+                            }
+                        )
+                except Exception as exc:
+                    raise RuntimeError(f"Tracker logging failed for {provider}: {exc}") from exc
 
         if stage_store and self._storage and hasattr(self._storage, "save_stage_events"):
+            emit_progress("Storing stage events...", status="finalizing")
             try:
                 prompt_metadata_entries = self._build_prompt_metadata_entries(prompt_bundle)
                 stage_event_builder = StageEventBuilder()
@@ -755,6 +813,7 @@ class WebUIAdapter:
 
         # 5. 결과 저장
         if self._storage:
+            emit_progress("Saving evaluation run...", status="finalizing")
             logger.info(f"Saving evaluation run: {result.run_id}")
             if prompt_bundle:
                 self._storage.save_prompt_set(prompt_bundle)

evalvault/adapters/inbound/api/main.py

@@ -15,6 +15,7 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from starlette.responses import JSONResponse
 
 from evalvault.adapters.inbound.api.adapter import WebUIAdapter, create_adapter
+from evalvault.config.runtime_services import ensure_local_observability
 from evalvault.config.settings import Settings, get_settings, is_production_profile
 
 logger = logging.getLogger(__name__)
@@ -63,6 +64,7 @@ async def lifespan(app: FastAPI):
     # Startup: Initialize adapter
     adapter = create_adapter()
     app.state.adapter = adapter
+    ensure_local_observability(get_settings())
     try:
         from evalvault.adapters.inbound.api.routers.chat import warm_rag_index
 

evalvault/adapters/inbound/api/routers/config.py

@@ -71,7 +71,9 @@ class ConfigUpdateRequest(BaseModel):
     phoenix_endpoint: str | None = None
     phoenix_enabled: bool | None = None
     phoenix_sample_rate: float | None = None
-    tracker_provider: Literal["langfuse", "mlflow", "phoenix", "none"] | None = None
+    phoenix_project_name: str | None = None
+    phoenix_annotations_enabled: bool | None = None
+    tracker_provider: str | None = None
     postgres_host: str | None = None
     postgres_port: int | None = None
     postgres_database: str | None = None

evalvault/adapters/inbound/cli/app.py

@@ -14,6 +14,8 @@ import typer
 from rich import print as rprint
 from rich.console import Console
 
+from evalvault.config.runtime_services import ensure_local_observability
+from evalvault.config.settings import get_settings
 from evalvault.domain.metrics.registry import list_metric_names
 
 from .commands import attach_sub_apps, register_all_commands
@@ -61,6 +63,7 @@ def main(
     ),
 ) -> None:
     """EvalVault - RAG evaluation system."""
+    ensure_local_observability(get_settings())
 
 
 if __name__ == "__main__":  # pragma: no cover

evalvault/adapters/inbound/cli/commands/analyze.py

@@ -358,6 +358,11 @@ def register_analyze_commands(app: typer.Typer, console: Console) -> None:
         profile: str | None = profile_option(
             help_text="비교 리포트용 LLM 프로필 (dev, prod, openai)",
         ),
+        use_llm_report: bool = typer.Option(
+            True,
+            "--use-llm-report/--no-llm-report",
+            help="LLM 보고서 사용 여부",
+        ),
     ) -> None:
         """두 실행을 통계적으로 비교합니다."""
 
@@ -461,7 +466,7 @@ def register_analyze_commands(app: typer.Typer, console: Console) -> None:
                 compare_metrics=metric_list,
                 test_type=test,
                 report_type="comparison",
-                use_llm_report=True,
+                use_llm_report=use_llm_report,
             )
 
         artifacts_dir = resolve_artifact_dir(

evalvault/adapters/inbound/cli/commands/method.py

@@ -31,7 +31,7 @@ from ..utils.validators import parse_csv_option, validate_choices
 from .run_helpers import (
     _display_results,
     _is_oss_open_model,
-    _log_to_tracker,
+    _log_to_trackers,
     _resolve_thresholds,
     _save_results,
     _save_to_db,
@@ -419,12 +419,12 @@ def create_method_app(console: Console) -> typer.Typer:
         _display_results(result, console)
 
         if tracker and tracker != "none":
-            _log_to_tracker(settings, result, console, tracker_type=tracker)
+            _log_to_trackers(settings, result, console, tracker_type=tracker)
 
         if eval_output:
             _save_results(eval_output, result, console)
 
-        _save_to_db(db_path, result, console)
+        _save_to_db(settings, db_path, result, console)
 
     return method_app