evalvault 1.75.0__py3-none-any.whl → 1.76.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -24,7 +24,7 @@ from evalvault.adapters.outbound.judge_calibration_reporter import JudgeCalibrat
 from evalvault.adapters.outbound.ops.report_renderer import render_json, render_markdown
 from evalvault.adapters.outbound.report import MarkdownReportAdapter
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
-from evalvault.config.settings import Settings
+from evalvault.config.settings import Settings, resolve_tracker_providers
 from evalvault.domain.entities import (
     CalibrationResult,
     FeedbackSummary,
@@ -217,56 +217,83 @@ class WebUIAdapter:
             logger.warning(f"Failed to create LLM adapter for {model_id}: {e}, using default")
             return self._llm_adapter
 
-    def _get_tracker(
+    def _get_trackers(
         self,
         settings: Settings,
         tracker_config: dict[str, Any] | None,
-    ) -> tuple[str | None, Any | None]:
-        provider = (tracker_config or {}).get("provider") or "none"
-        provider = provider.lower()
-
-        if provider in {"none", ""}:
-            return None, None
-
-        if provider == "langfuse":
-            if not settings.langfuse_public_key or not settings.langfuse_secret_key:
-                logger.warning("Langfuse credentials missing; skipping tracker logging.")
-                return None, None
-            from evalvault.adapters.outbound.tracker.langfuse_adapter import LangfuseAdapter
-
-            return provider, LangfuseAdapter(
-                public_key=settings.langfuse_public_key,
-                secret_key=settings.langfuse_secret_key,
-                host=settings.langfuse_host,
-            )
+    ) -> list[tuple[str, Any]]:
+        provider = (tracker_config or {}).get("provider") or settings.tracker_provider or "none"
+        providers = resolve_tracker_providers(provider)
+        if not providers or providers == ["none"]:
+            return []
+        required = {"mlflow", "phoenix"}
+        if not required.issubset(set(providers)):
+            raise RuntimeError("Tracker must include both mlflow and phoenix")
+
+        trackers: list[tuple[str, Any]] = []
+        for entry in providers:
+            if entry == "langfuse":
+                if not settings.langfuse_public_key or not settings.langfuse_secret_key:
+                    raise RuntimeError("Langfuse credentials missing")
+                from evalvault.adapters.outbound.tracker.langfuse_adapter import LangfuseAdapter
+
+                trackers.append(
+                    (
+                        entry,
+                        LangfuseAdapter(
+                            public_key=settings.langfuse_public_key,
+                            secret_key=settings.langfuse_secret_key,
+                            host=settings.langfuse_host,
+                        ),
+                    )
+                )
+                continue
 
-        if provider == "phoenix":
-            from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
+            if entry == "phoenix":
+                from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
 
-            ensure_phoenix_instrumentation(settings, force=True)
-            try:
-                from evalvault.adapters.outbound.tracker.phoenix_adapter import PhoenixAdapter
-            except ImportError as exc:
-                logger.warning("Phoenix extras not installed: %s", exc)
-                return None, None
-            return provider, PhoenixAdapter(endpoint=settings.phoenix_endpoint)
-
-        if provider == "mlflow":
-            if not settings.mlflow_tracking_uri:
-                logger.warning("MLflow tracking URI missing; skipping tracker logging.")
-                return None, None
-            try:
-                from evalvault.adapters.outbound.tracker.mlflow_adapter import MLflowAdapter
-            except ImportError as exc:
-                logger.warning("MLflow adapter unavailable: %s", exc)
-                return None, None
-            return provider, MLflowAdapter(
-                tracking_uri=settings.mlflow_tracking_uri,
-                experiment_name=settings.mlflow_experiment_name,
-            )
+                ensure_phoenix_instrumentation(settings, force=True)
+                try:
+                    from evalvault.adapters.outbound.tracker.phoenix_adapter import PhoenixAdapter
+                except ImportError as exc:
+                    raise RuntimeError("Phoenix extras not installed") from exc
+                trackers.append(
+                    (
+                        entry,
+                        PhoenixAdapter(
+                            endpoint=settings.phoenix_endpoint,
+                            project_name=getattr(settings, "phoenix_project_name", None),
+                            annotations_enabled=getattr(
+                                settings,
+                                "phoenix_annotations_enabled",
+                                True,
+                            ),
+                        ),
+                    )
+                )
+                continue
+
+            if entry == "mlflow":
+                if not settings.mlflow_tracking_uri:
+                    raise RuntimeError("MLflow tracking URI missing")
+                try:
+                    from evalvault.adapters.outbound.tracker.mlflow_adapter import MLflowAdapter
+                except ImportError as exc:
+                    raise RuntimeError("MLflow adapter unavailable") from exc
+                trackers.append(
+                    (
+                        entry,
+                        MLflowAdapter(
+                            tracking_uri=settings.mlflow_tracking_uri,
+                            experiment_name=settings.mlflow_experiment_name,
+                        ),
+                    )
+                )
+                continue
 
-        logger.warning("Unknown tracker provider: %s", provider)
-        return None, None
+            raise RuntimeError(f"Unknown tracker provider: {entry}")
+
+        return trackers
 
     @staticmethod
     def _build_phoenix_trace_url(endpoint: str, trace_id: str) -> str:
@@ -425,7 +452,11 @@ class WebUIAdapter:
             dataset.metadata["domain"] = requested_domain
 
         settings = self._settings or Settings()
-        tracker_provider, tracker = self._get_tracker(settings, request.tracker_config)
+        try:
+            trackers = self._get_trackers(settings, request.tracker_config)
+        except RuntimeError as exc:
+            raise RuntimeError(f"Tracker configuration error: {exc}") from exc
+        tracker_providers = [provider for provider, _ in trackers]
         stage_store = bool(request.stage_store)
 
         retriever_instance = None
@@ -484,7 +515,7 @@ class WebUIAdapter:
                 )
                 from evalvault.domain.services.memory_aware_evaluator import MemoryAwareEvaluator
 
-                tracer = PhoenixTracerAdapter() if tracker_provider == "phoenix" else None
+                tracer = PhoenixTracerAdapter() if "phoenix" in tracker_providers else None
                 memory_adapter = build_domain_memory_adapter(
                     settings=self._settings,
                     db_path=Path(memory_db_path) if memory_db_path else None,
@@ -696,22 +727,27 @@ class WebUIAdapter:
                 str(request.threshold_profile).strip().lower()
             )
 
-        if tracker and tracker_provider:
-            try:
-                trace_id = tracker.log_evaluation_run(result)
-                if tracker_provider == "phoenix":
-                    endpoint = settings.phoenix_endpoint or "http://localhost:6006/v1/traces"
-                    phoenix_meta = result.tracker_metadata.setdefault("phoenix", {})
-                    phoenix_meta.update(
-                        {
-                            "trace_id": trace_id,
-                            "endpoint": endpoint,
-                            "trace_url": self._build_phoenix_trace_url(endpoint, trace_id),
-                            "schema_version": 2,
-                        }
-                    )
-            except Exception as exc:
-                logger.warning("Tracker logging failed: %s", exc)
+        if trackers:
+            result.tracker_metadata.setdefault("tracker_providers", tracker_providers)
+            for provider, tracker in trackers:
+                try:
+                    trace_id = tracker.log_evaluation_run(result)
+                    provider_meta = result.tracker_metadata.setdefault(provider, {})
+                    if isinstance(provider_meta, dict):
+                        provider_meta.setdefault("trace_id", trace_id)
+                    if provider == "phoenix":
+                        endpoint = settings.phoenix_endpoint or "http://localhost:6006/v1/traces"
+                        phoenix_meta = result.tracker_metadata.setdefault("phoenix", {})
+                        phoenix_meta.update(
+                            {
+                                "trace_id": trace_id,
+                                "endpoint": endpoint,
+                                "trace_url": self._build_phoenix_trace_url(endpoint, trace_id),
+                                "schema_version": 2,
+                            }
+                        )
+                except Exception as exc:
+                    raise RuntimeError(f"Tracker logging failed for {provider}: {exc}") from exc
 
         if stage_store and self._storage and hasattr(self._storage, "save_stage_events"):
             try:

evalvault/adapters/inbound/api/routers/config.py

@@ -71,7 +71,9 @@ class ConfigUpdateRequest(BaseModel):
     phoenix_endpoint: str | None = None
     phoenix_enabled: bool | None = None
     phoenix_sample_rate: float | None = None
-    tracker_provider: Literal["langfuse", "mlflow", "phoenix", "none"] | None = None
+    phoenix_project_name: str | None = None
+    phoenix_annotations_enabled: bool | None = None
+    tracker_provider: str | None = None
     postgres_host: str | None = None
     postgres_port: int | None = None
     postgres_database: str | None = None

evalvault/adapters/inbound/cli/commands/method.py

@@ -31,7 +31,7 @@ from ..utils.validators import parse_csv_option, validate_choices
 from .run_helpers import (
     _display_results,
     _is_oss_open_model,
-    _log_to_tracker,
+    _log_to_trackers,
     _resolve_thresholds,
     _save_results,
     _save_to_db,
@@ -419,7 +419,7 @@ def create_method_app(console: Console) -> typer.Typer:
         _display_results(result, console)
 
         if tracker and tracker != "none":
-            _log_to_tracker(settings, result, console, tracker_type=tracker)
+            _log_to_trackers(settings, result, console, tracker_type=tracker)
 
         if eval_output:
             _save_results(eval_output, result, console)

evalvault/adapters/inbound/cli/commands/run.py

@@ -33,7 +33,7 @@ from evalvault.adapters.outbound.phoenix.sync_service import (
 from evalvault.adapters.outbound.storage.factory import build_storage_adapter
 from evalvault.adapters.outbound.tracer.phoenix_tracer_adapter import PhoenixTracerAdapter
 from evalvault.config.phoenix_support import ensure_phoenix_instrumentation
-from evalvault.config.settings import Settings, apply_profile
+from evalvault.config.settings import Settings, apply_profile, resolve_tracker_providers
 from evalvault.domain.entities.analysis_pipeline import AnalysisIntent
 from evalvault.domain.entities.multiturn import (
     MultiTurnConversationRecord,
@@ -86,7 +86,8 @@ from .run_helpers import (
     _display_results,
     _evaluate_streaming_run,
     _is_oss_open_model,
-    _log_to_tracker,
+    _log_analysis_artifacts,
+    _log_to_trackers,
     _option_was_provided,
     _print_run_mode_banner,
     _resolve_thresholds,
@@ -178,6 +179,14 @@ def _log_duration(
     _log_timestamp(console, verbose, f"{message} ({elapsed:.2f}s)")
 
 
+def _infer_phoenix_model_provider(model_name: str) -> str:
+    if not model_name:
+        return "OPENAI"
+    provider = model_name.split("/")[0].upper() if "/" in model_name else "OPENAI"
+    allowed = {"OPENAI", "AZURE_OPENAI", "ANTHROPIC", "GOOGLE", "DEEPSEEK", "XAI", "AWS", "OLLAMA"}
+    return provider if provider in allowed else "OPENAI"
+
+
 def register_run_commands(
     app: typer.Typer,
     console: Console,
@@ -358,10 +367,13 @@ def register_run_commands(
             help="Store stage events in the SQLite database (requires --db).",
         ),
         tracker: str = typer.Option(
-            "none",
+            "mlflow+phoenix",
             "--tracker",
             "-t",
-            help="Tracker to log results: 'langfuse', 'mlflow', 'phoenix', or 'none'.",
+            help=(
+                "Tracker to log results: 'langfuse', 'mlflow', 'phoenix', 'none', "
+                "or combinations like 'mlflow+phoenix'."
+            ),
             rich_help_panel="Simple mode preset",
         ),
         langfuse: bool = typer.Option(
@@ -667,13 +679,24 @@ def register_run_commands(
         tracker_override = _option_was_provided(ctx, "tracker") or langfuse
         selected_tracker = tracker
         if preset.default_tracker:
-            if tracker_override and tracker != preset.default_tracker:
-                print_cli_warning(
-                    console,
-                    f"Simple 모드는 tracker={preset.default_tracker}로 고정됩니다.",
-                    tips=["다른 Tracker를 사용하려면 --mode full을 사용하세요."],
-                )
-            selected_tracker = preset.default_tracker
+            if tracker_override:
+                try:
+                    providers = resolve_tracker_providers(tracker)
+                except ValueError as exc:
+                    print_cli_error(console, "Tracker 설정이 올바르지 않습니다.", details=str(exc))
+                    raise typer.Exit(2) from exc
+                if providers == ["none"]:
+                    selected_tracker = preset.default_tracker
+                elif preset.default_tracker not in providers:
+                    print_cli_warning(
+                        console,
+                        f"Simple 모드는 tracker에 {preset.default_tracker}가 포함되어야 합니다.",
+                        tips=["다른 Tracker를 사용하려면 --mode full을 사용하세요."],
+                    )
+                    providers.append(preset.default_tracker)
+                    selected_tracker = "+".join(providers)
+            else:
+                selected_tracker = preset.default_tracker
         tracker = selected_tracker
 
         prompt_manifest_value = prompt_manifest
@@ -1646,10 +1669,29 @@ def register_run_commands(
             )
             raise typer.Exit(2) from exc
 
+        effective_tracker = tracker
+        if langfuse and tracker == "none" and not preset.default_tracker:
+            effective_tracker = "langfuse"
+            print_cli_warning(
+                console,
+                "--langfuse 플래그는 곧 제거됩니다.",
+                tips=["대신 --tracker langfuse를 사용하세요."],
+            )
+
+        try:
+            effective_providers = resolve_tracker_providers(effective_tracker)
+        except ValueError as exc:
+            print_cli_error(console, "Tracker 설정이 올바르지 않습니다.", details=str(exc))
+            raise typer.Exit(2) from exc
+
         phoenix_dataset_name = phoenix_dataset
         if phoenix_experiment and not phoenix_dataset_name:
             phoenix_dataset_name = f"{ds.name}:{ds.version}"
 
+        auto_phoenix_sync = "phoenix" in effective_providers
+        if auto_phoenix_sync and not phoenix_dataset_name:
+            phoenix_dataset_name = f"{ds.name}:{ds.version}"
+
         phoenix_dataset_description_value = phoenix_dataset_description
         if phoenix_dataset_name and not phoenix_dataset_description_value:
             desc_source = ds.metadata.get("description") if isinstance(ds.metadata, dict) else None
@@ -1659,13 +1701,20 @@ def register_run_commands(
         phoenix_dataset_result: dict[str, Any] | None = None
         phoenix_experiment_result: dict[str, Any] | None = None
 
-        if phoenix_dataset_name or phoenix_experiment:
+        if phoenix_dataset_name or phoenix_experiment or auto_phoenix_sync:
             try:
                 phoenix_sync_service = PhoenixSyncService(
                     endpoint=settings.phoenix_endpoint,
                     api_token=getattr(settings, "phoenix_api_token", None),
                 )
             except PhoenixSyncError as exc:
+                if auto_phoenix_sync:
+                    print_cli_error(
+                        console,
+                        "Phoenix Sync 서비스를 초기화할 수 없습니다.",
+                        details=str(exc),
+                    )
+                    raise typer.Exit(2) from exc
                 print_cli_warning(
                     console,
                     "Phoenix Sync 서비스를 초기화할 수 없습니다.",
@@ -1673,19 +1722,10 @@ def register_run_commands(
                 )
                 phoenix_sync_service = None
 
-        effective_tracker = tracker
-        if langfuse and tracker == "none" and not preset.default_tracker:
-            effective_tracker = "langfuse"
-            print_cli_warning(
-                console,
-                "--langfuse 플래그는 곧 제거됩니다.",
-                tips=["대신 --tracker langfuse를 사용하세요."],
-            )
-
         config_wants_phoenix = getattr(settings, "phoenix_enabled", False)
         if not isinstance(config_wants_phoenix, bool):
             config_wants_phoenix = False
-        should_enable_phoenix = effective_tracker == "phoenix" or config_wants_phoenix
+        should_enable_phoenix = "phoenix" in effective_providers or config_wants_phoenix
         if should_enable_phoenix:
             ensure_phoenix_instrumentation(settings, console=console, force=True)
 
@@ -2032,6 +2072,9 @@ def register_run_commands(
             )
             if prompt_bundle:
                 result.tracker_metadata["prompt_set"] = build_prompt_summary(prompt_bundle)
+                result.tracker_metadata["prompt_set_detail"] = prompt_bundle.to_dict(
+                    include_content=True
+                )
 
         if retriever_instance or used_versioned_prefill:
             retriever_tracker_meta: dict[str, Any] = {
@@ -2105,13 +2148,29 @@ def register_run_commands(
                     )
                     console.print(f"[dim]View datasets: {dataset_info.url}[/dim]")
                 except PhoenixSyncError as exc:
+                    if auto_phoenix_sync:
+                        print_cli_error(
+                            console,
+                            "Phoenix Dataset 업로드에 실패했습니다.",
+                            details=str(exc),
+                        )
+                        raise typer.Exit(2) from exc
                     print_cli_warning(
                         console,
                         "Phoenix Dataset 업로드에 실패했습니다.",
                         tips=[str(exc)],
                     )
+            if auto_phoenix_sync and not phoenix_experiment:
+                phoenix_experiment = f"{result.model_name}-{result.run_id[:8]}"
             if phoenix_experiment:
                 if not phoenix_dataset_result:
+                    if auto_phoenix_sync:
+                        print_cli_error(
+                            console,
+                            "Dataset 업로드에 실패해 Phoenix Experiment 생성을 진행할 수 없습니다.",
+                            details="Phoenix dataset 업로드가 필요합니다.",
+                        )
+                        raise typer.Exit(2)
                     print_cli_warning(
                         console,
                         "Dataset 업로드에 실패해 Phoenix Experiment 생성을 건너뜁니다.",
@@ -2169,6 +2228,41 @@ def register_run_commands(
             phoenix_meta = result.tracker_metadata.setdefault("phoenix", {})
             phoenix_meta.setdefault("schema_version", 2)
             phoenix_meta["prompts"] = prompt_metadata_entries
+            if phoenix_sync_service and "phoenix" in effective_providers:
+                try:
+                    prompt_set_summary = result.tracker_metadata.get("prompt_set") or {}
+                    prompt_set_name = prompt_set_summary.get("prompt_set_name")
+                    prompt_entries = list(prompt_metadata_entries)
+                    prompt_set_detail = result.tracker_metadata.get("prompt_set_detail")
+                    if isinstance(prompt_set_detail, dict):
+                        for item in prompt_set_detail.get("items", []):
+                            prompt = item.get("prompt") or {}
+                            if not isinstance(prompt, dict):
+                                continue
+                            prompt_entries.append(
+                                {
+                                    "name": prompt.get("name"),
+                                    "role": item.get("role"),
+                                    "kind": prompt.get("kind"),
+                                    "checksum": prompt.get("checksum"),
+                                    "content": prompt.get("content"),
+                                    "source": prompt.get("source"),
+                                }
+                            )
+                    synced = phoenix_sync_service.sync_prompts(
+                        prompt_entries=prompt_entries,
+                        model_name=result.model_name,
+                        model_provider=_infer_phoenix_model_provider(result.model_name),
+                        prompt_set_name=prompt_set_name,
+                    )
+                    if synced:
+                        phoenix_meta["prompts"] = synced
+                except PhoenixSyncError as exc:
+                    print_cli_warning(
+                        console,
+                        "Phoenix Prompt 동기화에 실패했습니다.",
+                        tips=[str(exc)],
+                    )
 
         if stage_events or stage_store:
             stage_event_builder = StageEventBuilder()
@@ -2187,7 +2281,7 @@ def register_run_commands(
 
         if effective_tracker != "none":
             phoenix_opts = None
-            if effective_tracker == "phoenix":
+            if "phoenix" in effective_providers:
                 phoenix_opts = {
                     "max_traces": phoenix_max_traces,
                     "metadata": phoenix_trace_metadata or None,
@@ -2198,7 +2292,7 @@ def register_run_commands(
                 verbose,
                 f"Tracker 로깅 시작 ({effective_tracker})",
             )
-            _log_to_tracker(
+            _log_to_trackers(
                 settings,
                 result,
                 console,
@@ -2276,6 +2370,12 @@ def register_run_commands(
                     pipeline_result,
                     artifacts_dir=artifacts_dir,
                 )
+                result.tracker_metadata["analysis_artifacts"] = {
+                    "dir": artifact_index.get("dir"),
+                    "index": artifact_index.get("index"),
+                    "output": str(analysis_output_path),
+                    "report": str(analysis_report_path),
+                }
                 payload = serialize_pipeline_result(pipeline_result)
                 payload["run_id"] = result.run_id
                 payload["artifacts"] = artifact_index
@@ -2292,6 +2392,18 @@ def register_run_commands(
                     "[green]자동 분석 상세 결과 저장:[/green] "
                     f"{artifact_index['dir']} (index: {artifact_index['index']})\n"
                 )
+                if effective_tracker != "none":
+                    _log_analysis_artifacts(
+                        settings,
+                        result,
+                        console,
+                        effective_tracker,
+                        analysis_payload=payload,
+                        artifact_index=artifact_index,
+                        report_text=report_text,
+                        output_path=analysis_output_path,
+                        report_path=analysis_report_path,
+                    )
 
     @app.command(
         name="run-simple",
@@ -2395,10 +2507,13 @@ def register_run_commands(
             help="Store stage events in the SQLite database (requires --db).",
         ),
         tracker: str = typer.Option(
-            "none",
+            "mlflow+phoenix",
             "--tracker",
             "-t",
-            help="Tracker to log results: 'langfuse', 'mlflow', 'phoenix', or 'none'.",
+            help=(
+                "Tracker to log results: 'langfuse', 'mlflow', 'phoenix', 'none', "
+                "or combinations like 'mlflow+phoenix'."
+            ),
         ),
         langfuse: bool = typer.Option(
             False,
@@ -2687,10 +2802,13 @@ def register_run_commands(
             help="Store stage events in the SQLite database (requires --db).",
         ),
         tracker: str = typer.Option(
-            "none",
+            "mlflow+phoenix",
             "--tracker",
             "-t",
-            help="Tracker to log results: 'langfuse', 'mlflow', 'phoenix', or 'none'.",
+            help=(
+                "Tracker to log results: 'langfuse', 'mlflow', 'phoenix', 'none', "
+                "or combinations like 'mlflow+phoenix'."
+            ),
         ),
         langfuse: bool = typer.Option(
             False,