evalvault 1.76.0__py3-none-any.whl → 1.77.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -567,11 +567,31 @@ class WebUIAdapter:
 
         # 2. 진행률 초기화
         start_time = time.monotonic()
+        total_cases = len(dataset.test_cases)
+
+        def emit_progress(message: str, *, status: str = "running") -> None:
+            if not on_progress:
+                return
+            elapsed = time.monotonic() - start_time
+            rate = (total_cases / elapsed) if total_cases > 0 and elapsed > 0 else None
+            on_progress(
+                EvalProgress(
+                    current=total_cases,
+                    total=total_cases,
+                    current_metric=message,
+                    percent=100.0 if total_cases > 0 else 0.0,
+                    status=status,
+                    elapsed_seconds=elapsed,
+                    eta_seconds=0.0,
+                    rate=rate,
+                )
+            )
+
         if on_progress:
             on_progress(
                 EvalProgress(
                     current=0,
-                    total=len(dataset.test_cases),
+                    total=total_cases,
                     current_metric="",
                     percent=0.0,
                     status="running",
@@ -728,6 +748,7 @@ class WebUIAdapter:
             )
 
         if trackers:
+            emit_progress("Logging trackers...", status="finalizing")
             result.tracker_metadata.setdefault("tracker_providers", tracker_providers)
             for provider, tracker in trackers:
                 try:
@@ -750,6 +771,7 @@ class WebUIAdapter:
                     raise RuntimeError(f"Tracker logging failed for {provider}: {exc}") from exc
 
         if stage_store and self._storage and hasattr(self._storage, "save_stage_events"):
+            emit_progress("Storing stage events...", status="finalizing")
             try:
                 prompt_metadata_entries = self._build_prompt_metadata_entries(prompt_bundle)
                 stage_event_builder = StageEventBuilder()
@@ -791,6 +813,7 @@ class WebUIAdapter:
 
         # 5. 결과 저장
         if self._storage:
+            emit_progress("Saving evaluation run...", status="finalizing")
             logger.info(f"Saving evaluation run: {result.run_id}")
             if prompt_bundle:
                 self._storage.save_prompt_set(prompt_bundle)

evalvault/adapters/inbound/api/main.py

@@ -15,6 +15,7 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from starlette.responses import JSONResponse
 
 from evalvault.adapters.inbound.api.adapter import WebUIAdapter, create_adapter
+from evalvault.config.runtime_services import ensure_local_observability
 from evalvault.config.settings import Settings, get_settings, is_production_profile
 
 logger = logging.getLogger(__name__)
@@ -63,6 +64,7 @@ async def lifespan(app: FastAPI):
     # Startup: Initialize adapter
     adapter = create_adapter()
     app.state.adapter = adapter
+    ensure_local_observability(get_settings())
     try:
         from evalvault.adapters.inbound.api.routers.chat import warm_rag_index
 

evalvault/adapters/inbound/cli/app.py

@@ -14,6 +14,8 @@ import typer
 from rich import print as rprint
 from rich.console import Console
 
+from evalvault.config.runtime_services import ensure_local_observability
+from evalvault.config.settings import get_settings
 from evalvault.domain.metrics.registry import list_metric_names
 
 from .commands import attach_sub_apps, register_all_commands
@@ -61,6 +63,7 @@ def main(
     ),
 ) -> None:
     """EvalVault - RAG evaluation system."""
+    ensure_local_observability(get_settings())
 
 
 if __name__ == "__main__":  # pragma: no cover

evalvault/adapters/inbound/cli/commands/analyze.py

@@ -358,6 +358,11 @@ def register_analyze_commands(app: typer.Typer, console: Console) -> None:
         profile: str | None = profile_option(
             help_text="비교 리포트용 LLM 프로필 (dev, prod, openai)",
         ),
+        use_llm_report: bool = typer.Option(
+            True,
+            "--use-llm-report/--no-llm-report",
+            help="LLM 보고서 사용 여부",
+        ),
     ) -> None:
         """두 실행을 통계적으로 비교합니다."""
 
@@ -461,7 +466,7 @@ def register_analyze_commands(app: typer.Typer, console: Console) -> None:
                 compare_metrics=metric_list,
                 test_type=test,
                 report_type="comparison",
-                use_llm_report=True,
+                use_llm_report=use_llm_report,
             )
 
         artifacts_dir = resolve_artifact_dir(

evalvault/adapters/inbound/cli/commands/method.py

@@ -424,7 +424,7 @@ def create_method_app(console: Console) -> typer.Typer:
         if eval_output:
             _save_results(eval_output, result, console)
 
-        _save_to_db(db_path, result, console)
+        _save_to_db(settings, db_path, result, console)
 
     return method_app
 

evalvault/adapters/inbound/cli/commands/run.py

@@ -875,7 +875,7 @@ def register_run_commands(
         if profile_name:
             settings = apply_profile(settings, profile_name)
 
-        if db_path is None:
+        if db_path is None and settings.db_backend == "sqlite":
             db_path = Path(settings.evalvault_db_path)
 
         excel_output: Path | None = None
@@ -1221,6 +1221,7 @@ def register_run_commands(
                     for turn in conversation.turn_results:
                         turn_results.append(turn)
                 _save_multiturn_to_db(
+                    settings,
                     db_path,
                     run_record,
                     conversation_records,
@@ -1670,7 +1671,7 @@ def register_run_commands(
             raise typer.Exit(2) from exc
 
         effective_tracker = tracker
-        if langfuse and tracker == "none" and not preset.default_tracker:
+        if langfuse:
             effective_tracker = "langfuse"
             print_cli_warning(
                 console,
@@ -1688,7 +1689,7 @@ def register_run_commands(
         if phoenix_experiment and not phoenix_dataset_name:
             phoenix_dataset_name = f"{ds.name}:{ds.version}"
 
-        auto_phoenix_sync = "phoenix" in effective_providers
+        auto_phoenix_sync = "phoenix" in effective_providers and not stream
         if auto_phoenix_sync and not phoenix_dataset_name:
             phoenix_dataset_name = f"{ds.name}:{ds.version}"
 
@@ -1703,8 +1704,11 @@ def register_run_commands(
 
         if phoenix_dataset_name or phoenix_experiment or auto_phoenix_sync:
             try:
+                phoenix_endpoint = getattr(settings, "phoenix_endpoint", None)
+                if not isinstance(phoenix_endpoint, str) or not phoenix_endpoint.strip():
+                    phoenix_endpoint = "http://localhost:6006/v1/traces"
                 phoenix_sync_service = PhoenixSyncService(
-                    endpoint=settings.phoenix_endpoint,
+                    endpoint=phoenix_endpoint,
                     api_token=getattr(settings, "phoenix_api_token", None),
                 )
             except PhoenixSyncError as exc:
@@ -2304,6 +2308,7 @@ def register_run_commands(
         db_started_at = datetime.now()
         _log_timestamp(console, verbose, "DB 저장 시작")
         _save_to_db(
+            settings,
             db_path,
             result,
             console,

evalvault/adapters/inbound/cli/commands/run_helpers.py

@@ -345,20 +345,18 @@ def _get_tracker(
         )
 
     elif tracker_type == "mlflow":
-        if not settings.mlflow_tracking_uri:
-            message = "MLflow tracking URI가 설정되지 않았습니다."
-            tips = ["MLFLOW_TRACKING_URI 환경 변수를 설정하세요."]
-            if required:
-                print_cli_error(console, message, fixes=tips)
-                raise typer.Exit(2)
-            print_cli_warning(console, message + " 로깅을 건너뜁니다.", tips=tips)
-            return None
+        tracking_uri = getattr(settings, "mlflow_tracking_uri", None)
+        if not isinstance(tracking_uri, str) or not tracking_uri.strip():
+            tracking_uri = f"sqlite:///{Path.cwd() / 'mlruns.db'}"
+        experiment_name = getattr(settings, "mlflow_experiment_name", None)
+        if not isinstance(experiment_name, str) or not experiment_name.strip():
+            experiment_name = "evalvault"
         try:
             from evalvault.adapters.outbound.tracker.mlflow_adapter import MLflowAdapter
 
             return MLflowAdapter(
-                tracking_uri=settings.mlflow_tracking_uri,
-                experiment_name=settings.mlflow_experiment_name,
+                tracking_uri=tracking_uri,
+                experiment_name=experiment_name,
             )
         except ImportError:
             message = "MLflow extra가 설치되지 않았습니다."
@@ -373,8 +371,11 @@ def _get_tracker(
         try:
             from evalvault.adapters.outbound.tracker.phoenix_adapter import PhoenixAdapter
 
+            endpoint = getattr(settings, "phoenix_endpoint", None)
+            if not isinstance(endpoint, str) or not endpoint.strip():
+                endpoint = "http://localhost:6006/v1/traces"
             return PhoenixAdapter(
-                endpoint=settings.phoenix_endpoint,
+                endpoint=endpoint,
                 service_name="evalvault",
                 project_name=getattr(settings, "phoenix_project_name", None),
                 annotations_enabled=getattr(settings, "phoenix_annotations_enabled", True),
@@ -407,9 +408,6 @@ def _resolve_tracker_list(tracker_type: str) -> list[str]:
     unknown = [entry for entry in providers if entry not in supported]
     if unknown:
         raise ValueError(f"Unknown tracker provider(s): {', '.join(unknown)}")
-    required = {"mlflow", "phoenix"}
-    if not required.issubset(set(providers)):
-        raise ValueError("tracker must include both 'mlflow' and 'phoenix'")
     return providers
 
 
@@ -555,6 +553,7 @@ def _log_analysis_artifacts(
 
 
 def _save_to_db(
+    settings: Settings,
     db_path: Path | None,
     result,
     console: Console,
@@ -563,7 +562,7 @@ def _save_to_db(
     export_excel: bool = True,
 ) -> None:
     """Persist evaluation run (and optional prompt set) to database."""
-    storage = build_storage_adapter(settings=Settings(), db_path=db_path)
+    storage = build_storage_adapter(settings=settings, db_path=db_path)
     storage_label = (
         "PostgreSQL" if isinstance(storage, PostgreSQLStorageAdapter) else f"SQLite ({db_path})"
     )
@@ -607,6 +606,7 @@ def _save_to_db(
 
 
 def _save_multiturn_to_db(
+    settings: Settings,
     db_path: Path | None,
     run_record: MultiTurnRunRecord,
     conversations: list[MultiTurnConversationRecord],
@@ -618,7 +618,7 @@ def _save_multiturn_to_db(
     metric_thresholds: dict[str, float] | None = None,
 ) -> None:
     """Persist multiturn evaluation run to database."""
-    storage = build_storage_adapter(settings=Settings(), db_path=db_path)
+    storage = build_storage_adapter(settings=settings, db_path=db_path)
     storage_label = (
         "PostgreSQL" if isinstance(storage, PostgreSQLStorageAdapter) else f"SQLite ({db_path})"
     )
@@ -840,6 +840,8 @@ def log_phoenix_traces(
         return 0
 
     limit = max_traces if max_traces is not None else run.total_test_cases
+    if not isinstance(limit, int):
+        limit = None
 
     count = 0
     for result in run.results: