evalvault 1.75.0__py3-none-any.whl → 1.76.0__py3-none-any.whl

evalvault/config/settings.py

@@ -55,6 +55,33 @@ def _parse_cors_origins(cors_origins: str | None) -> list[str]:
     return [origin.strip() for origin in cors_origins.split(",") if origin.strip()]
 
 
+def resolve_tracker_providers(provider: str | None) -> list[str]:
+    if not provider:
+        return []
+    normalized = provider.strip().lower()
+    if normalized in {"none", "off", "disabled"}:
+        return ["none"]
+    aliases = {
+        "all": ["mlflow", "phoenix"],
+        "default": ["mlflow", "phoenix"],
+    }
+    if normalized in aliases:
+        return aliases[normalized]
+    separators = [",", "+", "/", "|"]
+    for sep in separators:
+        normalized = normalized.replace(sep, ",")
+    providers = [p.strip() for p in normalized.split(",") if p.strip()]
+    if not providers:
+        return []
+    if "none" in providers and len(providers) > 1:
+        raise ValueError("tracker_provider cannot combine 'none' with other providers")
+    deduped: list[str] = []
+    for entry in providers:
+        if entry not in deduped:
+            deduped.append(entry)
+    return deduped
+
+
 SECRET_REFERENCE_FIELDS = (
     "api_auth_tokens",
     "knowledge_read_tokens",
@@ -83,13 +110,14 @@ def _validate_production_settings(settings: "Settings") -> None:
     if settings.llm_provider == "openai" and not settings.openai_api_key:
         missing.append("OPENAI_API_KEY")
 
-    if settings.tracker_provider == "langfuse":
+    providers = resolve_tracker_providers(settings.tracker_provider)
+    if "langfuse" in providers:
         if not settings.langfuse_public_key:
             missing.append("LANGFUSE_PUBLIC_KEY")
         if not settings.langfuse_secret_key:
             missing.append("LANGFUSE_SECRET_KEY")
 
-    if settings.tracker_provider == "mlflow" and not settings.mlflow_tracking_uri:
+    if "mlflow" in providers and not settings.mlflow_tracking_uri:
         missing.append("MLFLOW_TRACKING_URI")
 
     if (
@@ -355,6 +383,14 @@ class Settings(BaseSettings):
         default="http://localhost:6006/v1/traces",
         description="Phoenix OTLP endpoint for traces",
     )
+    phoenix_project_name: str = Field(
+        default="evalvault",
+        description="Phoenix project name for grouping traces",
+    )
+    phoenix_annotations_enabled: bool = Field(
+        default=True,
+        description="Enable automatic Phoenix span annotations",
+    )
     phoenix_api_token: str | None = Field(
         default=None,
         description="Phoenix API token for cloud deployments (optional)",
@@ -372,8 +408,8 @@ class Settings(BaseSettings):
 
     # Tracker Provider Selection
     tracker_provider: str = Field(
-        default="langfuse",
-        description="Tracker provider: 'langfuse', 'mlflow', or 'phoenix'",
+        default="mlflow+phoenix",
+        description="Tracker provider: 'langfuse', 'mlflow', 'phoenix', 'none', or combinations",
     )
 
     # Cluster map configuration

evalvault/domain/services/evaluator.py

@@ -1934,6 +1934,8 @@ class RagasEvaluator:
 
     def _calculate_cost(self, model_name: str, prompt_tokens: int, completion_tokens: int) -> float:
         """Calculate estimated cost in USD based on model pricing."""
+        if "ollama" in model_name:
+            return 0.0
         # Find matching model key (exact or substring match)
         price_key = "openai/gpt-4o"  # Default fallback
         for key in self.MODEL_PRICING:

{evalvault-1.75.0.dist-info → evalvault-1.76.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.75.0
+Version: 1.76.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme

{evalvault-1.75.0.dist-info → evalvault-1.76.0.dist-info}/RECORD

@@ -5,13 +5,13 @@ evalvault/mkdocs_helpers.py,sha256=1AKVQ1W2_VO4qclhfyefyU9Dz1Hzkh1DWDwsFMe24jc,3
 evalvault/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalvault/adapters/inbound/__init__.py,sha256=SG1svel1PwqetnqVpKFLSv612_WwGwLTbFpYgwk6FMw,166
 evalvault/adapters/inbound/api/__init__.py,sha256=LeVVttCA3tLKoHA2PO4z3y8VkfVcf3Bq8CZSzo91lf4,34
-evalvault/adapters/inbound/api/adapter.py,sha256=AQ6-4_wzvszIPlq61y4V9MOZyuM-hvhrw0wVrFPHLUk,85145
+evalvault/adapters/inbound/api/adapter.py,sha256=wzudfkMALcX3IH7MfG-dYc5j-nzxBdDBT1X8_vJWick,86848
 evalvault/adapters/inbound/api/main.py,sha256=QgLxzHEy7aycGKIFLtN12tWTjnpWLtQ2XDXKV_2FDvg,7531
 evalvault/adapters/inbound/api/routers/__init__.py,sha256=q07_YF9TnBl68bqcRCvhPU4-zRTyvmPoHVehwO6W7QM,19
 evalvault/adapters/inbound/api/routers/benchmark.py,sha256=yevntbZcNtMvbVODsITUBgR1Ka4pdFQrXBJJ4K4Jyr4,4477
 evalvault/adapters/inbound/api/routers/calibration.py,sha256=FzEOe7KqDtfGDlOKREsAoYId1jTILyqkDm_Y3LyHk54,4130
 evalvault/adapters/inbound/api/routers/chat.py,sha256=N6lTO_rAswtpL0xusnZk-Gh_ShjlJgM8haRMynwNxVo,42023
-evalvault/adapters/inbound/api/routers/config.py,sha256=LygN0fVMr8NFtj5zuQXnVFhoafx56Txa98vpwtPa4Jc,4104
+evalvault/adapters/inbound/api/routers/config.py,sha256=U6dLwJTT2m54yTC5dK8Pi2FWJfclEGP947Mo24sBWZM,4155
 evalvault/adapters/inbound/api/routers/domain.py,sha256=Yc94E_lEEB5LERsBRwYeKXnou8PFaysLXauBbtTzfGM,3826
 evalvault/adapters/inbound/api/routers/knowledge.py,sha256=yb_e7OEPtwldOAzHTGiWe7jShHw2JdpOFnzGPMceRsg,7109
 evalvault/adapters/inbound/api/routers/mcp.py,sha256=yHANV7qIXig-7YSiQgXzSTuabqFStH5yT3URyQGY2W4,4764
@@ -39,15 +39,15 @@ evalvault/adapters/inbound/cli/commands/history.py,sha256=dGkoRz59FZCbs-Af29MxgV
 evalvault/adapters/inbound/cli/commands/init.py,sha256=7q86fUeBVA08fU_N0lAV6Lakxirq4val2jIyALlDy3E,8822
 evalvault/adapters/inbound/cli/commands/kg.py,sha256=ycV9Xj6SUUJLTyTfLZcjXDVLcZqwo7Gw878ZhZAeDoc,19155
 evalvault/adapters/inbound/cli/commands/langfuse.py,sha256=aExhZ5WYT0FzJI4v1sF-a1jqy9b1BF46_HBtfiQjVGI,4085
-evalvault/adapters/inbound/cli/commands/method.py,sha256=s4O4nM0IMT5-VAkpesSQlJWXvvkz_ztUUkNaGDwWNGA,19088
+evalvault/adapters/inbound/cli/commands/method.py,sha256=2DuMD1AMaQr9gl6xAQty_YKdq5MA9Y-FMYMmERnf9-4,19090
 evalvault/adapters/inbound/cli/commands/ops.py,sha256=jh0ryYIDvj2VDhOR6bESPHnyogtzAfSs81gKMGlOczs,4042
 evalvault/adapters/inbound/cli/commands/phoenix.py,sha256=LQi3KTLq1ybjjBuz92oQ6lYyBS3mHrCHk0qe-7bqB4U,15611
 evalvault/adapters/inbound/cli/commands/pipeline.py,sha256=nBsgSG9RvwDRy_X-01DqZe0jS9zpp1j4hkccni3sruc,12253
 evalvault/adapters/inbound/cli/commands/profile_difficulty.py,sha256=PF8MbYzufbNVlKKnBp6JF9PrBe-ncGDOe8JHWW0L278,6329
 evalvault/adapters/inbound/cli/commands/prompts.py,sha256=ExbTG_8yZupuw6em6OPOdTOMc4T-YPc7tUx5_cbXeiU,26708
 evalvault/adapters/inbound/cli/commands/regress.py,sha256=VzwTucxJav71Xmd18qCb93XG_mTvfjmW52SQRv8Sgu4,19082
-evalvault/adapters/inbound/cli/commands/run.py,sha256=BkuOFHZISJ_aO_0Sy-xNPdbqSEDbT2PXLllGhbeuggk,130710
-evalvault/adapters/inbound/cli/commands/run_helpers.py,sha256=-1d7GTnCnSz1DwmuwRy5E2_WNU_B0x0wvLnohU1kIWk,43739
+evalvault/adapters/inbound/cli/commands/run.py,sha256=H65pQ_bbUST_d6_OdUDU9ZS-FMLE8FMy4KXLWj11Qzg,136577
+evalvault/adapters/inbound/cli/commands/run_helpers.py,sha256=yFt7jzZX8NEClCJdwOt9LfrpIhD5J_m9HD-XSrSLP0I,47994
 evalvault/adapters/inbound/cli/commands/stage.py,sha256=XiPfJrdtUWrdSw5EzLgvz5WEEBW1qz1FALEhDYamMjE,23236
 evalvault/adapters/inbound/cli/utils/__init__.py,sha256=QPNKneZS-Z-tTnYYxtgJXgcJWY6puUlRQcKrn7Mlv1M,685
 evalvault/adapters/inbound/cli/utils/analysis_io.py,sha256=RHkKEq4e-PtbtRDlXAJWU80RYHNPw-O5V9_GujdaGfc,13393
@@ -162,7 +162,7 @@ evalvault/adapters/outbound/llm/__init__.py,sha256=MZODPJbwpMuuubG7cxXEh75V3e0r6
 evalvault/adapters/outbound/llm/anthropic_adapter.py,sha256=LTU3RhtgaKNLvMpwj1-GGCclsBZM2dzVJ1OIMO9BHqI,8162
 evalvault/adapters/outbound/llm/azure_adapter.py,sha256=FR9Gimy-jonzQjtILGHKXAY_ycbjGKbKLFiD-OZ8EVs,3033
 evalvault/adapters/outbound/llm/base.py,sha256=c4cJJTKwJiBlGNCV7qomq5mQCrbc48Yi7c3vjlat7Q0,7674
-evalvault/adapters/outbound/llm/factory.py,sha256=TTpbyDQH9TCr1wmajSkLgkF_fHGv6sX0hZOEqIWZudU,3509
+evalvault/adapters/outbound/llm/factory.py,sha256=Ee8VTIDK46JmfQ7nqxQd1onr0qizA7pzU8l9_7L3qsE,3496
 evalvault/adapters/outbound/llm/instructor_factory.py,sha256=6iJIjZbCJwDYEpV673R3lQOupzeWlvjCmhjYqthrueY,4497
 evalvault/adapters/outbound/llm/llm_relation_augmenter.py,sha256=B22CJB66Vfh_SK_11J8jlfIhjDmY_7FXp-bTL2YGWGw,4848
 evalvault/adapters/outbound/llm/ollama_adapter.py,sha256=HuPjX9rhBn95OzwNNXOgD-FIOKSUNuHEcNKDeHxw8kA,11481
@@ -186,7 +186,7 @@ evalvault/adapters/outbound/nlp/korean/toolkit.py,sha256=iMaY5VueC5P0rxm5SCdz69C
 evalvault/adapters/outbound/nlp/korean/toolkit_factory.py,sha256=x3v-AAkVInOabC4PtOtStsZrFnHun0IOqZDyQGaQVm8,586
 evalvault/adapters/outbound/ops/__init__.py,sha256=_QiDVPuiYWkIwW_ELEVKD_v6dLojjyvIJWs4qVNxehw,164
 evalvault/adapters/outbound/ops/report_renderer.py,sha256=mezVKdIsnJSNvBW6xkhpNG3MOFXHZLZspmHk5o-e8Cg,6354
-evalvault/adapters/outbound/phoenix/sync_service.py,sha256=i6gHpNiZXKQ5yzV9B2TPb-P1N45k_Ck5ruzh3oqp4d8,9122
+evalvault/adapters/outbound/phoenix/sync_service.py,sha256=3rNtolFzwbRbmsP7fCh6lMDcpEf428mLqpZuw1tYW3M,13271
 evalvault/adapters/outbound/report/__init__.py,sha256=8VeMrfj63mDR-xUHct-drNNBA5M-m-B7sgC1qUJF7g4,660
 evalvault/adapters/outbound/report/ci_report_formatter.py,sha256=5YD8BwtOjLnHcNbbG0HJziOifD9BDhBtZT1oItd6zJE,1233
 evalvault/adapters/outbound/report/dashboard_generator.py,sha256=g0SANOrOS_mSfLfKbfbxflxZmCNbZGrGzA0g03_Yb48,8356
@@ -213,17 +213,17 @@ evalvault/adapters/outbound/tracer/phoenix_tracer_adapter.py,sha256=inmTAolAVsm0
 evalvault/adapters/outbound/tracker/__init__.py,sha256=Suu5BznOK5uTuD5_jS8JMZd8RPfQNlddLxHCBvMTm_4,358
 evalvault/adapters/outbound/tracker/langfuse_adapter.py,sha256=uI-t5v9AC5VUMYsIc1FHYImourZeErGMXB0_prOMErc,18839
 evalvault/adapters/outbound/tracker/log_sanitizer.py,sha256=ilKTTSzsHslQYc-elnWu0Z3HKNNw1D1iI0_cCvYbo1M,2653
-evalvault/adapters/outbound/tracker/mlflow_adapter.py,sha256=6pSxbxSDZE7jN7uSMU6VFg0JlO7cBiMLYcd53NYpfcY,7350
-evalvault/adapters/outbound/tracker/phoenix_adapter.py,sha256=8p2qJeKn6OvIxNbD16h_QrhmCzKIBIf8_ej535MNn_A,26443
+evalvault/adapters/outbound/tracker/mlflow_adapter.py,sha256=8vo53d2ZpcYDz3J798iru7Sk-Wm3XhARJMVmS5d6pRE,13495
+evalvault/adapters/outbound/tracker/phoenix_adapter.py,sha256=EXEhoOFTFNSVuSnN2wFIH3H1kq4T14601sGIcRIy8hs,32947
 evalvault/config/__init__.py,sha256=UCgeDx62M2gOuFvdN29wWwny2fdH4bPY_uUC3-42eDw,1297
 evalvault/config/agent_types.py,sha256=EP2Pv3ZtOzDXIvIa-Hnd1to9JIbMUtGitrlwzZtx0Ys,13418
 evalvault/config/domain_config.py,sha256=rOgNA2T8NWlDzcEFC0shdUCCww0lI1E5fUm5QrKQSZI,9264
-evalvault/config/instrumentation.py,sha256=L8on9HjB6Ji8cSOJ6Pepsopfg9okDNMWF7LKZiCA6Ro,5266
+evalvault/config/instrumentation.py,sha256=BW2a3OqHH6aQNjjlL0oyEI6K3pXuORy6x__-Zfm0ocY,5403
 evalvault/config/langfuse_support.py,sha256=DEzVMfMGGf1V45W_2oUG-NCDfsYI4UUdnYJIgBSrN2o,582
 evalvault/config/model_config.py,sha256=KlzDbGyDLeOGE7ElekFFk5YjjT5u8i6KO2B4EyZkLnI,3542
-evalvault/config/phoenix_support.py,sha256=e6RPWd6Qb7KU6Q8pLaYTpJGWULtvEEU6B0xHWyVyOH0,13604
+evalvault/config/phoenix_support.py,sha256=w2W8UQTprac8BWdIfV9kPjr2sEj7EF9pNBXX2JB1x4M,13810
 evalvault/config/secret_manager.py,sha256=YjPMuNqeBrAR2BzCJvsBNUExaU4TBSFyZ8kVYZZifqA,4172
-evalvault/config/settings.py,sha256=yBntqR4PsznvARCRyP_1sMC53x2EziZhT5BWV_3qtGA,19455
+evalvault/config/settings.py,sha256=2EoO6WutkwD2F7rkyECxx5-VenivntOXPZwemVQ9ZtU,20696
 evalvault/config/playbooks/improvement_playbook.yaml,sha256=9F9WVVCydFfz6zUuGYzZ4PKdW1LLtcBKVF36T7xT764,26965
 evalvault/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalvault/domain/entities/__init__.py,sha256=7adWG4vwr67fC_ysT1rSvYG5U7qVc2CfWfUf7gDhx-k,4244
@@ -284,7 +284,7 @@ evalvault/domain/services/document_versioning.py,sha256=M1qZaMpQ2exVT1wkVAmvEPPu
 evalvault/domain/services/domain_learning_hook.py,sha256=fNbOQmQ7nsQJMpfWqEOcLxOiWG0deCkjBOOZZFOsIvk,7268
 evalvault/domain/services/embedding_overlay.py,sha256=ZTNxUPXpHGbQ3Uri5DD3feTUFn7qrhuNshhyCQEvRuM,3559
 evalvault/domain/services/entity_extractor.py,sha256=f3Rf5saK8QsgetLNK1Hbxzt8PtttJZCicSR63S8DJ5k,14141
-evalvault/domain/services/evaluator.py,sha256=Iud5KWompKyMm59qsaH0jWkNL4nl_A7UI3pES-rCjcA,78921
+evalvault/domain/services/evaluator.py,sha256=aJp-vgJxlXPHMMjcnjcVDoq8gcwdNb1IfVN8sup7u7M,78979
 evalvault/domain/services/experiment_comparator.py,sha256=IBrxIwux-8GucwlLx6e5lUqB9miSPvBLGJK9ctoW7Y0,3299
 evalvault/domain/services/experiment_manager.py,sha256=2k-qGiAUyZuqqmcp4P-M3Z9HTXwwcqW5HQYKNkcIHuI,4863
 evalvault/domain/services/experiment_reporter.py,sha256=QYlVmCFSx8hKTPMezc7QjJE07b3MSQ82Q4QVucSHLVY,1420
@@ -364,8 +364,8 @@ evalvault/reports/__init__.py,sha256=Bb1X4871msAN8I6PM6nKGED3psPwZt88hXZBAOdH06Y
 evalvault/reports/release_notes.py,sha256=pZj0PBFT-4F_Ty-Kv5P69BuoOnmTCn4kznDcORFJd0w,4011
 evalvault/scripts/__init__.py,sha256=NwEeIFQbkX4ml2R_PhtIoNtArDSX_suuoymgG_7Kwso,89
 evalvault/scripts/regression_runner.py,sha256=SxZori5BZ8jVQ057Mf5V5FPgIVDccrV5oRONmnhuk8w,8438
-evalvault-1.75.0.dist-info/METADATA,sha256=Jb-JFy3QXbPR4UpLR-TgNRhhnDTn604-zH6lFiq3Fvc,8752
-evalvault-1.75.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evalvault-1.75.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
-evalvault-1.75.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
-evalvault-1.75.0.dist-info/RECORD,,
+evalvault-1.76.0.dist-info/METADATA,sha256=HleQUSCdC3UNzeCVN4PdvcOG5Jx-dYfMgaq8VwRnlYQ,8752
+evalvault-1.76.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evalvault-1.76.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
+evalvault-1.76.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
+evalvault-1.76.0.dist-info/RECORD,,