evalvault 1.75.0__py3-none-any.whl → 1.77.0__py3-none-any.whl

evalvault/adapters/outbound/tracker/phoenix_adapter.py

@@ -52,6 +52,8 @@ class PhoenixAdapter(TrackerPort):
         self,
         endpoint: str = "http://localhost:6006/v1/traces",
         service_name: str = "evalvault",
+        project_name: str | None = None,
+        annotations_enabled: bool = True,
     ):
         """Initialize Phoenix adapter with OpenTelemetry.
 
@@ -61,11 +63,14 @@ class PhoenixAdapter(TrackerPort):
         """
         self._endpoint = endpoint
         self._service_name = service_name
+        self._project_name = project_name
+        self._annotations_enabled = annotations_enabled
         self._tracer: Any | None = None
         self._tracer_provider: TracerProvider | None = None
         self._active_spans: dict[str, Any] = {}
         self._tracer_any: Any | None = None
         self._initialized = False
+        self._annotations_client: Any | None = None
 
     def _ensure_initialized(self) -> None:
         """Lazy initialization of OpenTelemetry tracer."""
@@ -96,7 +101,10 @@ class PhoenixAdapter(TrackerPort):
                     return
 
             # Create resource with service name
-            resource = Resource.create({"service.name": self._service_name})
+            resource_attributes = {"service.name": self._service_name}
+            if self._project_name:
+                resource_attributes["project.name"] = self._project_name
+            resource = Resource.create(resource_attributes)
 
             # Create tracer provider
             self._tracer_provider = TracerProvider(resource=resource)
@@ -123,6 +131,64 @@ class PhoenixAdapter(TrackerPort):
                 "Failed to initialize Phoenix tracer. Check endpoint configuration and dependencies."
             ) from e
 
+    def _phoenix_base_url(self) -> str:
+        if "/v1/traces" in self._endpoint:
+            return self._endpoint.split("/v1/traces")[0]
+        return self._endpoint.rstrip("/")
+
+    def _get_annotations_client(self) -> Any | None:
+        if not self._annotations_enabled:
+            return None
+        if self._annotations_client is not None:
+            return self._annotations_client
+        try:
+            from phoenix.client import Client
+        except Exception:
+            return None
+        self._annotations_client = Client(base_url=self._phoenix_base_url())
+        return self._annotations_client
+
+    def _annotate_span(
+        self,
+        *,
+        span: Any,
+        name: str,
+        label: str,
+        score: float | None = None,
+        explanation: str | None = None,
+    ) -> None:
+        client = self._get_annotations_client()
+        if client is None or span is None:
+            return
+        try:
+            from opentelemetry.trace import format_span_id
+
+            span_id = format_span_id(span.get_span_context().span_id)
+            spans_client = getattr(client, "spans", None)
+            add_span_annotation = (
+                getattr(spans_client, "add_span_annotation", None) if spans_client else None
+            )
+            if callable(add_span_annotation):
+                add_span_annotation(
+                    annotation_name=name,
+                    annotator_kind="CODE",
+                    span_id=span_id,
+                    label=label,
+                    score=score,
+                    explanation=explanation,
+                )
+                return
+            client.annotations.add_span_annotation(
+                annotation_name=name,
+                annotator_kind="CODE",
+                span_id=span_id,
+                label=label,
+                score=score,
+                explanation=explanation,
+            )
+        except Exception:
+            return
+
     def start_trace(self, name: str, metadata: dict[str, Any] | None = None) -> str:
         """Start a new trace.
 
@@ -287,6 +353,7 @@ class PhoenixAdapter(TrackerPort):
 
         # Calculate per-metric summary
         metric_summary = {}
+        total_count = len(run.results) if run.results else 0
         for metric_name in run.metrics_evaluated:
             passed_count = sum(
                 1
@@ -299,9 +366,9 @@ class PhoenixAdapter(TrackerPort):
                 "average_score": round(avg_score, 4) if avg_score else 0.0,
                 "threshold": threshold,
                 "passed": passed_count,
-                "failed": len(run.results) - passed_count,
-                "total": len(run.results),
-                "pass_rate": round(passed_count / len(run.results), 4) if run.results else 0.0,
+                "failed": total_count - passed_count,
+                "total": total_count,
+                "pass_rate": round(passed_count / total_count, 4) if total_count else 0.0,
             }
 
         # Start root trace
@@ -328,8 +395,17 @@ class PhoenixAdapter(TrackerPort):
 
         # Set evaluation-specific attributes
         span = self._active_spans[trace_id]
+        span.set_attribute("openinference.span.kind", "EVALUATOR")
         span.set_attribute("evaluation.metrics", json.dumps(run.metrics_evaluated))
         span.set_attribute("evaluation.thresholds", json.dumps(run.thresholds))
+        span.set_attribute("evaluation.status", "pass" if run.pass_rate >= 1.0 else "fail")
+        if run.tracker_metadata:
+            project_name = run.tracker_metadata.get("project_name")
+            if project_name:
+                span.set_attribute("project.name", project_name)
+            project_kind = run.tracker_metadata.get("evaluation_task") or "evaluation"
+            span.set_attribute("project.kind", project_kind)
+            span.set_attribute("project.status", "pass" if run.pass_rate >= 1.0 else "fail")
 
         # Log average scores for each metric
         for metric_name, summary in metric_summary.items():
@@ -369,6 +445,8 @@ class PhoenixAdapter(TrackerPort):
             },
             "metrics": metric_summary,
             "custom_metrics": (run.tracker_metadata or {}).get("custom_metric_snapshot"),
+            "prompt_metadata": (run.tracker_metadata or {}).get("phoenix", {}).get("prompts"),
+            "tracker_metadata": run.tracker_metadata,
             "test_cases": [
                 {
                     "test_case_id": result.test_case_id,
@@ -420,6 +498,23 @@ class PhoenixAdapter(TrackerPort):
             f"test-case-{result.test_case_id}",
             context=context,
         ) as span:
+            try:
+                from opentelemetry.trace import Status, StatusCode
+
+                span.set_status(Status(StatusCode.OK if result.all_passed else StatusCode.ERROR))
+            except Exception:
+                pass
+            span.set_attribute("openinference.span.kind", "EVALUATOR")
+            span.set_attribute("evaluation.status", "pass" if result.all_passed else "fail")
+            self._annotate_span(
+                span=span,
+                name="evaluation_result",
+                label="pass" if result.all_passed else "fail",
+                score=1.0 if result.all_passed else 0.0,
+                explanation="All metrics passed"
+                if result.all_passed
+                else "One or more metrics failed",
+            )
             # Input data
             safe_question = sanitize_text(result.question, max_chars=MAX_LOG_CHARS) or ""
             safe_answer = sanitize_text(result.answer, max_chars=MAX_LOG_CHARS) or ""
@@ -439,6 +534,10 @@ class PhoenixAdapter(TrackerPort):
             # Metrics
             span.set_attribute("output.all_passed", result.all_passed)
             span.set_attribute("output.tokens_used", result.tokens_used)
+            if result.tokens_used:
+                span.set_attribute("llm.token_count.total", result.tokens_used)
+            if result.cost_usd is not None:
+                span.set_attribute("llm.cost.total", result.cost_usd)
 
             for metric in result.metrics:
                 span.set_attribute(f"metric.{metric.name}.score", metric.score)
@@ -486,6 +585,7 @@ class PhoenixAdapter(TrackerPort):
                 )
             if result.latency_ms:
                 span.set_attribute("timing.latency_ms", result.latency_ms)
+                span.set_attribute("evaluation.latency_ms", result.latency_ms)
 
     def log_retrieval(
         self,
@@ -528,6 +628,13 @@ class PhoenixAdapter(TrackerPort):
         if tracer is None:
             raise RuntimeError("Phoenix tracer is not initialized")
         with tracer.start_span("retrieval", context=context) as span:
+            try:
+                from opentelemetry.trace import Status, StatusCode
+
+                span.set_status(Status(StatusCode.OK))
+            except Exception:
+                pass
+            span.set_attribute("openinference.span.kind", "RETRIEVER")
             # Set retrieval attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)
@@ -541,14 +648,24 @@ class PhoenixAdapter(TrackerPort):
 
             span.set_attribute("spec.version", "0.1")
             span.set_attribute("rag.module", "retrieve")
+            if data.retrieval_time_ms:
+                span.set_attribute("retrieval.latency_ms", data.retrieval_time_ms)
 
             documents_payload = _build_retrieval_payload(data.candidates)
             span.set_attribute("custom.retrieval.doc_count", len(documents_payload))
             if documents_payload:
                 span.set_attribute("retrieval.documents_json", serialize_json(documents_payload))
-                doc_ids = _extract_doc_ids(documents_payload)
-                if doc_ids:
-                    span.set_attribute("output.value", doc_ids)
+                previews = [
+                    item.get("content_preview")
+                    for item in documents_payload
+                    if item.get("content_preview")
+                ]
+                if previews:
+                    span.set_attribute("output.value", previews)
+                else:
+                    doc_ids = _extract_doc_ids(documents_payload)
+                    if doc_ids:
+                        span.set_attribute("output.value", doc_ids)
 
             # Log each retrieved document as an event
             for i, doc in enumerate(data.candidates):
@@ -615,10 +732,31 @@ class PhoenixAdapter(TrackerPort):
         if tracer is None:
             raise RuntimeError("Phoenix tracer is not initialized")
         with tracer.start_span("generation", context=context) as span:
+            try:
+                from opentelemetry.trace import Status, StatusCode
+
+                span.set_status(Status(StatusCode.OK))
+            except Exception:
+                pass
+            span.set_attribute("openinference.span.kind", "LLM")
             # Set generation attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)
 
+            if data.model:
+                span.set_attribute("llm.model_name", data.model)
+                provider = data.model.split("/")[0] if "/" in data.model else ""
+                if provider:
+                    span.set_attribute("llm.provider", provider)
+            if data.input_tokens:
+                span.set_attribute("llm.token_count.prompt", data.input_tokens)
+            if data.output_tokens:
+                span.set_attribute("llm.token_count.completion", data.output_tokens)
+            if data.total_tokens:
+                span.set_attribute("llm.token_count.total", data.total_tokens)
+            if data.cost_usd is not None:
+                span.set_attribute("llm.cost.total", data.cost_usd)
+
             # Set prompt/response (truncate if too long)
             prompt = sanitize_text(data.prompt, max_chars=MAX_LOG_CHARS) or ""
             response = sanitize_text(data.response, max_chars=MAX_LOG_CHARS) or ""
@@ -637,6 +775,13 @@ class PhoenixAdapter(TrackerPort):
                 safe_template = sanitize_text(data.prompt_template, max_chars=MAX_LOG_CHARS)
                 if safe_template:
                     span.set_attribute("generation.prompt_template", safe_template)
+                    span.set_attribute("llm.prompt_template.template", safe_template)
+                    span.set_attribute("llm.prompt_template.version", "v1")
+            prompt_vars = data.metadata.get("prompt_variables") if data.metadata else None
+            if prompt_vars:
+                span.set_attribute(
+                    "llm.prompt_template.variables", json.dumps(prompt_vars, default=str)
+                )
 
     def log_rag_trace(self, data: RAGTraceData) -> str:
         """Log a full RAG trace (retrieval + generation) to Phoenix."""
@@ -660,6 +805,8 @@ class PhoenixAdapter(TrackerPort):
             span = self._active_spans[trace_id]
             should_end = True
 
+        span.set_attribute("openinference.span.kind", "CHAIN")
+
         for key, value in data.to_span_attributes().items():
             span.set_attribute(key, value)
 
@@ -667,11 +814,23 @@ class PhoenixAdapter(TrackerPort):
             self.log_retrieval(trace_id, data.retrieval)
         if data.generation:
             self.log_generation(trace_id, data.generation)
+        output_preview = ""
         if data.final_answer:
-            preview = sanitize_text(data.final_answer, max_chars=MAX_LOG_CHARS)
-            if preview:
-                span.set_attribute("rag.final_answer", preview)
-                span.set_attribute("output.value", preview)
+            output_preview = sanitize_text(data.final_answer, max_chars=MAX_LOG_CHARS)
+        if not output_preview and data.generation and data.generation.response:
+            output_preview = sanitize_text(data.generation.response, max_chars=MAX_LOG_CHARS)
+        if not output_preview and data.retrieval:
+            previews: list[str] = []
+            for doc in data.retrieval.candidates:
+                if not doc.content:
+                    continue
+                preview = sanitize_text(doc.content, max_chars=MAX_CONTEXT_CHARS)
+                if preview:
+                    previews.append(preview)
+            output_preview = "\n".join(previews[:3])
+        if output_preview:
+            span.set_attribute("rag.final_answer", output_preview)
+            span.set_attribute("output.value", output_preview)
 
         if safe_query:
             span.set_attribute("input.value", safe_query)
@@ -697,7 +856,14 @@ def _build_retrieval_payload(
     payload: list[dict[str, Any]] = []
     for index, doc in enumerate(documents, start=1):
         doc_id = doc.chunk_id or doc.source or doc.metadata.get("doc_id") or f"doc_{index}"
-        item: dict[str, Any] = {"doc_id": doc_id, "score": doc.score}
+        preview = ""
+        if doc.content:
+            preview = sanitize_text(doc.content, max_chars=MAX_CONTEXT_CHARS)
+        item: dict[str, Any] = {
+            "doc_id": doc_id,
+            "score": doc.score,
+            "content_preview": preview,
+        }
         if doc.source:
             item["source"] = doc.source
         if doc.rerank_score is not None:

evalvault/config/instrumentation.py

@@ -26,6 +26,7 @@ _tracer_provider: TracerProvider | None = None
 def setup_phoenix_instrumentation(
     endpoint: str = "http://localhost:6006/v1/traces",
     service_name: str = "evalvault",
+    project_name: str | None = None,
     enable_langchain: bool = True,
     enable_openai: bool = True,
     sample_rate: float = 1.0,
@@ -73,12 +74,13 @@ def setup_phoenix_instrumentation(
         return None
 
     # Create resource with service name
-    resource = Resource.create(
-        {
-            "service.name": service_name,
-            "service.version": "0.1.0",
-        }
-    )
+    resource_attributes = {
+        "service.name": service_name,
+        "service.version": "0.1.0",
+    }
+    if project_name:
+        resource_attributes["project.name"] = project_name
+    resource = Resource.create(resource_attributes)
 
     # Clamp sample rate between 0 and 1
     ratio = max(0.0, min(sample_rate, 1.0))

evalvault/config/phoenix_support.py

@@ -59,10 +59,15 @@ def ensure_phoenix_instrumentation(
     if api_token:
         headers = {"api-key": api_token}
 
+    project_name = getattr(settings, "phoenix_project_name", None)
+    if project_name is not None and not isinstance(project_name, str):
+        project_name = None
+
     try:
         setup_phoenix_instrumentation(
             endpoint=endpoint,
             service_name="evalvault",
+            project_name=project_name,
             sample_rate=sample_rate,
             headers=headers,
         )

evalvault/config/runtime_services.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import logging
+import socket
+import subprocess
+from dataclasses import dataclass
+from shutil import which
+from urllib.parse import urlparse
+
+from evalvault.config.settings import Settings, is_production_profile
+
+logger = logging.getLogger(__name__)
+
+_PHOENIX_CONTAINER = "evalvault-phoenix"
+
+
+@dataclass(frozen=True)
+class Endpoint:
+    host: str
+    port: int
+
+
+def _is_local_host(host: str | None) -> bool:
+    if not host:
+        return False
+    return host in {"localhost", "127.0.0.1", "0.0.0.0"}
+
+
+def _port_is_open(host: str, port: int) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=0.2):
+            return True
+    except OSError:
+        return False
+
+
+def _parse_http_endpoint(url: str | None, default_port: int) -> Endpoint | None:
+    if not url or not isinstance(url, str):
+        return None
+    parsed = urlparse(url)
+    if parsed.scheme and parsed.scheme not in {"http", "https"}:
+        return None
+    host = parsed.hostname or ""
+    port = parsed.port or default_port
+    if not host or port <= 0:
+        return None
+    return Endpoint(host=host, port=port)
+
+
+def _start_mlflow(port: int) -> bool:
+    if which("mlflow") is None:
+        logger.warning("MLflow CLI not found. Install with: uv sync --extra mlflow")
+        return False
+    try:
+        subprocess.Popen(
+            ["mlflow", "server", "--host", "0.0.0.0", "--port", str(port)],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        logger.info("Started MLflow server on port %s", port)
+        return True
+    except Exception as exc:  # pragma: no cover - safety net
+        logger.warning("Failed to start MLflow server: %s", exc)
+        return False
+
+
+def _start_phoenix(port: int) -> bool:
+    if which("docker") is None:
+        logger.warning("Docker not found. Phoenix auto-start skipped.")
+        return False
+    try:
+        start = subprocess.run(
+            ["docker", "start", _PHOENIX_CONTAINER],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+        if start.returncode != 0:
+            subprocess.run(
+                [
+                    "docker",
+                    "run",
+                    "-d",
+                    "-p",
+                    f"{port}:6006",
+                    "--name",
+                    _PHOENIX_CONTAINER,
+                    "arizephoenix/phoenix:latest",
+                ],
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+        logger.info("Ensured Phoenix container is running on port %s", port)
+        return True
+    except Exception as exc:  # pragma: no cover - safety net
+        logger.warning("Failed to start Phoenix container: %s", exc)
+        return False
+
+
+def ensure_local_observability(settings: Settings) -> None:
+    if is_production_profile(settings.evalvault_profile):
+        return
+
+    phoenix_endpoint = _parse_http_endpoint(
+        getattr(settings, "phoenix_endpoint", None) or "http://localhost:6006/v1/traces",
+        6006,
+    )
+    if (
+        phoenix_endpoint
+        and _is_local_host(phoenix_endpoint.host)
+        and not _port_is_open(phoenix_endpoint.host, phoenix_endpoint.port)
+    ):
+        _start_phoenix(phoenix_endpoint.port)
+
+    mlflow_endpoint = _parse_http_endpoint(getattr(settings, "mlflow_tracking_uri", None), 5000)
+    if (
+        mlflow_endpoint
+        and _is_local_host(mlflow_endpoint.host)
+        and not _port_is_open(mlflow_endpoint.host, mlflow_endpoint.port)
+    ):
+        _start_mlflow(mlflow_endpoint.port)

evalvault/config/settings.py

@@ -55,6 +55,33 @@ def _parse_cors_origins(cors_origins: str | None) -> list[str]:
     return [origin.strip() for origin in cors_origins.split(",") if origin.strip()]
 
 
+def resolve_tracker_providers(provider: str | None) -> list[str]:
+    if not provider:
+        return []
+    normalized = provider.strip().lower()
+    if normalized in {"none", "off", "disabled"}:
+        return ["none"]
+    aliases = {
+        "all": ["mlflow", "phoenix"],
+        "default": ["mlflow", "phoenix"],
+    }
+    if normalized in aliases:
+        return aliases[normalized]
+    separators = [",", "+", "/", "|"]
+    for sep in separators:
+        normalized = normalized.replace(sep, ",")
+    providers = [p.strip() for p in normalized.split(",") if p.strip()]
+    if not providers:
+        return []
+    if "none" in providers and len(providers) > 1:
+        raise ValueError("tracker_provider cannot combine 'none' with other providers")
+    deduped: list[str] = []
+    for entry in providers:
+        if entry not in deduped:
+            deduped.append(entry)
+    return deduped
+
+
 SECRET_REFERENCE_FIELDS = (
     "api_auth_tokens",
     "knowledge_read_tokens",
@@ -83,13 +110,14 @@ def _validate_production_settings(settings: "Settings") -> None:
     if settings.llm_provider == "openai" and not settings.openai_api_key:
         missing.append("OPENAI_API_KEY")
 
-    if settings.tracker_provider == "langfuse":
+    providers = resolve_tracker_providers(settings.tracker_provider)
+    if "langfuse" in providers:
         if not settings.langfuse_public_key:
             missing.append("LANGFUSE_PUBLIC_KEY")
         if not settings.langfuse_secret_key:
             missing.append("LANGFUSE_SECRET_KEY")
 
-    if settings.tracker_provider == "mlflow" and not settings.mlflow_tracking_uri:
+    if "mlflow" in providers and not settings.mlflow_tracking_uri:
         missing.append("MLFLOW_TRACKING_URI")
 
     if (
@@ -355,6 +383,14 @@ class Settings(BaseSettings):
         default="http://localhost:6006/v1/traces",
         description="Phoenix OTLP endpoint for traces",
     )
+    phoenix_project_name: str = Field(
+        default="evalvault",
+        description="Phoenix project name for grouping traces",
+    )
+    phoenix_annotations_enabled: bool = Field(
+        default=True,
+        description="Enable automatic Phoenix span annotations",
+    )
     phoenix_api_token: str | None = Field(
         default=None,
         description="Phoenix API token for cloud deployments (optional)",
@@ -372,8 +408,8 @@ class Settings(BaseSettings):
 
     # Tracker Provider Selection
     tracker_provider: str = Field(
-        default="langfuse",
-        description="Tracker provider: 'langfuse', 'mlflow', or 'phoenix'",
+        default="mlflow+phoenix",
+        description="Tracker provider: 'langfuse', 'mlflow', 'phoenix', 'none', or combinations",
     )
 
     # Cluster map configuration

evalvault/domain/services/evaluator.py

@@ -1934,6 +1934,8 @@ class RagasEvaluator:
 
     def _calculate_cost(self, model_name: str, prompt_tokens: int, completion_tokens: int) -> float:
         """Calculate estimated cost in USD based on model pricing."""
+        if "ollama" in model_name:
+            return 0.0
         # Find matching model key (exact or substring match)
         price_key = "openai/gpt-4o"  # Default fallback
         for key in self.MODEL_PRICING:

{evalvault-1.75.0.dist-info → evalvault-1.77.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.75.0
+Version: 1.77.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme
@@ -66,6 +66,7 @@ Requires-Dist: ijson>=3.3.0; extra == 'dev'
 Requires-Dist: kiwipiepy>=0.18.0; extra == 'dev'
 Requires-Dist: langchain-anthropic; extra == 'dev'
 Requires-Dist: lm-eval[api]>=0.4.0; extra == 'dev'
+Requires-Dist: manim>=0.18.0; extra == 'dev'
 Requires-Dist: mkdocs-material>=9.5.0; extra == 'dev'
 Requires-Dist: mkdocs>=1.5.0; extra == 'dev'
 Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'dev'