evalvault 1.75.0__py3-none-any.whl → 1.76.0__py3-none-any.whl

evalvault/adapters/outbound/tracker/mlflow_adapter.py

@@ -5,7 +5,7 @@ import tempfile
 from typing import Any
 
 from evalvault.adapters.outbound.tracker.log_sanitizer import MAX_LOG_CHARS, sanitize_payload
-from evalvault.domain.entities import EvaluationRun
+from evalvault.domain.entities import EvaluationRun, TestCaseResult
 from evalvault.ports.outbound.tracker_port import TrackerPort
 
 
@@ -29,6 +29,17 @@ class MLflowAdapter(TrackerPort):
             tracking_uri: MLflow tracking server URI
             experiment_name: MLflow experiment name
         """
+        try:
+            import torch  # type: ignore
+        except Exception:
+            torch = None  # type: ignore
+        if torch is not None and not hasattr(torch, "Tensor"):
+
+            class _TorchTensor:  # pragma: no cover - guard for namespace package
+                pass
+
+            torch.Tensor = _TorchTensor  # type: ignore[attr-defined]
+
         import mlflow
 
         mlflow.set_tracking_uri(tracking_uri)
@@ -36,6 +47,21 @@ class MLflowAdapter(TrackerPort):
         self._mlflow = mlflow
         self._active_runs: dict[str, Any] = {}  # trace_id -> mlflow run
 
+    def _enable_system_metrics(self) -> None:
+        try:
+            enable_fn = getattr(self._mlflow, "enable_system_metrics_logging", None)
+            if callable(enable_fn):
+                enable_fn()
+        except Exception:  # pragma: no cover - optional dependency
+            return
+
+    def _start_mlflow_run(self, name: str) -> Any:
+        try:
+            return self._mlflow.start_run(run_name=name, log_system_metrics=True)
+        except TypeError:
+            self._enable_system_metrics()
+            return self._mlflow.start_run(run_name=name)
+
     def start_trace(self, name: str, metadata: dict[str, Any] | None = None) -> str:
         """
         Start a new MLflow run (mapped to trace).
@@ -47,7 +73,7 @@ class MLflowAdapter(TrackerPort):
         Returns:
             trace_id: MLflow run ID
         """
-        run = self._mlflow.start_run(run_name=name)
+        run = self._start_mlflow_run(name)
         trace_id = run.info.run_id
 
         # Log metadata as MLflow parameters (only primitive types)
@@ -59,6 +85,12 @@ class MLflowAdapter(TrackerPort):
         self._active_runs[trace_id] = run
         return trace_id
 
+    def _write_temp_file(self, suffix: str, content: str) -> str:
+        with tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False) as f:
+            f.write(content)
+            f.flush()
+        return f.name
+
     def add_span(
         self,
         trace_id: str,
@@ -89,10 +121,9 @@ class MLflowAdapter(TrackerPort):
             "input": sanitize_payload(input_data, max_chars=MAX_LOG_CHARS),
             "output": sanitize_payload(output_data, max_chars=MAX_LOG_CHARS),
         }
-
-        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-            json.dump(span_data, f, default=str)
-            self._mlflow.log_artifact(f.name, f"spans/{name}")
+        payload = json.dumps(span_data, default=str)
+        path = self._write_temp_file(".json", payload)
+        self._mlflow.log_artifact(path, f"spans/{name}")
 
     def log_score(
         self,
@@ -145,9 +176,15 @@ class MLflowAdapter(TrackerPort):
             raise ValueError(f"Run not found: {trace_id}")
 
         if artifact_type == "json":
-            with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-                json.dump(data, f, default=str)
-                self._mlflow.log_artifact(f.name, f"artifacts/{name}")
+            payload = json.dumps(data, default=str)
+            path = self._write_temp_file(".json", payload)
+            self._mlflow.log_artifact(path, f"artifacts/{name}")
+        elif artifact_type == "text":
+            path = self._write_temp_file(".txt", str(data))
+            self._mlflow.log_artifact(path, f"artifacts/{name}")
+        else:
+            path = self._write_temp_file(".txt", str(data))
+            self._mlflow.log_artifact(path, f"artifacts/{name}")
 
     def end_trace(self, trace_id: str) -> None:
         """
@@ -180,53 +217,171 @@ class MLflowAdapter(TrackerPort):
         Returns:
             trace_id: ID of the created MLflow run
         """
-        # 1. Start MLflow run
-        trace_id = self.start_trace(
-            name=f"evaluation-{run.run_id[:8]}",
-            metadata={
-                "dataset_name": run.dataset_name,
-                "dataset_version": run.dataset_version,
-                "model_name": run.model_name,
-                "total_test_cases": run.total_test_cases,
-            },
-        )
-
-        # 2. Log average metric scores
-        for metric_name in run.metrics_evaluated:
-            avg_score = run.get_avg_score(metric_name)
-            if avg_score is not None:
-                self.log_score(trace_id, f"avg_{metric_name}", avg_score)
-
-        # 3. Log overall pass rate
-        self.log_score(trace_id, "pass_rate", run.pass_rate)
-
-        # 4. Log resource usage
-        self._mlflow.log_metric("total_tokens", run.total_tokens)
-
-        if run.duration_seconds:
-            self._mlflow.log_metric("duration_seconds", run.duration_seconds)
-
-        # 5. Save individual test results as artifact
-        results_data = []
-        for result in run.results:
-            result_dict = {
-                "test_case_id": result.test_case_id,
-                "all_passed": result.all_passed,
-                "tokens_used": result.tokens_used,
+
+        def _log_run() -> str:
+            trace_id = self.start_trace(
+                name=f"evaluation-{run.run_id[:8]}",
+                metadata={
+                    "dataset_name": run.dataset_name,
+                    "dataset_version": run.dataset_version,
+                    "model_name": run.model_name,
+                    "total_test_cases": run.total_test_cases,
+                },
+            )
+
+            self._mlflow.set_tag("run_id", run.run_id)
+            self._mlflow.set_tag("model_name", run.model_name)
+            self._mlflow.set_tag("dataset", f"{run.dataset_name}:{run.dataset_version}")
+            if run.tracker_metadata:
+                project_name = run.tracker_metadata.get("project_name")
+                if project_name:
+                    self._mlflow.set_tag("project_name", project_name)
+
+            for metric_name in run.metrics_evaluated:
+                avg_score = run.get_avg_score(metric_name)
+                if avg_score is not None:
+                    self.log_score(trace_id, f"avg_{metric_name}", avg_score)
+
+            self.log_score(trace_id, "pass_rate", run.pass_rate)
+            self._mlflow.log_metric("total_tokens", run.total_tokens)
+            if run.duration_seconds:
+                self._mlflow.log_metric("duration_seconds", run.duration_seconds)
+            if run.total_cost_usd is not None:
+                self._mlflow.log_metric("total_cost_usd", run.total_cost_usd)
+
+            results_data = []
+            for result in run.results:
+                result_dict = {
+                    "test_case_id": result.test_case_id,
+                    "all_passed": result.all_passed,
+                    "tokens_used": result.tokens_used,
+                    "metrics": [
+                        {"name": m.name, "score": m.score, "passed": m.passed}
+                        for m in result.metrics
+                    ],
+                }
+                results_data.append(result_dict)
+                self._trace_test_case(result)
+
+            self.save_artifact(trace_id, "test_results", results_data)
+            self.save_artifact(
+                trace_id,
+                "custom_metric_snapshot",
+                (run.tracker_metadata or {}).get("custom_metric_snapshot"),
+            )
+            if run.tracker_metadata:
+                self.save_artifact(trace_id, "tracker_metadata", run.tracker_metadata)
+                self._register_prompts(run)
+
+            self.end_trace(trace_id)
+            return trace_id
+
+        trace_name = f"evaluation-{run.run_id[:8]}"
+        trace_attrs = {
+            "dataset_name": run.dataset_name,
+            "dataset_version": run.dataset_version,
+            "model_name": run.model_name,
+        }
+        try:
+            traced = self._mlflow.trace(
+                name=trace_name, span_type="EVALUATION", attributes=trace_attrs
+            )
+            return traced(_log_run)()
+        except Exception:
+            return _log_run()
+
+    def _register_prompts(self, run: EvaluationRun) -> None:
+        genai = getattr(self._mlflow, "genai", None)
+        if genai is None:
+            return
+        register_fn = getattr(genai, "register_prompt", None)
+        if not callable(register_fn):
+            return
+
+        prompt_entries = self._extract_prompt_entries(run)
+        if not prompt_entries:
+            return
+
+        for entry in prompt_entries:
+            name = entry.get("name") or entry.get("role") or "prompt"
+            content = entry.get("content") or entry.get("content_preview") or ""
+            if not content:
+                continue
+            tags = {
+                "kind": str(entry.get("kind") or "custom"),
+                "role": str(entry.get("role") or ""),
+                "checksum": str(entry.get("checksum") or ""),
+                "run_id": run.run_id,
+            }
+            prompt_set_name = entry.get("prompt_set_name")
+            if prompt_set_name:
+                tags["prompt_set"] = str(prompt_set_name)
+            register_fn(
+                name=name,
+                template=content,
+                commit_message=entry.get("checksum"),
+                tags=tags,
+                model_config={
+                    "model_name": run.model_name,
+                },
+            )
+
+    def _extract_prompt_entries(self, run: EvaluationRun) -> list[dict[str, Any]]:
+        entries: list[dict[str, Any]] = []
+        metadata = run.tracker_metadata or {}
+        prompt_set_detail = metadata.get("prompt_set_detail")
+        if isinstance(prompt_set_detail, dict):
+            prompt_set_name = prompt_set_detail.get("name")
+            for item in prompt_set_detail.get("items", []):
+                prompt = item.get("prompt") or {}
+                if not isinstance(prompt, dict):
+                    continue
+                entries.append(
+                    {
+                        "name": prompt.get("name"),
+                        "role": item.get("role"),
+                        "kind": prompt.get("kind"),
+                        "checksum": prompt.get("checksum"),
+                        "content": prompt.get("content"),
+                        "prompt_set_name": prompt_set_name,
+                    }
+                )
+
+        phoenix_meta = metadata.get("phoenix") or {}
+        if isinstance(phoenix_meta, dict):
+            for entry in phoenix_meta.get("prompts", []) or []:
+                if not isinstance(entry, dict):
+                    continue
+                entries.append(entry)
+        return entries
+
+    def _trace_test_case(self, result: TestCaseResult) -> None:
+        trace_fn = getattr(self._mlflow, "trace", None)
+        if not callable(trace_fn):
+            return
+
+        attrs = {
+            "test_case_id": result.test_case_id,
+            "all_passed": result.all_passed,
+            "tokens_used": result.tokens_used,
+            "latency_ms": result.latency_ms,
+        }
+
+        def _emit() -> dict[str, Any]:
+            return {
                 "metrics": [
                     {"name": m.name, "score": m.score, "passed": m.passed} for m in result.metrics
                 ],
+                "tokens_used": result.tokens_used,
+                "latency_ms": result.latency_ms,
             }
-            results_data.append(result_dict)
-
-        self.save_artifact(trace_id, "test_results", results_data)
-        self.save_artifact(
-            trace_id,
-            "custom_metric_snapshot",
-            (run.tracker_metadata or {}).get("custom_metric_snapshot"),
-        )
 
-        # 6. End MLflow run
-        self.end_trace(trace_id)
-
-        return trace_id
+        try:
+            wrapped = trace_fn(
+                name=f"test_case_{result.test_case_id}",
+                span_type="EVALUATION",
+                attributes=attrs,
+            )
+            wrapped(_emit)()
+        except Exception:
+            return

evalvault/adapters/outbound/tracker/phoenix_adapter.py

@@ -52,6 +52,8 @@ class PhoenixAdapter(TrackerPort):
         self,
         endpoint: str = "http://localhost:6006/v1/traces",
         service_name: str = "evalvault",
+        project_name: str | None = None,
+        annotations_enabled: bool = True,
     ):
         """Initialize Phoenix adapter with OpenTelemetry.
 
@@ -61,11 +63,14 @@ class PhoenixAdapter(TrackerPort):
         """
         self._endpoint = endpoint
         self._service_name = service_name
+        self._project_name = project_name
+        self._annotations_enabled = annotations_enabled
         self._tracer: Any | None = None
         self._tracer_provider: TracerProvider | None = None
         self._active_spans: dict[str, Any] = {}
         self._tracer_any: Any | None = None
         self._initialized = False
+        self._annotations_client: Any | None = None
 
     def _ensure_initialized(self) -> None:
         """Lazy initialization of OpenTelemetry tracer."""
@@ -96,7 +101,10 @@ class PhoenixAdapter(TrackerPort):
                     return
 
             # Create resource with service name
-            resource = Resource.create({"service.name": self._service_name})
+            resource_attributes = {"service.name": self._service_name}
+            if self._project_name:
+                resource_attributes["project.name"] = self._project_name
+            resource = Resource.create(resource_attributes)
 
             # Create tracer provider
             self._tracer_provider = TracerProvider(resource=resource)
@@ -123,6 +131,50 @@ class PhoenixAdapter(TrackerPort):
                 "Failed to initialize Phoenix tracer. Check endpoint configuration and dependencies."
             ) from e
 
+    def _phoenix_base_url(self) -> str:
+        if "/v1/traces" in self._endpoint:
+            return self._endpoint.split("/v1/traces")[0]
+        return self._endpoint.rstrip("/")
+
+    def _get_annotations_client(self) -> Any | None:
+        if not self._annotations_enabled:
+            return None
+        if self._annotations_client is not None:
+            return self._annotations_client
+        try:
+            from phoenix.client import Client
+        except Exception:
+            return None
+        self._annotations_client = Client(base_url=self._phoenix_base_url())
+        return self._annotations_client
+
+    def _annotate_span(
+        self,
+        *,
+        span: Any,
+        name: str,
+        label: str,
+        score: float | None = None,
+        explanation: str | None = None,
+    ) -> None:
+        client = self._get_annotations_client()
+        if client is None or span is None:
+            return
+        try:
+            from opentelemetry.trace import format_span_id
+
+            span_id = format_span_id(span.get_span_context().span_id)
+            client.annotations.add_span_annotation(
+                annotation_name=name,
+                annotator_kind="CODE",
+                span_id=span_id,
+                label=label,
+                score=score,
+                explanation=explanation,
+            )
+        except Exception:
+            return
+
     def start_trace(self, name: str, metadata: dict[str, Any] | None = None) -> str:
         """Start a new trace.
 
@@ -328,8 +380,17 @@ class PhoenixAdapter(TrackerPort):
 
         # Set evaluation-specific attributes
         span = self._active_spans[trace_id]
+        span.set_attribute("openinference.span.kind", "EVALUATOR")
         span.set_attribute("evaluation.metrics", json.dumps(run.metrics_evaluated))
         span.set_attribute("evaluation.thresholds", json.dumps(run.thresholds))
+        span.set_attribute("evaluation.status", "pass" if run.pass_rate >= 1.0 else "fail")
+        if run.tracker_metadata:
+            project_name = run.tracker_metadata.get("project_name")
+            if project_name:
+                span.set_attribute("project.name", project_name)
+            project_kind = run.tracker_metadata.get("evaluation_task") or "evaluation"
+            span.set_attribute("project.kind", project_kind)
+            span.set_attribute("project.status", "pass" if run.pass_rate >= 1.0 else "fail")
 
         # Log average scores for each metric
         for metric_name, summary in metric_summary.items():
@@ -369,6 +430,8 @@ class PhoenixAdapter(TrackerPort):
             },
             "metrics": metric_summary,
             "custom_metrics": (run.tracker_metadata or {}).get("custom_metric_snapshot"),
+            "prompt_metadata": (run.tracker_metadata or {}).get("phoenix", {}).get("prompts"),
+            "tracker_metadata": run.tracker_metadata,
             "test_cases": [
                 {
                     "test_case_id": result.test_case_id,
@@ -420,6 +483,23 @@ class PhoenixAdapter(TrackerPort):
             f"test-case-{result.test_case_id}",
             context=context,
         ) as span:
+            try:
+                from opentelemetry.trace import Status, StatusCode
+
+                span.set_status(Status(StatusCode.OK if result.all_passed else StatusCode.ERROR))
+            except Exception:
+                pass
+            span.set_attribute("openinference.span.kind", "EVALUATOR")
+            span.set_attribute("evaluation.status", "pass" if result.all_passed else "fail")
+            self._annotate_span(
+                span=span,
+                name="evaluation_result",
+                label="pass" if result.all_passed else "fail",
+                score=1.0 if result.all_passed else 0.0,
+                explanation="All metrics passed"
+                if result.all_passed
+                else "One or more metrics failed",
+            )
             # Input data
             safe_question = sanitize_text(result.question, max_chars=MAX_LOG_CHARS) or ""
             safe_answer = sanitize_text(result.answer, max_chars=MAX_LOG_CHARS) or ""
@@ -439,6 +519,10 @@ class PhoenixAdapter(TrackerPort):
             # Metrics
             span.set_attribute("output.all_passed", result.all_passed)
             span.set_attribute("output.tokens_used", result.tokens_used)
+            if result.tokens_used:
+                span.set_attribute("llm.token_count.total", result.tokens_used)
+            if result.cost_usd is not None:
+                span.set_attribute("llm.cost.total", result.cost_usd)
 
             for metric in result.metrics:
                 span.set_attribute(f"metric.{metric.name}.score", metric.score)
@@ -486,6 +570,7 @@ class PhoenixAdapter(TrackerPort):
                 )
             if result.latency_ms:
                 span.set_attribute("timing.latency_ms", result.latency_ms)
+                span.set_attribute("evaluation.latency_ms", result.latency_ms)
 
     def log_retrieval(
         self,
@@ -528,6 +613,13 @@ class PhoenixAdapter(TrackerPort):
         if tracer is None:
             raise RuntimeError("Phoenix tracer is not initialized")
         with tracer.start_span("retrieval", context=context) as span:
+            try:
+                from opentelemetry.trace import Status, StatusCode
+
+                span.set_status(Status(StatusCode.OK))
+            except Exception:
+                pass
+            span.set_attribute("openinference.span.kind", "RETRIEVER")
             # Set retrieval attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)
@@ -541,14 +633,24 @@ class PhoenixAdapter(TrackerPort):
 
             span.set_attribute("spec.version", "0.1")
             span.set_attribute("rag.module", "retrieve")
+            if data.retrieval_time_ms:
+                span.set_attribute("retrieval.latency_ms", data.retrieval_time_ms)
 
             documents_payload = _build_retrieval_payload(data.candidates)
             span.set_attribute("custom.retrieval.doc_count", len(documents_payload))
             if documents_payload:
                 span.set_attribute("retrieval.documents_json", serialize_json(documents_payload))
-                doc_ids = _extract_doc_ids(documents_payload)
-                if doc_ids:
-                    span.set_attribute("output.value", doc_ids)
+                previews = [
+                    item.get("content_preview")
+                    for item in documents_payload
+                    if item.get("content_preview")
+                ]
+                if previews:
+                    span.set_attribute("output.value", previews)
+                else:
+                    doc_ids = _extract_doc_ids(documents_payload)
+                    if doc_ids:
+                        span.set_attribute("output.value", doc_ids)
 
             # Log each retrieved document as an event
             for i, doc in enumerate(data.candidates):
@@ -615,10 +717,31 @@ class PhoenixAdapter(TrackerPort):
         if tracer is None:
             raise RuntimeError("Phoenix tracer is not initialized")
         with tracer.start_span("generation", context=context) as span:
+            try:
+                from opentelemetry.trace import Status, StatusCode
+
+                span.set_status(Status(StatusCode.OK))
+            except Exception:
+                pass
+            span.set_attribute("openinference.span.kind", "LLM")
             # Set generation attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)
 
+            if data.model:
+                span.set_attribute("llm.model_name", data.model)
+                provider = data.model.split("/")[0] if "/" in data.model else ""
+                if provider:
+                    span.set_attribute("llm.provider", provider)
+            if data.input_tokens:
+                span.set_attribute("llm.token_count.prompt", data.input_tokens)
+            if data.output_tokens:
+                span.set_attribute("llm.token_count.completion", data.output_tokens)
+            if data.total_tokens:
+                span.set_attribute("llm.token_count.total", data.total_tokens)
+            if data.cost_usd is not None:
+                span.set_attribute("llm.cost.total", data.cost_usd)
+
             # Set prompt/response (truncate if too long)
             prompt = sanitize_text(data.prompt, max_chars=MAX_LOG_CHARS) or ""
             response = sanitize_text(data.response, max_chars=MAX_LOG_CHARS) or ""
@@ -637,6 +760,13 @@ class PhoenixAdapter(TrackerPort):
                 safe_template = sanitize_text(data.prompt_template, max_chars=MAX_LOG_CHARS)
                 if safe_template:
                     span.set_attribute("generation.prompt_template", safe_template)
+                    span.set_attribute("llm.prompt_template.template", safe_template)
+                    span.set_attribute("llm.prompt_template.version", "v1")
+            prompt_vars = data.metadata.get("prompt_variables") if data.metadata else None
+            if prompt_vars:
+                span.set_attribute(
+                    "llm.prompt_template.variables", json.dumps(prompt_vars, default=str)
+                )
 
     def log_rag_trace(self, data: RAGTraceData) -> str:
         """Log a full RAG trace (retrieval + generation) to Phoenix."""
@@ -660,6 +790,8 @@ class PhoenixAdapter(TrackerPort):
             span = self._active_spans[trace_id]
             should_end = True
 
+        span.set_attribute("openinference.span.kind", "CHAIN")
+
         for key, value in data.to_span_attributes().items():
             span.set_attribute(key, value)
 
@@ -667,11 +799,21 @@ class PhoenixAdapter(TrackerPort):
             self.log_retrieval(trace_id, data.retrieval)
         if data.generation:
             self.log_generation(trace_id, data.generation)
+        output_preview = ""
         if data.final_answer:
-            preview = sanitize_text(data.final_answer, max_chars=MAX_LOG_CHARS)
-            if preview:
-                span.set_attribute("rag.final_answer", preview)
-                span.set_attribute("output.value", preview)
+            output_preview = sanitize_text(data.final_answer, max_chars=MAX_LOG_CHARS)
+        if not output_preview and data.generation and data.generation.response:
+            output_preview = sanitize_text(data.generation.response, max_chars=MAX_LOG_CHARS)
+        if not output_preview and data.retrieval:
+            previews = [
+                sanitize_text(doc.content, max_chars=MAX_CONTEXT_CHARS)
+                for doc in data.retrieval.candidates
+                if doc.content
+            ]
+            output_preview = "\n".join(previews[:3])
+        if output_preview:
+            span.set_attribute("rag.final_answer", output_preview)
+            span.set_attribute("output.value", output_preview)
 
         if safe_query:
             span.set_attribute("input.value", safe_query)
@@ -697,7 +839,14 @@ def _build_retrieval_payload(
     payload: list[dict[str, Any]] = []
     for index, doc in enumerate(documents, start=1):
         doc_id = doc.chunk_id or doc.source or doc.metadata.get("doc_id") or f"doc_{index}"
-        item: dict[str, Any] = {"doc_id": doc_id, "score": doc.score}
+        preview = ""
+        if doc.content:
+            preview = sanitize_text(doc.content, max_chars=MAX_CONTEXT_CHARS)
+        item: dict[str, Any] = {
+            "doc_id": doc_id,
+            "score": doc.score,
+            "content_preview": preview,
+        }
         if doc.source:
             item["source"] = doc.source
         if doc.rerank_score is not None:

evalvault/config/instrumentation.py

@@ -26,6 +26,7 @@ _tracer_provider: TracerProvider | None = None
 def setup_phoenix_instrumentation(
     endpoint: str = "http://localhost:6006/v1/traces",
     service_name: str = "evalvault",
+    project_name: str | None = None,
     enable_langchain: bool = True,
     enable_openai: bool = True,
     sample_rate: float = 1.0,
@@ -73,12 +74,13 @@ def setup_phoenix_instrumentation(
         return None
 
     # Create resource with service name
-    resource = Resource.create(
-        {
-            "service.name": service_name,
-            "service.version": "0.1.0",
-        }
-    )
+    resource_attributes = {
+        "service.name": service_name,
+        "service.version": "0.1.0",
+    }
+    if project_name:
+        resource_attributes["project.name"] = project_name
+    resource = Resource.create(resource_attributes)
 
     # Clamp sample rate between 0 and 1
     ratio = max(0.0, min(sample_rate, 1.0))

evalvault/config/phoenix_support.py

@@ -59,10 +59,15 @@ def ensure_phoenix_instrumentation(
     if api_token:
         headers = {"api-key": api_token}
 
+    project_name = getattr(settings, "phoenix_project_name", None)
+    if project_name is not None and not isinstance(project_name, str):
+        project_name = None
+
     try:
         setup_phoenix_instrumentation(
             endpoint=endpoint,
             service_name="evalvault",
+            project_name=project_name,
             sample_rate=sample_rate,
             headers=headers,
         )