evalvault 1.75.0__py3-none-any.whl → 1.77.0__py3-none-any.whl

evalvault/adapters/outbound/llm/factory.py

@@ -97,7 +97,7 @@ def _resolve_faithfulness_fallback_config(
 
 def _default_faithfulness_fallback_model(provider: str) -> str | None:
     if provider == "ollama":
-        return "gpt-oss-safeguard:20b"
+        return "qwen3:8b"
     if provider == "vllm":
         return "gpt-oss-120b"
     return None

evalvault/adapters/outbound/phoenix/sync_service.py

@@ -15,7 +15,7 @@ class PhoenixSyncError(RuntimeError):
 def _normalize_base_url(endpoint: str) -> str:
     """Convert OTLP endpoint (…/v1/traces) to Phoenix REST base URL."""
 
-    if not endpoint:
+    if not isinstance(endpoint, str) or not endpoint:
         return "http://localhost:6006"
     base = endpoint.strip()
     suffix = "/v1/traces"
@@ -104,6 +104,22 @@ class PhoenixSyncService:
                 dataset_description=description,
             )
         except Exception as exc:  # pragma: no cover - HTTP/serialization errors
+            message = str(exc)
+            if "already exists" in message:
+                existing = self._find_dataset_by_name(dataset_name)
+                if existing:
+                    dataset_obj = self._client.datasets.get_dataset(dataset=existing["id"])
+                    dataset_url = self._client.experiments.get_dataset_experiments_url(
+                        dataset_obj.id
+                    )
+                    return PhoenixDatasetInfo(
+                        dataset_id=dataset_obj.id,
+                        dataset_name=dataset_obj.name,
+                        dataset_version_id=dataset_obj.version_id,
+                        url=dataset_url,
+                        description=description,
+                        example_count=getattr(dataset_obj, "examples", None),
+                    )
             raise PhoenixSyncError(f"Dataset upload failed: {exc}") from exc
 
         dataset_url = self._client.experiments.get_dataset_experiments_url(phoenix_dataset.id)
@@ -173,6 +189,74 @@ class PhoenixSyncService:
             )
         return examples
 
+    def _find_dataset_by_name(self, dataset_name: str) -> dict[str, Any] | None:
+        try:
+            datasets = self._client.datasets.list()
+        except Exception:
+            return None
+        for entry in datasets:
+            if entry.get("name") == dataset_name:
+                return entry
+        return None
+
+    def sync_prompts(
+        self,
+        *,
+        prompt_entries: list[dict[str, Any]],
+        model_name: str,
+        model_provider: str,
+        prompt_set_name: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """Create prompt versions in Phoenix Prompt Management."""
+
+        if not prompt_entries:
+            return []
+
+        try:
+            from phoenix.client.resources.prompts import PromptVersion
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise PhoenixSyncError("Phoenix prompt client unavailable") from exc
+
+        synced: list[dict[str, Any]] = []
+        for index, entry in enumerate(prompt_entries, start=1):
+            name = entry.get("name") or entry.get("role") or f"prompt_{index}"
+            content = entry.get("content") or entry.get("content_preview") or ""
+            if not content:
+                continue
+            prompt_version = PromptVersion(
+                [{"role": "system", "content": content}],
+                model_name=model_name,
+                model_provider=model_provider,
+                template_format="NONE",
+            )
+            prompt_metadata = {
+                "kind": entry.get("kind"),
+                "role": entry.get("role"),
+                "checksum": entry.get("checksum"),
+                "status": entry.get("status"),
+                "source": entry.get("source") or entry.get("path"),
+                "order": index,
+            }
+            if prompt_set_name:
+                prompt_metadata["prompt_set"] = prompt_set_name
+            try:
+                version = self._client.prompts.create(
+                    version=prompt_version,
+                    name=name,
+                    prompt_description=entry.get("notes"),
+                    prompt_metadata=_as_serializable(prompt_metadata),
+                )
+                synced.append(
+                    {
+                        **entry,
+                        "phoenix_prompt_version_id": getattr(version, "id", None),
+                    }
+                )
+            except Exception as exc:  # pragma: no cover - HTTP errors
+                raise PhoenixSyncError(f"Prompt sync failed: {exc}") from exc
+
+        return synced
+
     def _build_input_payload(self, test_case: TestCase) -> dict[str, Any]:
         return {
             "question": test_case.question,
@@ -258,6 +342,21 @@ def build_experiment_metadata(
         "total_test_cases": run.total_test_cases,
         "metrics": metrics,
     }
+    if run.results:
+        latencies = [r.latency_ms for r in run.results if r.latency_ms]
+        tokens = [r.tokens_used for r in run.results if r.tokens_used]
+        costs = [r.cost_usd for r in run.results if r.cost_usd is not None]
+        if latencies:
+            payload["avg_latency_ms"] = round(sum(latencies) / len(latencies), 2)
+        if tokens:
+            payload["avg_tokens"] = round(sum(tokens) / len(tokens), 2)
+        if costs:
+            payload["avg_cost_usd"] = round(sum(costs) / len(costs), 6)
+    if run.total_tokens:
+        payload["total_tokens"] = run.total_tokens
+    if run.total_cost_usd is not None:
+        payload["total_cost_usd"] = run.total_cost_usd
+    payload["error_rate"] = round(1 - run.pass_rate, 4)
     if reliability_snapshot:
         payload["reliability_snapshot"] = reliability_snapshot
     if dataset.metadata:

evalvault/adapters/outbound/report/markdown_adapter.py

@@ -50,6 +50,15 @@ class MarkdownReportAdapter:
         # 통계 분석
         if bundle.statistical:
             sections.append(self._generate_statistical_section(bundle.statistical))
+            sections.append(self._generate_reason_section(bundle.statistical))
+            sections.append(self._generate_meaning_section(bundle.statistical))
+            sections.append(self._generate_dataset_delta_section(bundle.statistical))
+            sections.append(self._generate_improvement_plan_section(bundle.statistical))
+        else:
+            sections.append(self._generate_reason_section(None))
+            sections.append(self._generate_meaning_section(None))
+            sections.append(self._generate_dataset_delta_section(None))
+            sections.append(self._generate_improvement_plan_section(None))
 
         # NLP 분석
         if include_nlp and bundle.has_nlp and bundle.nlp:
@@ -208,6 +217,89 @@ class MarkdownReportAdapter:
 
         return "\n".join(lines)
 
+    def _generate_reason_section(self, stat: StatisticalAnalysis | None) -> str:
+        lines = ["## 원인/근거"]
+        if stat is None:
+            lines.append(
+                "- 통계 분석 결과가 없어 원인/근거를 도출할 수 없습니다. (추가 데이터 필요)"
+            )
+            return "\n".join(lines)
+
+        if stat.low_performers:
+            for lp in stat.low_performers[:5]:
+                lines.append(
+                    f"- {lp.test_case_id}: {lp.metric_name} {lp.score:.2f} < {lp.threshold:.2f}"
+                )
+        elif stat.insights:
+            for insight in stat.insights[:5]:
+                lines.append(f"- {insight}")
+        else:
+            lines.append("- 추가 데이터 필요")
+        return "\n".join(lines)
+
+    def _generate_meaning_section(self, stat: StatisticalAnalysis | None) -> str:
+        lines = ["## 결과 의미"]
+        if stat is None:
+            lines.append("- 통계 분석 결과가 없어 의미를 해석할 수 없습니다. (추가 데이터 필요)")
+            return "\n".join(lines)
+
+        if stat.overall_pass_rate < 0.7:
+            lines.append("- 전체 통과율이 낮아 사용자 신뢰/정확성 리스크가 큽니다.")
+        else:
+            lines.append("- 전체 통과율이 기준 이상으로 기본 품질은 유지됩니다.")
+
+        low_metrics = [
+            metric for metric, rate in (stat.metric_pass_rates or {}).items() if rate < 0.7
+        ]
+        if low_metrics:
+            metrics_str = ", ".join(sorted(low_metrics)[:6])
+            lines.append(f"- 기준 미달 메트릭: {metrics_str}")
+        return "\n".join(lines)
+
+    def _generate_dataset_delta_section(self, stat: StatisticalAnalysis | None) -> str:
+        lines = ["## 데이터셋 차이"]
+        if stat is None:
+            lines.append("- 데이터셋 기준 차이를 판단할 수 없습니다. (추가 데이터 필요)")
+            return "\n".join(lines)
+
+        low_metrics = [
+            metric for metric, rate in (stat.metric_pass_rates or {}).items() if rate < 0.7
+        ]
+        if low_metrics:
+            lines.append("- 데이터셋 기준 미달 지표: " + ", ".join(sorted(low_metrics)[:6]))
+        else:
+            lines.append("- 데이터셋 기준 미달 지표가 없습니다.")
+        return "\n".join(lines)
+
+    def _generate_improvement_plan_section(self, stat: StatisticalAnalysis | None) -> str:
+        lines = ["## 개선 방향"]
+        if stat is None:
+            lines.append("- 개선 방향 도출을 위한 분석 결과가 부족합니다. (추가 데이터 필요)")
+            return "\n".join(lines)
+
+        action_map = {
+            "context_precision": "랭커/리랭커 도입 및 상위 문서 필터링 강화",
+            "context_recall": "검색 범위 확장 또는 하드 네거티브 추가",
+            "mrr": "상위 K 재정렬 및 쿼리 재작성 적용",
+            "ndcg": "랭킹 품질 지표 최적화(리랭킹/하이브리드 검색)",
+            "hit_rate": "검색 후보군 확대 또는 인덱싱 개선",
+            "answer_relevancy": "답변 포맷/질문 의도 정렬 프롬프트 강화",
+            "faithfulness": "근거 인용/검증 단계 추가",
+            "factual_correctness": "정답 검증 규칙 강화 및 근거 필터링",
+            "semantic_similarity": "정답 기준 문장 재정의 및 평가셋 보강",
+        }
+
+        low_metrics = [
+            metric for metric, rate in (stat.metric_pass_rates or {}).items() if rate < 0.7
+        ]
+        if low_metrics:
+            for metric in sorted(low_metrics)[:5]:
+                action = action_map.get(metric, "실험을 통해 개선 방향을 재검증")
+                lines.append(f"- {metric}: {action}")
+        else:
+            lines.append("- 개선 대상 지표가 명확하지 않습니다. (추가 데이터 필요)")
+        return "\n".join(lines)
+
     def _generate_nlp_section(self, nlp: NLPAnalysis) -> str:
         """NLP 분석 섹션 생성."""
         lines = ["## NLP 분석"]

evalvault/adapters/outbound/storage/factory.py

@@ -19,12 +19,9 @@ def build_storage_adapter(
 ) -> StoragePort:
     resolved_settings = settings or Settings()
 
-    if db_path is not None:
-        return SQLiteStorageAdapter(db_path=db_path)
-
     backend = getattr(resolved_settings, "db_backend", "postgres")
     if backend == "sqlite":
-        resolved_db_path = resolved_settings.evalvault_db_path
+        resolved_db_path = db_path or resolved_settings.evalvault_db_path
         if resolved_db_path is None:
             raise RuntimeError("SQLite backend selected but evalvault_db_path is not set.")
         return SQLiteStorageAdapter(db_path=resolved_db_path)

evalvault/adapters/outbound/tracker/mlflow_adapter.py

@@ -5,7 +5,7 @@ import tempfile
 from typing import Any
 
 from evalvault.adapters.outbound.tracker.log_sanitizer import MAX_LOG_CHARS, sanitize_payload
-from evalvault.domain.entities import EvaluationRun
+from evalvault.domain.entities import EvaluationRun, TestCaseResult
 from evalvault.ports.outbound.tracker_port import TrackerPort
 
 
@@ -29,6 +29,17 @@ class MLflowAdapter(TrackerPort):
             tracking_uri: MLflow tracking server URI
             experiment_name: MLflow experiment name
         """
+        try:
+            import torch  # type: ignore
+        except Exception:
+            torch = None  # type: ignore
+        if torch is not None and not hasattr(torch, "Tensor"):
+
+            class _TorchTensor:  # pragma: no cover - guard for namespace package
+                pass
+
+            torch.Tensor = _TorchTensor  # type: ignore[attr-defined]
+
         import mlflow
 
         mlflow.set_tracking_uri(tracking_uri)
@@ -36,6 +47,21 @@ class MLflowAdapter(TrackerPort):
         self._mlflow = mlflow
         self._active_runs: dict[str, Any] = {}  # trace_id -> mlflow run
 
+    def _enable_system_metrics(self) -> None:
+        try:
+            enable_fn = getattr(self._mlflow, "enable_system_metrics_logging", None)
+            if callable(enable_fn):
+                enable_fn()
+        except Exception:  # pragma: no cover - optional dependency
+            return
+
+    def _start_mlflow_run(self, name: str) -> Any:
+        try:
+            return self._mlflow.start_run(run_name=name, log_system_metrics=True)
+        except TypeError:
+            self._enable_system_metrics()
+            return self._mlflow.start_run(run_name=name)
+
     def start_trace(self, name: str, metadata: dict[str, Any] | None = None) -> str:
         """
         Start a new MLflow run (mapped to trace).
@@ -47,7 +73,7 @@ class MLflowAdapter(TrackerPort):
         Returns:
             trace_id: MLflow run ID
         """
-        run = self._mlflow.start_run(run_name=name)
+        run = self._start_mlflow_run(name)
         trace_id = run.info.run_id
 
         # Log metadata as MLflow parameters (only primitive types)
@@ -59,6 +85,12 @@ class MLflowAdapter(TrackerPort):
         self._active_runs[trace_id] = run
         return trace_id
 
+    def _write_temp_file(self, suffix: str, content: str) -> str:
+        with tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False) as f:
+            f.write(content)
+            f.flush()
+        return f.name
+
     def add_span(
         self,
         trace_id: str,
@@ -89,10 +121,9 @@ class MLflowAdapter(TrackerPort):
             "input": sanitize_payload(input_data, max_chars=MAX_LOG_CHARS),
             "output": sanitize_payload(output_data, max_chars=MAX_LOG_CHARS),
         }
-
-        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-            json.dump(span_data, f, default=str)
-            self._mlflow.log_artifact(f.name, f"spans/{name}")
+        payload = json.dumps(span_data, default=str)
+        path = self._write_temp_file(".json", payload)
+        self._mlflow.log_artifact(path, f"spans/{name}")
 
     def log_score(
         self,
@@ -145,9 +176,15 @@ class MLflowAdapter(TrackerPort):
             raise ValueError(f"Run not found: {trace_id}")
 
         if artifact_type == "json":
-            with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-                json.dump(data, f, default=str)
-                self._mlflow.log_artifact(f.name, f"artifacts/{name}")
+            payload = json.dumps(data, default=str)
+            path = self._write_temp_file(".json", payload)
+            self._mlflow.log_artifact(path, f"artifacts/{name}")
+        elif artifact_type == "text":
+            path = self._write_temp_file(".txt", str(data))
+            self._mlflow.log_artifact(path, f"artifacts/{name}")
+        else:
+            path = self._write_temp_file(".txt", str(data))
+            self._mlflow.log_artifact(path, f"artifacts/{name}")
 
     def end_trace(self, trace_id: str) -> None:
         """
@@ -180,53 +217,171 @@ class MLflowAdapter(TrackerPort):
         Returns:
             trace_id: ID of the created MLflow run
         """
-        # 1. Start MLflow run
-        trace_id = self.start_trace(
-            name=f"evaluation-{run.run_id[:8]}",
-            metadata={
-                "dataset_name": run.dataset_name,
-                "dataset_version": run.dataset_version,
-                "model_name": run.model_name,
-                "total_test_cases": run.total_test_cases,
-            },
-        )
-
-        # 2. Log average metric scores
-        for metric_name in run.metrics_evaluated:
-            avg_score = run.get_avg_score(metric_name)
-            if avg_score is not None:
-                self.log_score(trace_id, f"avg_{metric_name}", avg_score)
-
-        # 3. Log overall pass rate
-        self.log_score(trace_id, "pass_rate", run.pass_rate)
-
-        # 4. Log resource usage
-        self._mlflow.log_metric("total_tokens", run.total_tokens)
-
-        if run.duration_seconds:
-            self._mlflow.log_metric("duration_seconds", run.duration_seconds)
-
-        # 5. Save individual test results as artifact
-        results_data = []
-        for result in run.results:
-            result_dict = {
-                "test_case_id": result.test_case_id,
-                "all_passed": result.all_passed,
-                "tokens_used": result.tokens_used,
+
+        def _log_run() -> str:
+            trace_id = self.start_trace(
+                name=f"evaluation-{run.run_id[:8]}",
+                metadata={
+                    "dataset_name": run.dataset_name,
+                    "dataset_version": run.dataset_version,
+                    "model_name": run.model_name,
+                    "total_test_cases": run.total_test_cases,
+                },
+            )
+
+            self._mlflow.set_tag("run_id", run.run_id)
+            self._mlflow.set_tag("model_name", run.model_name)
+            self._mlflow.set_tag("dataset", f"{run.dataset_name}:{run.dataset_version}")
+            if run.tracker_metadata:
+                project_name = run.tracker_metadata.get("project_name")
+                if project_name:
+                    self._mlflow.set_tag("project_name", project_name)
+
+            for metric_name in run.metrics_evaluated:
+                avg_score = run.get_avg_score(metric_name)
+                if avg_score is not None:
+                    self.log_score(trace_id, f"avg_{metric_name}", avg_score)
+
+            self.log_score(trace_id, "pass_rate", run.pass_rate)
+            self._mlflow.log_metric("total_tokens", run.total_tokens)
+            if run.duration_seconds:
+                self._mlflow.log_metric("duration_seconds", run.duration_seconds)
+            if run.total_cost_usd is not None:
+                self._mlflow.log_metric("total_cost_usd", run.total_cost_usd)
+
+            results_data = []
+            for result in run.results:
+                result_dict = {
+                    "test_case_id": result.test_case_id,
+                    "all_passed": result.all_passed,
+                    "tokens_used": result.tokens_used,
+                    "metrics": [
+                        {"name": m.name, "score": m.score, "passed": m.passed}
+                        for m in result.metrics
+                    ],
+                }
+                results_data.append(result_dict)
+                self._trace_test_case(result)
+
+            self.save_artifact(trace_id, "test_results", results_data)
+            self.save_artifact(
+                trace_id,
+                "custom_metric_snapshot",
+                (run.tracker_metadata or {}).get("custom_metric_snapshot"),
+            )
+            if run.tracker_metadata:
+                self.save_artifact(trace_id, "tracker_metadata", run.tracker_metadata)
+                self._register_prompts(run)
+
+            self.end_trace(trace_id)
+            return trace_id
+
+        trace_name = f"evaluation-{run.run_id[:8]}"
+        trace_attrs = {
+            "dataset_name": run.dataset_name,
+            "dataset_version": run.dataset_version,
+            "model_name": run.model_name,
+        }
+        try:
+            traced = self._mlflow.trace(
+                name=trace_name, span_type="EVALUATION", attributes=trace_attrs
+            )
+            return traced(_log_run)()
+        except Exception:
+            return _log_run()
+
+    def _register_prompts(self, run: EvaluationRun) -> None:
+        genai = getattr(self._mlflow, "genai", None)
+        if genai is None:
+            return
+        register_fn = getattr(genai, "register_prompt", None)
+        if not callable(register_fn):
+            return
+
+        prompt_entries = self._extract_prompt_entries(run)
+        if not prompt_entries:
+            return
+
+        for entry in prompt_entries:
+            name = entry.get("name") or entry.get("role") or "prompt"
+            content = entry.get("content") or entry.get("content_preview") or ""
+            if not content:
+                continue
+            tags = {
+                "kind": str(entry.get("kind") or "custom"),
+                "role": str(entry.get("role") or ""),
+                "checksum": str(entry.get("checksum") or ""),
+                "run_id": run.run_id,
+            }
+            prompt_set_name = entry.get("prompt_set_name")
+            if prompt_set_name:
+                tags["prompt_set"] = str(prompt_set_name)
+            register_fn(
+                name=name,
+                template=content,
+                commit_message=entry.get("checksum"),
+                tags=tags,
+                model_config={
+                    "model_name": run.model_name,
+                },
+            )
+
+    def _extract_prompt_entries(self, run: EvaluationRun) -> list[dict[str, Any]]:
+        entries: list[dict[str, Any]] = []
+        metadata = run.tracker_metadata or {}
+        prompt_set_detail = metadata.get("prompt_set_detail")
+        if isinstance(prompt_set_detail, dict):
+            prompt_set_name = prompt_set_detail.get("name")
+            for item in prompt_set_detail.get("items", []):
+                prompt = item.get("prompt") or {}
+                if not isinstance(prompt, dict):
+                    continue
+                entries.append(
+                    {
+                        "name": prompt.get("name"),
+                        "role": item.get("role"),
+                        "kind": prompt.get("kind"),
+                        "checksum": prompt.get("checksum"),
+                        "content": prompt.get("content"),
+                        "prompt_set_name": prompt_set_name,
+                    }
+                )
+
+        phoenix_meta = metadata.get("phoenix") or {}
+        if isinstance(phoenix_meta, dict):
+            for entry in phoenix_meta.get("prompts", []) or []:
+                if not isinstance(entry, dict):
+                    continue
+                entries.append(entry)
+        return entries
+
+    def _trace_test_case(self, result: TestCaseResult) -> None:
+        trace_fn = getattr(self._mlflow, "trace", None)
+        if not callable(trace_fn):
+            return
+
+        attrs = {
+            "test_case_id": result.test_case_id,
+            "all_passed": result.all_passed,
+            "tokens_used": result.tokens_used,
+            "latency_ms": result.latency_ms,
+        }
+
+        def _emit() -> dict[str, Any]:
+            return {
                 "metrics": [
                     {"name": m.name, "score": m.score, "passed": m.passed} for m in result.metrics
                 ],
+                "tokens_used": result.tokens_used,
+                "latency_ms": result.latency_ms,
             }
-            results_data.append(result_dict)
-
-        self.save_artifact(trace_id, "test_results", results_data)
-        self.save_artifact(
-            trace_id,
-            "custom_metric_snapshot",
-            (run.tracker_metadata or {}).get("custom_metric_snapshot"),
-        )
 
-        # 6. End MLflow run
-        self.end_trace(trace_id)
-
-        return trace_id
+        try:
+            wrapped = trace_fn(
+                name=f"test_case_{result.test_case_id}",
+                span_type="EVALUATION",
+                attributes=attrs,
+            )
+            wrapped(_emit)()
+        except Exception:
+            return