evalvault 1.64.0__py3-none-any.whl → 1.66.0__py3-none-any.whl

evalvault/adapters/inbound/cli/commands/run_helpers.py

@@ -88,7 +88,25 @@ RUN_MODE_PRESETS: dict[str, RunModePreset] = {
     ),
 }
 
-SUMMARY_METRIC_ORDER = ("summary_faithfulness", "summary_score", "entity_preservation")
+SUMMARY_METRIC_ORDER = (
+    "summary_faithfulness",
+    "summary_score",
+    "entity_preservation",
+    "summary_accuracy",
+    "summary_risk_coverage",
+    "summary_non_definitive",
+    "summary_needs_followup",
+)
+
+SUMMARY_METRIC_SOURCE = {
+    "summary_faithfulness": "LLM",
+    "summary_score": "LLM",
+    "entity_preservation": "Rule",
+    "summary_accuracy": "Rule",
+    "summary_risk_coverage": "Rule",
+    "summary_non_definitive": "Rule",
+    "summary_needs_followup": "Rule",
+}
 
 
 def _display_results(result, console: Console, verbose: bool = False) -> None:
@@ -180,8 +198,9 @@ def _display_summary_guidance(result, console: Console) -> None:
         if score is None:
             continue
         recommended = SUMMARY_RECOMMENDED_THRESHOLDS[metric]
+        source = SUMMARY_METRIC_SOURCE.get(metric, "Rule")
         if score < recommended:
-            warnings.append(f"- {metric}: {score:.3f} < {recommended:.2f}")
+            warnings.append(f"- {metric} ({source}): {score:.3f} < {recommended:.2f}")
 
     if warnings:
         header = "[bold red]사용자 노출 기준 미달[/bold red]"

evalvault/adapters/outbound/analysis/comparison_pipeline_adapter.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import asyncio
+
+from evalvault.domain.entities.analysis_pipeline import AnalysisIntent, PipelineResult
+from evalvault.domain.services.pipeline_orchestrator import AnalysisPipelineService
+from evalvault.ports.outbound.comparison_pipeline_port import ComparisonPipelinePort
+
+
+class ComparisonPipelineAdapter(ComparisonPipelinePort):
+    def __init__(self, service: AnalysisPipelineService) -> None:
+        self._service = service
+
+    def run_comparison(
+        self,
+        *,
+        run_ids: list[str],
+        compare_metrics: list[str] | None,
+        test_type: str,
+        parallel: bool,
+        concurrency: int | None,
+        report_type: str,
+        use_llm_report: bool,
+    ) -> PipelineResult:
+        params = {
+            "run_ids": run_ids,
+            "compare_metrics": compare_metrics,
+            "test_type": test_type,
+            "report_type": report_type,
+            "use_llm_report": use_llm_report,
+        }
+        if parallel:
+            if concurrency is not None:
+                params["max_concurrency"] = concurrency
+            return asyncio.run(
+                self._service.analyze_intent_async(
+                    AnalysisIntent.GENERATE_COMPARISON,
+                    run_id=run_ids[0] if run_ids else None,
+                    **params,
+                )
+            )
+        return self._service.analyze_intent(
+            AnalysisIntent.GENERATE_COMPARISON,
+            run_id=run_ids[0] if run_ids else None,
+            **params,
+        )
+
+
+__all__ = ["ComparisonPipelineAdapter"]

evalvault/adapters/outbound/artifact_fs.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.ports.outbound.artifact_fs_port import ArtifactFileSystemPort
+
+
+class LocalArtifactFileSystemAdapter(ArtifactFileSystemPort):
+    def exists(self, path: Path) -> bool:
+        return path.exists()
+
+    def is_dir(self, path: Path) -> bool:
+        return path.is_dir()
+
+    def read_text(self, path: Path) -> str:
+        return path.read_text(encoding="utf-8")

evalvault/adapters/outbound/filesystem/__init__.py

@@ -0,0 +1,3 @@
+from evalvault.adapters.outbound.filesystem.ops_snapshot_writer import OpsSnapshotWriter
+
+__all__ = ["OpsSnapshotWriter"]

evalvault/adapters/outbound/filesystem/difficulty_profile_writer.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.ports.outbound.difficulty_profile_port import DifficultyProfileWriterPort
+
+
+class DifficultyProfileWriter(DifficultyProfileWriterPort):
+    def write_profile(
+        self,
+        *,
+        output_path: Path,
+        artifacts_dir: Path,
+        envelope: dict[str, object],
+        artifacts: dict[str, object],
+    ) -> dict[str, object]:
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+        breakdown_path = artifacts_dir / "difficulty_breakdown.json"
+        cases_path = artifacts_dir / "difficulty_cases.json"
+        breakdown_payload = artifacts.get("breakdown")
+        cases_payload = artifacts.get("cases")
+        write_json(
+            breakdown_path,
+            breakdown_payload if isinstance(breakdown_payload, dict) else {},
+        )
+        write_json(
+            cases_path,
+            {"cases": cases_payload} if isinstance(cases_payload, list) else {"cases": []},
+        )
+
+        index_payload = {
+            "files": {
+                "breakdown": str(breakdown_path),
+                "cases": str(cases_path),
+            }
+        }
+        index_path = artifacts_dir / "index.json"
+        write_json(index_path, index_payload)
+
+        artifacts_index = {
+            "dir": str(artifacts_dir),
+            "index": str(index_path),
+        }
+        envelope["artifacts"] = artifacts_index
+        write_json(output_path, envelope)
+
+        return artifacts_index

evalvault/adapters/outbound/filesystem/ops_snapshot_writer.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.ports.outbound.ops_snapshot_port import OpsSnapshotWriterPort
+
+
+class OpsSnapshotWriter(OpsSnapshotWriterPort):
+    def write_snapshot(self, path: Path, payload: dict[str, Any]) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        write_json(path, payload)

evalvault/adapters/outbound/judge_calibration_adapter.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from evalvault.domain.entities import EvaluationRun, SatisfactionFeedback
+from evalvault.domain.entities.judge_calibration import JudgeCalibrationResult
+from evalvault.domain.services.judge_calibration_service import JudgeCalibrationService
+from evalvault.ports.outbound.judge_calibration_port import JudgeCalibrationPort
+
+
+class JudgeCalibrationAdapter(JudgeCalibrationPort):
+    def __init__(self) -> None:
+        self._service = JudgeCalibrationService()
+
+    def calibrate(
+        self,
+        run: EvaluationRun,
+        feedbacks: list[SatisfactionFeedback],
+        *,
+        labels_source: str,
+        method: str,
+        metrics: list[str],
+        holdout_ratio: float,
+        seed: int,
+        parallel: bool = False,
+        concurrency: int = 8,
+    ) -> JudgeCalibrationResult:
+        return self._service.calibrate(
+            run,
+            feedbacks,
+            labels_source=labels_source,
+            method=method,
+            metrics=metrics,
+            holdout_ratio=holdout_ratio,
+            seed=seed,
+            parallel=parallel,
+            concurrency=concurrency,
+        )

evalvault/adapters/outbound/judge_calibration_reporter.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.entities.judge_calibration import JudgeCalibrationResult
+
+
+class JudgeCalibrationReporter:
+    def render_json(self, result: JudgeCalibrationResult) -> dict[str, Any]:
+        return {
+            "summary": asdict(result.summary),
+            "metrics": [asdict(metric) for metric in result.metrics],
+            "case_results": {
+                metric: [asdict(entry) for entry in entries]
+                for metric, entries in result.case_results.items()
+            },
+            "warnings": list(result.warnings),
+        }
+
+    def write_artifacts(
+        self,
+        *,
+        result: JudgeCalibrationResult,
+        artifacts_dir: Path,
+    ) -> dict[str, str]:
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+        index_path = artifacts_dir / "index.json"
+        payload = {
+            "run_id": result.summary.run_id,
+            "metrics": [metric.metric for metric in result.metrics],
+            "cases": {},
+        }
+        for metric, cases in result.case_results.items():
+            case_path = artifacts_dir / f"{metric}.json"
+            case_payload = [
+                {
+                    "test_case_id": case.test_case_id,
+                    "raw_score": case.raw_score,
+                    "calibrated_score": case.calibrated_score,
+                    "label": case.label,
+                    "label_source": case.label_source,
+                }
+                for case in cases
+            ]
+            case_path.write_text(
+                json.dumps(case_payload, ensure_ascii=False, indent=2),
+                encoding="utf-8",
+            )
+            payload["cases"][metric] = str(case_path)
+        index_path.write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        return {"dir": str(artifacts_dir), "index": str(index_path)}

evalvault/adapters/outbound/report/llm_report_generator.py

@@ -499,8 +499,20 @@ SUMMARY_RECOMMENDED_THRESHOLDS = {
     "summary_faithfulness": 0.90,
     "summary_score": 0.85,
     "entity_preservation": 0.90,
+    "summary_accuracy": 0.90,
+    "summary_risk_coverage": 0.90,
+    "summary_non_definitive": 0.80,
+    "summary_needs_followup": 0.80,
 }
-SUMMARY_METRIC_ORDER = ("summary_faithfulness", "summary_score", "entity_preservation")
+SUMMARY_METRIC_ORDER = (
+    "summary_faithfulness",
+    "summary_score",
+    "entity_preservation",
+    "summary_accuracy",
+    "summary_risk_coverage",
+    "summary_non_definitive",
+    "summary_needs_followup",
+)
 
 
 @dataclass

evalvault/adapters/outbound/storage/base_sql.py

@@ -664,6 +664,8 @@ class BaseSQLStorageAdapter(ABC):
     def export_run_to_excel(self, run_id: str, output_path) -> Path:
         from openpyxl import Workbook
 
+        from evalvault.domain.metrics.registry import get_metric_spec_map
+
         output = Path(output_path)
         output.parent.mkdir(parents=True, exist_ok=True)
 
@@ -837,6 +839,23 @@ class BaseSQLStorageAdapter(ABC):
 
         summary_rows: list[dict[str, Any]] = []
         run_payload = run_rows[0] if run_rows else {}
+        custom_metric_rows: list[dict[str, Any]] = []
+        run_metadata = self._deserialize_json(run_payload.get("metadata")) if run_payload else None
+        if isinstance(run_metadata, dict):
+            custom_snapshot = run_metadata.get("custom_metric_snapshot")
+            if isinstance(custom_snapshot, dict):
+                entries = custom_snapshot.get("metrics")
+                if isinstance(entries, list):
+                    for entry in entries:
+                        if isinstance(entry, dict):
+                            row = dict(entry)
+                            row["schema_version"] = custom_snapshot.get("schema_version")
+                            custom_metric_rows.append(row)
+        if custom_metric_rows:
+            custom_metric_rows = self._normalize_rows(
+                custom_metric_rows,
+                json_columns={"inputs", "rules"},
+            )
         prompt_set_id = None
         prompt_set_name = None
         if run_prompt_payloads:
@@ -878,14 +897,17 @@ class BaseSQLStorageAdapter(ABC):
                 if isinstance(threshold, (int, float)) and score >= threshold:
                     entry["pass_count"] += 1
 
+        metric_spec_map = get_metric_spec_map()
         for entry in metrics_index.values():
             count = entry["count"] or 0
+            spec = metric_spec_map.get(entry["metric_name"])
             metric_summary_rows.append(
                 {
                     "metric_name": entry["metric_name"],
                     "avg_score": (entry["score_sum"] / count) if count else None,
                     "pass_rate": (entry["pass_count"] / count) if count else None,
                     "samples": count,
+                    "source": spec.source if spec else None,
                 }
             )
 
@@ -956,7 +978,25 @@ class BaseSQLStorageAdapter(ABC):
             (
                 "MetricsSummary",
                 metric_summary_rows,
-                ["metric_name", "avg_score", "pass_rate", "samples"],
+                ["metric_name", "avg_score", "pass_rate", "samples", "source"],
+            ),
+            (
+                "CustomMetrics",
+                custom_metric_rows,
+                [
+                    "schema_version",
+                    "metric_name",
+                    "source",
+                    "description",
+                    "evaluation_method",
+                    "inputs",
+                    "output",
+                    "evaluation_process",
+                    "rules",
+                    "notes",
+                    "implementation_path",
+                    "implementation_hash",
+                ],
             ),
             (
                 "RunPromptSets",

evalvault/adapters/outbound/tracker/langfuse_adapter.py

@@ -63,13 +63,15 @@ class LangfuseAdapter(TrackerPort):
                 span.update_trace(name=name, metadata=metadata)
             self._traces[trace_id] = span
         else:
-            # Langfuse 2.x: use trace method
-            trace = self._client.trace(
+            trace_fn: Any = getattr(self._client, "trace", None)
+            if trace_fn is None:
+                raise RuntimeError("Langfuse client does not expose trace API")
+            trace_obj = trace_fn(
                 name=name,
                 metadata=metadata,
             )
-            trace_id = trace.id
-            self._traces[trace_id] = trace
+            trace_id = trace_obj.id
+            self._traces[trace_id] = trace_obj
         return trace_id
 
     def add_span(
@@ -240,7 +242,7 @@ class LangfuseAdapter(TrackerPort):
             passed_count = sum(
                 1
                 for r in run.results
-                if r.get_metric(metric_name) and r.get_metric(metric_name).passed
+                if (metric := r.get_metric(metric_name)) and metric.passed is True
             )
             avg_score = run.get_avg_score(metric_name)
             threshold = run.thresholds.get(metric_name, 0.7)
@@ -358,6 +360,7 @@ class LangfuseAdapter(TrackerPort):
             "summary": trace_output["summary"],
             "metrics": metric_summary,
             "phoenix_links": phoenix_links or {},
+            "custom_metrics": (run.tracker_metadata or {}).get("custom_metric_snapshot"),
             "test_cases": [
                 {
                     "test_case_id": result.test_case_id,
@@ -421,12 +424,15 @@ class LangfuseAdapter(TrackerPort):
             }
 
             # Span metadata: additional info
-            span_metadata = {
+            span_metadata: dict[str, float | int] = {
                 "tokens_used": result.tokens_used,
                 "latency_ms": result.latency_ms,
             }
             if result.cost_usd:
-                span_metadata["cost_usd"] = result.cost_usd
+                span_metadata = {
+                    **span_metadata,
+                    "cost_usd": float(result.cost_usd),
+                }
 
             if hasattr(root_span, "start_span"):
                 child_span = root_span.start_span(

evalvault/adapters/outbound/tracker/mlflow_adapter.py

@@ -220,6 +220,11 @@ class MLflowAdapter(TrackerPort):
             results_data.append(result_dict)
 
         self.save_artifact(trace_id, "test_results", results_data)
+        self.save_artifact(
+            trace_id,
+            "custom_metric_snapshot",
+            (run.tracker_metadata or {}).get("custom_metric_snapshot"),
+        )
 
         # 6. End MLflow run
         self.end_trace(trace_id)

evalvault/adapters/outbound/tracker/phoenix_adapter.py

@@ -26,8 +26,7 @@ from evalvault.domain.entities import (
 from evalvault.ports.outbound.tracker_port import TrackerPort
 
 if TYPE_CHECKING:
-    from opentelemetry.sdk.trace import Span, TracerProvider
-    from opentelemetry.trace import Tracer
+    from opentelemetry.sdk.trace import TracerProvider
 
 
 class PhoenixAdapter(TrackerPort):
@@ -62,9 +61,10 @@ class PhoenixAdapter(TrackerPort):
         """
         self._endpoint = endpoint
         self._service_name = service_name
-        self._tracer: Tracer | None = None
+        self._tracer: Any | None = None
         self._tracer_provider: TracerProvider | None = None
-        self._active_spans: dict[str, Span] = {}
+        self._active_spans: dict[str, Any] = {}
+        self._tracer_any: Any | None = None
         self._initialized = False
 
     def _ensure_initialized(self) -> None:
@@ -90,7 +90,8 @@ class PhoenixAdapter(TrackerPort):
                 provider = get_tracer_provider()
                 if provider:
                     self._tracer_provider = provider
-                    self._tracer = trace.get_tracer(__name__)
+                    self._tracer_any = trace.get_tracer(__name__)
+                    self._tracer = self._tracer_any
                     self._initialized = True
                     return
 
@@ -109,7 +110,8 @@ class PhoenixAdapter(TrackerPort):
             trace.set_tracer_provider(self._tracer_provider)
 
             # Get tracer
-            self._tracer = trace.get_tracer(__name__)
+            self._tracer_any = trace.get_tracer(__name__)
+            self._tracer = self._tracer_any
             self._initialized = True
 
         except ImportError as e:
@@ -134,7 +136,12 @@ class PhoenixAdapter(TrackerPort):
         self._ensure_initialized()
 
         # Start a new span as root
-        span = self._tracer.start_span(name)
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
+        span = tracer.start_span(name)
         trace_id = str(uuid.uuid4())
 
         # Set metadata as span attributes
@@ -173,10 +180,15 @@ class PhoenixAdapter(TrackerPort):
 
         from opentelemetry import trace
 
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span(name, context=context) as span:
+        with tracer.start_span(name, context=context) as span:
             if input_data is not None:
                 safe_input = sanitize_payload(input_data, max_chars=MAX_LOG_CHARS)
                 span.set_attribute("input", json.dumps(safe_input, default=str))
@@ -279,7 +291,7 @@ class PhoenixAdapter(TrackerPort):
             passed_count = sum(
                 1
                 for r in run.results
-                if r.get_metric(metric_name) and r.get_metric(metric_name).passed
+                if (metric := r.get_metric(metric_name)) and metric.passed is True
             )
             avg_score = run.get_avg_score(metric_name)
             threshold = run.thresholds.get(metric_name, 0.7)
@@ -340,13 +352,40 @@ class PhoenixAdapter(TrackerPort):
                 "version": run.dataset_version,
                 "total_test_cases": run.total_test_cases,
             },
+            "evaluation_config": {
+                "model": run.model_name,
+                "metrics": run.metrics_evaluated,
+                "thresholds": run.thresholds,
+            },
             "summary": {
-                "pass_rate": run.pass_rate,
+                "total_test_cases": run.total_test_cases,
+                "passed": run.passed_test_cases,
+                "failed": run.total_test_cases - run.passed_test_cases,
+                "pass_rate": round(run.pass_rate, 4),
+                "duration_seconds": round(run.duration_seconds, 2)
+                if run.duration_seconds
+                else None,
                 "total_tokens": run.total_tokens,
-                "duration_seconds": run.duration_seconds,
             },
             "metrics": metric_summary,
+            "custom_metrics": (run.tracker_metadata or {}).get("custom_metric_snapshot"),
+            "test_cases": [
+                {
+                    "test_case_id": result.test_case_id,
+                    "all_passed": result.all_passed,
+                    "metrics": {
+                        metric.name: {
+                            "score": metric.score,
+                            "threshold": metric.threshold,
+                            "passed": metric.passed,
+                        }
+                        for metric in result.metrics
+                    },
+                }
+                for result in run.results
+            ],
         }
+
         self.save_artifact(trace_id, "ragas_evaluation", structured_artifact)
 
         # End the trace
@@ -369,10 +408,15 @@ class PhoenixAdapter(TrackerPort):
         """
         from opentelemetry import trace
 
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span(
+        with tracer.start_span(
             f"test-case-{result.test_case_id}",
             context=context,
         ) as span:
@@ -478,7 +522,12 @@ class PhoenixAdapter(TrackerPort):
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span("retrieval", context=context) as span:
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
+        with tracer.start_span("retrieval", context=context) as span:
             # Set retrieval attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)
@@ -560,7 +609,12 @@ class PhoenixAdapter(TrackerPort):
         parent_span = self._active_spans[trace_id]
         context = trace.set_span_in_context(parent_span)
 
-        with self._tracer.start_span("generation", context=context) as span:
+        tracer = self._tracer_any
+        if tracer is None:
+            tracer = self._tracer
+        if tracer is None:
+            raise RuntimeError("Phoenix tracer is not initialized")
+        with tracer.start_span("generation", context=context) as span:
             # Set generation attributes
             for key, value in data.to_span_attributes().items():
                 span.set_attribute(key, value)

evalvault/config/settings.py

@@ -321,6 +321,27 @@ class Settings(BaseSettings):
         default="https://cloud.langfuse.com", description="Langfuse host URL"
     )
 
+    mcp_enabled: bool = Field(
+        default=False,
+        description="Enable MCP JSON-RPC endpoint over HTTP.",
+    )
+    mcp_protocol_version: str = Field(
+        default="2025-11-25",
+        description="MCP protocol version to advertise.",
+    )
+    mcp_server_version: str = Field(
+        default="0.1.0",
+        description="EvalVault MCP server version.",
+    )
+    mcp_auth_tokens: str | None = Field(
+        default=None,
+        description="Comma-separated bearer tokens for MCP endpoint (required).",
+    )
+    mcp_allowed_tools: str | None = Field(
+        default=None,
+        description="Comma-separated allowlist of MCP tool names.",
+    )
+
     # MLflow Configuration (optional)
     mlflow_tracking_uri: str | None = Field(default=None, description="MLflow tracking server URI")
     mlflow_experiment_name: str = Field(default="evalvault", description="MLflow experiment name")

evalvault/domain/entities/__init__.py

@@ -34,6 +34,12 @@ from evalvault.domain.entities.improvement import (
     RAGComponent,
     RAGImprovementGuide,
 )
+from evalvault.domain.entities.judge_calibration import (
+    JudgeCalibrationCase,
+    JudgeCalibrationMetric,
+    JudgeCalibrationResult,
+    JudgeCalibrationSummary,
+)
 from evalvault.domain.entities.kg import EntityModel, RelationModel
 from evalvault.domain.entities.method import MethodInput, MethodInputDataset, MethodOutput
 from evalvault.domain.entities.prompt import Prompt, PromptSet, PromptSetBundle, PromptSetItem
@@ -104,6 +110,10 @@ __all__ = [
     "PatternType",
     "RAGComponent",
     "RAGImprovementGuide",
+    "JudgeCalibrationCase",
+    "JudgeCalibrationMetric",
+    "JudgeCalibrationResult",
+    "JudgeCalibrationSummary",
     # KG
     "EntityModel",
     "RelationModel",