evalvault 1.63.1__py3-none-any.whl → 1.65.0__py3-none-any.whl

evalvault/adapters/outbound/analysis/comparison_pipeline_adapter.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import asyncio
+
+from evalvault.domain.entities.analysis_pipeline import AnalysisIntent, PipelineResult
+from evalvault.domain.services.pipeline_orchestrator import AnalysisPipelineService
+from evalvault.ports.outbound.comparison_pipeline_port import ComparisonPipelinePort
+
+
+class ComparisonPipelineAdapter(ComparisonPipelinePort):
+    def __init__(self, service: AnalysisPipelineService) -> None:
+        self._service = service
+
+    def run_comparison(
+        self,
+        *,
+        run_ids: list[str],
+        compare_metrics: list[str] | None,
+        test_type: str,
+        parallel: bool,
+        concurrency: int | None,
+        report_type: str,
+        use_llm_report: bool,
+    ) -> PipelineResult:
+        params = {
+            "run_ids": run_ids,
+            "compare_metrics": compare_metrics,
+            "test_type": test_type,
+            "report_type": report_type,
+            "use_llm_report": use_llm_report,
+        }
+        if parallel:
+            if concurrency is not None:
+                params["max_concurrency"] = concurrency
+            return asyncio.run(
+                self._service.analyze_intent_async(
+                    AnalysisIntent.GENERATE_COMPARISON,
+                    run_id=run_ids[0] if run_ids else None,
+                    **params,
+                )
+            )
+        return self._service.analyze_intent(
+            AnalysisIntent.GENERATE_COMPARISON,
+            run_id=run_ids[0] if run_ids else None,
+            **params,
+        )
+
+
+__all__ = ["ComparisonPipelineAdapter"]

evalvault/adapters/outbound/artifact_fs.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.ports.outbound.artifact_fs_port import ArtifactFileSystemPort
+
+
+class LocalArtifactFileSystemAdapter(ArtifactFileSystemPort):
+    def exists(self, path: Path) -> bool:
+        return path.exists()
+
+    def is_dir(self, path: Path) -> bool:
+        return path.is_dir()
+
+    def read_text(self, path: Path) -> str:
+        return path.read_text(encoding="utf-8")

evalvault/adapters/outbound/filesystem/__init__.py

@@ -0,0 +1,3 @@
+from evalvault.adapters.outbound.filesystem.ops_snapshot_writer import OpsSnapshotWriter
+
+__all__ = ["OpsSnapshotWriter"]

evalvault/adapters/outbound/filesystem/difficulty_profile_writer.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.ports.outbound.difficulty_profile_port import DifficultyProfileWriterPort
+
+
+class DifficultyProfileWriter(DifficultyProfileWriterPort):
+    def write_profile(
+        self,
+        *,
+        output_path: Path,
+        artifacts_dir: Path,
+        envelope: dict[str, object],
+        artifacts: dict[str, object],
+    ) -> dict[str, object]:
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+        breakdown_path = artifacts_dir / "difficulty_breakdown.json"
+        cases_path = artifacts_dir / "difficulty_cases.json"
+        breakdown_payload = artifacts.get("breakdown")
+        cases_payload = artifacts.get("cases")
+        write_json(
+            breakdown_path,
+            breakdown_payload if isinstance(breakdown_payload, dict) else {},
+        )
+        write_json(
+            cases_path,
+            {"cases": cases_payload} if isinstance(cases_payload, list) else {"cases": []},
+        )
+
+        index_payload = {
+            "files": {
+                "breakdown": str(breakdown_path),
+                "cases": str(cases_path),
+            }
+        }
+        index_path = artifacts_dir / "index.json"
+        write_json(index_path, index_payload)
+
+        artifacts_index = {
+            "dir": str(artifacts_dir),
+            "index": str(index_path),
+        }
+        envelope["artifacts"] = artifacts_index
+        write_json(output_path, envelope)
+
+        return artifacts_index

evalvault/adapters/outbound/filesystem/ops_snapshot_writer.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from evalvault.adapters.inbound.cli.utils.analysis_io import write_json
+from evalvault.ports.outbound.ops_snapshot_port import OpsSnapshotWriterPort
+
+
+class OpsSnapshotWriter(OpsSnapshotWriterPort):
+    def write_snapshot(self, path: Path, payload: dict[str, Any]) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        write_json(path, payload)

evalvault/adapters/outbound/judge_calibration_adapter.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from evalvault.domain.entities import EvaluationRun, SatisfactionFeedback
+from evalvault.domain.entities.judge_calibration import JudgeCalibrationResult
+from evalvault.domain.services.judge_calibration_service import JudgeCalibrationService
+from evalvault.ports.outbound.judge_calibration_port import JudgeCalibrationPort
+
+
+class JudgeCalibrationAdapter(JudgeCalibrationPort):
+    def __init__(self) -> None:
+        self._service = JudgeCalibrationService()
+
+    def calibrate(
+        self,
+        run: EvaluationRun,
+        feedbacks: list[SatisfactionFeedback],
+        *,
+        labels_source: str,
+        method: str,
+        metrics: list[str],
+        holdout_ratio: float,
+        seed: int,
+        parallel: bool = False,
+        concurrency: int = 8,
+    ) -> JudgeCalibrationResult:
+        return self._service.calibrate(
+            run,
+            feedbacks,
+            labels_source=labels_source,
+            method=method,
+            metrics=metrics,
+            holdout_ratio=holdout_ratio,
+            seed=seed,
+            parallel=parallel,
+            concurrency=concurrency,
+        )

evalvault/adapters/outbound/judge_calibration_reporter.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.entities.judge_calibration import JudgeCalibrationResult
+
+
+class JudgeCalibrationReporter:
+    def render_json(self, result: JudgeCalibrationResult) -> dict[str, Any]:
+        return {
+            "summary": asdict(result.summary),
+            "metrics": [asdict(metric) for metric in result.metrics],
+            "case_results": {
+                metric: [asdict(entry) for entry in entries]
+                for metric, entries in result.case_results.items()
+            },
+            "warnings": list(result.warnings),
+        }
+
+    def write_artifacts(
+        self,
+        *,
+        result: JudgeCalibrationResult,
+        artifacts_dir: Path,
+    ) -> dict[str, str]:
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+        index_path = artifacts_dir / "index.json"
+        payload = {
+            "run_id": result.summary.run_id,
+            "metrics": [metric.metric for metric in result.metrics],
+            "cases": {},
+        }
+        for metric, cases in result.case_results.items():
+            case_path = artifacts_dir / f"{metric}.json"
+            case_payload = [
+                {
+                    "test_case_id": case.test_case_id,
+                    "raw_score": case.raw_score,
+                    "calibrated_score": case.calibrated_score,
+                    "label": case.label,
+                    "label_source": case.label_source,
+                }
+                for case in cases
+            ]
+            case_path.write_text(
+                json.dumps(case_payload, ensure_ascii=False, indent=2),
+                encoding="utf-8",
+            )
+            payload["cases"][metric] = str(case_path)
+        index_path.write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        return {"dir": str(artifacts_dir), "index": str(index_path)}

evalvault/adapters/outbound/methods/external_command.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import os
 import subprocess
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
@@ -18,7 +19,9 @@ class ExternalCommandMethod(RagMethodPort):
 
     name = "external_command"
     version = "0.1.0"
-    description = "Execute a method in a separate process."
+    description = (
+        "Execute a method in a separate process (shell=True requires a trusted command string)."
+    )
     tags = ("external", "isolation")
 
     def __init__(
@@ -67,6 +70,7 @@ class ExternalCommandMethod(RagMethodPort):
         )
 
         command = self._build_command(runtime)
+        self._validate_shell_usage(command)
         result = subprocess.run(  # noqa: S603 - user-controlled command by design
             command,
             cwd=self._workdir,
@@ -104,6 +108,23 @@ class ExternalCommandMethod(RagMethodPort):
         except KeyError as exc:
             raise ValueError(f"Unknown command placeholder: {exc}") from exc
 
+    def _validate_shell_usage(self, command: list[str] | str) -> None:
+        if not self._shell:
+            return
+        if not isinstance(command, str):
+            raise ValueError(
+                "shell=True requires a single command string; list arguments are rejected."
+            )
+        if not command.strip():
+            raise ValueError("shell=True requires a non-empty command string.")
+        if "\n" in command or "\r" in command:
+            raise ValueError("shell=True command must not contain newlines.")
+        warnings.warn(
+            "shell=True executes through the system shell. Use only trusted commands.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
     @staticmethod
     def _load_payload(path: Path) -> Any:
         if not path.exists():

evalvault/adapters/outbound/tracker/langfuse_adapter.py

@@ -4,6 +4,13 @@ from typing import Any
 
 from langfuse import Langfuse
 
+from evalvault.adapters.outbound.tracker.log_sanitizer import (
+    MAX_CONTEXT_CHARS,
+    MAX_LOG_CHARS,
+    sanitize_payload,
+    sanitize_text,
+    sanitize_text_list,
+)
 from evalvault.config.phoenix_support import extract_phoenix_links
 from evalvault.domain.entities import EvaluationRun
 from evalvault.ports.outbound.tracker_port import TrackerPort
@@ -56,13 +63,15 @@ class LangfuseAdapter(TrackerPort):
                 span.update_trace(name=name, metadata=metadata)
             self._traces[trace_id] = span
         else:
-            # Langfuse 2.x: use trace method
-            trace = self._client.trace(
+            trace_fn: Any = getattr(self._client, "trace", None)
+            if trace_fn is None:
+                raise RuntimeError("Langfuse client does not expose trace API")
+            trace_obj = trace_fn(
                 name=name,
                 metadata=metadata,
             )
-            trace_id = trace.id
-            self._traces[trace_id] = trace
+            trace_id = trace_obj.id
+            self._traces[trace_id] = trace_obj
         return trace_id
 
     def add_span(
@@ -88,21 +97,31 @@ class LangfuseAdapter(TrackerPort):
             raise ValueError(f"Trace not found: {trace_id}")
 
         trace_or_span = self._traces[trace_id]
+        safe_input = (
+            sanitize_payload(input_data, max_chars=MAX_LOG_CHARS)
+            if input_data is not None
+            else None
+        )
+        safe_output = (
+            sanitize_payload(output_data, max_chars=MAX_LOG_CHARS)
+            if output_data is not None
+            else None
+        )
         # Support both old and new Langfuse API
         if hasattr(trace_or_span, "start_span"):
             # Langfuse 3.x: create nested span
             child_span = trace_or_span.start_span(
                 name=name,
-                input=input_data,
-                output=output_data,
+                input=safe_input,
+                output=safe_output,
             )
             child_span.end()
         else:
             # Langfuse 2.x: use span method on trace
             trace_or_span.span(
                 name=name,
-                input=input_data,
-                output=output_data,
+                input=safe_input,
+                output=safe_output,
             )
 
     def log_score(
@@ -223,7 +242,7 @@ class LangfuseAdapter(TrackerPort):
             passed_count = sum(
                 1
                 for r in run.results
-                if r.get_metric(metric_name) and r.get_metric(metric_name).passed
+                if (metric := r.get_metric(metric_name)) and metric.passed is True
             )
             avg_score = run.get_avg_score(metric_name)
             threshold = run.thresholds.get(metric_name, 0.7)
@@ -377,10 +396,13 @@ class LangfuseAdapter(TrackerPort):
             # Span input: test case data (question, answer, contexts, ground_truth)
             span_input = {
                 "test_case_id": result.test_case_id,
-                "question": result.question,
-                "answer": result.answer,
-                "contexts": result.contexts,
-                "ground_truth": result.ground_truth,
+                "question": sanitize_text(result.question, max_chars=MAX_LOG_CHARS),
+                "answer": sanitize_text(result.answer, max_chars=MAX_LOG_CHARS),
+                "contexts": sanitize_text_list(
+                    result.contexts,
+                    max_chars=MAX_CONTEXT_CHARS,
+                ),
+                "ground_truth": sanitize_text(result.ground_truth, max_chars=MAX_LOG_CHARS),
             }
 
             # Span output: evaluation results
@@ -401,12 +423,15 @@ class LangfuseAdapter(TrackerPort):
             }
 
             # Span metadata: additional info
-            span_metadata = {
+            span_metadata: dict[str, float | int] = {
                 "tokens_used": result.tokens_used,
                 "latency_ms": result.latency_ms,
             }
             if result.cost_usd:
-                span_metadata["cost_usd"] = result.cost_usd
+                span_metadata = {
+                    **span_metadata,
+                    "cost_usd": float(result.cost_usd),
+                }
 
             if hasattr(root_span, "start_span"):
                 child_span = root_span.start_span(

evalvault/adapters/outbound/tracker/log_sanitizer.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+MASK_TOKEN = "[REDACTED]"
+MAX_LOG_CHARS = 1000
+MAX_CONTEXT_CHARS = 500
+MAX_LIST_ITEMS = 20
+MAX_PAYLOAD_DEPTH = 2
+
+_EMAIL_PATTERN = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+_PHONE_PATTERN = re.compile(
+    r"\b(?:\+?\d{1,3}[-.\s]?)?(?:\(?\d{2,4}\)?[-.\s]?)?\d{3,4}[-.\s]?\d{4}\b"
+)
+_SSN_PATTERN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
+_CARD_PATTERN = re.compile(r"\b(?:\d[ -]*?){13,16}\b")
+
+
+def _mask_pii(text: str) -> str:
+    text = _EMAIL_PATTERN.sub(MASK_TOKEN, text)
+    text = _PHONE_PATTERN.sub(MASK_TOKEN, text)
+    text = _SSN_PATTERN.sub(MASK_TOKEN, text)
+    text = _CARD_PATTERN.sub(MASK_TOKEN, text)
+    return text
+
+
+def _truncate(text: str, max_chars: int) -> str:
+    if max_chars <= 0:
+        return ""
+    if len(text) <= max_chars:
+        return text
+    if max_chars <= 3:
+        return text[:max_chars]
+    return f"{text[: max_chars - 3]}..."
+
+
+def sanitize_text(value: str | None, *, max_chars: int = MAX_LOG_CHARS) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        value = str(value)
+    return _truncate(_mask_pii(value), max_chars)
+
+
+def sanitize_text_list(
+    values: list[str] | tuple[str, ...] | None,
+    *,
+    max_items: int = MAX_LIST_ITEMS,
+    max_chars: int = MAX_CONTEXT_CHARS,
+) -> list[str]:
+    if not values:
+        return []
+    trimmed = list(values)[:max_items]
+    return [sanitize_text(item, max_chars=max_chars) or "" for item in trimmed]
+
+
+def sanitize_payload(
+    value: Any,
+    *,
+    max_chars: int = MAX_LOG_CHARS,
+    max_items: int = MAX_LIST_ITEMS,
+    max_depth: int = MAX_PAYLOAD_DEPTH,
+) -> Any:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return sanitize_text(value, max_chars=max_chars)
+    if isinstance(value, bool | int | float):
+        return value
+    if max_depth <= 0:
+        return sanitize_text(str(value), max_chars=max_chars)
+    if isinstance(value, dict):
+        return {
+            key: sanitize_payload(
+                item,
+                max_chars=max_chars,
+                max_items=max_items,
+                max_depth=max_depth - 1,
+            )
+            for key, item in list(value.items())[:max_items]
+        }
+    if isinstance(value, list | tuple | set):
+        return [
+            sanitize_payload(
+                item,
+                max_chars=max_chars,
+                max_items=max_items,
+                max_depth=max_depth - 1,
+            )
+            for item in list(value)[:max_items]
+        ]
+    return sanitize_text(str(value), max_chars=max_chars)

evalvault/adapters/outbound/tracker/mlflow_adapter.py

@@ -4,6 +4,7 @@ import json
 import tempfile
 from typing import Any
 
+from evalvault.adapters.outbound.tracker.log_sanitizer import MAX_LOG_CHARS, sanitize_payload
 from evalvault.domain.entities import EvaluationRun
 from evalvault.ports.outbound.tracker_port import TrackerPort
 
@@ -85,8 +86,8 @@ class MLflowAdapter(TrackerPort):
         # Store span data as JSON artifact
         span_data = {
             "name": name,
-            "input": input_data,
-            "output": output_data,
+            "input": sanitize_payload(input_data, max_chars=MAX_LOG_CHARS),
+            "output": sanitize_payload(output_data, max_chars=MAX_LOG_CHARS),
         }
 
         with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: