evalvault 1.73.2__py3-none-any.whl → 1.75.0__py3-none-any.whl

evalvault/adapters/outbound/storage/factory.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from evalvault.adapters.outbound.storage.postgres_adapter import PostgreSQLStorageAdapter
+from evalvault.adapters.outbound.storage.sqlite_adapter import SQLiteStorageAdapter
+from evalvault.config.settings import Settings
+from evalvault.ports.outbound.storage_port import StoragePort
+
+logger = logging.getLogger(__name__)
+
+
+def build_storage_adapter(
+    *,
+    settings: Settings | None = None,
+    db_path: Path | None = None,
+    fallback_to_sqlite: bool = True,
+) -> StoragePort:
+    resolved_settings = settings or Settings()
+
+    if db_path is not None:
+        return SQLiteStorageAdapter(db_path=db_path)
+
+    backend = getattr(resolved_settings, "db_backend", "postgres")
+    if backend == "sqlite":
+        resolved_db_path = resolved_settings.evalvault_db_path
+        if resolved_db_path is None:
+            raise RuntimeError("SQLite backend selected but evalvault_db_path is not set.")
+        return SQLiteStorageAdapter(db_path=resolved_db_path)
+
+    conn_string = resolved_settings.postgres_connection_string
+    if not conn_string:
+        host = resolved_settings.postgres_host or "localhost"
+        port = resolved_settings.postgres_port
+        database = resolved_settings.postgres_database
+        user = resolved_settings.postgres_user or "postgres"
+        password = resolved_settings.postgres_password or ""
+        conn_string = f"host={host} port={port} dbname={database} user={user} password={password}"
+
+    try:
+        return PostgreSQLStorageAdapter(connection_string=conn_string)
+    except Exception as exc:
+        if not fallback_to_sqlite:
+            raise
+        logger.warning("PostgreSQL adapter failed (%s). Falling back to SQLite.", exc)
+        resolved_db_path = resolved_settings.evalvault_db_path
+        if resolved_db_path is None:
+            raise
+        return SQLiteStorageAdapter(db_path=resolved_db_path)
+
+
+__all__ = ["build_storage_adapter"]

evalvault/adapters/outbound/storage/postgres_adapter.py

@@ -1128,6 +1128,96 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
             )
         return reports
 
+    def save_ops_report(
+        self,
+        *,
+        report_id: str | None,
+        run_id: str | None,
+        report_type: str,
+        format: str,
+        content: str | None,
+        metadata: dict[str, Any] | None = None,
+        created_at: str | None = None,
+    ) -> str:
+        report_id = report_id or str(uuid.uuid4())
+        if created_at is None:
+            created_at_value = datetime.now(UTC)
+        else:
+            created_at_value = (
+                datetime.fromisoformat(created_at) if isinstance(created_at, str) else created_at
+            )
+
+        with self._get_connection() as conn:
+            conn.execute(
+                """
+                INSERT INTO ops_reports (
+                    report_id, run_id, report_type, format, content, metadata, created_at
+                ) VALUES (%s, %s, %s, %s, %s, %s, %s)
+                ON CONFLICT (report_id) DO UPDATE SET
+                    run_id = EXCLUDED.run_id,
+                    report_type = EXCLUDED.report_type,
+                    format = EXCLUDED.format,
+                    content = EXCLUDED.content,
+                    metadata = EXCLUDED.metadata,
+                    created_at = EXCLUDED.created_at
+                """,
+                (
+                    report_id,
+                    run_id,
+                    report_type,
+                    format,
+                    content,
+                    self._serialize_pipeline_json(metadata),
+                    created_at_value,
+                ),
+            )
+            conn.commit()
+
+        return report_id
+
+    def list_ops_reports(
+        self,
+        *,
+        run_id: str,
+        report_type: str | None = None,
+        format: str | None = None,
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        clauses = ["run_id = %s"]
+        params: list[Any] = [run_id]
+        if report_type:
+            clauses.append("report_type = %s")
+            params.append(report_type)
+        if format:
+            clauses.append("format = %s")
+            params.append(format)
+        params.append(limit)
+
+        query = (
+            "SELECT report_id, run_id, report_type, format, content, metadata, created_at "
+            "FROM ops_reports WHERE " + " AND ".join(clauses) + " ORDER BY created_at DESC LIMIT %s"
+        )
+
+        with self._get_connection() as conn:
+            rows = conn.execute(query, tuple(params)).fetchall()
+
+        reports: list[dict[str, Any]] = []
+        for row in rows:
+            reports.append(
+                {
+                    "report_id": row["report_id"],
+                    "run_id": row["run_id"],
+                    "report_type": row["report_type"],
+                    "format": row["format"],
+                    "content": row["content"],
+                    "metadata": self._deserialize_json(row["metadata"]),
+                    "created_at": row["created_at"].isoformat()
+                    if isinstance(row["created_at"], datetime)
+                    else row["created_at"],
+                }
+            )
+        return reports
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """

evalvault/adapters/outbound/storage/postgres_schema.sql

@@ -1,6 +1,8 @@
 -- EvalVault PostgreSQL Database Schema
 -- Stores evaluation runs, test case results, and metric scores
 
+CREATE EXTENSION IF NOT EXISTS vector;
+
 -- Main evaluation runs table
 CREATE TABLE IF NOT EXISTS evaluation_runs (
     run_id UUID PRIMARY KEY,
@@ -241,6 +243,19 @@ CREATE TABLE IF NOT EXISTS analysis_reports (
 CREATE INDEX IF NOT EXISTS idx_reports_run_id ON analysis_reports(run_id);
 CREATE INDEX IF NOT EXISTS idx_reports_experiment_id ON analysis_reports(experiment_id);
 
+-- Ops reports table
+CREATE TABLE IF NOT EXISTS ops_reports (
+    report_id UUID PRIMARY KEY,
+    run_id UUID REFERENCES evaluation_runs(run_id) ON DELETE SET NULL,
+    report_type VARCHAR(50) NOT NULL,
+    format VARCHAR(20) NOT NULL,
+    content TEXT,
+    metadata JSONB,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_ops_reports_run_id ON ops_reports(run_id);
+
 -- Analysis pipeline results table
 CREATE TABLE IF NOT EXISTS pipeline_results (
     result_id UUID PRIMARY KEY,

evalvault/adapters/outbound/storage/schema.sql

@@ -271,6 +271,20 @@ CREATE TABLE IF NOT EXISTS analysis_reports (
 CREATE INDEX IF NOT EXISTS idx_reports_run_id ON analysis_reports(run_id);
 CREATE INDEX IF NOT EXISTS idx_reports_experiment_id ON analysis_reports(experiment_id);
 
+-- Ops reports table
+CREATE TABLE IF NOT EXISTS ops_reports (
+    report_id TEXT PRIMARY KEY,
+    run_id TEXT,
+    report_type TEXT NOT NULL,  -- 'ops_report', 'ops_snapshot'
+    format TEXT NOT NULL,  -- 'markdown', 'json'
+    content TEXT,  -- Report content (markdown/json) or file path
+    metadata TEXT,  -- JSON metadata
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (run_id) REFERENCES evaluation_runs(run_id) ON DELETE SET NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_ops_reports_run_id ON ops_reports(run_id);
+
 -- Analysis pipeline results table
 CREATE TABLE IF NOT EXISTS pipeline_results (
     result_id TEXT PRIMARY KEY,

evalvault/adapters/outbound/storage/sqlite_adapter.py

@@ -1211,6 +1211,83 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             )
         return reports
 
+    def save_ops_report(
+        self,
+        *,
+        report_id: str | None,
+        run_id: str | None,
+        report_type: str,
+        format: str,
+        content: str | None,
+        metadata: dict[str, Any] | None = None,
+        created_at: str | None = None,
+    ) -> str:
+        report_id = report_id or str(uuid.uuid4())
+        created_at = created_at or datetime.now().isoformat()
+
+        with self._get_connection() as conn:
+            conn = cast(Any, conn)
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO ops_reports (
+                    report_id, run_id, report_type, format, content, metadata, created_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    report_id,
+                    run_id,
+                    report_type,
+                    format,
+                    content,
+                    self._serialize_json(metadata),
+                    created_at,
+                ),
+            )
+            conn.commit()
+
+        return report_id
+
+    def list_ops_reports(
+        self,
+        *,
+        run_id: str,
+        report_type: str | None = None,
+        format: str | None = None,
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        query = (
+            "SELECT report_id, run_id, report_type, format, content, metadata, created_at "
+            "FROM ops_reports WHERE run_id = ?"
+        )
+        params: list[Any] = [run_id]
+        if report_type:
+            query += " AND report_type = ?"
+            params.append(report_type)
+        if format:
+            query += " AND format = ?"
+            params.append(format)
+        query += " ORDER BY created_at DESC LIMIT ?"
+        params.append(limit)
+
+        with self._get_connection() as conn:
+            conn = cast(Any, conn)
+            rows = conn.execute(query, tuple(params)).fetchall()
+
+        reports: list[dict[str, Any]] = []
+        for row in rows:
+            reports.append(
+                {
+                    "report_id": row["report_id"],
+                    "run_id": row["run_id"],
+                    "report_type": row["report_type"],
+                    "format": row["format"],
+                    "content": row["content"],
+                    "metadata": self._deserialize_json(row["metadata"]),
+                    "created_at": row["created_at"],
+                }
+            )
+        return reports
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """

evalvault/config/settings.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from typing import Any
 
-from pydantic import Field, PrivateAttr
+from pydantic import AliasChoices, Field, PrivateAttr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from evalvault.config.secret_manager import (
@@ -179,6 +179,10 @@ class Settings(BaseSettings):
         default="data/db/evalvault.db",
         description="SQLite database path for API/CLI storage.",
     )
+    db_backend: str = Field(
+        default="postgres",
+        description="Storage backend: 'postgres' or 'sqlite'.",
+    )
     evalvault_memory_db_path: str = Field(
         default="data/db/evalvault_memory.db",
         description="SQLite database path for Domain Memory storage.",
@@ -242,7 +246,7 @@ class Settings(BaseSettings):
         description="Ollama server URL",
     )
     ollama_model: str = Field(
-        default="gpt-oss-safeguard:20b",
+        default="qwen3:14b",
         description="Ollama model name for evaluation",
     )
     ollama_embedding_model: str = Field(
@@ -395,11 +399,31 @@ class Settings(BaseSettings):
     )
 
     # PostgreSQL Configuration (optional)
-    postgres_host: str | None = Field(default=None, description="PostgreSQL server host")
-    postgres_port: int = Field(default=5432, description="PostgreSQL server port")
-    postgres_database: str = Field(default="evalvault", description="PostgreSQL database name")
-    postgres_user: str | None = Field(default=None, description="PostgreSQL user")
-    postgres_password: str | None = Field(default=None, description="PostgreSQL password")
+    postgres_host: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("POSTGRES_HOST", "EVALVAULT_DB_HOST"),
+        description="PostgreSQL server host",
+    )
+    postgres_port: int = Field(
+        default=5432,
+        validation_alias=AliasChoices("POSTGRES_PORT", "EVALVAULT_DB_PORT"),
+        description="PostgreSQL server port",
+    )
+    postgres_database: str = Field(
+        default="evalvault",
+        validation_alias=AliasChoices("POSTGRES_DATABASE", "EVALVAULT_DB_NAME"),
+        description="PostgreSQL database name",
+    )
+    postgres_user: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("POSTGRES_USER", "EVALVAULT_DB_USER"),
+        description="PostgreSQL user",
+    )
+    postgres_password: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("POSTGRES_PASSWORD", "EVALVAULT_DB_PASSWORD"),
+        description="PostgreSQL password",
+    )
     postgres_connection_string: str | None = Field(
         default=None, description="PostgreSQL connection string (overrides other postgres settings)"
     )

evalvault/domain/entities/ops_report.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from evalvault.domain.entities.stage import StageMetric, StageSummary
+
+
+@dataclass
+class OpsReport:
+    run_summary: dict[str, Any]
+    ops_kpis: dict[str, Any]
+    stage_summary: StageSummary | None
+    stage_metrics: list[StageMetric]
+    bottlenecks: list[dict[str, Any]]
+    recommendations: list[str]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_summary": self.run_summary,
+            "ops_kpis": self.ops_kpis,
+            "stage_summary": _stage_summary_to_dict(self.stage_summary),
+            "stage_metrics": [metric.to_dict() for metric in self.stage_metrics],
+            "bottlenecks": self.bottlenecks,
+            "recommendations": self.recommendations,
+            "metadata": self.metadata,
+        }
+
+
+def _stage_summary_to_dict(summary: StageSummary | None) -> dict[str, Any] | None:
+    if summary is None:
+        return None
+    return {
+        "run_id": summary.run_id,
+        "total_events": summary.total_events,
+        "stage_type_counts": summary.stage_type_counts,
+        "stage_type_avg_durations": summary.stage_type_avg_durations,
+        "missing_required_stage_types": summary.missing_required_stage_types,
+    }

evalvault/domain/services/domain_learning_hook.py

@@ -20,7 +20,8 @@ class DomainLearningHook:
     Formation dynamics를 구현합니다.
 
     사용 예시:
-        memory_adapter = SQLiteDomainMemoryAdapter()
+        from evalvault.adapters.outbound.domain_memory import build_domain_memory_adapter
+        memory_adapter = build_domain_memory_adapter()
         hook = DomainLearningHook(memory_adapter)
 
         # 평가 후 메모리 형성

evalvault/domain/services/ops_report_service.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from typing import Any
+
+from evalvault.config.langfuse_support import get_langfuse_trace_url
+from evalvault.config.phoenix_support import get_phoenix_trace_url
+from evalvault.domain.entities.ops_report import OpsReport
+from evalvault.domain.entities.stage import StageEvent, StageMetric, StageSummary
+from evalvault.domain.services.stage_metric_guide_service import StageMetricGuideService
+from evalvault.domain.services.stage_metric_service import StageMetricService
+from evalvault.domain.services.stage_summary_service import StageSummaryService
+from evalvault.ports.outbound.stage_storage_port import StageStoragePort
+from evalvault.ports.outbound.storage_port import StoragePort
+
+
+class OpsReportService:
+    """Build an operational report for an evaluation run."""
+
+    def __init__(
+        self,
+        *,
+        metric_service: StageMetricService | None = None,
+        summary_service: StageSummaryService | None = None,
+        guide_service: StageMetricGuideService | None = None,
+    ) -> None:
+        self._metric_service = metric_service or StageMetricService()
+        self._summary_service = summary_service or StageSummaryService()
+        self._guide_service = guide_service or StageMetricGuideService()
+
+    def build_report(
+        self,
+        run_id: str,
+        *,
+        storage: StoragePort,
+        stage_storage: StageStoragePort,
+    ) -> OpsReport:
+        run = storage.get_run(run_id)
+        run_summary = run.to_summary_dict()
+        phoenix_trace_url = get_phoenix_trace_url(run.tracker_metadata)
+        langfuse_trace_url = get_langfuse_trace_url(run.tracker_metadata)
+
+        events = stage_storage.list_stage_events(run_id)
+        stage_summary = self._summarize_events(events)
+
+        stage_metrics = stage_storage.list_stage_metrics(run_id)
+        if not stage_metrics and events:
+            stage_metrics = self._metric_service.build_metrics(events)
+
+        bottlenecks = self._build_bottlenecks(stage_summary)
+        recommendations = self._build_recommendations(stage_metrics)
+
+        ops_kpis = self._build_ops_kpis(run, events)
+
+        metadata = {
+            "phoenix_trace_url": phoenix_trace_url,
+            "langfuse_trace_url": langfuse_trace_url,
+        }
+
+        return OpsReport(
+            run_summary=run_summary,
+            ops_kpis=ops_kpis,
+            stage_summary=stage_summary,
+            stage_metrics=stage_metrics,
+            bottlenecks=bottlenecks,
+            recommendations=recommendations,
+            metadata=metadata,
+        )
+
+    def _summarize_events(self, events: list[StageEvent]) -> StageSummary | None:
+        if not events:
+            return None
+        return self._summary_service.summarize(events)
+
+    def _build_bottlenecks(self, summary: StageSummary | None) -> list[dict[str, Any]]:
+        if summary is None:
+            return []
+        bottlenecks: list[dict[str, Any]] = []
+
+        for stage_type in summary.missing_required_stage_types:
+            bottlenecks.append(
+                {
+                    "type": "missing_stage",
+                    "stage_type": stage_type,
+                    "detail": "required stage missing",
+                }
+            )
+
+        durations = summary.stage_type_avg_durations
+        if durations:
+            top = sorted(durations.items(), key=lambda item: item[1], reverse=True)[:3]
+            for stage_type, duration in top:
+                bottlenecks.append(
+                    {
+                        "type": "latency",
+                        "stage_type": stage_type,
+                        "avg_duration_ms": round(duration, 3),
+                    }
+                )
+        return bottlenecks
+
+    def _build_recommendations(self, metrics: list[StageMetric]) -> list[str]:
+        if not metrics:
+            return []
+        guides = self._guide_service.build_guides(metrics)
+        recommendations: list[str] = []
+        for guide in guides:
+            top_action = guide.top_action
+            if top_action is None:
+                continue
+            hint = top_action.implementation_hint or top_action.description
+            label = f"[{guide.priority.value}] {guide.component.value}"
+            if hint:
+                recommendations.append(f"{label}: {top_action.title} - {hint}")
+            else:
+                recommendations.append(f"{label}: {top_action.title}")
+        return recommendations
+
+    def _build_ops_kpis(self, run, events: list[StageEvent]) -> dict[str, Any]:
+        total_cases = run.total_test_cases
+        latencies = [r.latency_ms for r in run.results if r.latency_ms]
+        tokens_used = [r.tokens_used for r in run.results if r.tokens_used]
+        costs = [r.cost_usd for r in run.results if r.cost_usd is not None]
+
+        avg_latency = _average(latencies)
+        p95_latency = _percentile(latencies, 0.95)
+        avg_tokens = _average(tokens_used)
+        avg_cost = _average(costs)
+        pass_rate = run.pass_rate
+        failure_rate = None if pass_rate is None else max(0.0, 1.0 - pass_rate)
+
+        error_rate = _stage_error_rate(events)
+        error_severity = _stage_error_severity(error_rate)
+
+        return {
+            "total_test_cases": total_cases,
+            "pass_rate": pass_rate,
+            "failure_rate": failure_rate,
+            "stage_error_rate": error_rate,
+            "stage_error_severity": error_severity,
+            "duration_seconds": run.duration_seconds,
+            "total_tokens": run.total_tokens,
+            "total_cost_usd": run.total_cost_usd,
+            "avg_latency_ms": avg_latency,
+            "p95_latency_ms": p95_latency,
+            "avg_tokens_per_case": avg_tokens,
+            "avg_cost_per_case_usd": avg_cost,
+        }
+
+
+def _average(values: list[float | int]) -> float | None:
+    if not values:
+        return None
+    return float(sum(values)) / len(values)
+
+
+def _percentile(values: list[float | int], ratio: float) -> float | None:
+    if not values:
+        return None
+    if ratio <= 0:
+        return float(min(values))
+    if ratio >= 1:
+        return float(max(values))
+    sorted_values = sorted(values)
+    index = int(round((len(sorted_values) - 1) * ratio))
+    return float(sorted_values[index])
+
+
+def _stage_error_rate(events: list[StageEvent]) -> float | None:
+    if not events:
+        return None
+    total = len(events)
+    failure_statuses = {"failed", "error", "timeout", "aborted"}
+    success_statuses = {"success", "ok", "completed", "pass"}
+    failures = 0
+    for event in events:
+        status = str(event.status or "").strip().lower()
+        if status in failure_statuses:
+            failures += 1
+            continue
+        if status and status not in success_statuses:
+            failures += 1
+    return failures / total
+
+
+def _stage_error_severity(rate: float | None) -> str | None:
+    if rate is None:
+        return None
+    if rate >= 0.05:
+        return "critical"
+    if rate >= 0.02:
+        return "warning"
+    return "ok"

evalvault/ports/inbound/web_port.py

@@ -18,7 +18,7 @@ class EvalRequest:
 
     dataset_path: str
     metrics: list[str]
-    model_name: str = "ollama/gpt-oss-safeguard:20b"
+    model_name: str = "ollama/qwen3:14b"
     evaluation_task: str = "qa"
     langfuse_enabled: bool = False
     thresholds: dict[str, float] = field(default_factory=dict)
@@ -121,12 +121,14 @@ class WebUIPort(Protocol):
     def list_runs(
         self,
         limit: int = 50,
+        offset: int = 0,
         filters: RunFilters | None = None,
     ) -> list[RunSummary]:
         """평가 목록 조회.
 
         Args:
             limit: 최대 조회 개수
+            offset: 조회 시작 위치
             filters: 필터 조건
 
         Returns:

evalvault/ports/outbound/storage_port.py

@@ -83,6 +83,7 @@ class StoragePort(Protocol):
     def list_runs(
         self,
         limit: int = 100,
+        offset: int = 0,
         dataset_name: str | None = None,
         model_name: str | None = None,
     ) -> list[EvaluationRun]:
@@ -90,6 +91,7 @@ class StoragePort(Protocol):
 
         Args:
             limit: 최대 조회 개수
+            offset: 조회 시작 위치 (선택)
             dataset_name: 필터링할 데이터셋 이름 (선택)
             model_name: 필터링할 모델 이름 (선택)