evalvault 1.69.0__tar.gz → 1.70.0__tar.gz

{evalvault-1.69.0 → evalvault-1.70.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.69.0
+Version: 1.70.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme

evalvault-1.70.0/config/regressions/ci.json

@@ -0,0 +1,20 @@
+{
+  "suites": [
+    {
+      "name": "unit-cli-gate",
+      "description": "Quality gate related CLI behavior",
+      "command": ["pytest", "tests/unit/test_cli.py", "-k", "gate", "-q"],
+      "timeout": 600
+    },
+    {
+      "name": "integration-cli-e2e",
+      "description": "CLI E2E smoke tests without API keys",
+      "command": [
+        "pytest",
+        "tests/integration/test_e2e_scenarios.py::TestCLIIntegrationE2E",
+        "-vv"
+      ],
+      "timeout": 900
+    }
+  ]
+}

evalvault-1.70.0/docs/guides/CI_REGRESSION_GATE.md

@@ -0,0 +1,36 @@
+# CI 회귀 게이트 (Regression Gate)
+
+EvalVault의 회귀 게이트는 CI에서 **핵심 CLI 흐름이 깨지지 않았는지** 빠르게 확인하는 안전장치입니다.
+
+## 목적
+- PR/릴리즈마다 핵심 CLI 경로를 최소 비용으로 재검증
+- API 키 없이 실행 가능한 스위트만 사용
+
+## 구성
+
+### 설정 파일
+- `config/regressions/ci.json`
+  - `unit-cli-gate`: gate 관련 CLI 유닛 테스트
+  - `integration-cli-e2e`: API 키 없이 가능한 CLI e2e 스모크
+
+### 실행 스크립트
+- `scripts/ci/run_regression_gate.py`
+
+## 로컬 실행
+
+```bash
+uv run python scripts/ci/run_regression_gate.py \
+  --config config/regressions/ci.json \
+  --format text
+```
+
+## CI 통합
+
+- `.github/workflows/ci.yml`의 `regression-gate` job에서 실행
+- 실패 시 CI가 실패하며, GitHub Actions 로그에 실패 스위트가 표시됩니다.
+
+## 실패 기준
+- 어떤 스위트든 실패 시 게이트 실패
+
+## 요약 파일
+- `reports/regression/ci_gate.json`에 요약이 저장됩니다.

evalvault-1.70.0/docs/guides/MULTITURN_EVAL_GUIDE.md

@@ -0,0 +1,45 @@
+# 멀티턴 평가 가이드
+
+이 문서는 멀티턴(대화형) RAG 평가를 **단일 턴 데이터셋 구조** 안에서 운영하는 최소 기준을 정의합니다.
+
+## 핵심 원칙
+- 멀티턴은 `test_cases`를 평탄화(flatten)하고, 메타데이터로 세션/턴을 연결합니다.
+- 기존 로더/평가/분석 파이프라인을 변경하지 않고, 추가 메타데이터로 멀티턴 집계를 수행합니다.
+
+## 데이터셋 필드 규약 (필수)
+`test_cases[].metadata`에 아래 키를 넣습니다.
+
+```json
+{
+  "metadata": {
+    "conversation_id": "conv-001",
+    "turn_index": 1,
+    "turn_id": "t01"
+  }
+}
+```
+
+### 필드 정의
+- `conversation_id`: 동일 대화 세션 식별자
+- `turn_index`: 턴 순서(정수)
+- `turn_id`: 턴 고유 ID (선택적으로 문자열)
+
+## 실행/분석 흐름
+1. `evalvault run`으로 실행 후 `--auto-analyze` 또는 별도 분석 파이프라인 실행
+2. 분석 파이프라인의 `multiturn_analyzer` 모듈이 대화/턴 집계를 생성
+3. 산출물은 `reports/analysis/artifacts/analysis_<RUN_ID>/index.json`에 등록
+
+## 산출물 요약
+`multiturn_analyzer` 모듈 출력:
+- `summary`: 대화 수, 평균 턴 수, 대화 단위 통과율, 최초 실패 턴 분포
+- `conversations`: 대화별 요약(최악 턴, 메트릭 평균)
+- `turns`: 턴 단위 상세
+- `coverage`: conversation_id/turn_index 커버리지
+
+## 주의사항
+- `turn_index`가 누락되면 대화 순서를 정확히 복원할 수 없습니다.
+- `conversation_id`가 없는 케이스는 대화 집계에서 제외됩니다.
+
+## 예시 템플릿
+- `docs/templates/dataset_template.json`
+- `docs/templates/ragas_dataset_example_ko90_en10.json`

{evalvault-1.69.0 → evalvault-1.70.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalvault"
-version = "1.69.0"
+version = "1.70.0"
 description = "RAG evaluation system using Ragas with Phoenix/Langfuse tracing"
 readme = "README.md"
 requires-python = ">=3.12"

evalvault-1.70.0/scripts/ci/run_regression_gate.py

@@ -0,0 +1,97 @@
+"""Run regression suites for CI quality gate."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+from pathlib import Path
+
+from evalvault.scripts.regression_runner import (
+    append_issue_log,
+    format_summary,
+    load_regression_config,
+    run_regression_suites,
+    select_suites,
+    write_json_summary,
+)
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="EvalVault regression gate runner")
+    parser.add_argument("--config", type=Path, default=None, help="Regression config path")
+    parser.add_argument(
+        "--suites",
+        type=str,
+        default=None,
+        help="Comma-separated suite names to run",
+    )
+    parser.add_argument(
+        "--summary",
+        type=Path,
+        default=None,
+        help="Write JSON summary to a file",
+    )
+    parser.add_argument(
+        "--issue-log",
+        type=Path,
+        default=None,
+        help="Append summary to a markdown log",
+    )
+    parser.add_argument("--tag", type=str, default=None, help="Label for the run")
+    parser.add_argument(
+        "--format",
+        type=str,
+        default="text",
+        choices=["text", "github-actions"],
+        help="Output format",
+    )
+    parser.add_argument(
+        "--stop-on-failure",
+        action="store_true",
+        help="Stop on first suite failure",
+    )
+    return parser.parse_args()
+
+
+def _split_names(raw: str | None) -> Sequence[str] | None:
+    if not raw:
+        return None
+    return [name.strip() for name in raw.split(",") if name.strip()]
+
+
+def _emit_github_actions(results) -> None:
+    for result in results:
+        status = "✅" if result.succeeded else "❌"
+        print(f"{status} {result.name} — {result.status.upper()} ({result.duration:.1f}s)")
+        if not result.succeeded:
+            message = result.stderr.splitlines()[-1] if result.stderr else "Suite failed"
+            print(f"::error::Regression suite failed: {result.name} ({message})")
+    passed = all(result.succeeded for result in results)
+    print(f"::set-output name=passed::{str(passed).lower()}")
+
+
+def main() -> int:
+    args = _parse_args()
+
+    suites = load_regression_config(args.config)
+    selected = select_suites(suites, _split_names(args.suites))
+    results = run_regression_suites(selected, stop_on_failure=args.stop_on_failure)
+
+    summary = format_summary(results, tag=args.tag)
+    if args.summary:
+        write_json_summary(args.summary, results, tag=args.tag)
+    if args.issue_log:
+        append_issue_log(args.issue_log, summary)
+
+    if args.format == "github-actions":
+        _emit_github_actions(results)
+    else:
+        print(summary)
+
+    if any(not result.succeeded for result in results):
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evalvault-1.70.0/scripts/offline/bundle_datasets.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+cd "$ROOT_DIR"
+
+OUTPUT_TAR=${OUTPUT_TAR:-dist/evalvault_datasets.tar}
+INCLUDE_DATA=${INCLUDE_DATA:-1}
+INCLUDE_FIXTURES=${INCLUDE_FIXTURES:-1}
+INCLUDE_TEMPLATES=${INCLUDE_TEMPLATES:-1}
+
+ITEMS=()
+
+if [ "$INCLUDE_DATA" = "1" ] && [ -d "data" ]; then
+  ITEMS+=("data")
+fi
+if [ "$INCLUDE_FIXTURES" = "1" ] && [ -d "tests/fixtures" ]; then
+  ITEMS+=("tests/fixtures")
+fi
+if [ "$INCLUDE_TEMPLATES" = "1" ] && [ -d "dataset_templates" ]; then
+  ITEMS+=("dataset_templates")
+fi
+
+if [ ${#ITEMS[@]} -eq 0 ]; then
+  echo "No dataset assets to bundle." >&2
+  exit 1
+fi
+
+mkdir -p "$(dirname "$OUTPUT_TAR")"
+tar -cf "$OUTPUT_TAR" "${ITEMS[@]}"
+sha256sum "$OUTPUT_TAR" > "${OUTPUT_TAR}.sha256"
+
+echo "Saved: $OUTPUT_TAR"
+echo "SHA256: ${OUTPUT_TAR}.sha256"

evalvault-1.70.0/scripts/offline/restore_datasets.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ARCHIVE=${1:-dist/evalvault_datasets.tar}
+
+if [ ! -f "$ARCHIVE" ]; then
+  echo "Archive not found: $ARCHIVE" >&2
+  exit 1
+fi
+
+if [ -f "${ARCHIVE}.sha256" ]; then
+  sha256sum -c "${ARCHIVE}.sha256"
+fi
+
+tar -xf "$ARCHIVE"
+echo "Restored dataset assets from $ARCHIVE"

evalvault-1.70.0/src/evalvault/adapters/outbound/analysis/multiturn_analyzer_module.py

@@ -0,0 +1,212 @@
+"""
+멀티턴 평가 요약 모듈입니다.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+
+from evalvault.adapters.outbound.analysis.base_module import BaseAnalysisModule
+from evalvault.adapters.outbound.analysis.pipeline_helpers import get_upstream_output, safe_mean
+from evalvault.domain.entities import EvaluationRun
+
+
+class MultiTurnAnalyzerModule(BaseAnalysisModule):
+    """멀티턴(대화) 단위로 결과를 집계합니다."""
+
+    module_id = "multiturn_analyzer"
+    name = "멀티턴 분석"
+    description = "대화/턴 메타데이터를 기준으로 멀티턴 성능을 요약합니다."
+    input_types = ["run"]
+    output_types = ["multiturn_summary", "multiturn_conversations", "multiturn_turns"]
+    requires = ["data_loader"]
+    tags = ["analysis", "multiturn"]
+
+    def execute(
+        self,
+        inputs: dict[str, Any],
+        params: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        loader_output = get_upstream_output(inputs, "load_data", "data_loader") or {}
+        run = loader_output.get("run")
+        if not isinstance(run, EvaluationRun):
+            return {
+                "available": False,
+                "summary": {},
+                "conversations": [],
+                "turns": [],
+                "coverage": {},
+            }
+
+        retrieval_meta = run.retrieval_metadata or {}
+        cases = run.results
+        total_cases = len(cases)
+
+        coverage = {
+            "total_cases": total_cases,
+            "has_conversation_id": 0,
+            "has_turn_index": 0,
+        }
+
+        grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        turns: list[dict[str, Any]] = []
+
+        for result in cases:
+            case_meta = _resolve_case_metadata(retrieval_meta, result.test_case_id)
+            conversation_id = _coerce_text(case_meta.get("conversation_id"))
+            turn_index = _coerce_turn_index(case_meta.get("turn_index"))
+            turn_id = _coerce_text(case_meta.get("turn_id"))
+
+            if conversation_id:
+                coverage["has_conversation_id"] += 1
+            if turn_index is not None:
+                coverage["has_turn_index"] += 1
+
+            metrics = {
+                metric.name: metric.score for metric in result.metrics if metric.score is not None
+            }
+            avg_score = safe_mean(metrics.values()) if metrics else 0.0
+            failed_metrics = [metric.name for metric in result.metrics if not metric.passed]
+            entry = {
+                "test_case_id": result.test_case_id,
+                "conversation_id": conversation_id,
+                "turn_index": turn_index,
+                "turn_id": turn_id,
+                "avg_score": round(avg_score, 4),
+                "metrics": metrics,
+                "failed_metrics": failed_metrics,
+                "passed_all": result.all_passed,
+            }
+            turns.append(entry)
+            if conversation_id:
+                grouped[conversation_id].append(entry)
+
+        conversations: list[dict[str, Any]] = []
+        first_failure_hist: dict[str, int] = defaultdict(int)
+
+        for conversation_id, entries in grouped.items():
+            entries_sorted = _sort_turns(entries)
+            avg_scores = [item["avg_score"] for item in entries_sorted]
+            metric_scores: dict[str, list[float]] = defaultdict(list)
+            for item in entries_sorted:
+                for name, score in (item.get("metrics") or {}).items():
+                    metric_scores[name].append(float(score))
+
+            metric_means = {
+                name: round(safe_mean(values), 4) for name, values in metric_scores.items()
+            }
+            passed_all = all(item.get("passed_all") for item in entries_sorted)
+            failure_turn = _first_failure_turn(entries_sorted)
+            if failure_turn is not None:
+                first_failure_hist[str(failure_turn)] += 1
+
+            worst_turn = _select_worst_turn(entries_sorted)
+
+            conversations.append(
+                {
+                    "conversation_id": conversation_id,
+                    "turn_count": len(entries_sorted),
+                    "avg_score": round(safe_mean(avg_scores), 4),
+                    "passed_all_turns": passed_all,
+                    "first_failure_turn_index": failure_turn,
+                    "worst_turn": worst_turn,
+                    "metric_means": metric_means,
+                }
+            )
+
+        conversation_count = len(grouped)
+        turn_count = sum(len(items) for items in grouped.values())
+        summary = {
+            "conversation_count": conversation_count,
+            "turn_count": turn_count,
+            "avg_turns_per_conversation": round(
+                (turn_count / conversation_count) if conversation_count else 0.0, 3
+            ),
+            "conversation_pass_rate": round(
+                (
+                    sum(1 for item in conversations if item.get("passed_all_turns"))
+                    / conversation_count
+                )
+                if conversation_count
+                else 0.0,
+                4,
+            ),
+            "first_failure_turn_histogram": dict(first_failure_hist),
+        }
+
+        if total_cases:
+            coverage["has_conversation_id"] = round(
+                coverage["has_conversation_id"] / total_cases, 4
+            )
+            coverage["has_turn_index"] = round(coverage["has_turn_index"] / total_cases, 4)
+
+        return {
+            "available": True,
+            "summary": summary,
+            "conversations": conversations,
+            "turns": turns,
+            "coverage": coverage,
+        }
+
+
+def _resolve_case_metadata(
+    retrieval_metadata: dict[str, dict[str, Any]],
+    test_case_id: str,
+) -> dict[str, Any]:
+    meta = retrieval_metadata.get(test_case_id)
+    if isinstance(meta, dict):
+        nested = meta.get("test_case_metadata")
+        if isinstance(nested, dict):
+            merged = dict(nested)
+            merged.update({k: v for k, v in meta.items() if k != "test_case_metadata"})
+            return merged
+        return dict(meta)
+    return {}
+
+
+def _coerce_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        trimmed = value.strip()
+        return trimmed or None
+    return str(value)
+
+
+def _coerce_turn_index(value: Any) -> int | None:
+    if value is None:
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float) and value.is_integer():
+        return int(value)
+    if isinstance(value, str) and value.strip().isdigit():
+        return int(value.strip())
+    return None
+
+
+def _sort_turns(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    if all(item.get("turn_index") is None for item in entries):
+        return list(entries)
+    return sorted(
+        entries, key=lambda item: (item.get("turn_index") is None, item.get("turn_index") or 0)
+    )
+
+
+def _first_failure_turn(entries: list[dict[str, Any]]) -> int | None:
+    for item in entries:
+        if not item.get("passed_all"):
+            return item.get("turn_index")
+    return None
+
+
+def _select_worst_turn(entries: list[dict[str, Any]]) -> dict[str, Any] | None:
+    if not entries:
+        return None
+    worst = min(entries, key=lambda item: item.get("avg_score", 0.0))
+    return {
+        "test_case_id": worst.get("test_case_id"),
+        "avg_score": worst.get("avg_score"),
+        "failed_metrics": worst.get("failed_metrics", []),
+    }

{evalvault-1.69.0 → evalvault-1.70.0}/uv.lock

@@ -1163,7 +1163,7 @@ wheels = [
 
 [[package]]
 name = "evalvault"
-version = "1.69.0"
+version = "1.70.0"
 source = { editable = "." }
 dependencies = [
     { name = "chainlit" },