evalvault 1.73.2__tar.gz → 1.75.0__tar.gz

{evalvault-1.73.2 → evalvault-1.75.0}/.env.example

@@ -16,9 +16,17 @@
 # - prod: 운영용 고성능 모델 (gpt-oss-safeguard:20b, qwen3-embedding:8b)
 # - openai: OpenAI API 사용 (gpt-5-mini, text-embedding-3-small)
 EVALVAULT_PROFILE=dev
-# SQLite DB 경로 (API/CLI 공통)
+# 기본 스토리지: PostgreSQL + pgvector
+# POSTGRES_HOST=localhost
+# POSTGRES_PORT=5432
+# POSTGRES_DATABASE=evalvault
+# POSTGRES_USER=postgres
+# POSTGRES_PASSWORD=your-password
+# POSTGRES_CONNECTION_STRING=postgresql://user:pass@localhost:5432/evalvault
+# SQLite를 쓰려면 아래를 명시하세요 (API/CLI 공통)
+# DB_BACKEND=sqlite
 # EVALVAULT_DB_PATH=data/db/evalvault.db
-# 도메인 메모리 DB 경로
+# 도메인 메모리 DB 경로 (SQLite 전용)
 # EVALVAULT_MEMORY_DB_PATH=data/db/evalvault_memory.db
 
 # ================================================
@@ -32,6 +40,21 @@ OLLAMA_TIMEOUT=120
 # - 예시: OLLAMA_TOOL_MODELS=gpt-oss:120b,gpt-oss-safeguard:120b,gpt-oss-safeguard:20b
 # OLLAMA_TOOL_MODELS=
 
+# 라우팅/챗 모델 (선택)
+# OLLAMA_ROUTER_MODEL=gemma3:1b
+# OLLAMA_CHAT_MODEL=gemma3:1b
+# OLLAMA_CHAT_TIMEOUT_SECONDS=180
+
+# 간단 챗 모드 (RAG/도구 호출 없이 Ollama만 사용)
+# EVALVAULT_CHAT_SIMPLE_MODE=true
+
+# RAG 범위/성능 튜닝
+# EVALVAULT_RAG_USER_GUIDE_LIMIT=10
+# EVALVAULT_RAG_USE_HYBRID=false
+# EVALVAULT_RAG_VECTOR_STORE=none
+# EVALVAULT_RAG_EMBEDDING_PROFILE=
+# EVALVAULT_CHAT_RUN_CONTEXT_ENABLED=false
+
 # ================================================
 # OpenAI 설정 (외부망)
 # ================================================
@@ -88,13 +111,21 @@ OPENAI_API_KEY=sk-your-api-key-here
 # MLFLOW_EXPERIMENT_NAME=evalvault
 
 # ================================================
-# PostgreSQL 설정 (선택 - 프로덕션 스토리지)
-# ================================================
-# POSTGRES_HOST=localhost
-# POSTGRES_PORT=5432
-# POSTGRES_DATABASE=evalvault
-# POSTGRES_USER=postgres
-# POSTGRES_PASSWORD=your-password
+# RAG Retriever 설정
+# ================================================
+# 하이브리드 검색 사용 여부 (BM25 + Dense)
+# EVALVAULT_RAG_USE_HYBRID=true
+# 벡터 스토어 종류 (pgvector|memory)
+# EVALVAULT_RAG_VECTOR_STORE=pgvector
+# 임베딩 프로파일 (dev|prod)
+# EVALVAULT_RAG_EMBEDDING_PROFILE=dev
+# LLM 없이 컨텍스트만 반환
+# EVALVAULT_RAG_LLM_ENABLED=false
+# pgvector 인덱스 옵션 (성능 튜닝)
+# EVALVAULT_RAG_PGVECTOR_INDEX=hnsw  # hnsw|ivfflat|none
+# EVALVAULT_RAG_PGVECTOR_INDEX_LISTS=100
+# EVALVAULT_RAG_PGVECTOR_HNSW_M=16
+# EVALVAULT_RAG_PGVECTOR_HNSW_EF_CONSTRUCTION=64
 
 # ================================================
 # API 인증 / CORS / Frontend 설정

{evalvault-1.73.2 → evalvault-1.75.0}/.env.offline.example

@@ -14,6 +14,7 @@ EVALVAULT_PROFILE=dev
 # ================================================
 # PostgreSQL (core stack)
 # ================================================
+POSTGRES_IMAGE=pgvector/pgvector:0.8.0-pg16
 POSTGRES_USER=evalvault
 POSTGRES_PASSWORD=evalvault
 POSTGRES_DB=evalvault
@@ -29,6 +30,14 @@ EVALVAULT_MEMORY_DB_PATH=data/db/evalvault_memory.db
 # ================================================
 CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
 
+# ================================================
+# Docker Base Images (offline builds)
+# ================================================
+EVALVAULT_PYTHON_IMAGE=python:3.12.6-slim
+EVALVAULT_UV_IMAGE=ghcr.io/astral-sh/uv:0.4.28
+EVALVAULT_NODE_IMAGE=node:20.19-alpine
+EVALVAULT_NGINX_IMAGE=nginx:1.27.3-alpine
+
 # API_AUTH_TOKENS=
 # KNOWLEDGE_READ_TOKENS=
 # KNOWLEDGE_WRITE_TOKENS=

{evalvault-1.73.2 → evalvault-1.75.0}/.gitignore

@@ -54,6 +54,7 @@ coverage.xml
 reports/*.html
 reports/*.xml
 reports/*.json
+reports/*.csv
 reports/analysis/
 reports/analysis/**
 reports/comparison/
@@ -129,6 +130,7 @@ celerybeat-schedule
 
 # Environments
 .env
+.env.offline
 .venv
 env/
 venv/
@@ -160,3 +162,6 @@ dmypy.json
 .LSOverride
 scratch/
 .sisyphus/
+
+# Local artifacts
+MagicMock/

{evalvault-1.73.2 → evalvault-1.75.0}/Dockerfile

@@ -1,11 +1,16 @@
 # EvalVault Dockerfile
 # Multi-stage build for optimized production image
 
+# Stage 0: uv binary
+ARG PYTHON_IMAGE=python:3.12.6-slim
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.4.28
+FROM ${UV_IMAGE} AS uv
+
 # Stage 1: Build stage
-FROM python:3.12-slim AS builder
+FROM ${PYTHON_IMAGE} AS builder
 
 # Install uv
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+COPY --from=uv /uv /usr/local/bin/uv
 
 # Set working directory
 WORKDIR /app
@@ -25,7 +30,8 @@ RUN uv sync --frozen --no-dev
 
 
 # Stage 2: Runtime stage
-FROM python:3.12-slim AS runtime
+ARG PYTHON_IMAGE=python:3.12.6-slim
+FROM ${PYTHON_IMAGE} AS runtime
 
 # Create non-root user for security
 RUN useradd --create-home --shell /bin/bash evalvault

evalvault-1.75.0/PKG-INFO

@@ -0,0 +1,221 @@
+Metadata-Version: 2.4
+Name: evalvault
+Version: 1.75.0
+Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
+Project-URL: Homepage, https://github.com/ntts9990/EvalVault
+Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme
+Project-URL: Repository, https://github.com/ntts9990/EvalVault.git
+Project-URL: Issues, https://github.com/ntts9990/EvalVault/issues
+Project-URL: Changelog, https://github.com/ntts9990/EvalVault/releases
+Author: EvalVault Contributors
+Maintainer: EvalVault Contributors
+License: Apache-2.0
+License-File: LICENSE.md
+Keywords: ai,evaluation,langfuse,llm,machine-learning,nlp,observability,opentelemetry,phoenix,rag,ragas,retrieval-augmented-generation,testing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: chainlit>=2.9.5
+Requires-Dist: chardet
+Requires-Dist: fastapi>=0.128.0
+Requires-Dist: instructor
+Requires-Dist: langchain-openai
+Requires-Dist: langfuse
+Requires-Dist: matplotlib<3.9.0,>=3.8.0
+Requires-Dist: networkx
+Requires-Dist: openai
+Requires-Dist: openpyxl
+Requires-Dist: pandas
+Requires-Dist: pydantic
+Requires-Dist: pydantic-settings
+Requires-Dist: pypdf>=4.3.0
+Requires-Dist: python-multipart
+Requires-Dist: ragas==0.4.2
+Requires-Dist: rich
+Requires-Dist: truststore>=0.10.4
+Requires-Dist: typer
+Requires-Dist: uvicorn>=0.40.0
+Requires-Dist: xlrd
+Provides-Extra: analysis
+Requires-Dist: scikit-learn>=1.3.0; extra == 'analysis'
+Requires-Dist: xgboost>=2.0.0; extra == 'analysis'
+Provides-Extra: anthropic
+Requires-Dist: anthropic; extra == 'anthropic'
+Requires-Dist: langchain-anthropic; extra == 'anthropic'
+Provides-Extra: benchmark
+Requires-Dist: datasets>=2.0.0; extra == 'benchmark'
+Requires-Dist: lm-eval[api]>=0.4.0; extra == 'benchmark'
+Provides-Extra: dashboard
+Requires-Dist: matplotlib<3.9.0,>=3.8.0; extra == 'dashboard'
+Provides-Extra: dev
+Requires-Dist: anthropic; extra == 'dev'
+Requires-Dist: arize-phoenix>=8.0.0; extra == 'dev'
+Requires-Dist: datasets>=2.0.0; extra == 'dev'
+Requires-Dist: faiss-cpu>=1.8.0; extra == 'dev'
+Requires-Dist: ijson>=3.3.0; extra == 'dev'
+Requires-Dist: kiwipiepy>=0.18.0; extra == 'dev'
+Requires-Dist: langchain-anthropic; extra == 'dev'
+Requires-Dist: lm-eval[api]>=0.4.0; extra == 'dev'
+Requires-Dist: mkdocs-material>=9.5.0; extra == 'dev'
+Requires-Dist: mkdocs>=1.5.0; extra == 'dev'
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'dev'
+Requires-Dist: mlflow>=2.0.0; extra == 'dev'
+Requires-Dist: openinference-instrumentation-langchain>=0.1.0; extra == 'dev'
+Requires-Dist: opentelemetry-api>=1.20.0; extra == 'dev'
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == 'dev'
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == 'dev'
+Requires-Dist: pgvector>=0.2.5; extra == 'dev'
+Requires-Dist: psycopg[binary]>=3.0.0; extra == 'dev'
+Requires-Dist: pydeps>=3.0.1; extra == 'dev'
+Requires-Dist: pymdown-extensions>=10.7.0; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest-html; extra == 'dev'
+Requires-Dist: pytest-mock; extra == 'dev'
+Requires-Dist: pytest-rerunfailures; extra == 'dev'
+Requires-Dist: pytest-xdist; extra == 'dev'
+Requires-Dist: python-multipart; extra == 'dev'
+Requires-Dist: rank-bm25>=0.2.2; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: scikit-learn<1.4.0,>=1.3.0; extra == 'dev'
+Requires-Dist: sentence-transformers>=5.2.0; extra == 'dev'
+Requires-Dist: xgboost>=2.0.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
+Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'docs'
+Requires-Dist: pymdown-extensions>=10.7.0; extra == 'docs'
+Provides-Extra: korean
+Requires-Dist: kiwipiepy>=0.18.0; extra == 'korean'
+Requires-Dist: rank-bm25>=0.2.2; extra == 'korean'
+Requires-Dist: sentence-transformers>=5.2.0; extra == 'korean'
+Provides-Extra: mlflow
+Requires-Dist: mlflow>=2.0.0; extra == 'mlflow'
+Provides-Extra: perf
+Requires-Dist: faiss-cpu>=1.8.0; extra == 'perf'
+Requires-Dist: ijson>=3.3.0; extra == 'perf'
+Provides-Extra: phoenix
+Requires-Dist: arize-phoenix>=8.0.0; extra == 'phoenix'
+Requires-Dist: openinference-instrumentation-langchain>=0.1.0; extra == 'phoenix'
+Requires-Dist: opentelemetry-api>=1.20.0; extra == 'phoenix'
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == 'phoenix'
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == 'phoenix'
+Provides-Extra: postgres
+Requires-Dist: pgvector>=0.2.5; extra == 'postgres'
+Requires-Dist: psycopg[binary]>=3.0.0; extra == 'postgres'
+Provides-Extra: secrets
+Requires-Dist: boto3; extra == 'secrets'
+Requires-Dist: google-cloud-secret-manager; extra == 'secrets'
+Requires-Dist: hvac; extra == 'secrets'
+Provides-Extra: timeseries
+Requires-Dist: aeon>=1.3.0; extra == 'timeseries'
+Requires-Dist: numba>=0.55.0; extra == 'timeseries'
+Provides-Extra: web
+Description-Content-Type: text/markdown
+
+# EvalVault
+
+RAG(Retrieval-Augmented Generation) 시스템을 대상으로 **평가(Eval) → 분석(Analysis) → 추적(Tracing) → 개선 루프**를 하나의 워크플로로 묶는 CLI + Web UI 플랫폼입니다.
+
+[![PyPI](https://img.shields.io/pypi/v/evalvault.svg)](https://pypi.org/project/evalvault/)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ntts9990/EvalVault/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/ntts9990/EvalVault/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE.md)
+
+English version? See `README.en.md`.
+
+---
+
+## Quickstart (CLI)
+
+```bash
+uv sync --extra dev
+cp .env.example .env
+
+uv run evalvault run --mode simple tests/fixtures/e2e/insurance_qa_korean.json \
+  --metrics faithfulness,answer_relevancy \
+  --profile dev \
+  --auto-analyze
+```
+
+Tip: 기본 저장소는 Postgres+pgvector입니다. SQLite를 쓰려면 `--db` 또는 `DB_BACKEND=sqlite` + `EVALVAULT_DB_PATH`를 지정하세요.
+
+---
+
+## 핵심 기능
+
+- **End-to-End 평가 루프**: Eval → Analysis → Tracing → Improvement를 한 흐름으로 실행
+- **Dataset 중심 운영**: 합격 기준(threshold)을 데이터셋에 유지
+- **Artifacts-first**: 보고서뿐 아니라 모듈별 원본 결과를 구조화 저장
+- **옵션형 Observability**: Phoenix/Langfuse/MLflow는 필요할 때만 활성화
+- **CLI + Web UI**: 동일 run_id 기반으로 히스토리/비교/리포트 통합
+
+---
+
+## 문서 허브
+
+- 문서 인덱스: `docs/INDEX.md`
+- 핸드북(교과서형): `docs/handbook/INDEX.md`
+- 외부 요약본: `docs/handbook/EXTERNAL.md`
+- 운영 가이드(로컬/도커/관측/런북): `docs/handbook/CHAPTERS/04_operations.md`
+- 워크플로(실행/분석/비교/회귀): `docs/handbook/CHAPTERS/03_workflows.md`
+- 품질/테스트/CI: `docs/handbook/CHAPTERS/06_quality_and_testing.md`
+- 아키텍처: `docs/handbook/CHAPTERS/01_architecture.md`
+- 오프라인/폐쇄망(Docker/모델 캐시): `docs/guides/OFFLINE_DOCKER.md`, `docs/guides/OFFLINE_MODELS.md`
+
+참고(호환성): `docs/guides/USER_GUIDE.md`, `docs/guides/DEV_GUIDE.md` 등 일부 문서는 과거 링크 호환을 위한 deprecated 스텁이며, 최신 내용은 handbook을 따릅니다.
+
+---
+
+## Web UI
+
+```bash
+# API
+uv run evalvault serve-api --reload
+
+# Frontend
+cd frontend
+npm install
+npm run dev
+```
+
+브라우저에서 `http://localhost:5173` 접속 후, Evaluation Studio에서 실행/히스토리/리포트를 확인합니다.
+
+---
+
+## 오프라인/폐쇄망
+
+- Docker 이미지 번들: `docs/guides/OFFLINE_DOCKER.md`
+- NLP 모델 캐시 번들: `docs/guides/OFFLINE_MODELS.md`
+
+LLM 모델은 폐쇄망 내부 인프라가 관리하며, EvalVault는 **분석용 NLP 모델 캐시**만 번들에 포함합니다.
+
+---
+
+## 기여
+
+```bash
+uv run ruff check src/ tests/
+uv run ruff format src/ tests/
+uv run pytest tests -v
+```
+
+- 기여 가이드: `CONTRIBUTING.md`
+- 개발/테스트 루틴: `AGENTS.md`, `docs/handbook/CHAPTERS/06_quality_and_testing.md`
+
+---
+
+## License
+
+EvalVault is licensed under the [Apache 2.0](LICENSE.md) license.

{evalvault-1.73.2 → evalvault-1.75.0}/README.en.md

@@ -43,20 +43,19 @@ Open `http://localhost:5173`, run an evaluation in Evaluation Studio (for exampl
 and insights.
 
 - LLM report language: `/api/v1/runs/{run_id}/report?language=en` (default: ko)
-  - Details: `docs/guides/USER_GUIDE.md#보고서-언어-옵션`
+  - Details: `docs/handbook/CHAPTERS/00_overview.md`
 - Feedback aggregation: latest value per `rater_id` + `test_case_id` (cancellations excluded)
-  - Details: `docs/guides/USER_GUIDE.md#피드백-집계-규칙`
+  - Details: `docs/handbook/CHAPTERS/02_data_and_metrics.md`
 
 **CLI (terminal view)**
 ```bash
 uv run evalvault run tests/fixtures/e2e/insurance_qa_korean.json \
   --metrics faithfulness,answer_relevancy \
-  --profile dev \
-  --db data/db/evalvault.db
-uv run evalvault history --db data/db/evalvault.db
-uv run evalvault analyze <RUN_ID> --db data/db/evalvault.db
+  --profile dev
+uv run evalvault history
+uv run evalvault analyze <RUN_ID>
 ```
-Tip: keep the same `--db` (or `EVALVAULT_DB_PATH`) so the Web UI can read the run.
+Tip: Postgres is the default store. Use `--db` or `DB_BACKEND=sqlite` + `EVALVAULT_DB_PATH` for SQLite, and keep the same settings so the Web UI can read the run.
 
 ---
 
@@ -186,7 +185,7 @@ The core contract is **module-level spans (`rag.module`) + log events + shared a
   - Learn facts/behaviors from past runs to auto-tune thresholds and augment context
   - DAG-based analysis pipeline with statistical, NLP, and causal modules for multi-faceted interpretation
 
-See the [User Guide](docs/guides/USER_GUIDE.md) for end-to-end workflows, Phoenix/Langfuse integration, and troubleshooting.
+See the [Handbook](docs/handbook/INDEX.md) for end-to-end workflows, operations, and troubleshooting.
 
 ---
 
@@ -228,7 +227,7 @@ uv sync --extra dev
    cp .env.example .env
    # set OPENAI_API_KEY or OLLAMA settings, LANGFUSE/PHOENIX keys, etc.
    ```
-   Optional SQLite path override:
+Optional SQLite path override (when using SQLite):
    ```bash
    # .env
    EVALVAULT_DB_PATH=/path/to/data/db/evalvault.db
@@ -247,10 +246,9 @@ uv sync --extra dev
    ```bash
    cp .env.example .env
    ollama pull gemma3:1b
-   uv run evalvault run tests/fixtures/e2e/insurance_qa_korean.json \
-     --metrics faithfulness \
-     --db data/db/evalvault.db \
-     --profile dev
+uv run evalvault run tests/fixtures/e2e/insurance_qa_korean.json \
+  --metrics faithfulness \
+  --profile dev
    ```
    Tip: embedding metrics like `answer_relevancy` also need `qwen3-embedding:0.6b`.
 
@@ -258,9 +256,9 @@ uv sync --extra dev
    ```bash
    cp .env.example .env
    printf "\nEVALVAULT_PROFILE=vllm\nVLLM_BASE_URL=http://localhost:8001/v1\nVLLM_MODEL=gpt-oss-120b\n" >> .env
-   uv run evalvault run tests/fixtures/e2e/insurance_qa_korean.json \
-     --metrics faithfulness \
-     --db data/db/evalvault.db
+uv run evalvault run tests/fixtures/e2e/insurance_qa_korean.json \
+  --metrics faithfulness \
+  --profile dev
    ```
    Tip: embedding metrics require `VLLM_EMBEDDING_MODEL` and a `/v1/embeddings` endpoint.
    If you use Ollama models that support tool/function calling, list them in
@@ -295,20 +293,21 @@ uv sync --extra dev
 
 3. **Run an evaluation**
    ```bash
-   uv run evalvault run tests/fixtures/sample_dataset.json \
-     --metrics faithfulness,answer_relevancy \
-     --profile dev \
-     --db data/db/evalvault.db
+uv run evalvault run tests/fixtures/sample_dataset.json \
+  --metrics faithfulness,answer_relevancy \
+  --profile dev
    ```
-   Tip: `--db` stores results for `history/export/web`. Add `--tracker phoenix` only if
-   Phoenix is configured (and `uv sync --extra phoenix` is installed).
+   Tip: For SQLite, pass `--db` (or set `DB_BACKEND=sqlite` + `EVALVAULT_DB_PATH`).
+   For Postgres, set `POSTGRES_*` or `POSTGRES_CONNECTION_STRING` so the Web UI can
+   read the same DB. Add `--tracker phoenix` only if Phoenix is configured
+   (and `uv sync --extra phoenix` is installed).
 
 4. **Inspect history**
    ```bash
-   uv run evalvault history --db data/db/evalvault.db
+    uv run evalvault history
    ```
 
-More examples (parallel runs, dataset streaming, Langfuse logging, Phoenix dataset sync, prompt manifest diffs, etc.) live in the [User Guide](docs/guides/USER_GUIDE.md).
+More examples (parallel runs, dataset streaming, Langfuse logging, Phoenix dataset sync, prompt manifest diffs, etc.) live in the [Handbook](docs/handbook/INDEX.md) and `examples/`.
 
 ---
 
@@ -372,9 +371,8 @@ On top of these, `StageMetricService` derives **pipeline-stage metrics** such as
 
 ## Documentation
 - [Docs Index](docs/INDEX.md): documentation hub.
-- [User Guide](docs/guides/USER_GUIDE.md): installation, configuration, CLI recipes, Web UI, Phoenix, automation.
-- [Dev Guide](docs/guides/DEV_GUIDE.md): local dev/test/lint routines.
-- [Developer Whitepaper](docs/new_whitepaper/INDEX.md): architecture, operations, and engineering standards.
+- [Handbook](docs/handbook/INDEX.md): internal SSoT (architecture, workflows, ops, quality).
+- [External Summary](docs/handbook/EXTERNAL.md): shareable overview.
 - [Open RAG Trace Spec](docs/architecture/open-rag-trace-spec.md): tracing schema and integration guide.
 - [CHANGELOG](CHANGELOG.md) for release history.
 

evalvault-1.75.0/README.md

@@ -0,0 +1,95 @@
+# EvalVault
+
+RAG(Retrieval-Augmented Generation) 시스템을 대상으로 **평가(Eval) → 분석(Analysis) → 추적(Tracing) → 개선 루프**를 하나의 워크플로로 묶는 CLI + Web UI 플랫폼입니다.
+
+[![PyPI](https://img.shields.io/pypi/v/evalvault.svg)](https://pypi.org/project/evalvault/)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ntts9990/EvalVault/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/ntts9990/EvalVault/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE.md)
+
+English version? See `README.en.md`.
+
+---
+
+## Quickstart (CLI)
+
+```bash
+uv sync --extra dev
+cp .env.example .env
+
+uv run evalvault run --mode simple tests/fixtures/e2e/insurance_qa_korean.json \
+  --metrics faithfulness,answer_relevancy \
+  --profile dev \
+  --auto-analyze
+```
+
+Tip: 기본 저장소는 Postgres+pgvector입니다. SQLite를 쓰려면 `--db` 또는 `DB_BACKEND=sqlite` + `EVALVAULT_DB_PATH`를 지정하세요.
+
+---
+
+## 핵심 기능
+
+- **End-to-End 평가 루프**: Eval → Analysis → Tracing → Improvement를 한 흐름으로 실행
+- **Dataset 중심 운영**: 합격 기준(threshold)을 데이터셋에 유지
+- **Artifacts-first**: 보고서뿐 아니라 모듈별 원본 결과를 구조화 저장
+- **옵션형 Observability**: Phoenix/Langfuse/MLflow는 필요할 때만 활성화
+- **CLI + Web UI**: 동일 run_id 기반으로 히스토리/비교/리포트 통합
+
+---
+
+## 문서 허브
+
+- 문서 인덱스: `docs/INDEX.md`
+- 핸드북(교과서형): `docs/handbook/INDEX.md`
+- 외부 요약본: `docs/handbook/EXTERNAL.md`
+- 운영 가이드(로컬/도커/관측/런북): `docs/handbook/CHAPTERS/04_operations.md`
+- 워크플로(실행/분석/비교/회귀): `docs/handbook/CHAPTERS/03_workflows.md`
+- 품질/테스트/CI: `docs/handbook/CHAPTERS/06_quality_and_testing.md`
+- 아키텍처: `docs/handbook/CHAPTERS/01_architecture.md`
+- 오프라인/폐쇄망(Docker/모델 캐시): `docs/guides/OFFLINE_DOCKER.md`, `docs/guides/OFFLINE_MODELS.md`
+
+참고(호환성): `docs/guides/USER_GUIDE.md`, `docs/guides/DEV_GUIDE.md` 등 일부 문서는 과거 링크 호환을 위한 deprecated 스텁이며, 최신 내용은 handbook을 따릅니다.
+
+---
+
+## Web UI
+
+```bash
+# API
+uv run evalvault serve-api --reload
+
+# Frontend
+cd frontend
+npm install
+npm run dev
+```
+
+브라우저에서 `http://localhost:5173` 접속 후, Evaluation Studio에서 실행/히스토리/리포트를 확인합니다.
+
+---
+
+## 오프라인/폐쇄망
+
+- Docker 이미지 번들: `docs/guides/OFFLINE_DOCKER.md`
+- NLP 모델 캐시 번들: `docs/guides/OFFLINE_MODELS.md`
+
+LLM 모델은 폐쇄망 내부 인프라가 관리하며, EvalVault는 **분석용 NLP 모델 캐시**만 번들에 포함합니다.
+
+---
+
+## 기여
+
+```bash
+uv run ruff check src/ tests/
+uv run ruff format src/ tests/
+uv run pytest tests -v
+```
+
+- 기여 가이드: `CONTRIBUTING.md`
+- 개발/테스트 루틴: `AGENTS.md`, `docs/handbook/CHAPTERS/06_quality_and_testing.md`
+
+---
+
+## License
+
+EvalVault is licensed under the [Apache 2.0](LICENSE.md) license.

{evalvault-1.73.2 → evalvault-1.75.0}/agent/README.md

@@ -27,7 +27,7 @@ evalvault (PyPI Package)              agent/ (Development Only)
 
 ### Development Mode (This Folder)
 
-Agents for improving EvalVault codebase based on the current roadmap and engineering standards (see `docs/ROADMAP.md`, `docs/new_whitepaper/INDEX.md`):
+Agents for improving EvalVault codebase based on the current roadmap and engineering standards (see `docs/handbook/CHAPTERS/08_roadmap.md`, `docs/handbook/INDEX.md`):
 
 | Agent Type | Focus | P-Levels |
 |------------|-------|----------|
@@ -299,7 +299,7 @@ The agent system follows the project documentation and current engineering stand
 - [Claude Agent SDK Docs](https://platform.claude.com/docs/en/agent-sdk/overview)
 - [Effective Harnesses](https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents)
 - [Docs Index](../docs/INDEX.md)
-- [Developer Whitepaper](../docs/new_whitepaper/INDEX.md)
+- [Handbook](../docs/handbook/INDEX.md)
 - [Open RAG Trace Spec](../docs/architecture/open-rag-trace-spec.md)
 - [Agent Types Configuration](../src/evalvault/config/agent_types.py)
 - [nonstop-agent](https://github.com/seolcoding/nonstop-agent)

{evalvault-1.73.2 → evalvault-1.75.0}/agent/memory/shared/dependencies.md

@@ -8,7 +8,7 @@
 | 문서 | 용도 |
 |------|------|
 | [docs/INDEX.md](../../../docs/INDEX.md) | 프로젝트 문서 허브(최신 링크) |
-| [Developer Whitepaper](../../../docs/new_whitepaper/INDEX.md) | 설계/운영/품질 기준 |
+| [Handbook](../../../docs/handbook/INDEX.md) | 설계/운영/품질 기준(SSoT) |
 | [agent/README.md](../../README.md) | 에이전트 시스템 사용법 |
 
 ---
@@ -138,7 +138,7 @@ architecture (Storage Adapter)┘
 | `src/evalvault/ports/outbound/tracker_port.py` | `observability` | `rag-data` | 스키마 변경 공유 |
 | `src/evalvault/domain/entities/result.py` | `architecture` | - | 테스트 영향 체크 |
 | `docs/INDEX.md` | `coordinator` | All | 문서 구조/링크 변경 시 동기화 |
-| `docs/new_whitepaper/INDEX.md` | `coordinator` | All | 설계/운영 기준 변경 시 동기화 |
+| `docs/handbook/INDEX.md` | `coordinator` | All | 설계/운영 기준 변경 시 동기화 |
 | `agent/memory/shared/decisions.md` | All | - | ADR 형식 준수 |
 
 ### Shared Namespaces

{evalvault-1.73.2 → evalvault-1.75.0}/agent/prompts/improvement/coordinator_prompt.md

@@ -45,7 +45,7 @@ cat agent/memory/shared/decisions.md | tail -50
 
 ## PARALLEL EXECUTION GROUPS
 
-From the current roadmap/standards (`docs/ROADMAP.md`, `docs/new_whitepaper/INDEX.md`):
+From the current roadmap/standards (`docs/handbook/CHAPTERS/08_roadmap.md`, `docs/handbook/INDEX.md`):
 
 ### Group A: Fully Independent (Can Run Together)
 - `performance`: Caching, batch processing
@@ -141,7 +141,7 @@ agent/memory/shared/
 docs/
 ├── ROADMAP.md            # Public direction
 ├── STATUS.md             # One-page snapshot
-└── new_whitepaper/       # Engineering standards
+└── handbook/             # Engineering standards (SSoT)
 
 feature_list.json         # Task tracking
 claude-progress.txt       # Session progress