evalvault 1.74.0__py3-none-any.whl → 1.76.0__py3-none-any.whl

evalvault/config/settings.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from typing import Any
 
-from pydantic import Field, PrivateAttr
+from pydantic import AliasChoices, Field, PrivateAttr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from evalvault.config.secret_manager import (
@@ -55,6 +55,33 @@ def _parse_cors_origins(cors_origins: str | None) -> list[str]:
     return [origin.strip() for origin in cors_origins.split(",") if origin.strip()]
 
 
+def resolve_tracker_providers(provider: str | None) -> list[str]:
+    if not provider:
+        return []
+    normalized = provider.strip().lower()
+    if normalized in {"none", "off", "disabled"}:
+        return ["none"]
+    aliases = {
+        "all": ["mlflow", "phoenix"],
+        "default": ["mlflow", "phoenix"],
+    }
+    if normalized in aliases:
+        return aliases[normalized]
+    separators = [",", "+", "/", "|"]
+    for sep in separators:
+        normalized = normalized.replace(sep, ",")
+    providers = [p.strip() for p in normalized.split(",") if p.strip()]
+    if not providers:
+        return []
+    if "none" in providers and len(providers) > 1:
+        raise ValueError("tracker_provider cannot combine 'none' with other providers")
+    deduped: list[str] = []
+    for entry in providers:
+        if entry not in deduped:
+            deduped.append(entry)
+    return deduped
+
+
 SECRET_REFERENCE_FIELDS = (
     "api_auth_tokens",
     "knowledge_read_tokens",
@@ -83,13 +110,14 @@ def _validate_production_settings(settings: "Settings") -> None:
     if settings.llm_provider == "openai" and not settings.openai_api_key:
         missing.append("OPENAI_API_KEY")
 
-    if settings.tracker_provider == "langfuse":
+    providers = resolve_tracker_providers(settings.tracker_provider)
+    if "langfuse" in providers:
         if not settings.langfuse_public_key:
             missing.append("LANGFUSE_PUBLIC_KEY")
         if not settings.langfuse_secret_key:
             missing.append("LANGFUSE_SECRET_KEY")
 
-    if settings.tracker_provider == "mlflow" and not settings.mlflow_tracking_uri:
+    if "mlflow" in providers and not settings.mlflow_tracking_uri:
         missing.append("MLFLOW_TRACKING_URI")
 
     if (
@@ -179,6 +207,10 @@ class Settings(BaseSettings):
         default="data/db/evalvault.db",
         description="SQLite database path for API/CLI storage.",
     )
+    db_backend: str = Field(
+        default="postgres",
+        description="Storage backend: 'postgres' or 'sqlite'.",
+    )
     evalvault_memory_db_path: str = Field(
         default="data/db/evalvault_memory.db",
         description="SQLite database path for Domain Memory storage.",
@@ -242,7 +274,7 @@ class Settings(BaseSettings):
         description="Ollama server URL",
     )
     ollama_model: str = Field(
-        default="gpt-oss-safeguard:20b",
+        default="qwen3:14b",
         description="Ollama model name for evaluation",
     )
     ollama_embedding_model: str = Field(
@@ -351,6 +383,14 @@ class Settings(BaseSettings):
         default="http://localhost:6006/v1/traces",
         description="Phoenix OTLP endpoint for traces",
     )
+    phoenix_project_name: str = Field(
+        default="evalvault",
+        description="Phoenix project name for grouping traces",
+    )
+    phoenix_annotations_enabled: bool = Field(
+        default=True,
+        description="Enable automatic Phoenix span annotations",
+    )
     phoenix_api_token: str | None = Field(
         default=None,
         description="Phoenix API token for cloud deployments (optional)",
@@ -368,8 +408,8 @@ class Settings(BaseSettings):
 
     # Tracker Provider Selection
     tracker_provider: str = Field(
-        default="langfuse",
-        description="Tracker provider: 'langfuse', 'mlflow', or 'phoenix'",
+        default="mlflow+phoenix",
+        description="Tracker provider: 'langfuse', 'mlflow', 'phoenix', 'none', or combinations",
     )
 
     # Cluster map configuration
@@ -395,11 +435,31 @@ class Settings(BaseSettings):
     )
 
     # PostgreSQL Configuration (optional)
-    postgres_host: str | None = Field(default=None, description="PostgreSQL server host")
-    postgres_port: int = Field(default=5432, description="PostgreSQL server port")
-    postgres_database: str = Field(default="evalvault", description="PostgreSQL database name")
-    postgres_user: str | None = Field(default=None, description="PostgreSQL user")
-    postgres_password: str | None = Field(default=None, description="PostgreSQL password")
+    postgres_host: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("POSTGRES_HOST", "EVALVAULT_DB_HOST"),
+        description="PostgreSQL server host",
+    )
+    postgres_port: int = Field(
+        default=5432,
+        validation_alias=AliasChoices("POSTGRES_PORT", "EVALVAULT_DB_PORT"),
+        description="PostgreSQL server port",
+    )
+    postgres_database: str = Field(
+        default="evalvault",
+        validation_alias=AliasChoices("POSTGRES_DATABASE", "EVALVAULT_DB_NAME"),
+        description="PostgreSQL database name",
+    )
+    postgres_user: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("POSTGRES_USER", "EVALVAULT_DB_USER"),
+        description="PostgreSQL user",
+    )
+    postgres_password: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("POSTGRES_PASSWORD", "EVALVAULT_DB_PASSWORD"),
+        description="PostgreSQL password",
+    )
     postgres_connection_string: str | None = Field(
         default=None, description="PostgreSQL connection string (overrides other postgres settings)"
     )

evalvault/domain/services/domain_learning_hook.py

@@ -20,7 +20,8 @@ class DomainLearningHook:
     Formation dynamics를 구현합니다.
 
     사용 예시:
-        memory_adapter = SQLiteDomainMemoryAdapter()
+        from evalvault.adapters.outbound.domain_memory import build_domain_memory_adapter
+        memory_adapter = build_domain_memory_adapter()
         hook = DomainLearningHook(memory_adapter)
 
         # 평가 후 메모리 형성

evalvault/domain/services/evaluator.py

@@ -1934,6 +1934,8 @@ class RagasEvaluator:
 
     def _calculate_cost(self, model_name: str, prompt_tokens: int, completion_tokens: int) -> float:
         """Calculate estimated cost in USD based on model pricing."""
+        if "ollama" in model_name:
+            return 0.0
         # Find matching model key (exact or substring match)
         price_key = "openai/gpt-4o"  # Default fallback
         for key in self.MODEL_PRICING:

evalvault/ports/inbound/web_port.py

@@ -18,7 +18,7 @@ class EvalRequest:
 
     dataset_path: str
     metrics: list[str]
-    model_name: str = "ollama/gpt-oss-safeguard:20b"
+    model_name: str = "ollama/qwen3:14b"
     evaluation_task: str = "qa"
     langfuse_enabled: bool = False
     thresholds: dict[str, float] = field(default_factory=dict)
@@ -121,12 +121,14 @@ class WebUIPort(Protocol):
     def list_runs(
         self,
         limit: int = 50,
+        offset: int = 0,
         filters: RunFilters | None = None,
     ) -> list[RunSummary]:
         """평가 목록 조회.
 
         Args:
             limit: 최대 조회 개수
+            offset: 조회 시작 위치
             filters: 필터 조건
 
         Returns:

evalvault/ports/outbound/storage_port.py

@@ -83,6 +83,7 @@ class StoragePort(Protocol):
     def list_runs(
         self,
         limit: int = 100,
+        offset: int = 0,
         dataset_name: str | None = None,
         model_name: str | None = None,
     ) -> list[EvaluationRun]:
@@ -90,6 +91,7 @@ class StoragePort(Protocol):
 
         Args:
             limit: 최대 조회 개수
+            offset: 조회 시작 위치 (선택)
             dataset_name: 필터링할 데이터셋 이름 (선택)
             model_name: 필터링할 모델 이름 (선택)
 

evalvault-1.76.0.dist-info/METADATA

@@ -0,0 +1,221 @@
+Metadata-Version: 2.4
+Name: evalvault
+Version: 1.76.0
+Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
+Project-URL: Homepage, https://github.com/ntts9990/EvalVault
+Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme
+Project-URL: Repository, https://github.com/ntts9990/EvalVault.git
+Project-URL: Issues, https://github.com/ntts9990/EvalVault/issues
+Project-URL: Changelog, https://github.com/ntts9990/EvalVault/releases
+Author: EvalVault Contributors
+Maintainer: EvalVault Contributors
+License: Apache-2.0
+License-File: LICENSE.md
+Keywords: ai,evaluation,langfuse,llm,machine-learning,nlp,observability,opentelemetry,phoenix,rag,ragas,retrieval-augmented-generation,testing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: chainlit>=2.9.5
+Requires-Dist: chardet
+Requires-Dist: fastapi>=0.128.0
+Requires-Dist: instructor
+Requires-Dist: langchain-openai
+Requires-Dist: langfuse
+Requires-Dist: matplotlib<3.9.0,>=3.8.0
+Requires-Dist: networkx
+Requires-Dist: openai
+Requires-Dist: openpyxl
+Requires-Dist: pandas
+Requires-Dist: pydantic
+Requires-Dist: pydantic-settings
+Requires-Dist: pypdf>=4.3.0
+Requires-Dist: python-multipart
+Requires-Dist: ragas==0.4.2
+Requires-Dist: rich
+Requires-Dist: truststore>=0.10.4
+Requires-Dist: typer
+Requires-Dist: uvicorn>=0.40.0
+Requires-Dist: xlrd
+Provides-Extra: analysis
+Requires-Dist: scikit-learn>=1.3.0; extra == 'analysis'
+Requires-Dist: xgboost>=2.0.0; extra == 'analysis'
+Provides-Extra: anthropic
+Requires-Dist: anthropic; extra == 'anthropic'
+Requires-Dist: langchain-anthropic; extra == 'anthropic'
+Provides-Extra: benchmark
+Requires-Dist: datasets>=2.0.0; extra == 'benchmark'
+Requires-Dist: lm-eval[api]>=0.4.0; extra == 'benchmark'
+Provides-Extra: dashboard
+Requires-Dist: matplotlib<3.9.0,>=3.8.0; extra == 'dashboard'
+Provides-Extra: dev
+Requires-Dist: anthropic; extra == 'dev'
+Requires-Dist: arize-phoenix>=8.0.0; extra == 'dev'
+Requires-Dist: datasets>=2.0.0; extra == 'dev'
+Requires-Dist: faiss-cpu>=1.8.0; extra == 'dev'
+Requires-Dist: ijson>=3.3.0; extra == 'dev'
+Requires-Dist: kiwipiepy>=0.18.0; extra == 'dev'
+Requires-Dist: langchain-anthropic; extra == 'dev'
+Requires-Dist: lm-eval[api]>=0.4.0; extra == 'dev'
+Requires-Dist: mkdocs-material>=9.5.0; extra == 'dev'
+Requires-Dist: mkdocs>=1.5.0; extra == 'dev'
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'dev'
+Requires-Dist: mlflow>=2.0.0; extra == 'dev'
+Requires-Dist: openinference-instrumentation-langchain>=0.1.0; extra == 'dev'
+Requires-Dist: opentelemetry-api>=1.20.0; extra == 'dev'
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == 'dev'
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == 'dev'
+Requires-Dist: pgvector>=0.2.5; extra == 'dev'
+Requires-Dist: psycopg[binary]>=3.0.0; extra == 'dev'
+Requires-Dist: pydeps>=3.0.1; extra == 'dev'
+Requires-Dist: pymdown-extensions>=10.7.0; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest-html; extra == 'dev'
+Requires-Dist: pytest-mock; extra == 'dev'
+Requires-Dist: pytest-rerunfailures; extra == 'dev'
+Requires-Dist: pytest-xdist; extra == 'dev'
+Requires-Dist: python-multipart; extra == 'dev'
+Requires-Dist: rank-bm25>=0.2.2; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: scikit-learn<1.4.0,>=1.3.0; extra == 'dev'
+Requires-Dist: sentence-transformers>=5.2.0; extra == 'dev'
+Requires-Dist: xgboost>=2.0.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
+Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'docs'
+Requires-Dist: pymdown-extensions>=10.7.0; extra == 'docs'
+Provides-Extra: korean
+Requires-Dist: kiwipiepy>=0.18.0; extra == 'korean'
+Requires-Dist: rank-bm25>=0.2.2; extra == 'korean'
+Requires-Dist: sentence-transformers>=5.2.0; extra == 'korean'
+Provides-Extra: mlflow
+Requires-Dist: mlflow>=2.0.0; extra == 'mlflow'
+Provides-Extra: perf
+Requires-Dist: faiss-cpu>=1.8.0; extra == 'perf'
+Requires-Dist: ijson>=3.3.0; extra == 'perf'
+Provides-Extra: phoenix
+Requires-Dist: arize-phoenix>=8.0.0; extra == 'phoenix'
+Requires-Dist: openinference-instrumentation-langchain>=0.1.0; extra == 'phoenix'
+Requires-Dist: opentelemetry-api>=1.20.0; extra == 'phoenix'
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == 'phoenix'
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == 'phoenix'
+Provides-Extra: postgres
+Requires-Dist: pgvector>=0.2.5; extra == 'postgres'
+Requires-Dist: psycopg[binary]>=3.0.0; extra == 'postgres'
+Provides-Extra: secrets
+Requires-Dist: boto3; extra == 'secrets'
+Requires-Dist: google-cloud-secret-manager; extra == 'secrets'
+Requires-Dist: hvac; extra == 'secrets'
+Provides-Extra: timeseries
+Requires-Dist: aeon>=1.3.0; extra == 'timeseries'
+Requires-Dist: numba>=0.55.0; extra == 'timeseries'
+Provides-Extra: web
+Description-Content-Type: text/markdown
+
+# EvalVault
+
+RAG(Retrieval-Augmented Generation) 시스템을 대상으로 **평가(Eval) → 분석(Analysis) → 추적(Tracing) → 개선 루프**를 하나의 워크플로로 묶는 CLI + Web UI 플랫폼입니다.
+
+[![PyPI](https://img.shields.io/pypi/v/evalvault.svg)](https://pypi.org/project/evalvault/)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/ntts9990/EvalVault/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/ntts9990/EvalVault/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE.md)
+
+English version? See `README.en.md`.
+
+---
+
+## Quickstart (CLI)
+
+```bash
+uv sync --extra dev
+cp .env.example .env
+
+uv run evalvault run --mode simple tests/fixtures/e2e/insurance_qa_korean.json \
+  --metrics faithfulness,answer_relevancy \
+  --profile dev \
+  --auto-analyze
+```
+
+Tip: 기본 저장소는 Postgres+pgvector입니다. SQLite를 쓰려면 `--db` 또는 `DB_BACKEND=sqlite` + `EVALVAULT_DB_PATH`를 지정하세요.
+
+---
+
+## 핵심 기능
+
+- **End-to-End 평가 루프**: Eval → Analysis → Tracing → Improvement를 한 흐름으로 실행
+- **Dataset 중심 운영**: 합격 기준(threshold)을 데이터셋에 유지
+- **Artifacts-first**: 보고서뿐 아니라 모듈별 원본 결과를 구조화 저장
+- **옵션형 Observability**: Phoenix/Langfuse/MLflow는 필요할 때만 활성화
+- **CLI + Web UI**: 동일 run_id 기반으로 히스토리/비교/리포트 통합
+
+---
+
+## 문서 허브
+
+- 문서 인덱스: `docs/INDEX.md`
+- 핸드북(교과서형): `docs/handbook/INDEX.md`
+- 외부 요약본: `docs/handbook/EXTERNAL.md`
+- 운영 가이드(로컬/도커/관측/런북): `docs/handbook/CHAPTERS/04_operations.md`
+- 워크플로(실행/분석/비교/회귀): `docs/handbook/CHAPTERS/03_workflows.md`
+- 품질/테스트/CI: `docs/handbook/CHAPTERS/06_quality_and_testing.md`
+- 아키텍처: `docs/handbook/CHAPTERS/01_architecture.md`
+- 오프라인/폐쇄망(Docker/모델 캐시): `docs/guides/OFFLINE_DOCKER.md`, `docs/guides/OFFLINE_MODELS.md`
+
+참고(호환성): `docs/guides/USER_GUIDE.md`, `docs/guides/DEV_GUIDE.md` 등 일부 문서는 과거 링크 호환을 위한 deprecated 스텁이며, 최신 내용은 handbook을 따릅니다.
+
+---
+
+## Web UI
+
+```bash
+# API
+uv run evalvault serve-api --reload
+
+# Frontend
+cd frontend
+npm install
+npm run dev
+```
+
+브라우저에서 `http://localhost:5173` 접속 후, Evaluation Studio에서 실행/히스토리/리포트를 확인합니다.
+
+---
+
+## 오프라인/폐쇄망
+
+- Docker 이미지 번들: `docs/guides/OFFLINE_DOCKER.md`
+- NLP 모델 캐시 번들: `docs/guides/OFFLINE_MODELS.md`
+
+LLM 모델은 폐쇄망 내부 인프라가 관리하며, EvalVault는 **분석용 NLP 모델 캐시**만 번들에 포함합니다.
+
+---
+
+## 기여
+
+```bash
+uv run ruff check src/ tests/
+uv run ruff format src/ tests/
+uv run pytest tests -v
+```
+
+- 기여 가이드: `CONTRIBUTING.md`
+- 개발/테스트 루틴: `AGENTS.md`, `docs/handbook/CHAPTERS/06_quality_and_testing.md`
+
+---
+
+## License
+
+EvalVault is licensed under the [Apache 2.0](LICENSE.md) license.