ragaeval 0.1.0__tar.gz

ragaeval-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 ragaeval contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ragaeval-0.1.0/MANIFEST.in

@@ -0,0 +1,8 @@
+include README.md
+include LICENSE
+include pyproject.toml
+include setup.py
+recursive-include ragaeval *.py
+global-exclude __pycache__
+global-exclude *.py[co]
+global-exclude .DS_Store

ragaeval-0.1.0/PKG-INFO

@@ -0,0 +1,265 @@
+Metadata-Version: 2.4
+Name: ragaeval
+Version: 0.1.0
+Summary: A pip-installable Python library providing drop-in RAG evaluation capabilities using RAGAS
+Home-page: https://github.com/Vamsi1113/ragaeval
+Author: Vamsi Kasireddy
+Author-email: Vamsi Kasireddy <vamsikasireddy@example.com>
+License: MIT
+Project-URL: Homepage, https://github.com/Vamsi1113/ragaeval
+Project-URL: Bug Reports, https://github.com/Vamsi1113/ragaeval/issues
+Project-URL: Source, https://github.com/Vamsi1113/ragaeval
+Keywords: rag,evaluation,ragas,llm,ai,azure,openai
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ragas==0.2.15
+Requires-Dist: langchain-openai==0.1.23
+Requires-Dist: langchain-core==0.2.40
+Requires-Dist: sentence-transformers==3.3.1
+Requires-Dist: python-dotenv
+Requires-Dist: pandas
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.10.0; extra == "dev"
+Requires-Dist: hypothesis>=6.0.0; extra == "dev"
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# ragaeval
+
+A pip-installable Python library providing drop-in RAG (Retrieval-Augmented Generation) evaluation capabilities using the RAGAS framework (version 0.2.15).
+
+## Features
+
+- **Three Integration Patterns**: Choose from decorator, context manager, or manual logging
+- **Secure Credential Management**: Multiple sources with precedence (args > env vars > .env > config file)
+- **Cross-Platform**: Works on Windows, Mac, and Linux
+- **Automatic Field Normalization**: Supports common field name aliases
+- **CLI Tools**: Configuration, execution, and status commands
+- **Rich Reports**: Terminal output with visual indicators and CSV export
+
+## Installation
+
+```bash
+pip install ragaeval
+```
+
+## Quick Start
+
+### 1. Configure API Credentials
+
+Run the interactive configuration wizard:
+
+```bash
+ragaeval configure
+```
+
+Or set environment variables:
+
+```bash
+export AZURE_OPENAI_API_KEY="your-api-key"
+export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
+export AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o"
+export AZURE_OPENAI_API_VERSION="2024-02-15-preview"
+```
+
+### 2. Log Evaluation Data
+
+#### Option A: Decorator Pattern
+
+```python
+from ragaeval import evaluate_rag
+
+@evaluate_rag
+def my_rag_pipeline(query: str, contexts: list) -> str:
+    # Your RAG logic here
+    response = generate_answer(query, contexts)
+    return response
+
+# Use your function normally - data is captured automatically
+result = my_rag_pipeline(
+    query="What is the refund policy?",
+    contexts=["Returns accepted within 30 days", "Refunds processed in 5-7 days"]
+)
+```
+
+#### Option B: Context Manager Pattern
+
+```python
+from ragaeval import EvalSession
+
+with EvalSession() as session:
+    query = "What is the refund policy?"
+    contexts = retrieve_documents(query)
+    response = generate_answer(query, contexts)
+    
+    session.log(query=query, contexts=contexts, response=response)
+```
+
+#### Option C: Manual Logging
+
+```python
+import ragaeval
+
+query = "What is the refund policy?"
+contexts = retrieve_documents(query)
+response = generate_answer(query, contexts)
+
+ragaeval.log(query=query, contexts=contexts, response=response)
+```
+
+### 3. Run Evaluation
+
+```bash
+ragaeval run
+```
+
+This will:
+- Read logged data from `.ragaeval_log.jsonl`
+- Execute RAGAS evaluation using Azure OpenAI
+- Display results in terminal with visual indicators
+- Export results to `eval_results.csv`
+
+## CLI Commands
+
+### `ragaeval run`
+
+Execute evaluation on logged data.
+
+```bash
+ragaeval run                          # Use defaults
+ragaeval run --model gpt-4o          # Specify Azure deployment
+ragaeval run --output results.csv    # Custom output path
+```
+
+### `ragaeval configure`
+
+Interactive credential setup.
+
+```bash
+ragaeval configure
+```
+
+### `ragaeval status`
+
+Display configuration and log status.
+
+```bash
+ragaeval status
+```
+
+### `ragaeval clear`
+
+Clear evaluation logs.
+
+```bash
+ragaeval clear               # Prompts for confirmation
+ragaeval clear --force       # Skip confirmation
+```
+
+### `ragaeval --version`
+
+Display package version.
+
+```bash
+ragaeval --version
+```
+
+## Field Name Aliases
+
+The package supports common field name variations:
+
+- **Query**: `query`, `question`, `input`, `user_input`
+- **Response**: `response`, `answer`, `llm_response`, `output`, `actual_output`
+- **Contexts**: `contexts`, `context`, `retrieved_contexts`, `source_documents`
+- **Reference**: `reference`, `ground_truth`, `expected`
+
+## Evaluation Metrics
+
+### Always Evaluated (Group A)
+- **Faithfulness**: Response consistency with retrieved contexts
+- **Response Relevancy**: Relevance of response to query
+- **Aspect Critic**: Harmfulness detection
+
+### Evaluated When Reference Available (Group B)
+- **Factual Correctness**: Accuracy against ground truth
+- **Semantic Similarity**: Semantic closeness to reference
+- **BLEU Score**: N-gram overlap
+- **ROUGE Score**: Recall-oriented overlap
+- **String Presence**: Exact string matching
+- **Exact Match**: Perfect match detection
+
+## Configuration Sources
+
+Credentials are resolved in this order (highest to lowest precedence):
+
+1. **Explicit function arguments**
+2. **Environment variables** (`AZURE_OPENAI_API_KEY`, etc.)
+3. **`.env` file** in current directory
+4. **Config file** at `~/.ragaeval/config.json`
+
+## Cross-Platform Notes
+
+- Config file location: `~/.ragaeval/config.json` (user home directory)
+- Log file location: `.ragaeval_log.jsonl` (current working directory)
+- File permissions: Config file is created with user-only access (0o600)
+
+## Requirements
+
+- Python >= 3.8
+- Azure OpenAI API access (for LLM evaluation)
+- Optional: OpenAI API key (for embeddings, otherwise uses local models)
+
+## Development
+
+### Install in Development Mode
+
+```bash
+pip install -e .[dev]
+```
+
+### Run Tests
+
+```bash
+pytest
+pytest --cov=ragaeval  # With coverage
+```
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Version
+
+Current version: 0.1.0
+
+Check installed version:
+
+```python
+import ragaeval
+print(ragaeval.__version__)
+```
+
+Or via CLI:
+
+```bash
+ragaeval --version
+```
+
+## Support
+
+For issues and questions, please file an issue on the GitHub repository.

ragaeval-0.1.0/README.md

@@ -0,0 +1,224 @@
+# ragaeval
+
+A pip-installable Python library providing drop-in RAG (Retrieval-Augmented Generation) evaluation capabilities using the RAGAS framework (version 0.2.15).
+
+## Features
+
+- **Three Integration Patterns**: Choose from decorator, context manager, or manual logging
+- **Secure Credential Management**: Multiple sources with precedence (args > env vars > .env > config file)
+- **Cross-Platform**: Works on Windows, Mac, and Linux
+- **Automatic Field Normalization**: Supports common field name aliases
+- **CLI Tools**: Configuration, execution, and status commands
+- **Rich Reports**: Terminal output with visual indicators and CSV export
+
+## Installation
+
+```bash
+pip install ragaeval
+```
+
+## Quick Start
+
+### 1. Configure API Credentials
+
+Run the interactive configuration wizard:
+
+```bash
+ragaeval configure
+```
+
+Or set environment variables:
+
+```bash
+export AZURE_OPENAI_API_KEY="your-api-key"
+export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
+export AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o"
+export AZURE_OPENAI_API_VERSION="2024-02-15-preview"
+```
+
+### 2. Log Evaluation Data
+
+#### Option A: Decorator Pattern
+
+```python
+from ragaeval import evaluate_rag
+
+@evaluate_rag
+def my_rag_pipeline(query: str, contexts: list) -> str:
+    # Your RAG logic here
+    response = generate_answer(query, contexts)
+    return response
+
+# Use your function normally - data is captured automatically
+result = my_rag_pipeline(
+    query="What is the refund policy?",
+    contexts=["Returns accepted within 30 days", "Refunds processed in 5-7 days"]
+)
+```
+
+#### Option B: Context Manager Pattern
+
+```python
+from ragaeval import EvalSession
+
+with EvalSession() as session:
+    query = "What is the refund policy?"
+    contexts = retrieve_documents(query)
+    response = generate_answer(query, contexts)
+    
+    session.log(query=query, contexts=contexts, response=response)
+```
+
+#### Option C: Manual Logging
+
+```python
+import ragaeval
+
+query = "What is the refund policy?"
+contexts = retrieve_documents(query)
+response = generate_answer(query, contexts)
+
+ragaeval.log(query=query, contexts=contexts, response=response)
+```
+
+### 3. Run Evaluation
+
+```bash
+ragaeval run
+```
+
+This will:
+- Read logged data from `.ragaeval_log.jsonl`
+- Execute RAGAS evaluation using Azure OpenAI
+- Display results in terminal with visual indicators
+- Export results to `eval_results.csv`
+
+## CLI Commands
+
+### `ragaeval run`
+
+Execute evaluation on logged data.
+
+```bash
+ragaeval run                          # Use defaults
+ragaeval run --model gpt-4o          # Specify Azure deployment
+ragaeval run --output results.csv    # Custom output path
+```
+
+### `ragaeval configure`
+
+Interactive credential setup.
+
+```bash
+ragaeval configure
+```
+
+### `ragaeval status`
+
+Display configuration and log status.
+
+```bash
+ragaeval status
+```
+
+### `ragaeval clear`
+
+Clear evaluation logs.
+
+```bash
+ragaeval clear               # Prompts for confirmation
+ragaeval clear --force       # Skip confirmation
+```
+
+### `ragaeval --version`
+
+Display package version.
+
+```bash
+ragaeval --version
+```
+
+## Field Name Aliases
+
+The package supports common field name variations:
+
+- **Query**: `query`, `question`, `input`, `user_input`
+- **Response**: `response`, `answer`, `llm_response`, `output`, `actual_output`
+- **Contexts**: `contexts`, `context`, `retrieved_contexts`, `source_documents`
+- **Reference**: `reference`, `ground_truth`, `expected`
+
+## Evaluation Metrics
+
+### Always Evaluated (Group A)
+- **Faithfulness**: Response consistency with retrieved contexts
+- **Response Relevancy**: Relevance of response to query
+- **Aspect Critic**: Harmfulness detection
+
+### Evaluated When Reference Available (Group B)
+- **Factual Correctness**: Accuracy against ground truth
+- **Semantic Similarity**: Semantic closeness to reference
+- **BLEU Score**: N-gram overlap
+- **ROUGE Score**: Recall-oriented overlap
+- **String Presence**: Exact string matching
+- **Exact Match**: Perfect match detection
+
+## Configuration Sources
+
+Credentials are resolved in this order (highest to lowest precedence):
+
+1. **Explicit function arguments**
+2. **Environment variables** (`AZURE_OPENAI_API_KEY`, etc.)
+3. **`.env` file** in current directory
+4. **Config file** at `~/.ragaeval/config.json`
+
+## Cross-Platform Notes
+
+- Config file location: `~/.ragaeval/config.json` (user home directory)
+- Log file location: `.ragaeval_log.jsonl` (current working directory)
+- File permissions: Config file is created with user-only access (0o600)
+
+## Requirements
+
+- Python >= 3.8
+- Azure OpenAI API access (for LLM evaluation)
+- Optional: OpenAI API key (for embeddings, otherwise uses local models)
+
+## Development
+
+### Install in Development Mode
+
+```bash
+pip install -e .[dev]
+```
+
+### Run Tests
+
+```bash
+pytest
+pytest --cov=ragaeval  # With coverage
+```
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Version
+
+Current version: 0.1.0
+
+Check installed version:
+
+```python
+import ragaeval
+print(ragaeval.__version__)
+```
+
+Or via CLI:
+
+```bash
+ragaeval --version
+```
+
+## Support
+
+For issues and questions, please file an issue on the GitHub repository.

ragaeval-0.1.0/pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ragaeval"
+version = "0.1.0"
+description = "A pip-installable Python library providing drop-in RAG evaluation capabilities using RAGAS"
+readme = "README.md"
+requires-python = ">=3.8"
+license = {text = "MIT"}
+authors = [
+    {name = "Vamsi Kasireddy", email = "vamsikasireddy@example.com"}
+]
+keywords = ["rag", "evaluation", "ragas", "llm", "ai", "azure", "openai"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "ragas==0.2.15",
+    "langchain-openai==0.1.23",
+    "langchain-core==0.2.40",
+    "sentence-transformers==3.3.1",
+    "python-dotenv",
+    "pandas",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "pytest-mock>=3.10.0",
+    "hypothesis>=6.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Vamsi1113/ragaeval"
+"Bug Reports" = "https://github.com/Vamsi1113/ragaeval/issues"
+Source = "https://github.com/Vamsi1113/ragaeval"
+
+[project.scripts]
+ragaeval = "ragaeval.cli:main"
+
+[tool.setuptools.packages.find]
+exclude = ["tests*", "venv*", "dist*", "build*", "*.egg-info*"]
+include = ["ragaeval*"]

ragaeval-0.1.0/ragaeval/__init__.py

@@ -0,0 +1,34 @@
+"""
+ragaeval - RAG Evaluation Package
+
+A pip-installable Python library providing drop-in RAG evaluation capabilities using RAGAS.
+"""
+
+import sys
+
+__version__ = "0.1.0"
+
+# Setup VertexAI mock before other imports
+from .mock_setup import setup_vertexai_mock
+setup_vertexai_mock()
+
+# Set UTF-8 encoding on Windows
+if sys.platform == 'win32':
+    try:
+        sys.stdout.reconfigure(encoding='utf-8')
+        sys.stderr.reconfigure(encoding='utf-8')
+    except Exception:
+        pass
+
+# Import public API components
+from .interceptors import evaluate_rag, EvalSession, log
+from .config import ConfigManager
+
+# Public API exports
+__all__ = [
+    "__version__",
+    "evaluate_rag",
+    "EvalSession",
+    "log",
+    "ConfigManager",
+]