meta-evaluator 0.1.0__tar.gz

meta_evaluator-0.1.0/.env.example

@@ -0,0 +1,53 @@
+# Environment Variables for LLM Providers
+# All variables are optional - only set the ones for providers you plan to use
+# For detailed setup instructions, see: https://docs.litellm.ai/docs/providers
+
+# ================================================================================
+# OpenAI
+# ================================================================================
+# Required for OpenAI models (gpt-4, gpt-3.5-turbo, etc.)
+export OPENAI_API_KEY=sk-your-openai-api-key-here
+
+# ================================================================================
+# Azure OpenAI
+# ================================================================================
+# Required for Azure OpenAI models
+export AZURE_API_BASE=https://your-resource-name.openai.azure.com/
+export AZURE_API_KEY=your-azure-api-key-here
+export AZURE_API_VERSION=2024-02-15-preview
+
+# ================================================================================
+# Anthropic (Claude)
+# ================================================================================
+# Required for Claude models (claude-3-opus, claude-3-sonnet, etc.)
+export ANTHROPIC_API_KEY=sk-ant-your-anthropic-api-key-here
+export ANTHROPIC_DEFAULT_MODEL=claude-3-sonnet
+
+# ================================================================================
+# AWS Bedrock
+# ================================================================================
+# Standard AWS credentials for Bedrock models
+export AWS_ACCESS_KEY_ID=your-aws-access-key-id
+export AWS_SECRET_ACCESS_KEY=your-aws-secret-access-key
+export AWS_SESSION_TOKEN=your-aws-session-token
+
+# Alternative: Bearer token for Bedrock
+export AWS_BEARER_TOKEN_BEDROCK=your-bedrock-bearer-token
+
+# ================================================================================
+# Other Providers
+# ================================================================================
+# Hugging Face models
+export HF_TOKEN=hf_your-hugging-face-token
+
+# OpenRouter (provides access to multiple models)
+export OPENROUTER_API_KEY=sk-or-your-openrouter-key
+
+# xAI (Grok models)
+export XAI_API_KEY=xai-your-api-key-here
+
+# Groq (fast inference)
+export GROQ_API_KEY=gsk_your-groq-api-key
+
+# Fireworks AI
+export FIREWORKS_AI_API_KEY=your-fireworks-api-key

meta_evaluator-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,32 @@
+# .github/workflows/code-quality.yml
+name: Code Quality
+on: [push, pull_request]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+
+    steps:
+      # 1 — Check out the code
+      - uses: actions/checkout@v4
+
+      # 2 — Install uv, but turn off its built-in cache
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: false   # we will manage caching explicitly
+
+      # 3 — Restore (or create) a cache keyed to lock-file + pyproject
+      - name: Restore uv cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv                     # default uv cache dir :contentReference[oaicite:3]{index=3}
+          key: uv-${{ runner.os }}-${{ hashFiles('uv.lock', 'pyproject.toml') }}
+
+      # 4 — Install all declared dependencies every run
+      - name: Sync dependencies
+        run: uv sync --group dev --extra all
+
+      # 5 — Run the same hooks you use locally
+      - name: Run pre-commit hooks
+        run: uv run pre-commit run --all-files
+

meta_evaluator-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,24 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    name: Build and publish to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

meta_evaluator-0.1.0/.gitignore

@@ -0,0 +1,37 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.env
+node_modules/
+package-lock.json
+plan.md
+
+# OS generated files
+.DS_Store
+
+# MkDocs build output
+site/
+
+# Data directories
+*/examples/demo_annotations/
+
+# Ignore all examples except tracked ones
+examples/*
+!examples/rejection/
+!examples/rabakbench/run_evaluation.py
+!examples/rabakbench/run_human_annotation.py
+
+# Project directories - ignore contents but keep structure
+**/project_dir/*
+!**/project_dir/.gitkeep
+
+# Logs
+**/logs/*
+!**/logs/.gitkeep

meta_evaluator-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+  - repo: local
+    hooks:
+      - id: ruff-check
+        name: ruff check
+        entry: uv run ruff check --preview --fix
+        language: system
+        types: [python]
+        pass_filenames: false
+      - id: ruff-format
+        name: ruff format
+        entry: uv run ruff format
+        language: system
+        types: [python]
+        pass_filenames: false
+      - id: pyright
+        name: pyright
+        entry: uv run pyright
+        language: system
+        types: [python]
+        pass_filenames: false
+      - id: pytest
+        name: pytest
+        entry: uv run pytest -k "not integration"
+        language: system
+        types: [python]
+        pass_filenames: false

meta_evaluator-0.1.0/.python-version

@@ -0,0 +1 @@
+3.13

meta_evaluator-0.1.0/CLAUDE.md

@@ -0,0 +1,94 @@
+# Claude Development Notes
+
+## App Development Guide
+
+This is a Python application to develop a MetaEvaluator. 
+Given an evaluation task and dataset, the MetaEvaluator gathers results from LLM Judges and Human annotators, and calculates alignment metrics to measure the performance of different LLM-as-a-Judge.
+
+## Logging 
+
+For each main class, initialize a logger object with the main class name, and utilize the same logger through the class methods.
+
+## Error Handling 
+
+Always use custom exceptions. Define an exceptions file in each main functionality group, and define all custom exceptions within the file. 
+Before implementing specific error messaging within each exception class, refer to the common utility (meta_evaluator.common.error_constants.py) and utilize existing error messages where applicable.
+
+## Linting and Code Quality
+
+After every task, you must run both:
+```bash
+uv tool run ruff check --preview --fix
+uv tool run ruff format .
+```
+
+Always fix all errors from the ruff check. If there are unfixable errors, document them and ask for guidance.
+
+## Type Checking
+
+Streamlit is an optional dependency (`ui` extra). To ensure pyright can resolve all imports, sync with extras before type checking:
+```bash
+uv sync --extra all
+```
+
+After every task, run type checking:
+```bash
+uv run pyright
+```
+
+If there are type errors that cannot be resolved, document them and ask for guidance before proceeding.
+
+## Testing
+
+Always look for existing pytest fixtures in the corresponding conftest.py files before implementing new fixtures. 
+All pytest fixtures should be defined in the corresponding conftest.py files with clear documentation. 
+
+To run tests, always use:
+```bash
+uv run pytest
+```
+
+You can use regular pytest options after `uv run pytest`, for example:
+```bash
+uv run pytest tests/specific_test.py
+uv run pytest -v
+uv run pytest -k "test_name"
+```
+
+After every task, run tests excluding integration tests (integration tests require external services and are slower):
+```bash
+uv run pytest -m "not integration"
+```
+
+## Task Completion Workflow
+
+At the end of every task, run in order:
+
+1. **Linting and formatting:**
+   ```bash
+   uv tool run ruff check --preview --fix
+   uv tool run ruff format .
+   ```
+
+2. **Type checking:**
+   ```bash
+   uv run pyright
+   ```
+
+3. **Testing:**
+   ```bash
+   uv run pytest -m "not integration"
+   ```
+
+Ensure your last command is always `uv tool run ruff format .`
+
+## Packaging
+
+- The package version is defined in `src/meta_evaluator/__init__.py` (`__version__`) and read by hatchling via `[tool.hatch.version]`.
+- Optional dependency groups: `docs` (mkdocs), `ui` (streamlit), `all` (everything). Core deps include matplotlib.
+- Publishing is automated via `.github/workflows/publish.yml` on GitHub release creation using trusted publishing (OIDC).
+
+## Additional Reminders (VERY IMPORTANT!)
+
+- If you have any questions or doubts about the instructions, ask me before you begin. 
+- Do NOT make any assumptions about what the user is asking. ALWAYS CLARIFY.

meta_evaluator-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 GovTech Singapore AI Practice - Responsible AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

meta_evaluator-0.1.0/PKG-INFO

@@ -0,0 +1,349 @@
+Metadata-Version: 2.4
+Name: meta-evaluator
+Version: 0.1.0
+Summary: Evaluate LLM-as-a-Judge systems by measuring alignment between judge outputs and human annotations
+Project-URL: Homepage, https://github.com/govtech-responsibleai/meta-evaluator
+Project-URL: Repository, https://github.com/govtech-responsibleai/meta-evaluator
+Project-URL: Issues, https://github.com/govtech-responsibleai/meta-evaluator/issues
+Author: aipractice
+License-Expression: MIT
+License-File: LICENSE
+Keywords: alignment,evaluation,judge,llm,meta-evaluation,nlp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.13
+Requires-Dist: beartype>=0.21.0
+Requires-Dist: boto3>=1.40.10
+Requires-Dist: botocore>=1.40.10
+Requires-Dist: google-auth>=2.40.3
+Requires-Dist: instructor>=1.8.3
+Requires-Dist: litellm>=1.65.1
+Requires-Dist: matplotlib>=3.5.0
+Requires-Dist: openai>=1.82.1
+Requires-Dist: polars>=1.30.0
+Requires-Dist: pydantic>=2.11.5
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: scikit-learn>=1.5.0
+Requires-Dist: scipy>=1.9.0
+Requires-Dist: tqdm>=4.66.0
+Provides-Extra: all
+Requires-Dist: mkdocs-material>=9.6.17; extra == 'all'
+Requires-Dist: mkdocs>=1.6.1; extra == 'all'
+Requires-Dist: streamlit>=1.50.0; extra == 'all'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.6.17; extra == 'docs'
+Requires-Dist: mkdocs>=1.6.1; extra == 'docs'
+Provides-Extra: ui
+Requires-Dist: streamlit>=1.50.0; extra == 'ui'
+Description-Content-Type: text/markdown
+
+# MetaEvaluator
+
+Evaluate LLM-as-a-Judge systems by measuring alignment between judge outputs with human annotations.
+
+## Overview
+
+MetaEvaluator helps you assess LLM judges by:
+- 🤖 **Running multiple judges** (OpenAI, Anthropic, Google, AWS, etc.) using **LiteLLM integration**
+- 👥 **Collecting human annotations** through a built-in Streamlit interface
+- 📊 **Computing alignment metrics** (Accuracy, Cohen's Kappa, Alt-Test, text/semantic similarity) and **generating comprehensive reports** with visualizations and statistical analysis
+
+## Installation
+
+1. **Install the package:**
+   ```bash
+   # Requires Python 3.13+
+   pip install meta-evaluator
+   ```
+
+   **Optional dependencies:**
+   ```bash
+   pip install meta-evaluator[ui]    # streamlit for human annotation interface
+   pip install meta-evaluator[docs]  # mkdocs for documentation
+   pip install meta-evaluator[all]   # all optional dependencies
+   ```
+
+   Or install directly from GitHub:
+   ```bash
+   pip install git+https://github.com/govtech-responsibleai/meta-evaluator
+   ```
+
+2. **Set up environment variables:**
+   You can either:
+   - Copy the [.env.example](https://github.com/govtech-responsibleai/meta-evaluator/blob/main/.env.example) file from the GitHub repo, replace with your API keys, and use `dotenv.load_dotenv()` in your script
+   - Set the environment variables directly in your shell
+
+   See [LiteLLM providers documentation](https://docs.litellm.ai/docs/providers) for all supported providers.
+
+3. **(Optional) For developers: clone the repository and set up dev tools:**
+   ```bash
+   git clone https://github.com/govtech-responsibleai/meta-evaluator
+   cd meta-evaluator
+   uv sync
+   uv run pre-commit install
+   ```
+
+## Getting Started
+
+See our [**Tutorial**](docs/tutorial.md) for a complete walkthrough, or check out the full example at: [`examples/rejection/run_evaluation.py`](examples/rejection/run_evaluation.py)
+The sections below provide an overview of the main components.
+
+### 1. Initialize MetaEvaluator
+Start by creating a MetaEvaluator instance:
+
+```python
+from meta_evaluator import MetaEvaluator
+
+# Create new project
+evaluator = MetaEvaluator(project_dir="my_project")
+```
+
+### 2. Load Data
+Load your evaluation datasets from CSV, JSON, or Parquet files:
+
+```python
+from meta_evaluator.data import DataLoader
+
+data = DataLoader.load_csv(
+    name="evaluation_data",
+    file_path="data/samples.csv"
+)
+evaluator.add_data(data)
+```
+
+### 3. Define Task  
+Define what and how to evaluate using EvalTask:
+
+```python
+from meta_evaluator.eval_task import EvalTask
+
+task = EvalTask(
+    task_schemas={
+        "rejection": ["rejection", "not rejection"],  # Classification (required by default)
+        "explanation": None,  # Free-form text (not required by default)
+    },
+    # required_tasks not specified - only classification tasks required by default
+    prompt_columns=["prompt"],         # Context columns
+    response_columns=["llm_response"], # What to evaluate
+    answering_method="structured",     # JSON output parsing
+    structured_outputs_fallback=True   # Fallback support
+)
+evaluator.add_eval_task(task)
+```
+
+### 4. Collect Human Annotations
+Collect human ground truth using the built-in Streamlit interface:
+
+```python
+# Launch annotation interface
+evaluator.launch_annotator(port=8501)
+```
+
+For deployment options including remote access (ngrok) and Docker deployment for classified data, see the [Deployment Guide](docs/annotation_guide/deployment.md).
+
+### 5. Judge Configuration & Prompt Templates
+Configure multiple LLM judges using YAML and template-based prompts:
+
+```yaml
+judges:
+  - id: gpt_4_judge
+    llm_client: openai
+    model: gpt-4o-mini
+    prompt_file: ./prompt.md   # Filepath relative to YAML file
+    temperature: 0.0           # Optional: sampling temperature
+
+  - id: claude_judge
+    llm_client: anthropic
+    model: claude-3-5-haiku-latest
+    prompt_file: ./prompt.md   # Filepath relative to YAML file
+
+  - id: hf_judge
+    llm_client: huggingface/together
+    model: meta-llama/Llama-3.3-70B-Instruct
+    prompt_file: ./prompt.md   # Filepath relative to YAML file
+    extra_headers:             # Optional: e.g. bill to a HuggingFace organisation
+      X-HF-Bill-To: your-org-name
+```
+
+**Prompt Template System**: MetaEvaluator uses a template-based system where you can define placeholders in your `prompt.md` files using curly braces (`{variable_name}`). These variables are automatically substituted with columns found in your dataset during evaluation.
+
+```markdown
+## Instructions:
+Evaluate whether the given response is a rejection.
+
+## To Evaluate:
+Prompt: {prompt}
+Response: {llm_response}
+```
+
+### 6. Run LLM Judges
+Load and run your configured judges:
+
+```python
+# Load judges and run evaluation
+evaluator.load_judges_from_yaml("judges.yaml", async_mode=True)
+evaluator.run_judges_async(
+    skip_duplicates=True,
+    consistency=1,   # Optional: run each judge N times and aggregate results
+)
+```
+
+### 7. Scoring
+Compare judge performance against human annotations:
+
+```python
+# Configure metrics
+from meta_evaluator.scores import MetricConfig, MetricsConfig
+from meta_evaluator.scores.metrics import (
+    ClassificationScorer, CohensKappaScorer, SemanticSimilarityScorer
+)
+
+config = MetricsConfig(
+    metrics=[
+        MetricConfig(
+            scorer=ClassificationScorer("accuracy"),
+            task_names=["rejection"],
+            task_strategy="single",
+            annotator_aggregation="majority_vote"  # Use consensus approach
+        ),
+        MetricConfig(
+            scorer=SemanticSimilarityScorer(),  # This metric requires OPENAI_API_KEY
+            task_names=["explanation"],
+            task_strategy="single",
+            annotator_aggregation="individual_average"  # Individual averaging
+        ),
+    ]
+)
+
+# Add metrics configuration and run comparison
+evaluator.add_metrics_config(config)  # Creates evaluator.score_report automatically
+evaluator.compare_async()
+
+# Generate summary report
+evaluator.score_report.save("score_report.html", format="html")  # Save HTML report
+evaluator.score_report.save("score_report.csv", format="csv")    # Save CSV report
+evaluator.score_report.print()  # Print to console
+```
+
+## External Data Loading
+
+MetaEvaluator supports loading pre-existing judge and human annotation results for scoring-only workflows. This is useful when you:
+- Have results from previous evaluation runs
+- Want to compute metrics on externally generated judge/human data
+- Need to re-run scoring with different metrics without re-evaluating
+
+### Loading A Single External Judge Results
+```python
+# Load external judge results from CSV
+evaluator.add_external_judge_results(
+    file_path="path/to/judge1_results.csv",
+    judge_id="external_judge_1",
+    llm_client="openai",
+    model_used="gpt-4",
+    run_id="external_run_1"
+)
+```
+
+**Required CSV columns for judge results:**
+- `original_id`: Unique identifier for each sample
+- Task columns matching your `EvalTask.task_schemas`
+
+### Loading A Single External Annotation Results
+```python
+# Load external human annotations from CSV
+
+evaluator.add_external_annotation_results(
+    file_path="path/to/human_results_1.csv",
+    annotator_id="annotator_1",
+    run_id="human_run_1"
+)
+```
+
+**Required CSV columns for human results:**
+- `original_id`: Unique identifier for each sample  
+- Task columns matching your `EvalTask.task_schemas`
+
+For detailed data format requirements and examples, see the [Results Guide](docs/guides/results.md#external-data-loading).
+
+## Available Metrics
+
+MetaEvaluator supports comprehensive alignment metrics for evaluating judge performance:
+
+### Classification Metrics
+- **Accuracy/F1/Recall/Precision**: Classification metrics between judge and human labels
+- **Cohen's Kappa**: Inter-rater agreement accounting for chance agreement  
+- **Alt-Test**: Statistical significance testing with leave-one-annotator-out methodology
+
+### Text Similarity Metrics  
+- **Text Similarity**: String-based similarity using sequence matching algorithms
+- **Semantic Similarity**: OpenAI embedding-based semantic similarity (requires API key)
+
+### Custom Metrics
+- **Custom Scorers**: Implement domain-specific metrics by extending `BaseScorer`
+
+See [Scoring Guide](docs/guides/scoring.md) for detailed usage examples and configuration options.
+
+## Documentation
+
+Comprehensive documentation is available in the `docs/` directory:
+
+- **[Tutorial](docs/tutorial.md)** - Complete walkthrough
+- **[Data Loading](docs/guides/evaldata.md)** - Load and manage evaluation datasets
+- **[Task Definition](docs/guides/evaltask.md)** - Define evaluation schemas and parsing methods
+- **[Judge Configuration](docs/guides/judges_load.md)** - Set up LLM judges with YAML
+- **[Running Evaluations](docs/guides/judges_run.md)** - Execute judge evaluations
+- **[Scoring & Metrics](docs/guides/scoring.md)** - Compute alignment metrics
+- **[Human Annotations](docs/annotation_guide/annotation.md)** - Collect human ground truth
+- **[Deployment Guide for Annotation Platform](docs/annotation_guide/deployment.md)** - Deployment options (local, ngrok, Docker)
+
+## Project Structure (automatically generated)
+
+```
+project_dir/
+├── data/                           # Serialized evaluation data
+├── results/                        # Judge evaluation results
+├── annotations/                    # Human annotation data
+└── scores/                         # Computed alignment metrics
+    ├── classification_accuracy/    # Detailed accuracy results
+    ├── cohens_kappa/               # Detailed kappa results
+    ├── alt_test/                   # Detailed alt-test results
+    └── text_similarity/            # Detailed similarity results
+```
+
+## Examples
+
+See the `examples/` directory for complete working examples:
+
+### Rejection Detection Evaluation
+- **[`examples/rejection/run_evaluation.py`](examples/rejection/run_evaluation.py)** - Complete async evaluation with multiple metrics
+- **[`examples/rejection/run_human_annotation.py`](examples/rejection/run_human_annotation.py)** - Launch human annotation interface
+- **[`examples/rejection/data/sample_rejection.csv`](examples/rejection/data/sample_rejection.csv)** - Sample rejection detection dataset
+- **[`examples/rejection/judges.yaml`](examples/rejection/judges.yaml)** - Judge configuration example
+- **[`examples/rejection/prompt.md`](examples/rejection/prompt.md)** - Evaluation prompt template
+
+### Docker Templates
+- **[`docker/Dockerfile`](docker/Dockerfile)** - Basic Dockerfile template
+- **[`docker/docker-compose.yml`](docker/docker-compose.yml)** - Docker compose template
+
+### RabakBench Evaluation (data not included)
+- **[`examples/rabakbench/run_evaluation.py`](examples/rabakbench/run_evaluation.py)** - Complete async evaluation with multiple metrics
+- **[`examples/rabakbench/run_human_annotation.py`](examples/rabakbench/run_human_annotation.py)** - Launch human annotation interface
+
+### Scoring-Only Evaluation (load in external results)
+- **[`examples/rejection/run_scoring_only.py`](examples/rejection/run_scoring_only.py)** - Load external judge/human results and run scoring without re-evaluation
+
+## Development Commands
+
+**Requirements:** [uv](https://docs.astral.sh/uv/) package manager
+
+- **Run linting:** `uv tool run ruff check --preview --fix`
+- **Run formatting:** `uv tool run ruff format .`
+- **Run type checking:** `uv run pyright`
+- **Run tests:** `uv run pytest --skip-integration`
+