evalforge 0.2.0__tar.gz

evalforge-0.2.0/.env.example

@@ -0,0 +1,27 @@
+# Copy this file to .env and fill in your values
+# Never commit .env to git
+
+# ── Required (at least one) ─────────────────────────
+OPENAI_API_KEY=sk-your-openai-key-here
+ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here
+
+# ── Google Gemini ───────────────────────────────────
+GOOGLE_API_KEY=AIza-your-google-api-key-here
+
+# ── AWS Bedrock ─────────────────────────────────────
+# AWS_ACCESS_KEY_ID=your-access-key
+# AWS_SECRET_ACCESS_KEY=your-secret-key
+# AWS_DEFAULT_REGION=us-east-1
+
+# ── Ollama (local) ──────────────────────────────────
+# OLLAMA_BASE_URL=http://localhost:11434
+
+# ── OpenRouter ──────────────────────────────────────
+# OPENROUTER_API_KEY=sk-or-your-key-here
+
+# ── LiteLLM Proxy ──────────────────────────────────
+# LITELLM_API_KEY=your-litellm-key
+
+# ── EvalForge Settings ──────────────────────────────
+EVALFORGE_DEFAULT_MODEL=gpt-4o-mini
+EVALFORGE_HISTORY_DIR=~/.evalforge/history

evalforge-0.2.0/.gitignore

@@ -0,0 +1,18 @@
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+.env
+.env.*
+!.env.example
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+~/.evalforge/
+*.html
+htmlcov/
+.coverage
+node_modules/

evalforge-0.2.0/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## [0.1.0] - Unreleased
+
+### Added
+- Initial release
+- YAML-based eval suite format
+- Scorers: exact, contains, regex, cosine, BLEU, ROUGE, LLM-judge
+- Multi-model parallel evaluation (OpenAI, Anthropic, Gemini, Bedrock, Ollama)
+- Multi-provider routing via providers.py
+- Run history and comparison
+- HTML, Markdown, JUnit reporters
+- Auto test case generation from production logs
+- Cost tracking with 31-model pricing table
+- Session cost breakdown (eval vs overhead)
+- Budget alerts and cumulative tracking
+- Security hardening: API key masking, YAML sanitization, XSS prevention
+- CI/CD integration with exit codes
+- npm wrapper package for Node.js users

evalforge-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 EvalForge Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalforge-0.2.0/PKG-INFO

@@ -0,0 +1,256 @@
+Metadata-Version: 2.4
+Name: evalforge
+Version: 0.2.0
+Summary: The pytest of LLMs — build, version, and run LLM eval suites from the terminal
+Project-URL: Homepage, https://github.com/yourusername/evalforge
+Project-URL: Documentation, https://github.com/yourusername/evalforge#readme
+Project-URL: Issues, https://github.com/yourusername/evalforge/issues
+Project-URL: Changelog, https://github.com/yourusername/evalforge/blob/main/CHANGELOG.md
+License: MIT License
+        
+        Copyright (c) 2025 EvalForge Contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: ai,cli,data-science,evaluation,llm,mlops,prompt-testing,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: anthropic>=0.25
+Requires-Dist: httpx>=0.27
+Requires-Dist: jinja2>=3.1
+Requires-Dist: nltk>=3.8
+Requires-Dist: openai>=1.0
+Requires-Dist: pydantic>=2
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13
+Requires-Dist: rouge-score>=0.1.2
+Requires-Dist: scikit-learn>=1.4
+Requires-Dist: sentence-transformers>=3.0
+Requires-Dist: tenacity>=8.2
+Requires-Dist: tiktoken>=0.7
+Requires-Dist: typer>=0.12
+Provides-Extra: all
+Requires-Dist: boto3>=1.34; extra == 'all'
+Requires-Dist: google-generativeai>=0.5; extra == 'all'
+Requires-Dist: litellm>=1.40; extra == 'all'
+Provides-Extra: bedrock
+Requires-Dist: boto3>=1.34; extra == 'bedrock'
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pip-audit>=2.7; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Requires-Dist: twine>=5.0; extra == 'dev'
+Provides-Extra: gemini
+Requires-Dist: google-generativeai>=0.5; extra == 'gemini'
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.40; extra == 'litellm'
+Description-Content-Type: text/markdown
+
+# 🔨 EvalForge
+
+**The ultimate evaluation platform for LLMs and Autonomous AI Agents.**
+
+EvalForge simplifies the process of testing, evaluating, and iterating on Large Language Models and AI Agents. Whether you're benchmarking a simple LLM prompt block or evaluating a complex, multi-tool Kubernetes Agent using LangChain/OpenAI SDK, EvalForge provides deterministic scoring, conversational edge-case simulation, and detailed cost breakdowns directly in your terminal.
+
+---
+
+## 🚀 Quick Start
+
+EvalForge can be installed easily via pip or npm (using the wrapper).
+
+**Python (PyPI)**
+```bash
+pip install evalforge
+```
+
+**Node / JavaScript (NPM Wrapper)**
+```bash
+npm install -g evalforge
+```
+
+### Initialize a Project
+To scaffold a basic LLM or Agent project, run:
+```bash
+evalforge init
+```
+
+---
+
+## 🧠 Model Evals (Classic LLM Mode)
+
+Test standard Large Language Models for prompt coherence, factual extraction, and instruction adherence.
+
+1. **Define a Suite**: Create an `llm_suite.yaml`:
+   ```yaml
+   name: "Summarization Task"
+   description: "Tests a model's summarization capabilities"
+   type: llm
+   model: gpt-4o-mini
+   cases:
+     - id: "long_text_summary"
+       messages:
+         - role: "user"
+           content: "Summarize this long text: [text ...]"
+       scorers:
+         - type: "llm_judge"
+           threshold: 0.8
+           rubric: "Did the model summarize the text accurately in under 5 sentences?"
+         - type: "contains"
+           value: "summary"
+           threshold: 1.0
+   ```
+
+2. **Run the Suite**:
+   ```bash
+   evalforge run llm_suite.yaml
+   ```
+   You can easily override the model to benchmark across providers (OpenAI, Anthropic, Gemini, Ollama):
+   ```bash
+   evalforge run llm_suite.yaml -m claude-3-haiku-20240307
+   ```
+
+---
+
+## 🤖 Agent Evals (Autonomous Agent Mode)
+
+EvalForge v0.2 introduces native support for Autonomous Agents. It dynamically generates edge-cases from your System Prompt and interfaces with any Python codebase (LangChain, LangGraph, standard OpenAI functions).
+
+### 1. Build an Agent Suite
+Define an agent suite specifying the `system_prompt` and pointing EvalForge to your application code.
+
+```yaml
+name: "Kubernetes Expert Agent"
+description: "Tests a web-browsing agent's ability to document K8s topics."
+version: 1.0.0
+type: agent
+parallel: 1
+model: gpt-4o
+agent:
+  adapter: python
+  config:
+    module: "k8s_bot.main"
+    function: "run_agent"
+system_prompt: |
+  You are a Kubernetes Expert Agent. You have access to a web browsing tool to retrieve documentation.
+  Do not generate documentation yourself; strictly use the web tool to search for information.
+  Never modify or delete infrastructure without explicit user confirmation.
+auto_generate:
+  enabled: true
+  num_cases: 6
+cases: []
+```
+
+### 2. Auto-Generate Test Cases
+EvalForge comes with an internal **LLM Validation Persona**. Using the `auto_generate` config block, EvalForge reads your system prompt and generates robust **Happy Paths**, **Conversational Edge Cases**, and **Boundary Limits**, saving them directly into your `suite.yaml`.
+
+Run evaluation with generation:
+```bash
+evalforge run --regenerate suite.yaml
+```
+
+### 3. Reviewing The Agent Results
+EvalForge will display an elegant UI in the terminal tracking your agent's hallucination score (via NLI), tool invocations, and semantic understanding.
+
+***Example Output Console:***
+```text
+                            Category Breakdown                             
+                                                                           
+  Category                 Passed      Pass Rate     Progress              
+ ───────────────────────────────────────────────────────────────────────── 
+  Create                    0/1            0%        ░░░░░░░░░░░░░░░       
+  Edge                      0/1            0%        ░░░░░░░░░░░░░░░       
+  Infrastructure            0/1            0%        ░░░░░░░░░░░░░░░       
+  Kubernetes                0/1            0%        ░░░░░░░░░░░░░░░       
+  Node                      0/1            0%        ░░░░░░░░░░░░░░░       
+  Pods                      0/1            0%        ░░░░░░░░░░░░░░░       
+                                                                           
+
+❌ Failed Cases (6)
+
+  list_pods_happy_path (gpt-4o)
+    → llm_judge score=0.75: The response provides a relevant command to retrieve the desired information but...
+    → hallucination score=0.29
+    Output: I currently don’t have the capability to directly interact with or retrieve live data from your Kubernetes cluster. Howe…
+
+  check_node_health_happy_path (gpt-4o)
+    → llm_judge score=0.00: The response fails to provide any node names, health conditions, or warnings, ...
+    → hallucination score=0.39
+    Output: I don't have the ability to check the health status of your nodes directly. However, I can guide you on how to do it usi…
+
+╭─ 💰 Cost Breakdown ─────────────────────╮
+│ Eval Cost:     $0.00 per 100k runs (0%) │
+│ Overhead:     $0.00039 (100%)           │
+│ Total:        $0.00039                  │
+╰─────────────────────────────────────────╯
+
+╭───────────────────────────────────────────────────────────────────────╮
+│ ❌ SUITE FAILED  —  0.0% pass rate  •  6 of 6 failed  •  run_d08fd895 │
+╰───────────────────────────────────────────────────────────────────────╯
+```
+
+---
+
+## 🧪 Evaluators & Scorers
+
+EvalForge packages multiple deterministic and probabilistic scorers designed for accuracy tracking:
+
+- `exact` & `contains`: Basic deterministic string matching.
+- `regex`: Extracts pattern matching.
+- `cosine`: Compares embeddings for similarity semantic checks.
+- `bleu` / `rouge`: Machine-translation N-Gram comparisons using NLTK mechanisms.
+- `llm_judge`: Asks an LLM to evaluate the string output based on a strictly provided `rubric` parameter.
+- `hallucination`: Leverages multi-lingual Cross-Encoders and huggingface NLI models (e.g., `deberta-v3-small`) to verify if the model output hallucinates information not present in the reference documents. 
+
+---
+
+## 🔧 Extending the Adapter
+
+By default, the `PythonAdapter` executes any `def func(input_text: str, context: dict) -> AgentResponse:` compatible endpoint. You can integrate `OpenAI SDK`, `LangChainAgentRunners`, or `LangGraph` graphs simply by exposing their invoker to EvalForge! 
+
+1. Ensure your module imports and initializes properly. Address `sys.path` concerns.
+2. Structure your return payload strictly around the `AgentResponse` parameters (e.g. `output`, `steps`, `tool_calls`, `token_usage`).
+
+---
+
+## 🛠 Commands
+
+| Command | Description |
+|---------|-------------|
+| `evalforge --version` | Print the current EvalForge Version |
+| `evalforge init` | Bootstrap a scaffold suite structure |
+| `evalforge run <suite.yaml>` | Run a defined eval suite against connected code |
+| `evalforge run --regenerate` | Instruct the LLM generation persona to create new cases |
+| `evalforge cost [run_id]` | Fetch historical evaluation usage costs |
+
+---
+
+## Licensing
+This project is licensed under the MIT License.

evalforge-0.2.0/PUBLISHING.md

@@ -0,0 +1,113 @@
+# Publishing EvalForge
+
+This guide details the steps to publish EvalForge to both **PyPI** (Python Package Index) and **NPM** (Node Package Manager) to make it accessible to developers across different ecosystems.
+
+---
+
+## 🐍 1. Publishing to PyPI
+
+Since EvalForge is built in Python, PyPI is its native distribution channel. We use `poetry` (or standard `build` and `twine`) for publishing. Since we have a `pyproject.toml`, we can use standard build tools.
+
+### Prerequisites
+1. Ensure you have a registered account on [PyPI](https://pypi.org).
+2. Install build and deployment tools:
+   ```bash
+   pip install build twine
+   ```
+
+### Step-by-step Guide
+
+1. **Update Version**: Ensure the version in `pyproject.toml` and `evalforge/__init__.py` is updated (e.g., `0.2.0`).
+2. **Build the Package**:
+   Run the following command in the root directory (where `pyproject.toml` is located):
+   ```bash
+   python -m build
+   ```
+   This will generate the distribution archives in a `dist/` folder (`.tar.gz` and `.whl` files).
+3. **Check the Build**:
+   Verify the built package doesn't have formatting errors:
+   ```bash
+   twine check dist/*
+   ```
+4. **Publish to PyPI**:
+   Upload the exact build to PyPI:
+   ```bash
+   twine upload dist/*
+   ```
+   *You will be prompted for your username (usually `__token__`) and your PyPI API token.*
+
+Once completed, users can install via:
+```bash
+pip install evalforge
+```
+
+---
+
+## 📦 2. Publishing to NPM
+
+To make EvalForge available to the Node/JS ecosystem (so JS developers can use `npx evalforge`), we publish an NPM wrapper package that invokes the Python CLI. 
+
+### Prerequisites
+1. An account on [npmjs.com](https://npmjs.com).
+2. Node.js and `npm` installed.
+
+### Step-by-step Guide
+
+1. **Create the NPM Wrapper Directory**:
+   Create a folder named `npm` within your repo (or at the root if you prefer to mix Python/Node) and initialize it:
+   ```bash
+   mkdir npm_package && cd npm_package
+   npm init -y
+   ```
+2. **Configure `package.json`**:
+   Edit `package.json` to configure the binary script.
+   ```json
+   {
+     "name": "evalforge",
+     "version": "0.2.0",
+     "description": "EvalForge: The ultimate AI Agent Evaluation Platform",
+     "bin": {
+       "evalforge": "bin/evalforge.js"
+     },
+     "scripts": {},
+     "author": "Your Name",
+     "license": "MIT"
+   }
+   ```
+3. **Create the Wrapper Script**:
+   Create a directory `bin` and a file `bin/evalforge.js`:
+   ```javascript
+   #!/usr/bin/env node
+   const { spawnSync } = require('child_process');
+
+   // Check if Python and the evalforge PyPI package is installed
+   const check = spawnSync('python3', ['-m', 'evalforge', '--version']);
+   
+   if (check.error || check.status !== 0) {
+       console.error("EvalForge requires Python. Please install EvalForge via pip first:");
+       console.error("  pip install evalforge");
+       process.exit(1);
+   }
+
+   // Pass execution to the Python CLI
+   const args = process.argv.slice(2);
+   const result = spawnSync('python3', ['-m', 'evalforge'].concat(args), { stdio: 'inherit' });
+   process.exit(result.status);
+   ```
+   Make the script executable:
+   ```bash
+   chmod +x bin/evalforge.js
+   ```
+
+4. **Publish to NPM**:
+   Log in and publish:
+   ```bash
+   npm login
+   npm publish
+   ```
+
+Once published, frontend and full-stack developers can use EvalForge directly in their pipelines via:
+```bash
+npx evalforge init
+npx evalforge run
+```

evalforge-0.2.0/README.md

@@ -0,0 +1,180 @@
+# 🔨 EvalForge
+
+**The ultimate evaluation platform for LLMs and Autonomous AI Agents.**
+
+EvalForge simplifies the process of testing, evaluating, and iterating on Large Language Models and AI Agents. Whether you're benchmarking a simple LLM prompt block or evaluating a complex, multi-tool Kubernetes Agent using LangChain/OpenAI SDK, EvalForge provides deterministic scoring, conversational edge-case simulation, and detailed cost breakdowns directly in your terminal.
+
+---
+
+## 🚀 Quick Start
+
+EvalForge can be installed easily via pip or npm (using the wrapper).
+
+**Python (PyPI)**
+```bash
+pip install evalforge
+```
+
+**Node / JavaScript (NPM Wrapper)**
+```bash
+npm install -g evalforge
+```
+
+### Initialize a Project
+To scaffold a basic LLM or Agent project, run:
+```bash
+evalforge init
+```
+
+---
+
+## 🧠 Model Evals (Classic LLM Mode)
+
+Test standard Large Language Models for prompt coherence, factual extraction, and instruction adherence.
+
+1. **Define a Suite**: Create an `llm_suite.yaml`:
+   ```yaml
+   name: "Summarization Task"
+   description: "Tests a model's summarization capabilities"
+   type: llm
+   model: gpt-4o-mini
+   cases:
+     - id: "long_text_summary"
+       messages:
+         - role: "user"
+           content: "Summarize this long text: [text ...]"
+       scorers:
+         - type: "llm_judge"
+           threshold: 0.8
+           rubric: "Did the model summarize the text accurately in under 5 sentences?"
+         - type: "contains"
+           value: "summary"
+           threshold: 1.0
+   ```
+
+2. **Run the Suite**:
+   ```bash
+   evalforge run llm_suite.yaml
+   ```
+   You can easily override the model to benchmark across providers (OpenAI, Anthropic, Gemini, Ollama):
+   ```bash
+   evalforge run llm_suite.yaml -m claude-3-haiku-20240307
+   ```
+
+---
+
+## 🤖 Agent Evals (Autonomous Agent Mode)
+
+EvalForge v0.2 introduces native support for Autonomous Agents. It dynamically generates edge-cases from your System Prompt and interfaces with any Python codebase (LangChain, LangGraph, standard OpenAI functions).
+
+### 1. Build an Agent Suite
+Define an agent suite specifying the `system_prompt` and pointing EvalForge to your application code.
+
+```yaml
+name: "Kubernetes Expert Agent"
+description: "Tests a web-browsing agent's ability to document K8s topics."
+version: 1.0.0
+type: agent
+parallel: 1
+model: gpt-4o
+agent:
+  adapter: python
+  config:
+    module: "k8s_bot.main"
+    function: "run_agent"
+system_prompt: |
+  You are a Kubernetes Expert Agent. You have access to a web browsing tool to retrieve documentation.
+  Do not generate documentation yourself; strictly use the web tool to search for information.
+  Never modify or delete infrastructure without explicit user confirmation.
+auto_generate:
+  enabled: true
+  num_cases: 6
+cases: []
+```
+
+### 2. Auto-Generate Test Cases
+EvalForge comes with an internal **LLM Validation Persona**. Using the `auto_generate` config block, EvalForge reads your system prompt and generates robust **Happy Paths**, **Conversational Edge Cases**, and **Boundary Limits**, saving them directly into your `suite.yaml`.
+
+Run evaluation with generation:
+```bash
+evalforge run --regenerate suite.yaml
+```
+
+### 3. Reviewing The Agent Results
+EvalForge will display an elegant UI in the terminal tracking your agent's hallucination score (via NLI), tool invocations, and semantic understanding.
+
+***Example Output Console:***
+```text
+                            Category Breakdown                             
+                                                                           
+  Category                 Passed      Pass Rate     Progress              
+ ───────────────────────────────────────────────────────────────────────── 
+  Create                    0/1            0%        ░░░░░░░░░░░░░░░       
+  Edge                      0/1            0%        ░░░░░░░░░░░░░░░       
+  Infrastructure            0/1            0%        ░░░░░░░░░░░░░░░       
+  Kubernetes                0/1            0%        ░░░░░░░░░░░░░░░       
+  Node                      0/1            0%        ░░░░░░░░░░░░░░░       
+  Pods                      0/1            0%        ░░░░░░░░░░░░░░░       
+                                                                           
+
+❌ Failed Cases (6)
+
+  list_pods_happy_path (gpt-4o)
+    → llm_judge score=0.75: The response provides a relevant command to retrieve the desired information but...
+    → hallucination score=0.29
+    Output: I currently don’t have the capability to directly interact with or retrieve live data from your Kubernetes cluster. Howe…
+
+  check_node_health_happy_path (gpt-4o)
+    → llm_judge score=0.00: The response fails to provide any node names, health conditions, or warnings, ...
+    → hallucination score=0.39
+    Output: I don't have the ability to check the health status of your nodes directly. However, I can guide you on how to do it usi…
+
+╭─ 💰 Cost Breakdown ─────────────────────╮
+│ Eval Cost:     $0.00 per 100k runs (0%) │
+│ Overhead:     $0.00039 (100%)           │
+│ Total:        $0.00039                  │
+╰─────────────────────────────────────────╯
+
+╭───────────────────────────────────────────────────────────────────────╮
+│ ❌ SUITE FAILED  —  0.0% pass rate  •  6 of 6 failed  •  run_d08fd895 │
+╰───────────────────────────────────────────────────────────────────────╯
+```
+
+---
+
+## 🧪 Evaluators & Scorers
+
+EvalForge packages multiple deterministic and probabilistic scorers designed for accuracy tracking:
+
+- `exact` & `contains`: Basic deterministic string matching.
+- `regex`: Extracts pattern matching.
+- `cosine`: Compares embeddings for similarity semantic checks.
+- `bleu` / `rouge`: Machine-translation N-Gram comparisons using NLTK mechanisms.
+- `llm_judge`: Asks an LLM to evaluate the string output based on a strictly provided `rubric` parameter.
+- `hallucination`: Leverages multi-lingual Cross-Encoders and huggingface NLI models (e.g., `deberta-v3-small`) to verify if the model output hallucinates information not present in the reference documents. 
+
+---
+
+## 🔧 Extending the Adapter
+
+By default, the `PythonAdapter` executes any `def func(input_text: str, context: dict) -> AgentResponse:` compatible endpoint. You can integrate `OpenAI SDK`, `LangChainAgentRunners`, or `LangGraph` graphs simply by exposing their invoker to EvalForge! 
+
+1. Ensure your module imports and initializes properly. Address `sys.path` concerns.
+2. Structure your return payload strictly around the `AgentResponse` parameters (e.g. `output`, `steps`, `tool_calls`, `token_usage`).
+
+---
+
+## 🛠 Commands
+
+| Command | Description |
+|---------|-------------|
+| `evalforge --version` | Print the current EvalForge Version |
+| `evalforge init` | Bootstrap a scaffold suite structure |
+| `evalforge run <suite.yaml>` | Run a defined eval suite against connected code |
+| `evalforge run --regenerate` | Instruct the LLM generation persona to create new cases |
+| `evalforge cost [run_id]` | Fetch historical evaluation usage costs |
+
+---
+
+## Licensing
+This project is licensed under the MIT License.

evalforge-0.2.0/evalforge/__init__.py

@@ -0,0 +1,4 @@
+"""evalforge — The pytest of LLMs."""
+
+__version__ = "0.2.0"
+__author__ = "EvalForge Contributors"

evalforge-0.2.0/evalforge/adapters/__init__.py

@@ -0,0 +1,16 @@
+from evalforge.schema import AgentConfig
+from evalforge.adapters.base import BaseAdapter
+
+def get_adapter(config: AgentConfig) -> BaseAdapter:
+    """Factory to get the right agent adapter based on config."""
+    if config.adapter == "python":
+        from evalforge.adapters.python_adapter import PythonAdapter
+        return PythonAdapter(config)
+    elif config.adapter == "http":
+        from evalforge.adapters.http_adapter import HttpAdapter
+        return HttpAdapter(config)
+    elif config.adapter == "command":
+        from evalforge.adapters.command_adapter import CommandAdapter
+        return CommandAdapter(config)
+    else:
+        raise ValueError(f"Unknown adapter type: {config.adapter}")