evalrun 0.4.0__tar.gz → 0.4.2__tar.gz

evalrun-0.4.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shivam Bhardwaj
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{evalrun-0.4.0/evalrun.egg-info → evalrun-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalrun
-Version: 0.4.0
+Version: 0.4.2
 Summary: Provider-agnostic evaluation and regression-testing toolkit for tool-using AI agents.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -10,8 +10,16 @@ Requires-Dist: requests>=2.28.0
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: python-frontmatter>=1.0.0
 Requires-Dist: PyYAML>=6.0
-Requires-Dist: langfuse>=2.0.0
-Requires-Dist: fastmcp>=0.1.0
+Requires-Dist: markdown-it-py>=3.0.0
+Provides-Extra: observability
+Requires-Dist: langfuse>=2.0.0; extra == "observability"
+Provides-Extra: mcp
+Requires-Dist: fastmcp>=0.1.0; extra == "mcp"
+Requires-Dist: mcp<2,>=1.27; extra == "mcp"
+Provides-Extra: all
+Requires-Dist: langfuse>=2.0.0; extra == "all"
+Requires-Dist: fastmcp>=0.1.0; extra == "all"
+Requires-Dist: mcp<2,>=1.27; extra == "all"
 Dynamic: license-file
 
 # evalrun
@@ -43,12 +51,30 @@ The toolkit works with hosted APIs and models running on your own computer. It d
 
 ### 1. Installation
 
+Install the released package from PyPI:
+
+```bash
+python -m pip install evalrun
+```
+
+The package installs the `evalrun` command and all runtime dependencies. You do
+not need to clone this repository to use the toolkit. To contribute or run the
+latest unreleased source instead, clone the repository and use an editable
+install:
+
 ```bash
 git clone https://github.com/imshivamb/agent-eval-platform.git
 cd agent-eval-platform
 pip install -e .
 ```
 
+Optional integrations are deliberately not required for the core CLI and SDK:
+
+```bash
+pip install "evalrun[observability]"  # optional Langfuse tracing
+pip install "evalrun[mcp]"            # optional MCP validation tools
+```
+
 If your shell says `evalrun: command not found`, activate the virtual environment and install the repository first:
 
 ```bash
@@ -66,7 +92,38 @@ evalrun demo
 open results/demo/report.html  # macOS
 ```
 
-### 3. Single Scenario Run (Hosted Model)
+The offline demo is the safest way to confirm that installation works. It makes
+no model requests, needs no API key, and executes only EvalRun's built-in demo
+agent.
+
+### 3. Create and check your own evaluation workspace
+
+Create starter files in a new folder:
+
+```bash
+evalrun init my-evaluation
+cd my-evaluation
+evalrun validate scenario.md
+```
+
+Edit `scenario.md` to describe the user request, hard constraints, expected
+behavior, scoring dimensions, pass criteria, and failure conditions. The
+scenario is ordinary Markdown; you do not need to change EvalRun's source code.
+Use `evalrun validate` before spending money on a model run. It reports the
+exact missing heading or unsupported dimension and returns exit code `2` when
+the file needs fixing.
+
+Run environment diagnostics at any time with:
+
+```bash
+evalrun doctor
+```
+
+`doctor` does not require an API key. Missing keys and unavailable model servers
+are reported as information, while invalid Python or unwritable output paths are
+reported as failures.
+
+### 4. Single Scenario Run (Hosted Model)
 
 ```bash
 export OPENAI_API_KEY="sk-proj-..."
@@ -92,6 +149,21 @@ EvalRun does not lock you to one model provider. The `--base-url` value is the a
 
 The judge normally uses the same endpoint and API key as the target model. You only need `--judge-base-url` or `--judge-api-key` when the judge is hosted somewhere different. The endpoint URL is not a credential; it simply tells EvalRun where to send the request.
 
+For a target and judge at different providers, configure both explicitly:
+
+```bash
+evalrun run \
+  --scenario path/to/scenario.md \
+  --agent my_agent:MyAgent \
+  --model gemini-3.7-flash \
+  --base-url https://generativelanguage.googleapis.com/v1beta/openai/ \
+  --api-key "$GEMINI_API_KEY" \
+  --judge-model stealth/ox-alpha \
+  --judge-base-url https://openrouter.ai/api/v1 \
+  --judge-api-key "$OPENROUTER_API_KEY" \
+  --output results/mixed-provider-run
+```
+
 For example, a Gemini run can be written as:
 
 ```bash
@@ -107,7 +179,7 @@ evalrun run \
   --output results/gemini-run
 ```
 
-### 4. Local Model Server Run (vLLM / Ollama)
+### 5. Local Model Server Run (vLLM / Ollama)
 
 ```bash
 # Users host their own local OpenAI-compatible server at http://localhost:8000/v1
@@ -121,7 +193,7 @@ evalrun run \
   --output results/local-run
 ```
 
-### 5. Guided Local Web UI
+### 6. Guided Local Web UI
 
 Launch the zero-dependency local web interface:
 
@@ -131,6 +203,17 @@ evalrun ui --port 8501
 
 Open `http://127.0.0.1:8501` in your browser to configure endpoints, select scenarios, run evaluations, view pass/fail/block verdicts, and launch interactive HTML reports.
 
+The UI is a local convenience layer. It supports built-in agents and workspace
+scenario paths. API keys are held in browser memory for the current request and
+are not saved by EvalRun. Do not expose this server to the public internet and
+do not paste untrusted Python code into an agent field. Complex or third-party
+agents should be run through the CLI or SDK in the environment where their code
+and dependencies are installed.
+
+The UI intentionally permits only the built-in travel agent. Custom Python,
+HTTP, and CLI agent adapters remain CLI/SDK-only because accepting arbitrary
+agent specifications from a browser would allow local code execution.
+
 ---
 
 ## 🐍 Python SDK Usage
@@ -138,7 +221,7 @@ Open `http://127.0.0.1:8501` in your browser to configure endpoints, select scen
 Integrate `evalrun` programmatically into Python automation pipelines:
 
 ```python
-from framework.sdk import evaluate, compare
+from framework import evaluate, compare
 
 # 1. Execute Benchmark Evaluation
 results = evaluate(
@@ -153,7 +236,7 @@ results = evaluate(
 report = compare(
     candidate_results=results,
     baseline="results/run-001",
-    max_overall_drop=5.0,
+    max_regression=5.0,
 )
 
 if report.release_blocked:
@@ -192,13 +275,26 @@ There is no central service to deploy for the current product. The recommended p
 
 1. Publish the repository on GitHub.
 2. Add tagged releases and a clear quick-start guide.
-3. Users install it locally with `pip install -e .` from a clone.
-4. Later publish the package to PyPI so users can run `pip install evalrun`.
+3. Users install the released package with `pip install evalrun`.
+4. Contributors install the repository with `pip install -e .` when working on source changes.
 5. Host documentation on GitHub Pages if useful; the evaluation engine itself remains local.
 6. Users provide their own hosted-model API keys or run their own local model server.
 
 The local UI is intended for local use, not public internet deployment. A shared hosted deployment would be a separate project requiring authentication, secret management, isolation, and hosted execution.
 
+### TestPyPI (maintainers only)
+
+Before a release, maintainers may validate a candidate package from TestPyPI:
+
+```bash
+python -m pip install \
+  --index-url https://test.pypi.org/simple/ \
+  --extra-index-url https://pypi.org/simple/ \
+  evalrun
+```
+
+End users should use the normal PyPI install shown above.
+
 ### Credential promise
 
 EvalRun does not provide, collect, or store model credentials. A key is read by the local process and passed to the selected model endpoint for that run. Keys are never written to `manifest.json`, `report.html`, or `regression_report.json`; reports contain only `[REDACTED]`. Prefer environment variables such as `OPENAI_API_KEY`, `GEMINI_API_KEY`, or `OPENROUTER_API_KEY`. Avoid putting real keys directly in shell commands because your terminal may save command history.
@@ -250,6 +346,48 @@ evalrun run \
 
 Or place multiple `.md` files in a folder and use `--suite path/to/folder`.
 
+Validate a suite before running it:
+
+```bash
+for scenario in evals/scenarios/my-domain/*.md; do
+  evalrun validate "$scenario" || exit 2
+done
+```
+
+## Connect your own agent
+
+EvalRun does not require agents to use a particular framework. The `--agent`
+value selects one of three adapters:
+
+| Agent form | Example | When to use |
+| --- | --- | --- |
+| Python import | `my_agent:MyAgent` | Your agent is installed/importable in the current environment. |
+| HTTP endpoint | `http://127.0.0.1:8080/predict` | Your agent is running as a local service. |
+| CLI process | `cli:python my_agent.py` | Your agent reads the scenario request from stdin and writes its answer to stdout. |
+
+Python agents receive the target model adapter from EvalRun. HTTP and CLI agents
+must expose the adapter contract described in
+[`docs/architecture.md`](docs/architecture.md). Run `evalrun --help` for the
+complete syntax and examples.
+
+For repeatable runs, put the same values in a JSON or TOML configuration file:
+
+```bash
+evalrun run --config evalrun.json
+```
+
+Command-line flags override values from the configuration file. Keep API keys
+in environment variables and reference them from your shell; never commit them
+to the config file.
+
+## Troubleshooting
+
+- `evalrun: command not found`: activate the virtual environment and run `python -m pip install evalrun` (or `pip install -e .` from a checkout), then run `rehash` in zsh.
+- `Missing required section`: run `evalrun validate path/to/scenario.md` and add the named Markdown heading.
+- Model authentication or endpoint errors: run `evalrun doctor`, confirm the provider's base URL, and check the matching environment variable.
+- A high evaluator score with `BLOCK`: inspect the independent-auditor findings in `report.html`; a quality score is not a release decision.
+- `report.html` is not created after an error: inspect the terminal error and `manifest.json`; failed executions are recorded as failed results rather than silently treated as a passing empty suite.
+
 ## 🏗️ Architecture & Documentation
 
 For detailed system component diagrams, dual-pass auditor sequence flows, and release gate decision trees, see [`docs/architecture.md`](file:///Users/shivam/Projects/AI/agent-eval-platform/docs/architecture.md).

evalrun-0.4.0/PKG-INFO → evalrun-0.4.2/README.md

@@ -1,19 +1,3 @@
-Metadata-Version: 2.4
-Name: evalrun
-Version: 0.4.0
-Summary: Provider-agnostic evaluation and regression-testing toolkit for tool-using AI agents.
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: openai>=1.0.0
-Requires-Dist: requests>=2.28.0
-Requires-Dist: pydantic>=2.0.0
-Requires-Dist: python-frontmatter>=1.0.0
-Requires-Dist: PyYAML>=6.0
-Requires-Dist: langfuse>=2.0.0
-Requires-Dist: fastmcp>=0.1.0
-Dynamic: license-file
-
 # evalrun
 
 > **Quality Score $\neq$ Release Decision.** An LLM evaluator can rate an itinerary 95/100 while an independent auditor blocks it for hard financial violations.
@@ -43,12 +27,30 @@ The toolkit works with hosted APIs and models running on your own computer. It d
 
 ### 1. Installation
 
+Install the released package from PyPI:
+
+```bash
+python -m pip install evalrun
+```
+
+The package installs the `evalrun` command and all runtime dependencies. You do
+not need to clone this repository to use the toolkit. To contribute or run the
+latest unreleased source instead, clone the repository and use an editable
+install:
+
 ```bash
 git clone https://github.com/imshivamb/agent-eval-platform.git
 cd agent-eval-platform
 pip install -e .
 ```
 
+Optional integrations are deliberately not required for the core CLI and SDK:
+
+```bash
+pip install "evalrun[observability]"  # optional Langfuse tracing
+pip install "evalrun[mcp]"            # optional MCP validation tools
+```
+
 If your shell says `evalrun: command not found`, activate the virtual environment and install the repository first:
 
 ```bash
@@ -66,7 +68,38 @@ evalrun demo
 open results/demo/report.html  # macOS
 ```
 
-### 3. Single Scenario Run (Hosted Model)
+The offline demo is the safest way to confirm that installation works. It makes
+no model requests, needs no API key, and executes only EvalRun's built-in demo
+agent.
+
+### 3. Create and check your own evaluation workspace
+
+Create starter files in a new folder:
+
+```bash
+evalrun init my-evaluation
+cd my-evaluation
+evalrun validate scenario.md
+```
+
+Edit `scenario.md` to describe the user request, hard constraints, expected
+behavior, scoring dimensions, pass criteria, and failure conditions. The
+scenario is ordinary Markdown; you do not need to change EvalRun's source code.
+Use `evalrun validate` before spending money on a model run. It reports the
+exact missing heading or unsupported dimension and returns exit code `2` when
+the file needs fixing.
+
+Run environment diagnostics at any time with:
+
+```bash
+evalrun doctor
+```
+
+`doctor` does not require an API key. Missing keys and unavailable model servers
+are reported as information, while invalid Python or unwritable output paths are
+reported as failures.
+
+### 4. Single Scenario Run (Hosted Model)
 
 ```bash
 export OPENAI_API_KEY="sk-proj-..."
@@ -92,6 +125,21 @@ EvalRun does not lock you to one model provider. The `--base-url` value is the a
 
 The judge normally uses the same endpoint and API key as the target model. You only need `--judge-base-url` or `--judge-api-key` when the judge is hosted somewhere different. The endpoint URL is not a credential; it simply tells EvalRun where to send the request.
 
+For a target and judge at different providers, configure both explicitly:
+
+```bash
+evalrun run \
+  --scenario path/to/scenario.md \
+  --agent my_agent:MyAgent \
+  --model gemini-3.7-flash \
+  --base-url https://generativelanguage.googleapis.com/v1beta/openai/ \
+  --api-key "$GEMINI_API_KEY" \
+  --judge-model stealth/ox-alpha \
+  --judge-base-url https://openrouter.ai/api/v1 \
+  --judge-api-key "$OPENROUTER_API_KEY" \
+  --output results/mixed-provider-run
+```
+
 For example, a Gemini run can be written as:
 
 ```bash
@@ -107,7 +155,7 @@ evalrun run \
   --output results/gemini-run
 ```
 
-### 4. Local Model Server Run (vLLM / Ollama)
+### 5. Local Model Server Run (vLLM / Ollama)
 
 ```bash
 # Users host their own local OpenAI-compatible server at http://localhost:8000/v1
@@ -121,7 +169,7 @@ evalrun run \
   --output results/local-run
 ```
 
-### 5. Guided Local Web UI
+### 6. Guided Local Web UI
 
 Launch the zero-dependency local web interface:
 
@@ -131,6 +179,17 @@ evalrun ui --port 8501
 
 Open `http://127.0.0.1:8501` in your browser to configure endpoints, select scenarios, run evaluations, view pass/fail/block verdicts, and launch interactive HTML reports.
 
+The UI is a local convenience layer. It supports built-in agents and workspace
+scenario paths. API keys are held in browser memory for the current request and
+are not saved by EvalRun. Do not expose this server to the public internet and
+do not paste untrusted Python code into an agent field. Complex or third-party
+agents should be run through the CLI or SDK in the environment where their code
+and dependencies are installed.
+
+The UI intentionally permits only the built-in travel agent. Custom Python,
+HTTP, and CLI agent adapters remain CLI/SDK-only because accepting arbitrary
+agent specifications from a browser would allow local code execution.
+
 ---
 
 ## 🐍 Python SDK Usage
@@ -138,7 +197,7 @@ Open `http://127.0.0.1:8501` in your browser to configure endpoints, select scen
 Integrate `evalrun` programmatically into Python automation pipelines:
 
 ```python
-from framework.sdk import evaluate, compare
+from framework import evaluate, compare
 
 # 1. Execute Benchmark Evaluation
 results = evaluate(
@@ -153,7 +212,7 @@ results = evaluate(
 report = compare(
     candidate_results=results,
     baseline="results/run-001",
-    max_overall_drop=5.0,
+    max_regression=5.0,
 )
 
 if report.release_blocked:
@@ -192,13 +251,26 @@ There is no central service to deploy for the current product. The recommended p
 
 1. Publish the repository on GitHub.
 2. Add tagged releases and a clear quick-start guide.
-3. Users install it locally with `pip install -e .` from a clone.
-4. Later publish the package to PyPI so users can run `pip install evalrun`.
+3. Users install the released package with `pip install evalrun`.
+4. Contributors install the repository with `pip install -e .` when working on source changes.
 5. Host documentation on GitHub Pages if useful; the evaluation engine itself remains local.
 6. Users provide their own hosted-model API keys or run their own local model server.
 
 The local UI is intended for local use, not public internet deployment. A shared hosted deployment would be a separate project requiring authentication, secret management, isolation, and hosted execution.
 
+### TestPyPI (maintainers only)
+
+Before a release, maintainers may validate a candidate package from TestPyPI:
+
+```bash
+python -m pip install \
+  --index-url https://test.pypi.org/simple/ \
+  --extra-index-url https://pypi.org/simple/ \
+  evalrun
+```
+
+End users should use the normal PyPI install shown above.
+
 ### Credential promise
 
 EvalRun does not provide, collect, or store model credentials. A key is read by the local process and passed to the selected model endpoint for that run. Keys are never written to `manifest.json`, `report.html`, or `regression_report.json`; reports contain only `[REDACTED]`. Prefer environment variables such as `OPENAI_API_KEY`, `GEMINI_API_KEY`, or `OPENROUTER_API_KEY`. Avoid putting real keys directly in shell commands because your terminal may save command history.
@@ -250,6 +322,48 @@ evalrun run \
 
 Or place multiple `.md` files in a folder and use `--suite path/to/folder`.
 
+Validate a suite before running it:
+
+```bash
+for scenario in evals/scenarios/my-domain/*.md; do
+  evalrun validate "$scenario" || exit 2
+done
+```
+
+## Connect your own agent
+
+EvalRun does not require agents to use a particular framework. The `--agent`
+value selects one of three adapters:
+
+| Agent form | Example | When to use |
+| --- | --- | --- |
+| Python import | `my_agent:MyAgent` | Your agent is installed/importable in the current environment. |
+| HTTP endpoint | `http://127.0.0.1:8080/predict` | Your agent is running as a local service. |
+| CLI process | `cli:python my_agent.py` | Your agent reads the scenario request from stdin and writes its answer to stdout. |
+
+Python agents receive the target model adapter from EvalRun. HTTP and CLI agents
+must expose the adapter contract described in
+[`docs/architecture.md`](docs/architecture.md). Run `evalrun --help` for the
+complete syntax and examples.
+
+For repeatable runs, put the same values in a JSON or TOML configuration file:
+
+```bash
+evalrun run --config evalrun.json
+```
+
+Command-line flags override values from the configuration file. Keep API keys
+in environment variables and reference them from your shell; never commit them
+to the config file.
+
+## Troubleshooting
+
+- `evalrun: command not found`: activate the virtual environment and run `python -m pip install evalrun` (or `pip install -e .` from a checkout), then run `rehash` in zsh.
+- `Missing required section`: run `evalrun validate path/to/scenario.md` and add the named Markdown heading.
+- Model authentication or endpoint errors: run `evalrun doctor`, confirm the provider's base URL, and check the matching environment variable.
+- A high evaluator score with `BLOCK`: inspect the independent-auditor findings in `report.html`; a quality score is not a release decision.
+- `report.html` is not created after an error: inspect the terminal error and `manifest.json`; failed executions are recorded as failed results rather than silently treated as a passing empty suite.
+
 ## 🏗️ Architecture & Documentation
 
 For detailed system component diagrams, dual-pass auditor sequence flows, and release gate decision trees, see [`docs/architecture.md`](file:///Users/shivam/Projects/AI/agent-eval-platform/docs/architecture.md).

{evalrun-0.4.0 → evalrun-0.4.2}/agents/reflection/agent.py

@@ -1,10 +1,10 @@
 """Reflection agent implementation for auditing and critiquing planned itineraries."""
 
 from typing import Optional
-from langfuse import observe
 from framework.llms import BaseLLM, Message
 from framework.models import AgentOutput
 from framework.memory import BaseSessionMemory
+from framework.observability import observe
 from agents.base import BaseAgent
 from .prompts import REFLECTION_SYSTEM_PROMPT
 

{evalrun-0.4.0 → evalrun-0.4.2}/agents/research/agent.py

@@ -1,7 +1,6 @@
-from langfuse import observe
-
 from framework.llms import BaseLLM, Message
 from framework.models import AgentOutput
+from framework.observability import observe
 from agents.base import BaseAgent
 from .prompts import RESEARCH_SYSTEM_PROMPT
 

{evalrun-0.4.0 → evalrun-0.4.2}/agents/research/planner.py

@@ -2,8 +2,7 @@
 
 from framework.llms import BaseLLM, Message
 from framework.utils import parse_json_markdown
-
-from langfuse import observe
+from framework.observability import observe
 
 
 class ResearchPlanner:

{evalrun-0.4.0 → evalrun-0.4.2}/agents/support/triage_agent.py

@@ -1,10 +1,9 @@
 """Customer-support triage agent used by the second-domain benchmark."""
 
-from langfuse import observe
-
 from agents.base import BaseAgent
 from framework.llms import BaseLLM, Message
 from framework.models import AgentOutput
+from framework.observability import observe
 
 
 SUPPORT_TRIAGE_SYSTEM_PROMPT = """You are a production customer-support triage agent.

{evalrun-0.4.0 → evalrun-0.4.2}/agents/travel/agent.py

@@ -1,5 +1,3 @@
-from langfuse import observe
-
 import asyncio
 import json
 from typing import Any, Dict, Optional
@@ -7,6 +5,7 @@ from framework.llms import BaseLLM, Message
 from framework.models import AgentOutput
 from framework.utils import parse_json_markdown
 from framework.memory import BaseSessionMemory
+from framework.observability import observe
 from agents.base import BaseAgent
 from agents.research.planner import ResearchPlanner
 from framework.mcp.revision_summary import (

evalrun-0.4.2/cli/__init__.py

@@ -0,0 +1,6 @@
+"""evalrun CLI package exports."""
+
+from cli.main import create_parser
+from cli.resolver import resolve_agent
+
+__all__ = ["create_parser", "resolve_agent"]