evalforge 0.1.0__tar.gz

evalforge-0.1.0/.gitignore

@@ -0,0 +1,42 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug
+target
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+# Generated by cargo mutants
+# Contains mutation testing data
+**/mutants.out*/
+
+# RustRover
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.env
+*.egg
+
+# macOS
+.DS_Store
+
+# IDE
+.vscode/
+.idea/
+*.swp
+
+# EvalForge specific
+*.trace.json.bak

evalforge-0.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: evalforge
+Version: 0.1.0
+Summary: Framework-agnostic LLM agent evaluation harness
+Project-URL: Homepage, https://github.com/YOUR_USERNAME/evalforge
+License: MIT
+Requires-Python: >=3.11
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# EvalForge Python SDK
+
+pip install evalforge
+
+## Quick Start
+import evalforge
+result = evalforge.run("trace.json", metrics=["faithfulness"])
+print(result.passed)
+print(result.metrics[0].score)

evalforge-0.1.0/README.md

@@ -0,0 +1,9 @@
+# EvalForge Python SDK
+
+pip install evalforge
+
+## Quick Start
+import evalforge
+result = evalforge.run("trace.json", metrics=["faithfulness"])
+print(result.passed)
+print(result.metrics[0].score)

evalforge-0.1.0/evalforge/__init__.py

@@ -0,0 +1,3 @@
+from .client import run, EvalResult, MetricResult
+
+__all__ = ["run", "EvalResult", "MetricResult"]

evalforge-0.1.0/evalforge/client.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class MetricResult:
+    metric: str
+    score: float
+    passed: bool
+    reason: str
+
+
+@dataclass
+class EvalResult:
+    trace_id: str
+    framework: str
+    metrics: list[MetricResult] = field(default_factory=list)
+    passed: bool = False
+
+
+def _find_binary() -> str:
+    # 1. Explicit override via environment variable
+    env_bin = os.environ.get("EVALFORGE_BIN")
+    if env_bin and Path(env_bin).is_file():
+        return env_bin
+
+    # 2. Walk up from client.py looking for target/debug/evalforge (up to 5 levels)
+    current = Path(__file__).resolve().parent
+    for _ in range(5):
+        candidate = current / "target" / "debug" / "evalforge"
+        if candidate.exists():
+            return str(candidate)
+        current = current.parent
+
+    # 3. System PATH
+    import shutil
+    path_bin = shutil.which("evalforge")
+    if path_bin:
+        return path_bin
+
+    raise RuntimeError(
+        f"EvalForge binary not found. "
+        f"Searched up to 5 parent dirs from {Path(__file__).resolve()}. "
+        f"Set EVALFORGE_BIN env var to the binary path, or run 'cargo build' first."
+    )
+
+
+def run(
+    trace: str,
+    metrics: list[str],
+    threshold: float = 0.7,
+    mock: bool = False,
+    api_key: str | None = None,
+) -> EvalResult:
+    binary = _find_binary()
+
+    cmd = [
+        binary,
+        "run",
+        "--trace", trace,
+        "--metrics", ",".join(metrics),
+        "--threshold", str(threshold),
+    ]
+    if mock:
+        cmd.append("--mock")
+
+    env = os.environ.copy()
+    if api_key is not None:
+        env["ANTHROPIC_API_KEY"] = api_key
+
+    proc = subprocess.run(cmd, capture_output=True, text=True, env=env)
+
+    if proc.returncode not in (0, 1):
+        raise RuntimeError(
+            f"evalforge exited with code {proc.returncode}.\n"
+            f"stdout: {proc.stdout}\nstderr: {proc.stderr}"
+        )
+
+    output = proc.stdout
+
+    # Parse trace summary fields
+    trace_id = _extract_field(output, "Trace ID")
+    framework = _extract_field(output, "Framework")
+
+    # Parse scoring result lines and the Reason: line that follows each.
+    # e.g.:
+    #   faithfulness     0.91   PASS
+    #   Reason: Mock score — skipping live API call
+    metric_results: list[MetricResult] = []
+    pattern = re.compile(
+        r"^(\w+)\s+([\d.]+)\s+(PASS|FAIL)\s*\nReason:\s*(.+)$",
+        re.MULTILINE,
+    )
+    for m in pattern.finditer(output):
+        metric_results.append(
+            MetricResult(
+                metric=m.group(1),
+                score=float(m.group(2)),
+                passed=m.group(3) == "PASS",
+                reason=m.group(4).strip(),
+            )
+        )
+
+    overall_passed = proc.returncode == 0 and all(r.passed for r in metric_results)
+
+    return EvalResult(
+        trace_id=trace_id,
+        framework=framework,
+        metrics=metric_results,
+        passed=overall_passed,
+    )
+
+
+def _extract_field(output: str, label: str) -> str:
+    pattern = re.compile(rf"^{re.escape(label)}:\s+(.+)$", re.MULTILINE)
+    m = pattern.search(output)
+    return m.group(1).strip() if m else ""

evalforge-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "evalforge"
+version = "0.1.0"
+description = "Framework-agnostic LLM agent evaluation harness"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/YOUR_USERNAME/evalforge"
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["evalforge"]

evalforge-0.1.0/tests/test_client.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+import evalforge
+from evalforge.client import EvalResult, MetricResult
+
+
+# Resolve fixture paths relative to this file so tests work regardless of
+# which directory pytest is invoked from.
+_WORKSPACE_ROOT = Path(__file__).parent.parent.parent.parent
+SAMPLE_TRACE = str(_WORKSPACE_ROOT / "tests/fixtures/sample_trace.json")
+SIMPLE_TRACE = str(_WORKSPACE_ROOT / "tests/fixtures/simple_trace.json")
+
+
+def test_mock_run():
+    result = evalforge.run(SAMPLE_TRACE, metrics=["faithfulness"], mock=True)
+    assert result.passed is True
+    assert len(result.metrics) == 1
+    assert result.metrics[0].score == 0.91
+
+
+def test_simple_trace_mock():
+    result = evalforge.run(SIMPLE_TRACE, metrics=["faithfulness"], mock=True)
+    assert result.passed is True
+    assert len(result.metrics) == 1
+    assert result.metrics[0].score == 0.91
+
+
+def test_threshold_boundary():
+    mr = MetricResult(metric="faithfulness", score=0.7, passed=True, reason="at threshold")
+    assert mr.passed is True
+
+
+def test_metric_result_fields():
+    mr = MetricResult(metric="faithfulness", score=0.85, passed=True, reason="looks good")
+    assert mr.metric == "faithfulness"
+    assert mr.score == 0.85
+    assert mr.passed is True
+    assert mr.reason == "looks good"
+
+    er = EvalResult(
+        trace_id="trace-001",
+        framework="langchain",
+        metrics=[mr],
+        passed=True,
+    )
+    assert er.trace_id == "trace-001"
+    assert er.framework == "langchain"
+    assert er.passed is True
+    assert er.metrics[0].metric == "faithfulness"

evalforge-0.1.0/uv.lock

@@ -0,0 +1,78 @@
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
+[[package]]
+name = "evalforge"
+version = "0.1.0"
+source = { editable = "." }
+
+[package.optional-dependencies]
+dev = [
+    { name = "pytest" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" }]
+provides-extras = ["dev"]
+
+[[package]]
+name = "iniconfig"
+version = "2.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
+]
+
+[[package]]
+name = "packaging"
+version = "26.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
+]
+
+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.20.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
+]
+
+[[package]]
+name = "pytest"
+version = "9.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "iniconfig" },
+    { name = "packaging" },
+    { name = "pluggy" },
+    { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" },
+]