agentevals-cli 0.9.0__tar.gz → 0.9.2__tar.gz

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/.github/workflows/ci.yml

@@ -22,6 +22,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
 
       - uses: astral-sh/setup-uv@v7
         with:
@@ -46,6 +48,8 @@ jobs:
         python-version: ["3.11", "3.12", "3.13"]
     steps:
       - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
 
       - uses: astral-sh/setup-uv@v7
         with:

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/.github/workflows/publish-evaluator-sdk.yml

@@ -18,7 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@main
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
 
       - name: Install uv
         uses: astral-sh/setup-uv@v6

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/.github/workflows/release.yml

@@ -20,6 +20,7 @@ jobs:
       - uses: actions/checkout@v6
         with:
           ref: ${{ github.event.inputs.tag || github.ref_name }}
+          fetch-depth: 0
 
       - uses: astral-sh/setup-uv@v7
         with:
@@ -31,9 +32,6 @@ jobs:
           cache: npm
           cache-dependency-path: ui/package-lock.json
 
-      - name: Set version from tag
-        run: uv version "${{ github.event.inputs.tag || github.ref_name }}" --package agentevals-cli
-
       - name: Build core and bundled wheels
         run: make release
 
@@ -51,6 +49,7 @@ jobs:
       - uses: actions/checkout@v6
         with:
           ref: ${{ github.event.inputs.tag || github.ref_name }}
+          fetch-depth: 0
 
       - uses: astral-sh/setup-uv@v7
         with:
@@ -65,11 +64,8 @@ jobs:
       # Same bundle as `make release` / `build-bundle`: wheel must include ui/dist in src/agentevals/_static
       # (see [tool.hatch.build] artifacts in pyproject.toml).
       - name: Release Python package (wheel + sdist with bundled UI)
-        env:
-          VERSION: ${{ github.event.inputs.tag || github.ref_name }}
         run: |
           uv sync --package agentevals-cli --all-extras
-          uv version "$VERSION" --package agentevals-cli
 
           make build-ui
           rm -rf src/agentevals/_static

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/Makefile

@@ -1,4 +1,4 @@
-VERSION := $(shell grep '^version' pyproject.toml | cut -d'"' -f2)
+VERSION := $(shell uv run --with hatch hatch version 2>/dev/null)
 WHEEL := dist/agentevals_cli-$(VERSION)-py3-none-any.whl
 
 DOCKER_REGISTRY ?= soloio

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentevals-cli
-Version: 0.9.0
+Version: 0.9.2
 Summary: Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces
 License-File: LICENSE
 Requires-Python: >=3.11

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/docs/custom-evaluators.md

@@ -317,6 +317,26 @@ The `grader.evaluation_metric` field selects the similarity algorithm:
 | `rouge_1` through `rouge_5` | Unigram through 5-gram overlap (F-measure) |
 | `rouge_l` | Longest common subsequence overlap (F-measure) |
 
+### Label Model Grader
+
+Scores responses without a golden set. The model reads each response and assigns a label from a fixed list. Passing labels are defined in the config.
+
+```yaml
+evaluators:
+  - name: quality_check
+    type: openai_eval
+    grader:
+      type: label_model
+      model: gpt-4o-mini
+      input:
+        - role: user
+          content: "Rate this response: {{ item.actual_response }}"
+      labels: [good, bad]
+      passing_labels: [good]
+```
+
+The `threshold` field is not used for `label_model`. A response passes if its assigned label is in `passing_labels`.
+
 ### How it works
 
 Under the hood, agentevals creates an ephemeral eval on OpenAI, submits the actual and expected responses as JSONL items, polls for results, and cleans up. The agent's response and the golden reference are both placed in the `item` namespace (with `include_sample_schema: false`), so OpenAI only grades the provided text without generating any model outputs.

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/examples/custom_evaluators/eval_config.yaml

@@ -32,3 +32,4 @@ evaluators:
     ref: evaluators/random_evaluator/random_evaluator.py
     threshold: 0.110
     executor: local
+

agentevals_cli-0.9.2/examples/custom_evaluators/eval_config_openai_eval.yaml

@@ -0,0 +1,18 @@
+# Eval config using OpenAI Evals API graders.
+# Requires OPENAI_API_KEY to be set.
+#
+# Run with:
+#   agentevals run samples/helm.json \
+#     --config examples/custom_evaluators/eval_config_openai_eval.yaml
+
+evaluators:
+  - name: quality_check
+    type: openai_eval
+    grader:
+      type: label_model
+      model: gpt-4o-mini
+      input:
+        - role: user
+          content: "Rate this response: {{ item.actual_response }}"
+      labels: [good, bad]
+      passing_labels: [good]

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [project]
 name = "agentevals-cli"
-version = "0.9.0"
+dynamic = ["version"]
 description = "Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -40,6 +40,9 @@ postgres = [
 [project.scripts]
 agentevals = "agentevals.cli:main"
 
+[tool.hatch.version]
+source = "vcs"
+
 [tool.hatch.build]
 artifacts = ["src/agentevals/_static/**"]
 

{agentevals_cli-0.9.0 → agentevals_cli-0.9.2}/src/agentevals/__init__.py

@@ -3,7 +3,7 @@
 from importlib.metadata import PackageNotFoundError, version
 
 try:
-    __version__ = version("agentevals")
+    __version__ = version("agentevals-cli")
 except PackageNotFoundError:
     __version__ = "0.0.0-dev"