agentevals-cli 0.9.8__tar.gz → 0.9.10__tar.gz

agentevals_cli-0.9.10/.github/dependabot.yml

@@ -0,0 +1,101 @@
+version: 2
+
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 2
+    labels: ["dependencies", "CI"]
+    commit-message:
+      prefix: "chore(deps):"
+    cooldown:
+      default-days: 5
+    groups:
+      github-actions:
+        applies-to: version-updates
+        patterns: ["*"]
+      github-actions-security:
+        applies-to: security-updates
+        patterns: ["*"]
+
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 3
+    labels: ["dependencies", "CLI"]
+    commit-message:
+      prefix: "chore(deps):"
+      prefix-development: "chore(deps-dev):"
+    cooldown:
+      default-days: 5
+      semver-major-days: 14
+    groups:
+      python-google-ai:
+        applies-to: version-updates
+        patterns:
+          - "google-adk"
+          - "google-genai"
+          - "google-auth*"
+      python:
+        applies-to: version-updates
+        patterns: ["*"]
+      python-security:
+        applies-to: security-updates
+        patterns: ["*"]
+
+  - package-ecosystem: "npm"
+    directory: "/ui"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 2
+    labels: ["dependencies", "UI"]
+    commit-message:
+      prefix: "chore(deps):"
+      prefix-development: "chore(deps-dev):"
+    cooldown:
+      default-days: 5
+      semver-major-days: 14
+    ignore:
+      # Majors need a migration pass; bump these by hand.
+      - dependency-name: "react"
+        update-types: ["version-update:semver-major"]
+      - dependency-name: "react-dom"
+        update-types: ["version-update:semver-major"]
+      - dependency-name: "antd"
+        update-types: ["version-update:semver-major"]
+      - dependency-name: "vite"
+        update-types: ["version-update:semver-major"]
+      # Plugin majors track vite majors: @vitejs/plugin-react 6 peers on vite 8.
+      - dependency-name: "@vitejs/*"
+        update-types: ["version-update:semver-major"]
+      # typescript-eslint declares a typescript upper bound, so TS majors move with it.
+      - dependency-name: "typescript"
+        update-types: ["version-update:semver-major"]
+    groups:
+      npm:
+        applies-to: version-updates
+        patterns: ["*"]
+      npm-security:
+        applies-to: security-updates
+        patterns: ["*"]
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    open-pull-requests-limit: 1
+    labels: ["dependencies"]
+    commit-message:
+      prefix: "chore(deps):"
+    cooldown:
+      default-days: 5
+    groups:
+      docker-base-images:
+        applies-to: version-updates
+        patterns: ["*"]

agentevals_cli-0.9.10/.github/workflows/ci.yml

@@ -0,0 +1,139 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  helm-chart:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Lint + template
+        run: make helm-test
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+        with:
+          enable-cache: true
+
+      - name: Verify uv.lock matches pyproject.toml
+        run: uv lock --check
+
+      - name: Install dependencies
+        run: |
+          uv venv
+          uv pip install setuptools
+          uv sync --dev
+
+      - name: Ruff check
+        run: uv run ruff check .
+
+      - name: Ruff format check
+        run: uv run ruff format --check .
+
+  ui:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22'
+          cache: npm
+          cache-dependency-path: ui/package-lock.json
+
+      # tsc -b && vite build, same as the release workflow bundles.
+      - name: Build UI
+        run: make build-ui
+
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      IMAGE_REF: agentevals:ci
+      IMAGE_VERSION: 0.0.0.dev0
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+
+      # The release workflow is the only other place this Dockerfile gets built, and it
+      # runs after the GitHub release is already published. Build it here so a break
+      # lands on the PR instead of halfway through a release.
+      # Single-arch keeps it native; the multi-arch build stays release-only because
+      # emulating arm64 through the node and uv layers is an order of magnitude slower.
+      - name: Build image
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          platforms: linux/amd64
+          build-args: VERSION=${{ env.IMAGE_VERSION }}
+          tags: ${{ env.IMAGE_REF }}
+          push: false
+          load: true
+          cache-from: type=gha
+          # Pull requests read main's layers but never write, so a branch cannot grow
+          # or displace the cache the other jobs share.
+          cache-to: ${{ github.event_name == 'push' && 'type=gha,mode=max' || '' }}
+
+      - name: Smoke test
+        run: |
+          docker run -d --name agentevals-ci -p 127.0.0.1:8001:8001 "$IMAGE_REF"
+          for _ in $(seq 1 30); do
+            if curl -fsS --max-time 3 http://127.0.0.1:8001/api/health -o "$RUNNER_TEMP/health.json"; then
+              break
+            fi
+            sleep 2
+          done
+          test -s "$RUNNER_TEMP/health.json" || { echo "server never became healthy"; exit 1; }
+          cat "$RUNNER_TEMP/health.json"
+          grep -q '"status":"ok"' "$RUNNER_TEMP/health.json"
+          # Proves the SETUPTOOLS_SCM_PRETEND_VERSION path still resolves; a silent
+          # break there would ship a mis-versioned image.
+          grep -q "\"version\":\"$IMAGE_VERSION\"" "$RUNNER_TEMP/health.json"
+          curl -fsS http://127.0.0.1:8001/ | grep -q '<div id="root">'
+          test "$(docker exec agentevals-ci id -u)" = "1000"
+
+      - name: Container logs
+        if: failure()
+        run: docker logs agentevals-ci
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+        with:
+          enable-cache: true
+
+      - name: Install Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          uv venv --python ${{ matrix.python-version }}
+          uv pip install setuptools
+          uv sync --dev --all-extras --frozen --python ${{ matrix.python-version }}
+
+      - name: Run tests
+        run: uv run pytest -m "not integration and not e2e" --tb=short -q

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/.github/workflows/publish-evaluator-sdk.yml

@@ -18,12 +18,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v6
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
 
       # Repo root cwd: uv build puts artifacts in ./dist; uv publish looks for dist/* relative to cwd.
       - name: 'Release Python Packages'

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/.github/workflows/release.yml

@@ -17,16 +17,16 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.event.inputs.tag || github.ref_name }}
           fetch-depth: 0
 
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
           enable-cache: true
 
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: '22'
           cache: npm
@@ -35,7 +35,7 @@ jobs:
       - name: Build core and bundled wheels
         run: make release
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: wheels
           path: |
@@ -46,16 +46,16 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.event.inputs.tag || github.ref_name }}
           fetch-depth: 0
 
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
           enable-cache: true
 
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: '22'
           cache: npm
@@ -82,12 +82,12 @@ jobs:
       contents: write
 
     steps:
-      - uses: actions/download-artifact@v8
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: wheels
           path: dist/
 
-      - uses: softprops/action-gh-release@v2.5.0
+      - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2
         with:
           tag_name: ${{ github.event.inputs.tag || github.ref_name }}
           files: dist/**/*.whl
@@ -100,22 +100,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.event.inputs.tag || github.ref_name }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v4
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v4
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
 
       - name: Build and push
         run: |
@@ -134,12 +134,12 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.event.inputs.tag || github.ref_name }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v4
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM node:25-bookworm-slim AS ui
+FROM node:26-bookworm-slim AS ui
 WORKDIR /build/ui
 COPY ui/package.json ui/package-lock.json ./
 # Skip lifecycle scripts during ci, then rebuild esbuild in its own layer — avoids ETXTBSY when

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/Makefile

@@ -9,13 +9,17 @@ DOCKER_IMAGE_REF := $(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY:%/=%)/$(DOCKER_IMA
 # Multi-arch build (requires docker buildx). Manifest lists must be pushed — use build-docker-local for a single-arch --load.
 PLATFORMS ?= linux/amd64,linux/arm64
 
+# LOCAL_VERSION falls back to a stub so build-docker-local works without uv or git history.
+LOCAL_VERSION ?= $(if $(VERSION),$(VERSION),0.0.0.dev0)
+LOCAL_TAG ?= local
+
 HELM_REPO ?= oci://ghcr.io/agentevals-dev/agentevals
 HELM_DIST_FOLDER ?= dist/helm
 HELM_CHART_DIR ?= charts/agentevals
 HELM_CHART_OCI_URL ?= $(HELM_REPO)/helm
 HELM_CHART_VERSION ?= $(VERSION)
 
-.PHONY: build build-bundle build-docker build-ui release clean dev-backend dev-backend-pg dev-frontend dev-bundle pg-up pg-down migrate test test-unit test-integration test-e2e helm-lint helm-template helm-test helm-cleanup helm-package helm-publish
+.PHONY: build build-bundle build-docker build-docker-local build-ui release clean dev-backend dev-backend-pg dev-frontend dev-bundle pg-up pg-down migrate test test-unit test-integration test-e2e helm-lint helm-template helm-test helm-cleanup helm-package helm-publish
 
 PG_CONTAINER ?= agentevals-pg
 PG_PORT      ?= 5432
@@ -31,6 +35,9 @@ build-docker:
 	@test -n "$(VERSION)" || { echo "ERROR: VERSION is empty. Pass VERSION=x.y.z explicitly, or install uv so hatch-vcs can resolve it."; exit 1; }
 	docker buildx build --platform $(PLATFORMS) --build-arg VERSION=$(VERSION) -t $(DOCKER_IMAGE_REF):$(DOCKER_TAG) --push .
 
+build-docker-local:
+	docker buildx build --build-arg VERSION=$(LOCAL_VERSION) -t $(DOCKER_IMAGE_REF):$(LOCAL_TAG) --load .
+
 build-ui:
 	cd ui && npm ci && npm run build
 

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/PKG-INFO

@@ -1,20 +1,21 @@
 Metadata-Version: 2.5
 Name: agentevals-cli
-Version: 0.9.8
+Version: 0.9.10
 Summary: Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces
 License-File: LICENSE
 Requires-Python: >=3.11
 Requires-Dist: aiohttp>=3.14.3
 Requires-Dist: authlib>=1.7.2
-Requires-Dist: click>=8.0
+Requires-Dist: click>=8.3.3
 Requires-Dist: fastapi>=0.115.0
-Requires-Dist: google-adk[eval]<2.2,>=2.1.0
+Requires-Dist: google-adk[eval]<3,>=2.5.0
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: idna>=3.18
 Requires-Dist: litellm>=1.85.7
 Requires-Dist: nltk>=3.10.2
 Requires-Dist: opentelemetry-proto>=1.36.0
 Requires-Dist: pyasn1>=0.6.4
+Requires-Dist: pygments>=2.20.0
 Requires-Dist: python-dotenv>=1.2.2
 Requires-Dist: python-multipart>=0.0.32
 Requires-Dist: pyyaml>=6.0

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/docs/custom-evaluators.md

@@ -37,6 +37,7 @@ pip install agentevals-evaluator-sdk
 # evaluators/response_quality.py
 from agentevals_evaluator_sdk import evaluator, EvalInput, EvalResult
 
+
 @evaluator
 def response_quality(input: EvalInput) -> EvalResult:
     scores = []
@@ -53,6 +54,7 @@ def response_quality(input: EvalInput) -> EvalResult:
         per_invocation_scores=scores,
     )
 
+
 if __name__ == "__main__":
     response_quality.run()
 ```
@@ -436,7 +438,7 @@ Then register it:
 _RUNTIMES: list[Runtime] = [
     PythonRuntime(),
     NodeRuntime(),
-    GoRuntime(),       # new
+    GoRuntime(),  # new
 ]
 ```
 
@@ -484,6 +486,7 @@ To support a different evaluator registry (e.g., a custom API), implement `Evalu
 ```python
 from agentevals.evaluator.sources import EvaluatorSource, EvaluatorInfo, register_source
 
+
 class MyRegistrySource(EvaluatorSource):
     @property
     def source_name(self) -> str:
@@ -492,6 +495,7 @@ class MyRegistrySource(EvaluatorSource):
     async def list_evaluators(self) -> list[EvaluatorInfo]: ...
     async def fetch_evaluator(self, ref: str, dest: Path) -> Path: ...
 
+
 register_source(MyRegistrySource())
 ```
 

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/examples/README.md

@@ -69,10 +69,12 @@ For simple prompt-to-response agents, there's also a decorator shorthand:
 ```python
 app = AgentEvals(eval_set_id="my-eval")
 
+
 @app.agent
 def my_agent(prompt):
     return llm.invoke(prompt).content
 
+
 app.run(["Hello!", "Tell me a joke"])
 ```
 

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/examples/dice_agent/README.md

@@ -89,13 +89,8 @@ Add a new tool in `agent.py`:
 def roll_multiple(count: int, sides: int = 6) -> dict:
     """Roll multiple dice at once."""
     results = [random.randint(1, sides) for _ in range(count)]
-    return {
-        "count": count,
-        "sides": sides,
-        "results": results,
-        "total": sum(results),
-        "average": sum(results) / count
-    }
+    return {"count": count, "sides": sides, "results": results, "total": sum(results), "average": sum(results) / count}
+
 
 dice_agent = Agent(
     name="dice_agent",

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/examples/langchain_agent/README.md

@@ -145,7 +145,7 @@ os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
 
 ```python
 tracer_provider = TracerProvider()  # For spans
-logger_provider = LoggerProvider()   # For logs
+logger_provider = LoggerProvider()  # For logs
 ```
 
 **Both are required** - spans provide metadata, logs provide message content.
@@ -164,10 +164,12 @@ The log processor shares the WebSocket connection from the span processor.
 ```python
 loop = asyncio.new_event_loop()
 
+
 def run_loop_in_background():
     asyncio.set_event_loop(loop)
     loop.run_forever()
 
+
 thread = threading.Thread(target=run_loop_in_background, daemon=True)
 thread.start()
 ```

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/packages/evaluator-sdk-py/README.md

@@ -15,6 +15,7 @@ pip install agentevals-evaluator-sdk
 ```python
 from agentevals_evaluator_sdk import evaluator, EvalInput, EvalResult
 
+
 @evaluator
 def my_evaluator(input: EvalInput) -> EvalResult:
     scores = []
@@ -27,6 +28,7 @@ def my_evaluator(input: EvalInput) -> EvalResult:
         per_invocation_scores=scores,
     )
 
+
 if __name__ == "__main__":
     my_evaluator.run()
 ```

{agentevals_cli-0.9.8 → agentevals_cli-0.9.10}/pyproject.toml

@@ -9,8 +9,8 @@ description = "Standalone framework to evaluate agent correctness based on porta
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "google-adk[eval]>=2.1.0,<2.2",
-    "click>=8.0",
+    "google-adk[eval]>=2.5.0,<3",
+    "click>=8.3.3",
     "tabulate>=0.9.0",
     "fastapi>=0.115.0",
     "uvicorn[standard]>=0.32.0",
@@ -28,6 +28,7 @@ dependencies = [
     "litellm>=1.85.7",
     "nltk>=3.10.2",
     "pyasn1>=0.6.4",
+    "pygments>=2.20.0",
     "requests>=2.34.2",
     "urllib3>=2.7.0",
 ]
@@ -83,7 +84,7 @@ asyncio_mode = "auto"
 
 [dependency-groups]
 dev = [
-    "pytest>=9.0.2",
+    "pytest>=9.0.3",
     "pytest-asyncio>=0.24.0",
     "httpx>=0.27.0",
     "ruff>=0.11.0",