agentevals-cli 0.6.0__tar.gz → 0.6.1__tar.gz

agentevals_cli-0.6.1/.dockerignore

@@ -0,0 +1,16 @@
+.venv
+**/__pycache__
+*.py[cod]
+.git
+.gitignore
+tests
+.pytest_cache
+.ruff_cache
+htmlcov
+.coverage
+ui/node_modules
+dist
+*.egg-info
+**/*.egg-info
+agents.md
+.cursor

{agentevals_cli-0.6.0 → agentevals_cli-0.6.1}/.github/workflows/release.yml

@@ -29,6 +29,9 @@ jobs:
           cache: npm
           cache-dependency-path: ui/package-lock.json
 
+      - name: Set version from tag
+        run: uv version "${{ github.event.inputs.tag || github.ref_name }}" --package agentevals-cli
+
       - name: Build core and bundled wheels
         run: make release
 
@@ -89,3 +92,40 @@ jobs:
           uv build --package agentevals-cli
           uv publish dist/* --token ${{ secrets.PYPI_TOKEN }}
           rm -rf src/agentevals/_static
+
+  push-docker:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Set appVersion in Chart.yaml
+        run: |
+          VERSION="${TAG#v}"
+          sed -i "s/^appVersion:.*/appVersion: \"$VERSION\"/" charts/agentevals/Chart.yaml
+        env:
+          TAG: ${{ github.event.inputs.tag || github.ref_name }}
+
+      - name: Build and push
+        run: |
+          VERSION="${TAG#v}"
+          make build-docker \
+            DOCKER_REGISTRY="ghcr.io/${{ github.repository_owner }}" \
+            DOCKER_TAG="$VERSION"
+        env:
+          TAG: ${{ github.event.inputs.tag || github.ref_name }}

agentevals_cli-0.6.1/Dockerfile

@@ -0,0 +1,38 @@
+# syntax=docker/dockerfile:1
+
+FROM node:25-bookworm-slim AS ui
+WORKDIR /build/ui
+COPY ui/package.json ui/package-lock.json ./
+# Skip lifecycle scripts during ci, then rebuild esbuild in its own layer — avoids ETXTBSY when
+# install.js execs the binary while overlayfs still has the file busy (common with BuildKit).
+RUN npm ci --ignore-scripts
+RUN npm rebuild esbuild
+COPY ui/ ./
+RUN npm run build
+
+FROM python:3.14-slim-bookworm
+
+WORKDIR /app
+
+# Install uv binary only (no pip); same approach as astral-sh/uv's Dockerfile.
+# https://github.com/astral-sh/uv/blob/6d889fd53d5c108d304c5a4085eb3140ec6a9cdb/Dockerfile#L21
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+COPY pyproject.toml uv.lock README.md ./
+COPY packages ./packages
+COPY src ./src
+
+COPY --from=ui /build/ui/dist ./src/agentevals/_static
+
+RUN uv sync --frozen --no-dev --extra live \
+    && groupadd --gid 1000 app \
+    && useradd --uid 1000 --gid app --home-dir /app --no-log-init app \
+    && chown -R app:app /app
+
+USER app
+ENV PATH="/app/.venv/bin:$PATH"
+ENV AGENTEVALS_SERVER_URL=http://127.0.0.1:8001
+
+EXPOSE 8001 4318 8080
+
+CMD ["agentevals", "serve", "--host", "0.0.0.0", "--port", "8001", "--otlp-port", "4318", "--mcp-port", "8080"]

{agentevals_cli-0.6.0 → agentevals_cli-0.6.1}/Makefile

@@ -1,11 +1,22 @@
 VERSION := $(shell grep '^version' pyproject.toml | cut -d'"' -f2)
 WHEEL := dist/agentevals_cli-$(VERSION)-py3-none-any.whl
 
-.PHONY: build build-bundle build-ui release clean dev-backend dev-frontend dev-bundle test test-unit test-integration test-e2e
+DOCKER_REGISTRY ?= soloio
+DOCKER_IMAGE ?= agentevals
+DOCKER_TAG ?= $(VERSION)
+DOCKER_IMAGE_REF := $(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY:%/=%)/$(DOCKER_IMAGE),$(DOCKER_IMAGE))
+
+# Multi-arch build (requires docker buildx). Manifest lists must be pushed — use build-docker-local for a single-arch --load.
+PLATFORMS ?= linux/amd64,linux/arm64
+
+.PHONY: build build-bundle build-docker build-ui release clean dev-backend dev-frontend dev-bundle test test-unit test-integration test-e2e
 
 build:
 	uv build
 
+build-docker:
+	docker buildx build --platform $(PLATFORMS) -t $(DOCKER_IMAGE_REF):$(DOCKER_TAG) --push .
+
 build-ui:
 	cd ui && npm ci && npm run build
 

{agentevals_cli-0.6.0 → agentevals_cli-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentevals-cli
-Version: 0.6.0
+Version: 0.6.1
 Summary: Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces
 License-File: LICENSE
 Requires-Python: >=3.11

agentevals_cli-0.6.1/charts/agentevals/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: agentevals
+description: agentevals web UI, OTLP HTTP receiver, and MCP (Streamable HTTP)
+type: application
+version: 0.1.0
+appVersion: "0.5.2"

agentevals_cli-0.6.1/charts/agentevals/templates/NOTES.txt

@@ -0,0 +1,12 @@
+1. UI and API are available at port {{ .Values.service.http.port }} (Service port name: http).
+2. OTLP HTTP receiver: port {{ .Values.service.otlpHttp.port }} (OTEL_EXPORTER_OTLP_ENDPOINT=http://<service>:{{ .Values.service.otlpHttp.port }}).
+3. MCP (Streamable HTTP): port {{ .Values.service.mcp.port }}, path /mcp (e.g. http://<service>:{{ .Values.service.mcp.port }}/mcp).
+{{- if .Values.ephemeralVolume.enabled }}
+4. An emptyDir is mounted at /tmp with HOME=/tmp/agentevals-home (ephemeral; lost on pod restart). Set ephemeralVolume.enabled=false and readOnlyRootFilesystem=false if you need a writable root without this mount.
+{{- end }}
+
+Get the Service URL:
+  export POD_NAME=$(kubectl get pods --namespace {{ include "agentevals.namespace" . }} -l "app.kubernetes.io/name={{ include "agentevals.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  kubectl --namespace {{ include "agentevals.namespace" . }} port-forward $POD_NAME {{ .Values.service.http.port }}:{{ .Values.service.http.port }}
+
+Health check: GET http://<pod-ip>:{{ .Values.service.http.containerPort }}/api/health

agentevals_cli-0.6.1/charts/agentevals/templates/_helpers.tpl

@@ -0,0 +1,57 @@
+{{- define "agentevals.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "agentevals.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "agentevals.namespace" -}}
+{{- default .Release.Namespace .Values.namespaceOverride }}
+{{- end }}
+
+{{- define "agentevals.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "agentevals.image" -}}
+{{- $registry := .Values.image.registry | default .Values.registry -}}
+{{- $tag := .Values.image.tag | default .Values.tag | default .Chart.AppVersion -}}
+{{- if $registry -}}
+{{- printf "%s/%s:%s" $registry .Values.image.repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .Values.image.repository $tag -}}
+{{- end -}}
+{{- end }}
+
+{{- define "agentevals.labels" -}}
+helm.sh/chart: {{ include "agentevals.chart" . }}
+{{ include "agentevals.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/part-of: agentevals
+{{- end }}
+
+{{- define "agentevals.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "agentevals.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{- define "agentevals.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "agentevals.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

agentevals_cli-0.6.1/charts/agentevals/templates/deployment.yaml

@@ -0,0 +1,128 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "agentevals.fullname" . }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "agentevals.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "agentevals.selectorLabels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      serviceAccountName: {{ include "agentevals.serviceAccountName" . }}
+      {{- if .Values.ephemeralVolume.enabled }}
+      volumes:
+        - name: agentevals-tmp
+          {{- if or .Values.ephemeralVolume.sizeLimit (eq .Values.ephemeralVolume.medium "Memory") }}
+          emptyDir:
+            {{- if eq .Values.ephemeralVolume.medium "Memory" }}
+            medium: Memory
+            {{- end }}
+            {{- with .Values.ephemeralVolume.sizeLimit }}
+            sizeLimit: {{ . }}
+            {{- end }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+      {{- end }}
+      containers:
+        - name: agentevals
+          image: {{ include "agentevals.image" . | quote }}
+          imagePullPolicy: {{ .Values.image.pullPolicy | default .Values.imagePullPolicy }}
+          {{- if .Values.command }}
+          command:
+            {{- toYaml .Values.command | nindent 12 }}
+          {{- end }}
+          {{- if .Values.args }}
+          args:
+            {{- toYaml .Values.args | nindent 12 }}
+          {{- end }}
+          env:
+            - name: AGENTEVALS_SERVER_URL
+              value: "http://127.0.0.1:{{ .Values.service.http.containerPort }}"
+            {{- if .Values.ephemeralVolume.enabled }}
+            - name: TMPDIR
+              value: "/tmp"
+            - name: HOME
+              value: "/tmp/agentevals-home"
+            {{- end }}
+            {{- with .Values.env }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with .Values.envFrom }}
+          envFrom:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.http.containerPort }}
+              protocol: TCP
+            - name: otlp-http
+              containerPort: {{ .Values.service.otlpHttp.containerPort }}
+              protocol: TCP
+            - name: mcp
+              containerPort: {{ .Values.service.mcp.containerPort }}
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          securityContext:
+            {{- $sc := deepCopy .Values.securityContext }}
+            {{- if not .Values.ephemeralVolume.enabled }}
+            {{- $_ := set $sc "readOnlyRootFilesystem" false }}
+            {{- end }}
+            {{- toYaml $sc | nindent 12 }}
+          startupProbe:
+            httpGet:
+              path: /api/health
+              port: http
+            failureThreshold: 60
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /api/health
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /api/health
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          {{- if .Values.ephemeralVolume.enabled }}
+          volumeMounts:
+            - name: agentevals-tmp
+              mountPath: /tmp
+          {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

agentevals_cli-0.6.1/charts/agentevals/templates/service.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "agentevals.fullname" . }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - name: http
+      port: {{ .Values.service.http.port }}
+      targetPort: http
+      protocol: TCP
+    - name: otlp-http
+      port: {{ .Values.service.otlpHttp.port }}
+      targetPort: otlp-http
+      protocol: TCP
+    - name: mcp
+      port: {{ .Values.service.mcp.port }}
+      targetPort: mcp
+      protocol: TCP
+  selector:
+    {{- include "agentevals.selectorLabels" . | nindent 4 }}

agentevals_cli-0.6.1/charts/agentevals/templates/serviceaccount.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "agentevals.serviceAccountName" . }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

agentevals_cli-0.6.1/charts/agentevals/values.yaml

@@ -0,0 +1,153 @@
+# ==============================================================================
+# Global
+# ==============================================================================
+
+# -- Number of replicas. Only 1 is supported (no shared job state across pods).
+replicaCount: 1
+
+# -- Global container image registry (prepended to image.repository)
+registry: ghcr.io
+
+# -- Global image tag override (defaults to Chart.appVersion)
+tag: ""
+
+# -- Global image pull policy
+imagePullPolicy: IfNotPresent
+
+# -- Image pull secrets
+imagePullSecrets: []
+
+# -- Override the chart name
+nameOverride: ""
+
+# -- Override the full resource name
+fullnameOverride: ""
+
+# -- Override the release namespace
+namespaceOverride: ""
+
+# ==============================================================================
+# Image
+# ==============================================================================
+
+image:
+  # -- Container image registry (overrides global registry)
+  registry: ""
+  # -- Container image repository (org/name, without registry prefix)
+  repository: agentevals-dev/agentevals
+  # -- Container image tag (defaults to global tag, then Chart.appVersion)
+  tag: ""
+  # -- Container image pull policy (defaults to global imagePullPolicy)
+  pullPolicy: ""
+
+# ==============================================================================
+# Service Account
+# ==============================================================================
+
+serviceAccount:
+  # -- Create a ServiceAccount
+  create: false
+  # -- Automount the service account token
+  automount: true
+  # -- ServiceAccount annotations
+  annotations: {}
+  # -- ServiceAccount name override
+  name: ""
+
+# ==============================================================================
+# Pod
+# ==============================================================================
+
+# -- Pod annotations
+podAnnotations: {}
+
+# -- Additional pod labels
+podLabels: {}
+
+# -- Pod security context
+podSecurityContext:
+  fsGroup: 1000
+
+# -- Container security context.
+# When ephemeralVolume.enabled is true, emptyDir at /tmp keeps the root
+# filesystem read-only safely. When ephemeralVolume.enabled is false the chart
+# forces readOnlyRootFilesystem to false so /tmp stays writable.
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+  readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 1000
+
+# ==============================================================================
+# Ephemeral Volume
+# ==============================================================================
+
+# -- Writable scratch space at /tmp (trace uploads, MCP temp files, streaming
+# JSONL). HOME is set to /tmp/agentevals-home so Path.home()/.cache stays
+# writable. When disabled the chart sets readOnlyRootFilesystem to false.
+ephemeralVolume:
+  # -- Enable emptyDir mount at /tmp
+  enabled: true
+  # -- Size limit for the emptyDir (Kubernetes 1.22+), e.g. "2Gi"
+  sizeLimit: ""
+  # -- Use "Memory" for tmpfs (faster, counts against memory limits); leave "" for node disk
+  medium: ""
+
+# ==============================================================================
+# Service
+# ==============================================================================
+
+service:
+  # -- Service type
+  type: ClusterIP
+  # -- UI / API HTTP port
+  http:
+    port: 8001
+    containerPort: 8001
+  # -- OTLP HTTP receiver port
+  otlpHttp:
+    port: 4318
+    containerPort: 4318
+  # -- MCP (Streamable HTTP) port
+  mcp:
+    port: 8080
+    containerPort: 8080
+
+# ==============================================================================
+# Resources
+# ==============================================================================
+
+# -- Container resource requests and limits
+resources: {}
+
+# ==============================================================================
+# Scheduling
+# ==============================================================================
+
+# -- Node selector
+nodeSelector: {}
+
+# -- Tolerations
+tolerations: []
+
+# -- Affinity rules
+affinity: {}
+
+# ==============================================================================
+# Overrides
+# ==============================================================================
+
+# -- Override the image entrypoint
+command: []
+
+# -- Override the image arguments
+args: []
+
+# -- Extra environment variables appended to the container env block
+env: []
+
+# -- Extra envFrom sources (ConfigMapRef, SecretRef)
+envFrom: []

agentevals_cli-0.6.1/examples/zero-code-examples/openai-agents/requirements.txt

@@ -0,0 +1,6 @@
+openai-agents>=0.3.3
+opentelemetry-instrumentation-openai-agents-v2>=0.1.0
+
+opentelemetry-sdk>=1.36.0
+opentelemetry-exporter-otlp-proto-http>=1.36.0
+python-dotenv>=1.0.0

agentevals_cli-0.6.1/examples/zero-code-examples/openai-agents/run.py

@@ -0,0 +1,105 @@
+"""Run a dice-rolling OpenAI Agents SDK agent with OTLP export — no agentevals SDK.
+
+Demonstrates zero-code integration: any OTel-instrumented agent streams
+traces to agentevals by pointing the OTLP exporter at the receiver.
+
+Unlike the LangChain and Strands examples, this one is fully self-contained:
+the agent code lives inline with no cross-folder imports.
+
+Prerequisites:
+    1. pip install -r requirements.txt
+    2. agentevals serve --dev
+    3. export OPENAI_API_KEY="your-key-here"
+
+Usage:
+    python examples/zero-code-examples/openai-agents/run.py
+"""
+
+import os
+import random
+
+from agents import Agent, Runner, function_tool
+from dotenv import load_dotenv
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.openai_agents import OpenAIAgentsInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+load_dotenv(override=True)
+
+
+@function_tool
+def roll_die(sides: int) -> int:
+    """Roll a die with the given number of sides and return the result."""
+    return random.randint(1, sides)
+
+
+@function_tool
+def check_prime(number: int) -> bool:
+    """Return True if the number is prime, False otherwise."""
+    if number < 2:
+        return False
+    for i in range(2, int(number**0.5) + 1):
+        if number % i == 0:
+            return False
+    return True
+
+
+def main():
+    if not os.getenv("OPENAI_API_KEY"):
+        print("OPENAI_API_KEY not set.")
+        return
+
+    endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
+    print(f"OTLP endpoint: {endpoint}")
+
+    os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "span_and_event")
+
+    os.environ.setdefault(
+        "OTEL_RESOURCE_ATTRIBUTES",
+        "agentevals.eval_set_id=openai_agents_eval,agentevals.session_name=openai-agents-zero-code",
+    )
+
+    resource = Resource.create()
+
+    tracer_provider = TracerProvider(resource=resource)
+    tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000))
+    trace.set_tracer_provider(tracer_provider)
+
+    OpenAIAgentsInstrumentor().instrument()
+
+    agent = Agent(
+        name="Dice Agent",
+        instructions="You are a helpful assistant. You can roll dice and check if numbers are prime.",
+        tools=[roll_die, check_prime],
+    )
+
+    test_queries = [
+        "Hi! Can you help me?",
+        "Roll a 20-sided die for me",
+        "Is the number you rolled prime?",
+    ]
+
+    conversation_input: list = []
+
+    try:
+        for i, query in enumerate(test_queries, 1):
+            print(f"\n[{i}/{len(test_queries)}] User: {query}")
+
+            conversation_input.append({"role": "user", "content": query})
+            result = Runner.run_sync(agent, conversation_input)
+
+            agent_response = result.final_output or ""
+            print(f"     Agent: {agent_response}")
+
+            conversation_input = result.to_input_list()
+    finally:
+        print()
+        tracer_provider.force_flush()
+        print("All traces flushed to OTLP receiver.")
+
+
+if __name__ == "__main__":
+    main()

{agentevals_cli-0.6.0 → agentevals_cli-0.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agentevals-cli"
-version = "0.6.0"
+version = "0.6.1"
 description = "Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces"
 readme = "README.md"
 requires-python = ">=3.11"