agentevals-cli 0.7.3__tar.gz → 0.8.0__tar.gz

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/DEVELOPMENT.md

@@ -23,10 +23,38 @@ make dev-frontend      # start Vite dev server (port 5173) with HMR
 make dev-bundle        # build UI, serve full bundled experience at port 8001 via uv run
 ```
 
-Standard development uses `dev-backend` + `dev-frontend` in separate terminals. The Vite dev server proxies nothing — the frontend calls the backend at `http://localhost:8001` directly via CORS.
+Standard development uses `dev-backend` + `dev-frontend` in separate terminals. The Vite dev server proxies nothing; the frontend calls the backend at `http://localhost:8001` directly via CORS.
 
 `dev-bundle` is useful for testing the bundled UI experience without building a wheel. It copies `ui/dist` into the source tree temporarily and cleans up when the server exits.
 
+### Postgres backend (optional, for `/api/runs`)
+
+> **Preview.** The schema, the CLI surface, and `/api/runs` shape are still
+> stabilizing. Recreate the agentevals schema between minor version upgrades
+> until further notice; do not depend on persisted data surviving a
+> `git pull` of agentevals itself.
+
+The default in-memory backend keeps `make dev-backend` zero-config. To exercise the async run pipeline locally, bring up a Postgres alongside the app:
+
+```bash
+make pg-up             # start postgres:18.3-alpine in a docker container (port 5432, ephemeral via --rm)
+make migrate           # apply the agentevals schema
+make dev-backend-pg    # pg-up + migrate + serve --dev with backend=postgres wired up
+make pg-down           # stop the container; data is discarded with --rm
+```
+
+Override the defaults via `PG_PORT=5433 make pg-up` etc. The `migrate` target is idempotent (a second invocation is a no-op).
+
+Once running, submit a run with:
+
+```bash
+curl -X POST http://localhost:8001/api/runs \
+    -H 'content-type: application/json' \
+    -d '{"spec": {"approach": "trace_replay", "target": {"kind": "inline", "inline": {...}}, "evalConfig": {"metrics": ["tool_trajectory_avg_score"]}}}'
+```
+
+Then poll `GET /api/runs/{runId}` and `GET /api/runs/{runId}/results`. Without `storage.backend=postgres`, the `/api/runs` endpoints return 503 with a hint pointing at the env var.
+
 ### Building
 
 ```bash

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/Dockerfile

@@ -24,7 +24,7 @@ COPY src ./src
 
 COPY --from=ui /build/ui/dist ./src/agentevals/_static
 
-RUN uv sync --frozen --no-dev --extra live \
+RUN uv sync --frozen --no-dev --extra live --extra postgres \
     && groupadd --gid 1000 app \
     && useradd --uid 1000 --gid app --home-dir /app --no-log-init app \
     && chown -R app:app /app

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/Makefile

@@ -15,7 +15,14 @@ HELM_CHART_DIR ?= charts/agentevals
 HELM_CHART_OCI_URL ?= $(HELM_REPO)/helm
 HELM_CHART_VERSION ?= $(VERSION)
 
-.PHONY: build build-bundle build-docker build-ui release clean dev-backend dev-frontend dev-bundle test test-unit test-integration test-e2e helm-lint helm-template helm-test helm-cleanup helm-package helm-publish
+.PHONY: build build-bundle build-docker build-ui release clean dev-backend dev-backend-pg dev-frontend dev-bundle pg-up pg-down migrate test test-unit test-integration test-e2e helm-lint helm-template helm-test helm-cleanup helm-package helm-publish
+
+PG_CONTAINER ?= agentevals-pg
+PG_PORT      ?= 5432
+PG_USER      ?= agentevals
+PG_PASSWORD  ?= agentevals
+PG_DATABASE  ?= agentevals
+PG_DSN       ?= postgresql://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/$(PG_DATABASE)
 
 build:
 	uv build
@@ -53,6 +60,30 @@ release: clean build-ui
 dev-backend:
 	uv run agentevals serve --dev
 
+pg-up:
+	@if [ -z "$$(docker ps -q -f name=^/$(PG_CONTAINER)$$)" ]; then \
+		docker run -d --rm --name $(PG_CONTAINER) \
+			-e POSTGRES_USER=$(PG_USER) \
+			-e POSTGRES_PASSWORD=$(PG_PASSWORD) \
+			-e POSTGRES_DB=$(PG_DATABASE) \
+			-p $(PG_PORT):5432 postgres:18.3-alpine; \
+	else \
+		echo "container $(PG_CONTAINER) already running"; \
+	fi
+	@until docker exec $(PG_CONTAINER) pg_isready -U $(PG_USER) >/dev/null 2>&1; do sleep 1; done
+	@echo "Postgres ready at $(PG_DSN)"
+
+pg-down:
+	-docker stop $(PG_CONTAINER)
+
+migrate:
+	AGENTEVALS_DATABASE_URL=$(PG_DSN) uv run agentevals migrate up
+
+dev-backend-pg: pg-up migrate
+	AGENTEVALS_STORAGE_BACKEND=postgres \
+	AGENTEVALS_DATABASE_URL=$(PG_DSN) \
+	uv run agentevals serve --dev
+
 dev-frontend:
 	cd ui && npm run dev
 

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentevals-cli
-Version: 0.7.3
+Version: 0.8.0
 Summary: Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces
 License-File: LICENSE
 Requires-Python: >=3.11
@@ -19,6 +19,8 @@ Requires-Dist: httpx>=0.27.0; extra == 'live'
 Requires-Dist: mcp>=1.26.0; extra == 'live'
 Provides-Extra: openai
 Requires-Dist: openai>=2.0; extra == 'openai'
+Provides-Extra: postgres
+Requires-Dist: asyncpg>=0.30.0; extra == 'postgres'
 Provides-Extra: streaming
 Requires-Dist: opentelemetry-sdk>=1.20.0; extra == 'streaming'
 Requires-Dist: websockets>=12.0; extra == 'streaming'
@@ -26,9 +28,9 @@ Description-Content-Type: text/markdown
 
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="docs/assets/logo-color-on-transparent.svg">
-    <source media="(prefers-color-scheme: light)" srcset="docs/assets/logo-dark-on-transparent.svg">
-    <img src="docs/assets/logo-color-on-transparent.svg" alt="agentevals" width="420" />
+    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/agentevals-dev/agentevals/main/docs/assets/logo-color-on-transparent.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/agentevals-dev/agentevals/main/docs/assets/logo-dark-on-transparent.svg">
+    <img src="https://raw.githubusercontent.com/agentevals-dev/agentevals/main/docs/assets/logo-color-on-transparent.svg" alt="agentevals" width="420" />
   </picture>
 </p>
 
@@ -312,6 +314,31 @@ The source for the chart lives in [`charts/agentevals/`](charts/agentevals/) if
 
 See the [Kubernetes example](examples/kubernetes/README.md) for an end-to-end walkthrough deploying agentevals alongside kagent and an OTel Collector on Kubernetes.
 
+#### Postgres backend (`/api/runs`)
+
+> **Preview.** Persistent run history backed by Postgres is under active
+> development. The `storage.*` and `database.postgres.*` chart values, the
+> `/api/runs` HTTP surface, and the database schema may change incompatibly
+> in upcoming releases. Operators evaluating this feature should plan to
+> recreate the agentevals schema when upgrading between minor versions.
+> Default in-memory mode is unaffected.
+
+By default the chart deploys agentevals with an in-memory backend; runs and results are not persisted. To enable the async `POST /api/runs` pipeline with durable Postgres-backed state:
+
+```bash
+# Bundled Postgres (dev / evaluation only):
+helm install agentevals oci://ghcr.io/agentevals-dev/agentevals/helm/agentevals \
+    --set storage.backend=postgres \
+    --set database.postgres.bundled.enabled=true
+
+# Or supply an external Postgres DSN:
+helm install agentevals oci://ghcr.io/agentevals-dev/agentevals/helm/agentevals \
+    --set storage.backend=postgres \
+    --set database.postgres.url='postgresql://user:pass@host:5432/dbname'
+```
+
+When `storage.backend=postgres` the app applies any pending schema migrations on startup (advisory-lock protected, safe across replicas) and starts an in-process worker that processes the run queue. Without `storage.backend=postgres` the `/api/runs` endpoints return 503 with a hint pointing at the env var.
+
 ## MCP Server
 
 Exposes evaluation tools to MCP clients. A `.mcp.json` at the project root lets Claude Code pick it up automatically.

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/README.md

@@ -1,8 +1,8 @@
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="docs/assets/logo-color-on-transparent.svg">
-    <source media="(prefers-color-scheme: light)" srcset="docs/assets/logo-dark-on-transparent.svg">
-    <img src="docs/assets/logo-color-on-transparent.svg" alt="agentevals" width="420" />
+    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/agentevals-dev/agentevals/main/docs/assets/logo-color-on-transparent.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/agentevals-dev/agentevals/main/docs/assets/logo-dark-on-transparent.svg">
+    <img src="https://raw.githubusercontent.com/agentevals-dev/agentevals/main/docs/assets/logo-color-on-transparent.svg" alt="agentevals" width="420" />
   </picture>
 </p>
 
@@ -286,6 +286,31 @@ The source for the chart lives in [`charts/agentevals/`](charts/agentevals/) if
 
 See the [Kubernetes example](examples/kubernetes/README.md) for an end-to-end walkthrough deploying agentevals alongside kagent and an OTel Collector on Kubernetes.
 
+#### Postgres backend (`/api/runs`)
+
+> **Preview.** Persistent run history backed by Postgres is under active
+> development. The `storage.*` and `database.postgres.*` chart values, the
+> `/api/runs` HTTP surface, and the database schema may change incompatibly
+> in upcoming releases. Operators evaluating this feature should plan to
+> recreate the agentevals schema when upgrading between minor versions.
+> Default in-memory mode is unaffected.
+
+By default the chart deploys agentevals with an in-memory backend; runs and results are not persisted. To enable the async `POST /api/runs` pipeline with durable Postgres-backed state:
+
+```bash
+# Bundled Postgres (dev / evaluation only):
+helm install agentevals oci://ghcr.io/agentevals-dev/agentevals/helm/agentevals \
+    --set storage.backend=postgres \
+    --set database.postgres.bundled.enabled=true
+
+# Or supply an external Postgres DSN:
+helm install agentevals oci://ghcr.io/agentevals-dev/agentevals/helm/agentevals \
+    --set storage.backend=postgres \
+    --set database.postgres.url='postgresql://user:pass@host:5432/dbname'
+```
+
+When `storage.backend=postgres` the app applies any pending schema migrations on startup (advisory-lock protected, safe across replicas) and starts an in-process worker that processes the run queue. Without `storage.backend=postgres` the `/api/runs` endpoints return 503 with a hint pointing at the env var.
+
 ## MCP Server
 
 Exposes evaluation tools to MCP clients. A `.mcp.json` at the project root lets Claude Code pick it up automatically.

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/charts/agentevals/templates/NOTES.txt

@@ -11,3 +11,11 @@ Get the Service URL:
   kubectl --namespace {{ include "agentevals.namespace" . }} port-forward $POD_NAME {{ .Values.service.http.port }}:{{ .Values.service.http.port }}
 
 Health check: GET http://<pod-ip>:{{ .Values.service.http.containerPort }}/api/health
+
+{{- if eq .Values.storage.backend "postgres" }}
+
+NOTE: Postgres-backed storage is a preview feature. The storage.* and
+database.postgres.* values, the /api/runs HTTP surface, and the database
+schema may change incompatibly in upcoming releases. Recreate the
+agentevals schema when upgrading between minor versions.
+{{- end }}

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/charts/agentevals/templates/_helpers.tpl

@@ -48,6 +48,17 @@ app.kubernetes.io/name: {{ include "agentevals.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{- /*
+Selector labels scoped to the main app Pod and its Service. Carries the
+``app.kubernetes.io/component: agentevals`` discriminator so the agentevals
+Service does not also match the bundled Postgres Pod (which carries
+``app.kubernetes.io/component: database`` instead).
+*/ -}}
+{{- define "agentevals.app.selectorLabels" -}}
+{{ include "agentevals.selectorLabels" . }}
+app.kubernetes.io/component: agentevals
+{{- end }}
+
 {{- define "agentevals.serviceAccountName" -}}
 {{- if .Values.serviceAccount.create }}
 {{- default (include "agentevals.fullname" .) .Values.serviceAccount.name }}
@@ -55,3 +66,25 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Service name for the bundled Postgres instance.
+*/}}
+{{- define "agentevals.postgresqlServiceName" -}}
+{{- printf "%s-postgresql" (include "agentevals.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Bundled Postgres image reference (registry/repository/name:tag).
+*/}}
+{{- define "agentevals.postgresql.image" -}}
+{{- $pg := .Values.database.postgres.bundled -}}
+{{- printf "%s/%s/%s:%s" $pg.image.registry $pg.image.repository $pg.image.name $pg.image.tag -}}
+{{- end -}}
+
+{{/*
+Secret name holding POSTGRES_PASSWORD for the bundled Postgres instance.
+*/}}
+{{- define "agentevals.passwordSecretName" -}}
+{{- printf "%s-postgresql" (include "agentevals.fullname" .) -}}
+{{- end -}}

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/charts/agentevals/templates/deployment.yaml

@@ -9,7 +9,7 @@ spec:
   replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
-      {{- include "agentevals.selectorLabels" . | nindent 6 }}
+      {{- include "agentevals.app.selectorLabels" . | nindent 6 }}
   template:
     metadata:
       {{- with .Values.podAnnotations }}
@@ -17,7 +17,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "agentevals.selectorLabels" . | nindent 8 }}
+        {{- include "agentevals.app.selectorLabels" . | nindent 8 }}
         {{- with .Values.podLabels }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -65,6 +65,29 @@ spec:
             - name: HOME
               value: "/tmp/agentevals-home"
             {{- end }}
+            {{- if eq .Values.storage.backend "postgres" }}
+            - name: AGENTEVALS_STORAGE_BACKEND
+              value: "postgres"
+            - name: AGENTEVALS_DATABASE_SCHEMA
+              value: {{ .Values.database.postgres.schema | quote }}
+            {{- if .Values.database.postgres.urlFile }}
+            - name: AGENTEVALS_DATABASE_URL_FILE
+              value: {{ .Values.database.postgres.urlFile | quote }}
+            {{- else if .Values.database.postgres.url }}
+            - name: AGENTEVALS_DATABASE_URL
+              value: {{ .Values.database.postgres.url | quote }}
+            {{- else if .Values.database.postgres.bundled.enabled }}
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "agentevals.passwordSecretName" . }}
+                  key: POSTGRES_PASSWORD
+            - name: AGENTEVALS_DATABASE_URL
+              value: {{ printf "postgresql://agentevals:$(POSTGRES_PASSWORD)@%s.%s.svc.cluster.local:5432/agentevals?sslmode=disable" (include "agentevals.postgresqlServiceName" .) (include "agentevals.namespace" .) | quote }}
+            {{- else }}
+            {{ fail "storage.backend=postgres requires database.postgres.url, database.postgres.urlFile, or database.postgres.bundled.enabled=true" }}
+            {{- end }}
+            {{- end }}
             {{- with .Values.env }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

agentevals_cli-0.8.0/charts/agentevals/templates/postgresql-secret.yaml

@@ -0,0 +1,13 @@
+{{- if and (eq .Values.storage.backend "postgres") .Values.database.postgres.bundled.enabled (not .Values.database.postgres.url) (not .Values.database.postgres.urlFile) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "agentevals.passwordSecretName" . }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+    app.kubernetes.io/component: database
+type: Opaque
+data:
+  POSTGRES_PASSWORD: {{ "agentevals" | b64enc | quote }}
+{{- end }}

agentevals_cli-0.8.0/charts/agentevals/templates/postgresql.yaml

@@ -0,0 +1,142 @@
+{{- if and (eq .Values.storage.backend "postgres") .Values.database.postgres.bundled.enabled (not .Values.database.postgres.url) (not .Values.database.postgres.urlFile) }}
+{{- $pg := .Values.database.postgres.bundled }}
+{{- $fullname := include "agentevals.postgresqlServiceName" . }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $fullname }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+    app.kubernetes.io/component: database
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ $fullname }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+    app.kubernetes.io/component: database
+spec:
+  accessModes:
+    - ReadWriteOnce
+  {{- if $pg.storageClassName }}
+  storageClassName: {{ $pg.storageClassName | quote }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ $pg.storage | quote }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $fullname }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+    app.kubernetes.io/component: database
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      {{- include "agentevals.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: database
+  template:
+    metadata:
+      labels:
+        {{- include "agentevals.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: database
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ $fullname }}
+      securityContext:
+        fsGroup: 999
+        runAsUser: 999
+        runAsGroup: 999
+        runAsNonRoot: true
+      containers:
+        - name: postgresql
+          image: {{ include "agentevals.postgresql.image" . }}
+          imagePullPolicy: {{ $pg.image.pullPolicy }}
+          securityContext:
+            allowPrivilegeEscalation: false
+          ports:
+            - name: postgresql
+              containerPort: 5432
+              protocol: TCP
+          env:
+            - name: POSTGRES_DB
+              value: "agentevals"
+            - name: POSTGRES_USER
+              value: "agentevals"
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "agentevals.passwordSecretName" . }}
+                  key: POSTGRES_PASSWORD
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+          livenessProbe:
+            exec:
+              command:
+                - pg_isready
+                - -U
+                - agentevals
+                - -d
+                - agentevals
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+            successThreshold: 1
+          readinessProbe:
+            exec:
+              command:
+                - pg_isready
+                - -U
+                - agentevals
+                - -d
+                - agentevals
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+            successThreshold: 1
+          {{- with $pg.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: {{ $fullname }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $fullname }}
+  namespace: {{ include "agentevals.namespace" . }}
+  labels:
+    {{- include "agentevals.labels" . | nindent 4 }}
+    app.kubernetes.io/component: database
+spec:
+  type: ClusterIP
+  ports:
+    - name: postgresql
+      port: 5432
+      targetPort: postgresql
+      protocol: TCP
+  selector:
+    {{- include "agentevals.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: database
+{{- end }}

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/charts/agentevals/templates/service.yaml

@@ -25,4 +25,4 @@ spec:
       targetPort: mcp
       protocol: TCP
   selector:
-    {{- include "agentevals.selectorLabels" . | nindent 4 }}
+    {{- include "agentevals.app.selectorLabels" . | nindent 4 }}

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/charts/agentevals/values.yaml

@@ -2,7 +2,10 @@
 # Global
 # ==============================================================================
 
-# -- Number of replicas. Only 1 is supported (no shared job state across pods).
+# -- Number of replicas. The default in-memory backend has no shared state, so
+# scale beyond 1 only when storage.backend is "postgres" (durable runs/results
+# in Postgres are safe to share across replicas via SELECT FOR UPDATE SKIP
+# LOCKED claim semantics).
 replicaCount: 1
 
 # -- Global container image registry (prepended to image.repository)
@@ -155,3 +158,73 @@ env: []
 
 # -- Extra envFrom sources (ConfigMapRef, SecretRef)
 envFrom: []
+
+# ==============================================================================
+# STORAGE  (preview feature)
+#
+# Persistent run history backed by Postgres is under active development.
+# storage.* and database.postgres.* keys, and the underlying schema, may
+# change incompatibly in upcoming releases. Treat persisted runs and
+# results as ephemeral; recreate the agentevals schema when upgrading
+# between minor versions. Default in-memory backend is unaffected.
+# ==============================================================================
+
+storage:
+  # -- Storage backend. "memory" (default) keeps the developer experience
+  # zero-config: nothing persisted, restarts lose in-flight state. "postgres"
+  # enables /api/runs and persists runs + results in Postgres (preview).
+  backend: memory
+
+# ==============================================================================
+# DATABASE CONFIGURATION
+# ==============================================================================
+# Used only when storage.backend is "postgres". Priority order (first match wins):
+#   1. database.postgres.urlFile  -- file-based DSN (workload identity friendly)
+#   2. database.postgres.url      -- literal DSN
+#   3. database.postgres.bundled  -- chart-bundled Postgres (dev/eval only)
+# If none is configured the chart fails to render.
+
+database:
+  postgres:
+    # -- External Postgres connection string.
+    # When set, takes precedence over the bundled instance regardless of
+    # database.postgres.bundled.enabled.
+    url: ""
+    # -- Path to a file containing the connection string. Takes precedence
+    # over url when set. Useful for projected workload-identity tokens.
+    urlFile: ""
+    # -- Postgres schema to use for agentevals tables.
+    schema: agentevals
+    # -- Bundled Postgres instance for development and evaluation only.
+    # Not suitable for production. Deployed when enabled is true and url /
+    # urlFile are not set.
+    bundled:
+      # -- Set to true to deploy a chart-managed Postgres alongside the app.
+      # Off by default so the zero-config install stays in-memory.
+      enabled: false
+      image:
+        # -- Bundled Postgres image registry
+        registry: docker.io
+        # -- Bundled Postgres image repository (org/namespace)
+        repository: library
+        # -- Bundled Postgres image name
+        name: postgres
+        # -- Bundled Postgres image tag
+        tag: "18.3-alpine"
+        # -- Bundled Postgres image pull policy
+        pullPolicy: IfNotPresent
+      # -- PersistentVolumeClaim size for the bundled Postgres data
+      storage: 1Gi
+      # -- StorageClass for the PVC. Defaults to the cluster default when empty.
+      storageClassName: ""
+      # The database name, user, and password are hardcoded for the bundled
+      # instance (all: "agentevals"). This is intentional for a dev/eval
+      # setup. Switch to an external database for production.
+      # -- Resource requests/limits for the bundled Postgres container
+      resources:
+        requests:
+          cpu: 250m
+          memory: 256Mi
+        limits:
+          cpu: 500m
+          memory: 512Mi

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agentevals-cli"
-version = "0.7.3"
+version = "0.8.0"
 description = "Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -33,6 +33,9 @@ streaming = [
 openai = [
     "openai>=2.0",
 ]
+postgres = [
+    "asyncpg>=0.30.0",
+]
 
 [project.scripts]
 agentevals = "agentevals.cli:main"
@@ -43,6 +46,9 @@ artifacts = ["src/agentevals/_static/**"]
 [tool.hatch.build.targets.wheel]
 packages = ["src/agentevals"]
 
+[tool.hatch.build.targets.wheel.force-include]
+"src/agentevals/storage/postgres/migrations" = "agentevals/storage/postgres/migrations"
+
 [tool.uv.workspace]
 members = ["packages/evaluator-sdk-py"]
 

{agentevals_cli-0.7.3 → agentevals_cli-0.8.0}/src/agentevals/api/app.py

@@ -16,13 +16,20 @@ from fastapi.responses import StreamingResponse
 
 from agentevals import __version__
 
+from ..run.service import RunService
+from ..run.worker import AsyncRunWorker
+from ..storage import StorageSettings, build_repos
+from ..storage.postgres.migrator import Migrator
 from ..utils.log_buffer import log_buffer
 from .debug_routes import debug_router
 from .routes import router
+from .runs_routes import runs_router
 
 if TYPE_CHECKING:
     from ..streaming.ws_server import StreamingTraceManager
 
+logger = logging.getLogger(__name__)
+
 try:
     from dotenv import load_dotenv
 
@@ -51,7 +58,39 @@ def _build_lifespan():
         mgr = getattr(app.state, "trace_manager", None)
         if mgr:
             mgr.start_cleanup_task()
+
+        storage_settings: StorageSettings | None = None
+        worker: AsyncRunWorker | None = None
+        try:
+            storage_settings = StorageSettings.from_env()
+        except Exception as exc:
+            logger.error("Storage configuration invalid; /api/runs will not be available: %s", exc)
+
+        if storage_settings is not None and storage_settings.backend == "postgres":
+            logger.info("Applying any pending migrations to schema '%s'", storage_settings.schema_name)
+            migrator = Migrator(
+                dsn=storage_settings.database_url or "",
+                schema=storage_settings.schema_name,
+                lock_timeout_s=storage_settings.migrate_lock_timeout_s,
+            )
+            await migrator.up()
+
+            repos = await build_repos(storage_settings)
+            app.state.storage_settings = storage_settings
+            app.state.repos = repos
+            app.state.run_service = RunService(repos.runs, repos.results)
+
+            worker = AsyncRunWorker(runs=repos.runs, results=repos.results, settings=storage_settings)
+            await worker.start()
+            app.state.run_worker = worker
+
         yield
+
+        if worker is not None:
+            await worker.stop()
+        repos = getattr(app.state, "repos", None)
+        if repos is not None:
+            await repos.close()
         if mgr:
             await mgr.shutdown()
         ae_logger.removeHandler(log_buffer)
@@ -70,6 +109,18 @@ def create_app(
         version=__version__,
         description="REST API for evaluating agent traces using ADK's scoring framework",
         lifespan=_build_lifespan(),
+        openapi_tags=[
+            {
+                "name": "runs",
+                "description": (
+                    "**Preview.** Async run pipeline backed by Postgres. The shape of "
+                    "submission, responses, and persisted run / result data may change "
+                    "incompatibly in upcoming releases. Operators evaluating this "
+                    "surface should plan to recreate persisted data when upgrading "
+                    "agentevals between minor versions."
+                ),
+            },
+        ],
     )
 
     app.add_middleware(
@@ -83,6 +134,7 @@ def create_app(
 
     app.include_router(router, prefix="/api")
     app.include_router(debug_router, prefix="/api/debug")
+    app.include_router(runs_router, prefix="/api")
 
     if trace_manager is not None:
         app.state.trace_manager = trace_manager