evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/bundled/skills/evalkit-deployment/try/cases/acme/income-statement/cases.yaml

@@ -0,0 +1,43 @@
+cases:
+- id: acme-revenue
+  input:
+    question: What was Acme Widgets' revenue in FY2025?
+  evaluators:
+    output:
+      answer_rubrics:
+        rubrics:
+        - rubric: States revenue of 48.2 million for FY2025.
+          type: factual_retrieval
+    trajectory:
+      required_tools_called:
+        tools: [lookup_financials]
+
+- id: acme-revenue-growth
+  input:
+    question: How much did Acme Widgets' revenue grow from FY2024 to FY2025?
+  evaluators:
+    output:
+      answer_rubrics:
+        rubrics:
+        - rubric: States FY2024 revenue of 41.5 million.
+          type: factual_retrieval
+        - rubric: States revenue growth of 16.1% (±0.1pp), or 6.7 million.
+          type: derived_computation
+    trajectory:
+      required_tools_called:
+        tools: [lookup_financials]
+
+- id: acme-gross-margin
+  input:
+    question: What were Acme Widgets' gross profit and gross margin in FY2025?
+  evaluators:
+    output:
+      answer_rubrics:
+        rubrics:
+        - rubric: States gross profit of 18.5 million.
+          type: factual_retrieval
+        - rubric: States a gross margin of 38.4% (±0.1pp).
+          type: derived_computation
+    trajectory:
+      required_tools_called:
+        tools: [lookup_financials]

evalkit/bundled/skills/evalkit-deployment/try/config.yaml

@@ -0,0 +1,17 @@
+# Server config for the try-it stack. See ../config.example.yaml for a real one.
+server:
+  base_url: http://localhost:8020
+
+db:
+  url: postgresql+psycopg://evalkit:evalkit@postgres/evalkit
+  migrate_on_start: true
+
+blob:
+  url: file:///data/evalkit
+
+cost_map:
+  source: bundled
+
+allow_private_endpoints: true # your LLM may run on this machine (host.docker.internal)
+
+encryption_key: env:EVALKIT_ENCRYPTION_KEY # seals the LLM provider key the setup service stores

evalkit/bundled/skills/evalkit-deployment/try/docker-compose.yml

@@ -0,0 +1,82 @@
+# evalkit try-it stack: the evalkit server and Postgres on this machine,
+# judging with your LLM. Walkthrough: docs/user/try-locally.md in the evalkit
+# repository, or /docs/try-locally on the server once it is up.
+#
+#   cp -n .env.example .env     # fill in LLM_BASE_URL, LLM_API_KEY, LLM_MODEL
+#   docker compose up -d --wait # add --build in an evalkit checkout
+#   docker compose down -v      # remove the stack and its data
+#
+# The image is EVALKIT_IMAGE when set. In an evalkit checkout, leave it unset
+# and `--build` builds it from the repo.
+#
+# Not a deployment: the server migrates on every start (db.migrate_on_start),
+# the encryption key is a fixed demo value, and so is the admin password.
+
+name: evalkit-try
+
+x-image: &image
+  image: ${EVALKIT_IMAGE:-evalkit:try}
+  build:
+    context: ../../..
+    dockerfile: deploy/Dockerfile
+
+services:
+  postgres:
+    image: postgres:17
+    environment:
+      POSTGRES_USER: evalkit
+      POSTGRES_PASSWORD: evalkit
+      POSTGRES_DB: evalkit
+    volumes:
+      - pg-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U evalkit -d evalkit"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+
+  server:
+    <<: *image
+    ports:
+      - "127.0.0.1:8020:8020"
+    volumes:
+      - ./config.yaml:/etc/evalkit/config.yaml:ro
+      - blob-data:/data/evalkit
+    environment:
+      EVALKIT_ADMIN_PASSWORD: ${EVALKIT_ADMIN_PASSWORD:?set in .env}
+      EVALKIT_ENCRYPTION_KEY: Hc7MWSn0ThmIx7XEO8V2W3u6aUbBXrOcmxHTMlVf1rE=
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8020/api/health')"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+      start_period: 60s
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  # One-shot: registers your LLM as the LLM provider "try" and makes it the
+  # default judge. Safe to rerun; it only creates what is missing.
+  setup:
+    <<: *image
+    environment:
+      EVALKIT_ADMIN_PASSWORD: ${EVALKIT_ADMIN_PASSWORD:?set in .env}
+      LLM_BASE_URL: ${LLM_BASE_URL:?set in .env}
+      LLM_API_KEY: ${LLM_API_KEY:?set in .env}
+      LLM_MODEL: ${LLM_MODEL:?set in .env}
+    command:
+      - sh
+      - -ec
+      - |
+        evalkit login --url http://server:8020 --username admin --password "$$EVALKIT_ADMIN_PASSWORD"
+        evalkit llm-providers show try >/dev/null 2>&1 ||
+          evalkit llm-providers create try --base-url "$$LLM_BASE_URL" --api-key-env LLM_API_KEY
+        evalkit judge-default set --provider try --model "$$LLM_MODEL" --concurrency 4
+    restart: "no"
+    depends_on:
+      server:
+        condition: service_healthy
+
+volumes:
+  pg-data:
+  blob-data:

evalkit/bundled/skills/evalkit-deployment/try/evalkit.yaml

@@ -0,0 +1,4 @@
+cases: cases
+credentials:
+  llm_providers:
+    env:LLM_BASE_URL: env:LLM_API_KEY

evalkit/bundled/skills/evalkit-deployment/try/mapping.json

@@ -0,0 +1,42 @@
+{
+  "schema_version": "1",
+  "kind": {
+    "attribute": "gen_ai.operation.name",
+    "values": {
+      "chat": "llm", "text_completion": "llm", "generate_content": "llm",
+      "embeddings": "other", "execute_tool": "tool", "invoke_agent": "agent",
+      "create_agent": "other"
+    }
+  },
+  "session_id": {"attribute": "gen_ai.conversation.id"},
+  "model": {"attribute": "gen_ai.request.model"},
+  "system_instructions": {"attribute": "gen_ai.system_instructions", "json": true},
+  "input_messages": {"attribute": "gen_ai.input.messages", "message_format": "gen_ai"},
+  "output_messages": {"attribute": "gen_ai.output.messages", "message_format": "gen_ai"},
+  "tokens": {
+    "prompt": {"attribute": "gen_ai.usage.input_tokens"},
+    "completion": {"attribute": "gen_ai.usage.output_tokens"},
+    "cached": {"attribute": "gen_ai.usage.cache_read.input_tokens"},
+    "cache_write": {"attribute": "gen_ai.usage.cache_creation.input_tokens"}
+  },
+  "cost_usd": null,
+  "tool": {
+    "name": {"attribute": "gen_ai.tool.name"},
+    "call_id": {"attribute": "gen_ai.tool.call.id"},
+    "arguments": {"attribute": "gen_ai.tool.call.arguments", "json": true},
+    "result": {"attribute": "gen_ai.tool.call.result"}
+  },
+  "agent_name": {"attribute": "gen_ai.agent.name"},
+  "input": {"attribute": "gen_ai.input.messages", "json": true},
+  "output": {"attribute": "gen_ai.output.messages", "json": true},
+  "known_attributes": [
+    "gen_ai.system", "gen_ai.provider.name", "gen_ai.agent.*", "gen_ai.main_agent.*",
+    "gen_ai.conversation.id", "gen_ai.conversation.compacted", "gen_ai.data_source.id",
+    "gen_ai.operation.name", "gen_ai.output.type", "gen_ai.prompt.*", "gen_ai.request.*",
+    "gen_ai.response.*", "gen_ai.skill.*", "gen_ai.system_instructions",
+    "gen_ai.token.modality", "gen_ai.tool.definitions", "gen_ai.tool.description",
+    "gen_ai.tool.type", "gen_ai.usage.reasoning.output_tokens", "gen_ai.usage.audio.*",
+    "gen_ai.usage.image.*", "gen_ai.usage.text.*", "gen_ai.workflow.name",
+    "server.address", "server.port", "error.type"
+  ]
+}

evalkit/bundled/skills/evalkit-deployment/try/run.py

@@ -0,0 +1,33 @@
+# /// script
+# requires-python = ">=3.13"
+# dependencies = ["evalkit-py[otel]"]
+# ///
+import argparse
+from pathlib import Path
+
+from agent import Analyst
+from dotenv import load_dotenv
+
+from evalkit.sdk import run_experiment, tracing
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run the acme dataset against the sample agent.")
+    parser.add_argument(
+        "--dry-run", action="store_true", help="Run one case without creating an experiment."
+    )
+    args = parser.parse_args()
+    load_dotenv(Path(__file__).parent / ".env")
+    tracing.register(agent="analyst")
+    analyst = Analyst()
+
+    def task(input: dict) -> str:
+        return analyst.ask(input["question"])
+
+    result = run_experiment(dataset="acme", agent="analyst", task=task, dry_run=args.dry_run)
+    if result.url:
+        print(result.url)
+
+
+if __name__ == "__main__":
+    main()