agentops-accelerator 0.3.0__py3-none-any.whl

agentops/templates/.gitignore

@@ -0,0 +1,15 @@
+# Generated by agentops init
+# Keep generated runtime artifacts out of source control
+.env
+.env.*
+!.env.example
+results/**
+official-eval/**
+.resolved/**
+
+# Keep source-of-truth AgentOps configuration under version control
+!config.yaml
+!run.yaml
+!bundles/**
+!datasets/**
+!data/**

agentops/templates/__init__.py

@@ -0,0 +1 @@
+"""Template assets used by AgentOps services."""

agentops/templates/agent-server/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+WORKDIR /app
+
+# Install AgentOps with the agent extras (FastAPI + Azure Monitor + crypto).
+ARG AGENTOPS_VERSION=
+RUN if [ -n "$AGENTOPS_VERSION" ]; then \
+      pip install "agentops-accelerator[agent]==${AGENTOPS_VERSION}"; \
+    else \
+      pip install "agentops-accelerator[agent] @ git+https://github.com/Azure/agentops.git@main"; \
+    fi
+
+# Optional: copy a workspace into the image so analyze can find historic
+# runs. In production you would mount this as a volume instead.
+COPY .agentops /app/.agentops
+
+EXPOSE 8080
+
+CMD ["agentops", "agent", "serve", "--host", "0.0.0.0", "--port", "8080"]

agentops/templates/agent-server/README.md

@@ -0,0 +1,61 @@
+# AgentOps Watchdog - deploy scaffold
+
+This folder contains the minimum bits to host `agentops agent serve`
+on **Azure Container Apps** as a GitHub Copilot Extension.
+
+## Files
+
+- `Dockerfile` - installs `agentops-accelerator[agent]` and runs
+  `agentops agent serve --host 0.0.0.0 --port 8080`.
+- `main.bicep` - single-resource ACA app with HTTPS ingress, a
+  user-assigned managed identity, and a `/healthz` liveness probe.
+
+## Quickstart
+
+```bash
+# 1. Build & push the image (server-side build avoids local Docker).
+az acr build \
+  --registry <your-acr> \
+  --image agentops-doctor:1.0.0 \
+  --file Dockerfile .
+
+# 2. Provision the Container App.
+az deployment group create \
+  --resource-group <rg> \
+  --template-file main.bicep \
+  --parameters \
+      environmentName=<aca-environment> \
+      image=<your-acr>.azurecr.io/agentops-doctor:1.0.0 \
+      userAssignedIdentityId=<umi-resource-id> \
+      appInsightsResourceId=<app-insights-resource-id> \
+      foundryProjectEndpoint=<https://...>
+```
+
+The user-assigned identity needs read access on the Application
+Insights resource (`Monitoring Reader`) and on the Foundry project
+(`Azure AI Developer`).
+
+## Wire to Copilot Chat
+
+Once the app is running, register a GitHub App that points its Copilot
+Extension webhook at:
+
+```
+https://<app-fqdn>/agents/messages
+```
+
+Local development bypasses the GitHub signature check via
+`agentops agent serve --no-verify`. **Never** deploy with
+`--no-verify` to a public endpoint.
+
+## What the agent does
+
+The container runs the Doctor analyzer on every chat turn,
+combining:
+
+1. AgentOps eval history (mounted at `/app/.agentops` or pulled at
+   runtime).
+2. Application Insights traces (Foundry telemetry).
+3. Foundry control plane (`azure-ai-projects`).
+
+It returns a Markdown reply with severity-ranked findings.

agentops/templates/agent-server/main.bicep

@@ -0,0 +1,94 @@
+// Minimal Bicep to deploy the AgentOps Watchdog as a Copilot Extension
+// hosted on Azure Container Apps. Pair with a pre-built image (e.g. from
+// `az acr build`).
+
+@description('Resource location.')
+param location string = resourceGroup().location
+
+@description('Container Apps environment name.')
+param environmentName string
+
+@description('Container app name.')
+param appName string = 'agentops-doctor'
+
+@description('Fully qualified image reference, e.g. myacr.azurecr.io/agentops-doctor:1.0.0.')
+param image string
+
+@description('Application Insights resource id consumed by the Doctor.')
+param appInsightsResourceId string = ''
+
+@description('Foundry project endpoint.')
+param foundryProjectEndpoint string = ''
+
+@description('User-assigned managed identity resource id with reader access on App Insights and Foundry.')
+param userAssignedIdentityId string
+
+resource env 'Microsoft.App/managedEnvironments@2024-03-01' existing = {
+  name: environmentName
+}
+
+resource app 'Microsoft.App/containerApps@2024-03-01' = {
+  name: appName
+  location: location
+  identity: {
+    type: 'UserAssigned'
+    userAssignedIdentities: {
+      '${userAssignedIdentityId}': {}
+    }
+  }
+  properties: {
+    managedEnvironmentId: env.id
+    configuration: {
+      activeRevisionsMode: 'Single'
+      ingress: {
+        external: true
+        targetPort: 8080
+        transport: 'auto'
+        allowInsecure: false
+      }
+    }
+    template: {
+      containers: [
+        {
+          name: 'doctor'
+          image: image
+          resources: {
+            cpu: json('0.5')
+            memory: '1.0Gi'
+          }
+          env: [
+            {
+              name: 'AZURE_CLIENT_ID'
+              value: userAssignedIdentityId
+            }
+            {
+              name: 'AGENTOPS_APP_INSIGHTS_RESOURCE_ID'
+              value: appInsightsResourceId
+            }
+            {
+              name: 'AZURE_AI_FOUNDRY_PROJECT_ENDPOINT'
+              value: foundryProjectEndpoint
+            }
+          ]
+          probes: [
+            {
+              type: 'Liveness'
+              httpGet: {
+                path: '/healthz'
+                port: 8080
+              }
+              initialDelaySeconds: 10
+              periodSeconds: 30
+            }
+          ]
+        }
+      ]
+      scale: {
+        minReplicas: 1
+        maxReplicas: 3
+      }
+    }
+  }
+}
+
+output appFqdn string = app.properties.configuration.ingress.fqdn

agentops/templates/agent.yaml

@@ -0,0 +1,87 @@
+# AgentOps Watchdog Agent configuration.
+#
+# Every section is optional — sources auto-skip with a diagnostic note when
+# they are disabled, missing required config, or missing the optional Azure
+# extras (`pip install "agentops-accelerator[agent] @ git+https://github.com/Azure/agentops.git@main"`).
+#
+# Run:
+#
+#   agentops doctor
+#   agentops agent serve --no-verify   # local dev
+#
+version: 1
+
+# Lookback window applied to production telemetry queries.
+lookback_days: 7
+
+sources:
+  # Reads local `.agentops/results/*/results.json` first. If local history
+  # is missing or too short, Doctor falls back to Foundry cloud evaluations
+  # when `foundry_control` has a reachable project endpoint.
+  results_history:
+    enabled: true
+    path: .agentops/results
+    lookback_runs: 10
+
+  # Azure Monitor / Application Insights for the deployed agent.
+  azure_monitor:
+    enabled: true
+    # Use either app_insights_resource_id (preferred) OR
+    # log_analytics_workspace_id, depending on how telemetry is wired.
+    app_insights_resource_id: ""
+    log_analytics_workspace_id: ""
+
+  # Foundry control plane (agent metadata / recent runs).
+  foundry_control:
+    enabled: true
+    # Inline value, or read from this env var.
+    project_endpoint: ""
+    project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT
+    # Empty list = consider all agents in the project.
+    agent_ids: []
+
+  # Azure management plane — read-only resource posture audits.
+  # Enabled by default and fail-open: Doctor reads `.azure/<env>/.env` when
+  # AZD metadata exists, then falls back to the Foundry project endpoint /
+  # Azure resource scan, then to the fields below. If discovery is not possible,
+  # the source is marked skipped with guidance in the report.
+  # Required RBAC when used: `Reader` on the resource group.
+  azure_resources:
+    enabled: true
+    # subscription_id: "00000000-0000-0000-0000-000000000000"
+    subscription_id_env: AZURE_SUBSCRIPTION_ID
+    # resource_group: "rg-myproject"
+    # cognitive_services_account: "ai-services-myproject"
+
+checks:
+  regression:
+    metrics: [coherence, fluency, similarity, f1_score, groundedness, tool_call_accuracy]
+    threshold_drop: 0.10
+    min_runs: 3
+
+  latency:
+    p95_threshold_seconds: 5.0
+
+  errors:
+    rate_threshold: 0.05
+
+  safety:
+    severity_floor: Medium  # Low | Medium | High
+
+  # WAF-AI posture audit. Enabled by default; it emits findings only when
+  # the `azure_resources` source is successfully discovered/read. When the
+  # source cannot be used, the reason appears in the report's Sources table.
+  # Findings are produced under the `security` category, filterable with
+  # `agentops doctor --categories security`.
+  posture:
+    enabled: true
+    pillar: security
+    # Skip individual rules without disabling the whole check, e.g.
+    # exclude_rules:
+    #   - waf.security.diagnostic_settings
+    exclude_rules: []
+
+server:
+  # GitHub App client id, used to distinguish the Copilot Extension that
+  # is allowed to call this server. Optional for v1.
+  github_app_client_id: ""

agentops/templates/agentops.yaml

@@ -0,0 +1,58 @@
+# AgentOps configuration — 1.0 flat schema.
+#
+# The four required fields are 'version', 'agent', 'dataset', and (optionally)
+# 'thresholds'. AgentOps infers evaluators from the agent type and dataset
+# columns, so most users only need this much.
+#
+# Examples:
+#
+#   agent: "my-rag:3"                      # Foundry prompt agent (name:version)
+#   agent: "https://...foundry.../agents/" # Foundry hosted endpoint
+#   agent: "https://api.example.com/chat"  # any HTTP/JSON agent (ACA, AKS, custom)
+#   agent: "model:gpt-4o"                  # raw Foundry model deployment
+
+# Schema version of this agentops.yaml file (not the agent version).
+version: 1
+
+# Foundry agent identifier as name:version, where the trailing number is
+# the *agent* version published in Foundry (not the schema version above).
+agent: "my-agent:1"
+
+dataset: .agentops/data/smoke.jsonl
+
+# Optional. Source-controlled instructions file used by prompt-agent CI/CD.
+# Generated prompt-agent deploy workflows create a candidate Foundry version
+# from this file, evaluate that exact version, then record it as deployed only
+# after the gate passes.
+#
+# prompt_file: .agentops/prompts/agent-instructions.md
+
+# Optional. Override the auto-selected pass/fail thresholds. AgentOps fills in
+# sensible defaults for the metrics that are auto-selected.
+#
+# thresholds:
+#   coherence: ">=3"
+#   groundedness: ">=3"
+#   avg_latency_seconds: "<=30"
+
+# Optional. Foundry prompt agents and Foundry publishing need a project
+# endpoint. If both this value and AZURE_AI_FOUNDRY_PROJECT_ENDPOINT are set,
+# this file wins for target invocation and publishing.
+#
+# project_endpoint: "https://<resource>.services.ai.azure.com/api/projects/<project>"
+#
+# Optional. Where the agent and evaluators execute.
+#   local (default) — AgentOps runs the agent row-by-row locally.
+#   cloud           — Foundry runs the agent and evaluators server-side
+#                     (only valid for 'name:version' Foundry prompt agents;
+#                     implicitly publishes to the New Foundry panel).
+#
+# execution: local
+#
+# Optional. Publish local results to the Classic Foundry Evaluations panel.
+# Only meaningful when execution: local. With execution: cloud the run is
+# always published (Foundry hosts the run by definition).
+#   false (default) — only local results.json / report.md are written.
+#   true            — execution: local + publish: true → upload to Classic Foundry.
+#
+# publish: false

agentops/templates/foundry.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 18 18">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: none;
+      }
+
+      .cls-1, .cls-2, .cls-3, .cls-4 {
+        stroke-width: 0px;
+      }
+
+      .cls-2 {
+        fill: url(#linear-gradient);
+      }
+
+      .cls-2, .cls-3 {
+        fill-rule: evenodd;
+      }
+
+      .cls-3 {
+        fill: url(#linear-gradient-3);
+      }
+
+      .cls-4 {
+        fill: url(#linear-gradient-2);
+      }
+    </style>
+    <linearGradient id="linear-gradient" x1="12.76" y1="11.97" x2="10.26" y2="1.96" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#712575"/>
+      <stop offset=".09" stop-color="#9a2884"/>
+      <stop offset=".18" stop-color="#bf2c92"/>
+      <stop offset=".27" stop-color="#da2e9c"/>
+      <stop offset=".34" stop-color="#eb30a2"/>
+      <stop offset=".4" stop-color="#f131a5"/>
+      <stop offset=".5" stop-color="#ec30a3"/>
+      <stop offset=".61" stop-color="#df2f9e"/>
+      <stop offset=".72" stop-color="#c92d96"/>
+      <stop offset=".83" stop-color="#aa2a8a"/>
+      <stop offset=".95" stop-color="#83267c"/>
+      <stop offset="1" stop-color="#712575"/>
+    </linearGradient>
+    <linearGradient id="linear-gradient-2" x1="13.71" y1="1.8" x2="13.71" y2="15.92" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#da7ed0"/>
+      <stop offset=".08" stop-color="#b17bd5"/>
+      <stop offset=".19" stop-color="#8778db"/>
+      <stop offset=".3" stop-color="#6276e1"/>
+      <stop offset=".41" stop-color="#4574e5"/>
+      <stop offset=".54" stop-color="#2e72e8"/>
+      <stop offset=".67" stop-color="#1d71eb"/>
+      <stop offset=".81" stop-color="#1471ec"/>
+      <stop offset="1" stop-color="#1171ed"/>
+    </linearGradient>
+    <linearGradient id="linear-gradient-3" x1="12.86" y1="2.12" x2="3.5" y2="17.13" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#da7ed0"/>
+      <stop offset=".05" stop-color="#b77bd4"/>
+      <stop offset=".11" stop-color="#9079da"/>
+      <stop offset=".18" stop-color="#6e77df"/>
+      <stop offset=".25" stop-color="#5175e3"/>
+      <stop offset=".33" stop-color="#3973e7"/>
+      <stop offset=".42" stop-color="#2772e9"/>
+      <stop offset=".54" stop-color="#1a71eb"/>
+      <stop offset=".68" stop-color="#1371ec"/>
+      <stop offset="1" stop-color="#1171ed"/>
+    </linearGradient>
+  </defs>
+  <path class="cls-2" d="m11.52,1.59c.44,0,.83.33.97.81s.96,3.45.96,3.45v5.9h-2.97l.06-10.17h.98Z"/>
+  <path class="cls-4" d="m15.88,6.2c0-.21-.17-.37-.37-.37h-1.75c-1.23,0-2.23,1-2.23,2.23v3.7h2.12c1.23,0,2.23-1,2.23-2.23v-3.33Z"/>
+  <path class="cls-3" d="m11.52,1.59c-.34,0-.61.27-.61.61l-.06,11.23c0,1.64-1.33,2.97-2.97,2.97H2.49c-.26,0-.43-.25-.35-.49L6.46,3.58c.42-1.19,1.54-1.99,2.8-1.99h2.27Z"/>
+  <rect class="cls-1" width="18" height="18"/>
+</svg>

agentops/templates/pipelines/azuredevops/agentops-deploy-dev-azd.yml

@@ -0,0 +1,118 @@
+# AgentOps - Deploy to dev with azd (Azure DevOps Pipelines)
+#
+# agentops:deploy-mode=azd
+#
+# Delegates provision/deploy to Azure Developer CLI. AgentOps only runs
+# the eval gate; azd owns infra, packaging, deploy, and azure.yaml hooks.
+#
+# Generated by `agentops workflow generate --platform azure-devops --deploy-mode azd`.
+
+trigger:
+  branches:
+    include:
+      - develop
+
+pr: none
+
+pool:
+  vmImage: ubuntu-latest
+
+variables:
+  - group: agentops
+  - name: AZURE_SERVICE_CONNECTION
+    value: agentops-azure
+  - name: AGENTOPS_CONFIG
+    value: agentops.yaml
+  - name: AZURE_ENV_NAME
+    value: dev
+  - name: RUN_AZD_PROVISION
+    value: "true"
+
+stages:
+  - stage: provision
+    displayName: azd provision (dev)
+    jobs:
+      - deployment: azd_provision
+        environment: dev
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+                - checkout: self
+                - task: AzureCLI@2
+                  displayName: Prepare azd env and provision
+                  inputs:
+                    azureSubscription: $(AZURE_SERVICE_CONNECTION)
+                    scriptType: bash
+                    scriptLocation: inlineScript
+                    inlineScript: |
+                      test -f azure.yaml || {
+                        echo "##vso[task.logissue type=error;sourcepath=azure.yaml]No azure.yaml found. Ask your coding agent to generate a zero-trust azd deployment, then re-run this pipeline."
+                        exit 1
+                      }
+                      curl -fsSL https://aka.ms/install-azd.sh | bash
+                      azd env new "$(AZURE_ENV_NAME)" --no-prompt --subscription "$(AZURE_SUBSCRIPTION_ID)" ${AZURE_LOCATION:+--location "$AZURE_LOCATION"} \
+                        || azd env select "$(AZURE_ENV_NAME)"
+                      azd env set AZURE_SUBSCRIPTION_ID "$(AZURE_SUBSCRIPTION_ID)"
+                      if [ -n "$AZURE_LOCATION" ]; then
+                        azd env set AZURE_LOCATION "$AZURE_LOCATION"
+                      fi
+                      if [ "$(RUN_AZD_PROVISION)" = "true" ]; then
+__AILZ_PREFLIGHT_COMMAND__
+                        azd provision --no-prompt
+                      fi
+
+  - stage: eval
+    displayName: AgentOps eval (pre-deploy)
+    dependsOn: provision
+    jobs:
+      - job: eval_gate
+        steps:
+          - checkout: self
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: "3.11"
+          - bash: |
+              python -m pip install --upgrade pip
+              python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main"
+            displayName: Install AgentOps Toolkit
+
+__EVAL_TASKS__
+          - task: PublishPipelineArtifact@1
+            condition: always()
+            inputs:
+              targetPath: __EVAL_ARTIFACT_TARGET__
+              artifact: agentops-deploy-dev-results
+              publishLocation: pipeline
+
+  - stage: deploy
+    displayName: azd deploy (dev)
+    dependsOn: eval
+    jobs:
+      - deployment: azd_deploy
+        environment: dev
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+                - checkout: self
+                - task: AzureCLI@2
+                  displayName: Run azd deploy
+                  inputs:
+                    azureSubscription: $(AZURE_SERVICE_CONNECTION)
+                    scriptType: bash
+                    scriptLocation: inlineScript
+                    inlineScript: |
+                      curl -fsSL https://aka.ms/install-azd.sh | bash
+                      test -f azure.yaml || {
+                        echo "##vso[task.logissue type=error;sourcepath=azure.yaml]No azure.yaml found. Ask your coding agent to generate a zero-trust azd deployment, then re-run this pipeline."
+                        exit 1
+                      }
+                      azd env new "$(AZURE_ENV_NAME)" --no-prompt --subscription "$(AZURE_SUBSCRIPTION_ID)" ${AZURE_LOCATION:+--location "$AZURE_LOCATION"} \
+                        || azd env select "$(AZURE_ENV_NAME)"
+                      azd env set AZURE_SUBSCRIPTION_ID "$(AZURE_SUBSCRIPTION_ID)"
+                      if [ -n "$AZURE_LOCATION" ]; then
+                        azd env set AZURE_LOCATION "$AZURE_LOCATION"
+                      fi
+                      azd env refresh "$(AZURE_ENV_NAME)" --no-prompt
+                      azd deploy --no-prompt

agentops/templates/pipelines/azuredevops/agentops-deploy-dev.yml

@@ -0,0 +1,73 @@
+# AgentOps — Deploy to dev (Azure DevOps Pipelines)
+#
+# Runs on push to `develop`. Executes the AgentOps eval against the dev
+# environment; on PASS, calls `agentops/deploy.sh` (your project hook)
+# to promote the agent. Adjust the deploy step to match your tooling.
+#
+# Setup mirrors agentops-pr.yml (variable group `agentops`, service
+# connection `agentops-azure`).
+#
+# Generated by `agentops workflow generate --platform azure-devops`.
+
+trigger:
+  branches:
+    include:
+      - develop
+
+pr: none
+
+pool:
+  vmImage: ubuntu-latest
+
+variables:
+  - group: agentops
+  - name: AZURE_SERVICE_CONNECTION
+    value: agentops-azure
+  - name: AGENTOPS_CONFIG
+    value: agentops.yaml
+  - name: TARGET_ENVIRONMENT
+    value: dev
+
+stages:
+  - stage: eval
+    displayName: AgentOps eval (pre-deploy)
+    jobs:
+      - deployment: agentops_eval_and_deploy
+        displayName: Eval + deploy to $(TARGET_ENVIRONMENT)
+        environment: $(TARGET_ENVIRONMENT)
+        timeoutInMinutes: 45
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+                - checkout: self
+                  fetchDepth: 1
+
+                - task: UsePythonVersion@0
+                  displayName: Set up Python 3.11
+                  inputs:
+                    versionSpec: "3.11"
+
+                - bash: |
+                    python -m pip install --upgrade pip
+                    python -m pip install "agentops-accelerator[foundry] @ git+https://github.com/Azure/agentops.git@main"
+                  displayName: Install AgentOps Toolkit
+
+__EVAL_TASKS__
+
+                - task: PublishPipelineArtifact@1
+                  displayName: Upload AgentOps results
+                  condition: always()
+                  inputs:
+                    targetPath: __EVAL_ARTIFACT_TARGET__
+                    artifact: agentops-deploy-$(TARGET_ENVIRONMENT)-results
+                    publishLocation: pipeline
+
+                - bash: |
+                    if [ -x ./agentops/deploy.sh ]; then
+                      ./agentops/deploy.sh "$(TARGET_ENVIRONMENT)"
+                    else
+                      echo "##[warning]No ./agentops/deploy.sh hook found; skipping deploy step."
+                      echo "Drop a script there to publish your agent/prompt/model to $(TARGET_ENVIRONMENT)."
+                    fi
+                  displayName: Deploy to $(TARGET_ENVIRONMENT)