agentops-accelerator 0.3.0__tar.gz

agentops_accelerator-0.3.0/.claude-plugin/marketplace.json

@@ -0,0 +1,29 @@
+{
+  "name": "agentops",
+  "metadata": {
+    "description": "AgentOps Toolkit plugin marketplace — evaluation skills for Microsoft Foundry agents",
+    "version": "1.0.0"
+  },
+  "owner": {
+    "name": "AgentOps Toolkit",
+    "email": "agentops@microsoft.com"
+  },
+  "plugins": [
+    {
+      "name": "agentops-accelerator",
+      "source": "../../plugins/agentops",
+      "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.",
+      "version": "0.3.0",
+      "keywords": [
+        "agentops",
+        "evaluation",
+        "foundry",
+        "copilot",
+        "agent-skills",
+        "ai-evaluation"
+      ],
+      "license": "MIT",
+      "repository": "https://github.com/Azure/agentops"
+    }
+  ]
+}

agentops_accelerator-0.3.0/.github/actions/azure-oidc-login/action.yml

@@ -0,0 +1,75 @@
+name: Azure OIDC login (composite, no node20 deps)
+description: |
+  Drop-in replacement for azure/login that performs the OIDC federated token
+  exchange entirely in bash, so it does not pull JavaScript action runtime
+  dependencies into the login step.
+
+  After this step runs, the az CLI is authenticated and AZURE_* environment
+  variables are exported for downstream tools (azure-identity etc).
+
+inputs:
+  client-id:
+    description: "Microsoft Entra application (client) ID"
+    required: true
+  tenant-id:
+    description: "Microsoft Entra tenant ID"
+    required: true
+  subscription-id:
+    description: "Azure subscription ID"
+    required: true
+  audience:
+    description: "Federated identity audience"
+    required: false
+    default: "api://AzureADTokenExchange"
+
+runs:
+  using: composite
+  steps:
+    - name: Federated OIDC login (bash)
+      shell: bash
+      env:
+        AZURE_CLIENT_ID: ${{ inputs.client-id }}
+        AZURE_TENANT_ID: ${{ inputs.tenant-id }}
+        AZURE_SUBSCRIPTION_ID: ${{ inputs.subscription-id }}
+        OIDC_AUDIENCE: ${{ inputs.audience }}
+      run: |
+        set -euo pipefail
+
+        : "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?id-token permission missing on the job}"
+        : "${ACTIONS_ID_TOKEN_REQUEST_URL:?id-token permission missing on the job}"
+
+        echo "::group::Requesting OIDC ID token from GitHub"
+        ID_TOKEN_JSON=$(curl -sS \
+          -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
+          -H "Accept: application/json" \
+          "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${OIDC_AUDIENCE}")
+        ID_TOKEN=$(printf '%s' "$ID_TOKEN_JSON" | python3 -c 'import sys,json;print(json.load(sys.stdin)["value"])')
+        if [[ -z "${ID_TOKEN}" || "${ID_TOKEN}" == "null" ]]; then
+          echo "Failed to obtain GitHub OIDC ID token. Response was:" >&2
+          echo "$ID_TOKEN_JSON" >&2
+          exit 1
+        fi
+        echo "::endgroup::"
+
+        echo "::group::az login --federated-token"
+        az login \
+          --service-principal \
+          --username "${AZURE_CLIENT_ID}" \
+          --tenant "${AZURE_TENANT_ID}" \
+          --federated-token "${ID_TOKEN}" \
+          --allow-no-subscriptions \
+          --output none
+        az account set --subscription "${AZURE_SUBSCRIPTION_ID}"
+        echo "::endgroup::"
+
+        # Export the same env vars azure/login sets for DefaultAzureCredential
+        # and other downstream Azure SDKs.
+        {
+          echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}"
+          echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}"
+          echo "AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}"
+          echo "AZURE_FEDERATED_TOKEN=${ID_TOKEN}"
+        } >> "${GITHUB_ENV}"
+
+        # The federated token is short-lived and a secret; mask it.
+        echo "::add-mask::${ID_TOKEN}"

agentops_accelerator-0.3.0/.github/code-quality-py.instructions.md

@@ -0,0 +1,103 @@
+---
+applyTo: '**/*.py'
+---
+# Systematic Code-Quality Pass Instructions for Python Codebase
+
+> Should check copilot-instructions.md for general guidelines first, but these are specific to Python files and should be applied in addition to any general instructions.
+
+You are performing a systematic code-quality pass on a Python codebase. Work through every folder one at a time. For each Python file, apply ALL of the following rules, then compile-check every edited file before moving to the next folder.
+
+## 1. Copyright & Module Docstring
+- Every `.py` file (except empty `__init__.py`) must start with:
+  ```
+  # Copyright (c) Microsoft Corporation.
+  # Licensed under the MIT License.
+  ```
+- Immediately after, add or replace the module-level docstring. It must:
+  - Describe what the module does in 1-2 sentences.
+  - Mention its role in the broader system (e.g., which pipeline stage, what it depends on).
+  - NOT contain generic filler like "This module provides utilities for…"
+
+## 2. Package `__init__.py`
+- If empty, add the copyright header and a package docstring listing sub-modules with one-line descriptions.
+
+## 3. Class Docstrings
+- Replace generic class docstrings with structured ones:
+  ```
+  """One-line summary.
+
+  Responsibilities:
+      1. First responsibility.
+      2. Second responsibility.
+
+  Attributes:
+      attr_name: Description.
+  """
+  ```
+- For dataclasses / Pydantic models, list all fields under "Attributes:".
+
+## 4. Method / Function Docstrings
+- Every public and non-trivial private method must have a docstring.
+- Use this structure:
+  ```
+  """One-line summary.
+
+  Steps:                    ← only for complex multi-step methods
+      1. First step.
+      2. Second step.
+
+  Args:
+      param: Description.
+
+  Returns:
+      Description.
+
+  Raises:
+      ExceptionType: When condition.
+  """
+  ```
+- Simple one-line methods (getters, delegates) get a single-line docstring.
+
+## 5. Comment Cleanup - REMOVE These
+- **Redundant inline comments** that just restate the code:
+  `# Create Claim_Process entry in Cosmos DB` above `new_claim_process = Claim_Process(...)`
+- **Banner comments** / section dividers:
+  `############################################################`
+  `## Initialize AgentFrameworkHelper and add it to the app  ##`
+  `############################################################`
+- **Commented-out code** (dead imports, print statements, old logic).
+- **Heritage/provenance comments** referencing deleted files:
+  `Replaces create_quiet_logger() from quiet_logging.py`
+- **Placeholder comments** that describe unimplemented intent:
+  `# Placeholder for document processing logic`
+- **"For demonstration" / "Here you would typically"** comments.
+
+## 6. Comment Cleanup - KEEP These
+- **Actionable TODOs** with clear intent: `# TODO: Make configurable if needed`
+- **Non-obvious "why" comments** that explain a design decision:
+  `# Avoid unbounded growth on very chatty endpoints.`
+- **Contract/protocol comments** that document external API behavior:
+  `# Image files bypass the 'extract' step.`
+
+## 7. Fix Stale References
+- Search for outdated terminology (old project names, old class names, old pipeline descriptions) and correct them to match the current code.
+
+## 8. Remove Dead Code
+- Delete unused imports.
+- Delete `pass` in `else` blocks that only existed to hold a now-deleted comment.
+- Delete redundant assignments like `claim_id = claim_id`.
+- Delete duplicate imports (e.g., `import os` at module level AND inside a function).
+
+## 9. Compile-Check
+- After finishing each folder, run `python -m py_compile <file>` on every edited file.
+- Fix any errors before proceeding to the next folder.
+
+## Working Process
+1. List the directory tree of the target folder.
+2. Read all Python files in the folder.
+3. Create a TODO list for the folder (one item per file + one for compile-check).
+4. Edit files, marking each TODO as you go.
+5. Compile-check all edited files.
+6. Move to the next folder.
+
+Start with the folder I specify and work through it completely before asking what to do next.

agentops_accelerator-0.3.0/.github/copilot-instructions.md

@@ -0,0 +1,93 @@
+# Copilot Instructions for AgentOps
+
+AgentOps Toolkit is a Python 3.11+ CLI, local Cockpit, and coding-agent skill
+set that helps teams answer two release questions for Microsoft Foundry agents:
+can we ship it, and where is the proof? Foundry runs the agent. AgentOps proves
+the release is ready with eval gates, Doctor readiness, CI/CD, release evidence,
+and trace-driven regression loops.
+
+Read these first for detailed architecture and product context:
+
+- `AGENTS.md`
+- `docs/how-it-works.md`
+- `CONTRIBUTING.md`
+- `README.md`
+
+## Current public CLI surface
+
+- `agentops --version`
+- `agentops explain [COMMAND...] [--no-pager] [--format text|markdown|html] [--out PATH] [--open]`
+- `agentops init [--force] [--dir PATH] [--no-prompt] [--no-appinsights] [--azd-env NAME] [--project-endpoint URL] [--agent REF] [--dataset PATH] [--appinsights-connection-string STR]`
+- `agentops init show [--dir PATH] [--reveal-secrets]`
+- `agentops eval analyze [--dir PATH] [--format text|markdown|json] [--out PATH]`
+- `agentops eval run [--config PATH] [--baseline PATH] [--output DIR]`
+- `agentops eval promote-traces --source PATH [--out PATH] [--max-rows N] [--label-mode self-similarity|pending] [--apply]`
+- `agentops report generate [--in PATH] [--out PATH]`
+- `agentops workflow analyze [--dir PATH] [--format text|markdown|json] [--out PATH]`
+- `agentops workflow generate [--force] [--dir PATH] [--kinds pr,dev,qa,prod,watchdog] [--platform github|azure-devops] [--deploy-mode auto|placeholder|azd|prompt-agent] [--doctor-gate critical|warning|none]`
+- `agentops skills install [--platform copilot|claude] [--from SOURCE] [--prompt] [--force] [--dir PATH]`
+- `agentops mcp serve`
+- `agentops doctor [--workspace PATH] [--config PATH] [--out PATH] [--lookback-days N] [--severity-fail SEVERITY] [--evidence-pack] [--evidence-out PATH]`
+- `agentops doctor explain [--no-pager] [--format text|markdown|html] [--out PATH] [--open]`
+- `agentops cockpit [--host HOST] [--port PORT] [--workspace PATH] [--no-preflight]`
+- `agentops agent serve [--host HOST] [--port PORT] [--config PATH] [--no-verify] [--workers N]`
+
+Do not add commands or flags unless the product discussion explicitly calls for
+them. Keep `--help` terse and put long-form manual content under `explain`.
+
+## Architecture rules
+
+- Keep `src/agentops/cli/app.py` thin: parse arguments, print concise output,
+  and delegate to `pipeline/` or `services/`.
+- Keep `src/agentops/core/` pure: no Azure SDK imports, no network calls.
+- Use `pathlib.Path` for paths.
+- Prefer small focused functions and explicit user-friendly errors.
+- Azure SDK imports must be lazy inside runtime functions.
+- No import-time side effects beyond the existing dotenv loader behavior in the
+  CLI entry point.
+
+Where to put common changes:
+
+| Change | Location |
+|---|---|
+| `agentops.yaml` schema | `src/agentops/core/agentops_config.py` |
+| `results.json` schema | `src/agentops/core/results.py` |
+| Release evidence schema | `src/agentops/core/release_evidence.py` |
+| Eval execution | `src/agentops/pipeline/` |
+| Doctor checks | `src/agentops/agent/checks/` |
+| Release evidence writer | `src/agentops/services/evidence_pack.py` |
+| Trace export to dataset candidates | `src/agentops/services/trace_promotion.py` |
+| Workflow generation | `src/agentops/services/cicd.py` and `src/agentops/templates/` |
+
+## Product principles
+
+- Use Foundry for hosted agents, cloud evaluations, traces, monitoring,
+  red teaming, datasets, and operations.
+- Use AgentOps for repo-controlled release readiness: eval config, CI gates,
+  normalized artifacts, Doctor diagnostics, release evidence,
+  trace-to-dataset promotion, and Cockpit links to Foundry/Azure Monitor.
+- `agentops doctor --evidence-pack` is a readiness projection over existing
+  signals; do not create a second exit-code contract.
+- Trace promotion is review-first. `self-similarity` labels support drift
+  detection, not human-verified correctness.
+- Doctor and Cockpit stay read-only; data promotion belongs in services/CLI.
+
+## Exit codes
+
+- `0` = execution succeeded and all thresholds passed
+- `2` = execution succeeded but one or more thresholds or configured finding
+  gates failed
+- `1` = runtime or configuration error
+
+Do not reinterpret these codes.
+
+## Testing
+
+Use existing tests only. The standard full-suite command is:
+
+```bash
+python -m pytest tests/ -x -q
+```
+
+Add focused unit tests for new schema, services, Doctor checks, CLI flags, and
+workflow template behavior. Azure SDK calls must be mocked or avoided in tests.

agentops_accelerator-0.3.0/.github/dependabot.yml

@@ -0,0 +1,30 @@
+version: 2
+
+updates:
+  # Python dependencies (pyproject.toml)
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "develop"
+    labels:
+      - "dependencies"
+
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "develop"
+    labels:
+      - "dependencies"
+      - "github-actions"
+
+  # VS Code extension (plugins/agentops/package.json)
+  - package-ecosystem: "npm"
+    directory: "/plugins/agentops"
+    schedule:
+      interval: "weekly"
+    target-branch: "develop"
+    labels:
+      - "dependencies"

agentops_accelerator-0.3.0/.github/extensions/agentops-skills/extension.mjs

@@ -0,0 +1,149 @@
+// Extension: agentops-skills
+// Injects AgentOps workflow skills as context when relevant prompts are detected.
+
+import { joinSession } from "@github/copilot-sdk/extension";
+
+const SKILLS = {
+    "run-evals": {
+        keywords: [
+            "run eval", "start agentops", "run.yaml", "regenerate report",
+            "evaluation results", "agentops init", "agentops eval", "agentops report generate",
+            "run an evaluation", "initialize agentops", "results.json", "report.md",
+            "eval run", "run config", "evaluation output",
+        ],
+        context: `## Skill: Run Evaluations
+
+### Purpose
+Guide through the implemented AgentOps evaluation workflow from workspace setup to report interpretation.
+
+### Available Commands
+- agentops init [--path <dir>] — Initialize workspace
+- agentops eval run — Execute evaluation
+- agentops report generate — Regenerate report from results.json
+
+### Typical Workflow
+1. Initialize workspace: agentops init
+2. Confirm run config exists (.agentops/run.yaml)
+3. Execute evaluation: agentops eval run
+4. Regenerate markdown report: agentops report generate
+5. Inspect outputs under .agentops/results/latest/
+
+### Outputs
+- results.json (machine-readable normalized results)
+- report.md (human-readable summary)
+- cloud_evaluation.json (cloud evaluation flows only)
+- Latest pointers: .agentops/results/latest/
+
+### Interpretation
+- Start with report.md for quick pass/fail narrative and threshold view.
+- Use results.json for metric-level details, row-level checks, and automation.
+- Distinguish: thresholds passing, threshold failures, runtime/config errors.
+
+### Guardrails
+- Do not invent commands or flags beyond documented CLI behavior.
+- Planned commands (compare, run-history) are stubbed — pivot to artifact inspection.`,
+    },
+
+    "investigate-regression": {
+        keywords: [
+            "regression", "score dropped", "threshold started failing",
+            "compare runs", "eval got worse", "debug evaluation",
+            "evaluation drift", "quality drop", "pass rate dropped",
+            "ci failing", "scores lower", "metrics degraded",
+        ],
+        context: `## Skill: Investigate Regression
+
+### Purpose
+Guide through regression investigation using currently available AgentOps outputs.
+
+### Available Commands
+- agentops eval run — Generate fresh artifacts
+- agentops report generate — Regenerate report
+
+### Planned (not implemented)
+- agentops eval compare --runs ID1,ID2
+
+### Investigation Steps
+1. Run fresh evaluation: agentops eval run
+2. Regenerate report: agentops report generate
+3. Compare current artifacts to baseline manually
+4. Report factual deltas, then propose controlled next steps
+
+### Required Inputs
+- At least one recent artifact set (results.json + report.md)
+- Preferably a baseline for side-by-side comparison
+- Context about what changed (prompt, model, dataset, bundle, backend, environment)
+
+### Interpretation
+- Separate observations (artifact-backed) from hypotheses (plausible causes).
+- Prioritize impact: which thresholds flipped, which metrics degraded most, broad vs concentrated failures.
+- End with actionable next checks (rerun with controlled changes, validate dataset, verify config).
+
+### Guardrails
+- agentops eval compare is NOT implemented — use manual artifact comparison.
+- Do not infer causality from correlation alone.
+- Keep remediation tied to reproducible checks.`,
+    },
+
+    "observability-triage": {
+        keywords: [
+            "tracing", "monitoring", "cockpit", "alerts", "triage",
+            "observability", "run health", "production triage",
+            "monitor evals", "set up tracing", "failed evaluation",
+            "quality monitoring",
+        ],
+        context: `## Skill: Observability Triage
+
+### Purpose
+Provide honest observability guidance: use current reporting artifacts today, frame tracing/monitoring as planned future work.
+
+### Available Commands (for triage today)
+- agentops eval run
+- agentops report generate
+
+### Planned/Stubbed (NOT implemented)
+- agentops trace init
+- agentops monitor setup
+- agentops monitor show
+- agentops monitor configure
+
+### Current Triage Approach
+- Use report.md for quick operational triage (what failed, severity).
+- Use results.json for detailed metric and threshold inspection.
+- Keep run artifacts organized for future compare/monitor automation.
+
+### When Users Ask for Unimplemented Features
+1. State explicitly: planned/stubbed, not available yet.
+2. Provide immediate fallback: artifact-based troubleshooting.
+3. Suggest preparation: organize artifacts for future tooling.
+
+### Guardrails
+- Do not present tracing or monitoring commands as available.
+- Do not imply real-time cockpits/alerts exist in CLI.
+- Always pivot to concrete available outputs (results.json, report.md).`,
+    },
+};
+
+function matchSkills(prompt) {
+    const lower = prompt.toLowerCase();
+    const matched = [];
+    for (const [name, skill] of Object.entries(SKILLS)) {
+        if (skill.keywords.some((kw) => lower.includes(kw))) {
+            matched.push(skill.context);
+        }
+    }
+    return matched;
+}
+
+const session = await joinSession({
+    hooks: {
+        onUserPromptSubmitted: async (input) => {
+            const matched = matchSkills(input.prompt);
+            if (matched.length > 0) {
+                return {
+                    additionalContext: `<agentops_skills>\n${matched.join("\n\n---\n\n")}\n</agentops_skills>`,
+                };
+            }
+        },
+    },
+});

agentops_accelerator-0.3.0/.github/plugin/marketplace.json

@@ -0,0 +1,29 @@
+{
+  "name": "agentops",
+  "metadata": {
+    "description": "AgentOps Toolkit plugin marketplace — evaluation skills for Microsoft Foundry agents",
+    "version": "1.0.0"
+  },
+  "owner": {
+    "name": "AgentOps Toolkit",
+    "email": "agentops@microsoft.com"
+  },
+  "plugins": [
+    {
+      "name": "agentops-accelerator",
+      "source": "../../plugins/agentops",
+      "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.",
+      "version": "0.3.0",
+      "keywords": [
+        "agentops",
+        "evaluation",
+        "foundry",
+        "copilot",
+        "agent-skills",
+        "ai-evaluation"
+      ],
+      "license": "MIT",
+      "repository": "https://github.com/Azure/agentops"
+    }
+  ]
+}