agentops-accelerator 0.5.0__tar.gz → 0.5.2__tar.gz

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/CHANGELOG.md

@@ -5,6 +5,41 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres
 
 ## [Unreleased]
 
+## [0.5.2] - 2026-06-20
+
+### Added
+- **Per-evaluator input remapping via `evaluators[].input_mapping`.** Evaluator
+  overrides in `agentops.yaml` now accept an optional `input_mapping` map that is
+  merged onto the preset's default inputs, so you only list the keys you want to
+  change. This is what lets a grey-box HTTP/JSON target point a RAG evaluator at
+  the live retrieved context captured by `response_fields`, for example
+  `context: $response.context` on `GroundednessEvaluator` and
+  `RetrievalEvaluator`. The mapping applies to both explicitly listed overrides
+  and auto-selected presets. A bare evaluator name string (`- GroundednessEvaluator`)
+  is still accepted as shorthand for `{ name: GroundednessEvaluator }`, so
+  existing configs are unchanged.
+
+
+
+### Added
+- **Rendered gate results in GitHub Actions job summaries.** When AgentOps runs
+  inside GitHub Actions (`GITHUB_STEP_SUMMARY` set), `agentops eval run` now
+  appends the full rendered `report.md` to the workflow run summary, and
+  `agentops assert run` / `agentops redteam run` append a concise pass/fail
+  summary (suite, cases, pass rate, per-dimension and per-risk-category
+  breakdowns). Reviewers can read the report directly on the run page without
+  downloading the uploaded artifacts. Writes are best-effort and a no-op outside
+  GitHub Actions, so local runs are unaffected.
+
+### Changed
+- **Generated workflows use Node24-ready action versions.** The prompt-agent and
+  watchdog workflow templates now pin `actions/download-artifact@v7` (instead of
+  the Node20 `@v4`), so generated pipelines no longer emit the "Node.js 20
+  actions are deprecated" warning. A regression guard checks every workflow
+  template against the known Node20 action majors.
+
+## [0.5.0] - 2026-06-19
+
 ### Added
 - **Grey-box retrieval capture for HTTP JSON targets.** An HTTP target can now
   capture extra named fields from a JSON response via a `response_fields` map

{agentops_accelerator-0.5.0/src/agentops_accelerator.egg-info → agentops_accelerator-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentops-accelerator
-Version: 0.5.0
+Version: 0.5.2
 Summary: Release readiness gates and evidence for Microsoft Foundry agents
 License: MIT License
         

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops/cli/app.py

@@ -2354,6 +2354,70 @@ def _resolve_eval_config_path(config: Path | None) -> Path:
     return Path("agentops.yaml")
 
 
+def _append_assert_step_summary(result, *, scored_cases, pass_rate) -> None:
+    """Append an ASSERT gate summary to the GitHub Actions step summary."""
+    from agentops.core.step_summary import append_step_summary, is_active
+
+    if not is_active():
+        return
+    status = "❌ VIOLATIONS" if result.has_violations else "✅ PASS"
+    lines = [
+        "## AgentOps ASSERT gate",
+        "",
+        f"**Result:** {status}",
+        "",
+        f"- **Suite:** `{result.suite}`",
+        f"- **Run:** `{result.run_id}`",
+        f"- **Cases:** {result.total_cases} "
+        f"(scored={scored_cases}, passed={result.passed_cases}, "
+        f"failed={result.failed_cases}, skipped={result.skipped_cases})",
+        f"- **Pass rate:** {pass_rate}",
+    ]
+    if result.dimension_summary:
+        lines += ["", "| Dimension | Result |", "| --- | --- |"]
+        for name, bucket in sorted(result.dimension_summary.items()):
+            violations = bucket.get("violations", 0)
+            total = bucket.get("total", 0)
+            skipped = bucket.get("skipped", 0)
+            if violations == 0:
+                clean = max(total - skipped, 0)
+                cell = f"{clean}/{total} clean ✅"
+            else:
+                cell = f"{violations}/{total} violating ❌"
+            if skipped:
+                cell += f" (skipped={skipped})"
+            lines.append(f"| {name} | {cell} |")
+    append_step_summary("\n".join(lines))
+
+
+def _append_redteam_step_summary(result, *, asr_pct) -> None:
+    """Append a Red Team gate summary to the GitHub Actions step summary."""
+    from agentops.core.step_summary import append_step_summary, is_active
+
+    if not is_active():
+        return
+    status = "❌ HIGH" if result.has_violations else "✅ PASS"
+    lines = [
+        "## AgentOps Red Team gate",
+        "",
+        f"**Result:** {status}",
+        "",
+        f"- **Attempts:** {result.total_attempts} "
+        f"(successful={result.successful_attacks})",
+        f"- **Attack success rate:** {asr_pct}",
+    ]
+    if result.fail_threshold is not None:
+        lines.append(f"- **Gate threshold:** {result.fail_threshold:.1%}")
+    if result.per_category:
+        lines += ["", "| Risk category | Successful | Rate |", "| --- | --- | --- |"]
+        for name, bucket in sorted(result.per_category.items()):
+            total = bucket.get("total", 0)
+            successful = bucket.get("successful", 0)
+            rate = bucket.get("attack_success_rate", 0.0)
+            lines.append(f"| {name} | {successful}/{total} | {rate:.1%} |")
+    append_step_summary("\n".join(lines))
+
+
 @assert_app.command("run")
 def cmd_assert_run(
     config: Annotated[
@@ -2564,6 +2628,8 @@ def cmd_assert_run(
             "reaching the target; try a less restrictive tester deployment.)"
         )
 
+    _append_assert_step_summary(result, scored_cases=scored_cases, pass_rate=pass_rate)
+
     if result.has_violations:
         msg = (
             f"{_cli_error('FAIL')}: ASSERT reported {result.failed_cases} "
@@ -2805,6 +2871,8 @@ def cmd_redteam_run(
             rate = bucket.get("attack_success_rate", 0.0)
             typer.echo(f"  {name}: {successful}/{total} ({rate:.1%})")
 
+    _append_redteam_step_summary(result, asr_pct=asr_pct)
+
     if result.has_violations:
         msg = (
             f"{_cli_error('FAIL')}: Red Team attack success rate "
@@ -2946,6 +3014,9 @@ def _run_flat_schema_eval(
     report_url = azd_eval.get("report_url") if isinstance(azd_eval, dict) else None
     if isinstance(report_url, str) and report_url.strip():
         typer.echo(f"{_cli_label('Foundry run')}:  {report_url.strip()}")
+    from agentops.core.step_summary import append_report_file
+
+    append_report_file(output_dir / "report.md")
     if result.summary.overall_passed:
         typer.echo(f"{_cli_label('Threshold status')}: {style('PASSED', 'bold', 'green')}")
         return

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops/core/agentops_config.py

@@ -152,12 +152,51 @@ class EvaluatorOverride(BaseModel):
         evaluators:
           - GroundednessEvaluator
           - CoherenceEvaluator
+
+    Each entry may instead be a mapping to remap the evaluator inputs. This is
+    how an HTTP/JSON target scores the *live* retrieved context (grey-box):
+    capture the extra fields on the target with ``response_fields`` and point
+    the evaluator at them via ``$response.<name>`` tokens::
+
+        response_fields:
+          context: context
+          retrieved_documents: retrieved_documents
+        evaluators:
+          - name: GroundednessEvaluator
+            input_mapping:
+              query: $prompt
+              response: $prediction
+              context: $response.context
+          - name: RetrievalEvaluator
+            input_mapping:
+              query: $prompt
+              context: $response.context
+
+    ``input_mapping`` is merged onto the preset's default mapping, so you only
+    list the keys you want to change.
     """
 
     name: str
+    input_mapping: Optional[Dict[str, str]] = Field(
+        None,
+        description=(
+            "Optional per-evaluator input remap merged onto the preset "
+            "defaults. Values use the resolver tokens $prompt, $prediction, "
+            "$expected, $context, $row.<col>, and $response.<name> (the last "
+            "reads a field captured by the target's response_fields)."
+        ),
+    )
 
     model_config = ConfigDict(frozen=True)
 
+    @model_validator(mode="before")
+    @classmethod
+    def _coerce_bare_name(cls, data: Any) -> Any:
+        """Accept a bare evaluator name string as shorthand for ``{name: ...}``."""
+        if isinstance(data, str):
+            return {"name": data}
+        return data
+
     @field_validator("name")
     @classmethod
     def _name_non_empty(cls, value: str) -> str:
@@ -165,6 +204,22 @@ class EvaluatorOverride(BaseModel):
             raise ValueError("evaluator name must be non-empty")
         return value
 
+    @field_validator("input_mapping")
+    @classmethod
+    def _mapping_non_empty(
+        cls, value: Optional[Dict[str, str]]
+    ) -> Optional[Dict[str, str]]:
+        if value is None:
+            return None
+        for key, token in value.items():
+            if not str(key).strip():
+                raise ValueError("input_mapping keys must be non-empty")
+            if not str(token).strip():
+                raise ValueError(
+                    f"input_mapping value for {key!r} must be non-empty"
+                )
+        return value
+
 
 # ---------------------------------------------------------------------------
 # Dataset sync configuration

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops/core/evaluators.py

@@ -32,7 +32,7 @@ final word - no auto-detection runs.
 from __future__ import annotations
 
 import json
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from pathlib import Path
 from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
 
@@ -348,6 +348,7 @@ def select_evaluators(
     shape: DatasetShape,
     *,
     overrides: Optional[List[str]] = None,
+    override_mappings: Optional[Dict[str, Dict[str, str]]] = None,
     threshold_metrics: Optional[Iterable[str]] = None,
 ) -> List[EvaluatorPreset]:
     """Return the ordered list of evaluators to run.
@@ -356,6 +357,12 @@ def select_evaluators(
     bypassed. Each name must exist in :data:`CATALOG` or a ``ValueError`` is
     raised.
 
+    ``override_mappings`` maps an evaluator name to an ``input_mapping`` patch
+    that is merged onto that preset's default mapping. This lets a grey-box
+    HTTP/JSON target point an evaluator at a live response field captured via
+    ``response_fields`` (e.g. ``context: $response.context``). It applies to
+    both the explicit-override path and the auto-selected presets.
+
     Otherwise the rules are:
 
     * Always include the baseline quality evaluators.
@@ -369,6 +376,16 @@ def select_evaluators(
       evaluators.
     * Always append the runtime ``avg_latency_seconds`` evaluator.
     """
+
+    def _apply_mappings(preset: EvaluatorPreset) -> EvaluatorPreset:
+        if not override_mappings:
+            return preset
+        patch = override_mappings.get(preset.name)
+        if not patch:
+            return preset
+        merged = {**preset.input_mapping, **patch}
+        return replace(preset, input_mapping=merged)
+
     if overrides:
         resolved: List[EvaluatorPreset] = []
         for name in overrides:
@@ -379,7 +396,7 @@ def select_evaluators(
                     f"unknown evaluator override {name!r}. "
                     f"Known evaluators: {known}"
                 )
-            resolved.append(preset)
+            resolved.append(_apply_mappings(preset))
         return resolved
 
     selected: List[EvaluatorPreset] = list(_QUALITY_BASELINE)
@@ -411,7 +428,7 @@ def select_evaluators(
         )
 
     selected.append(_LATENCY)
-    return selected
+    return [_apply_mappings(preset) for preset in selected]
 
 
 def _is_agent_target(kind: TargetKind) -> bool:

agentops_accelerator-0.5.2/src/agentops/core/step_summary.py

@@ -0,0 +1,65 @@
+"""Helpers for surfacing AgentOps gate output in CI job summaries.
+
+When AgentOps runs inside GitHub Actions, the ``GITHUB_STEP_SUMMARY``
+environment variable points at a Markdown file whose contents render on the
+workflow run page. These helpers append AgentOps output there so reviewers can
+read the rendered report directly on the run page, without downloading the
+uploaded artifacts.
+
+All writes are best-effort and never raise, so command handlers can call them
+unconditionally: outside GitHub Actions they simply do nothing.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+
+def _summary_path() -> Optional[Path]:
+    raw = os.environ.get("GITHUB_STEP_SUMMARY")
+    if not raw or not raw.strip():
+        return None
+    return Path(raw)
+
+
+def is_active() -> bool:
+    """Return True when a GitHub Actions step summary target is available."""
+    return _summary_path() is not None
+
+
+def append_step_summary(markdown: str) -> bool:
+    """Append a Markdown block to the GitHub Actions step summary.
+
+    Returns ``True`` when the block was written, ``False`` when not running
+    under GitHub Actions or when the write failed. Never raises.
+    """
+    path = _summary_path()
+    if path is None:
+        return False
+    try:
+        text = markdown if markdown.endswith("\n") else markdown + "\n"
+        with path.open("a", encoding="utf-8") as handle:
+            handle.write(text)
+            handle.write("\n")
+        return True
+    except Exception:
+        return False
+
+
+def append_report_file(report_path: Path, *, heading: Optional[str] = None) -> bool:
+    """Append the contents of a rendered report file to the step summary.
+
+    Used for ``agentops eval run`` so the full ``report.md`` renders inline on
+    the workflow run page. Returns ``False`` when not under GitHub Actions or
+    when the report cannot be read. Never raises.
+    """
+    if _summary_path() is None:
+        return False
+    try:
+        body = Path(report_path).read_text(encoding="utf-8")
+    except Exception:
+        return False
+    block = body if heading is None else f"{heading}\n\n{body}"
+    return append_step_summary(block)

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops/pipeline/orchestrator.py

@@ -132,10 +132,16 @@ def _run_evaluation_local(
     overrides = (
         [override.name for override in config.evaluators] if config.evaluators else None
     )
+    override_mappings = (
+        {o.name: dict(o.input_mapping) for o in config.evaluators if o.input_mapping}
+        if config.evaluators
+        else None
+    ) or None
     presets = select_evaluators(
         target,
         shape,
         overrides=overrides,
+        override_mappings=override_mappings,
         threshold_metrics=config.thresholds.keys(),
     )
     user_thresholds = [
@@ -277,10 +283,16 @@ def _run_evaluation_cloud(
     overrides = (
         [override.name for override in config.evaluators] if config.evaluators else None
     )
+    override_mappings = (
+        {o.name: dict(o.input_mapping) for o in config.evaluators if o.input_mapping}
+        if config.evaluators
+        else None
+    ) or None
     all_presets = select_evaluators(
         target,
         shape,
         overrides=overrides,
+        override_mappings=override_mappings,
         threshold_metrics=config.thresholds.keys(),
     )
 

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops/templates/workflows/agentops-deploy-prompt-agent.yml

@@ -93,7 +93,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Download candidate deployment record
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: foundry-agent-__ENV_KEY__-candidate
           path: .agentops/deployments
@@ -169,7 +169,7 @@ __EVAL_STEPS__
     timeout-minutes: 10
     steps:
       - name: Download candidate deployment record
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: foundry-agent-__ENV_KEY__-candidate
           path: .agentops/deployments

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml

@@ -110,7 +110,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Download candidate deployment record
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: foundry-agent-__ENV_KEY__-candidate
           path: .agentops/deployments

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops/templates/workflows/agentops-watchdog.yml

@@ -69,7 +69,7 @@ jobs:
       # downloads when the artifact does not exist yet, instead of
       # surfacing a red error annotation on every first run.
       - name: Restore Doctor history (if any)
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           pattern: agentops-doctor-history
           path: .agentops/agent/

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2/src/agentops_accelerator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentops-accelerator
-Version: 0.5.0
+Version: 0.5.2
 Summary: Release readiness gates and evidence for Microsoft Foundry agents
 License: MIT License
         

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/src/agentops_accelerator.egg-info/SOURCES.txt

@@ -154,6 +154,7 @@ src/agentops/core/evaluators.py
 src/agentops/core/governance.py
 src/agentops/core/release_evidence.py
 src/agentops/core/results.py
+src/agentops/core/step_summary.py
 src/agentops/mcp/__init__.py
 src/agentops/mcp/server.py
 src/agentops/pipeline/__init__.py
@@ -313,6 +314,7 @@ tests/unit/test_setup_wizard.py
 tests/unit/test_shared_credentials.py
 tests/unit/test_skills.py
 tests/unit/test_skills_sync.py
+tests/unit/test_step_summary.py
 tests/unit/test_telemetry.py
 tests/unit/test_time_range.py
 tests/unit/test_trace_promotion.py

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/tests/unit/test_cicd.py

@@ -31,8 +31,14 @@ _DEPRECATED_NODE20_ACTION_REFS = (
     "actions/checkout@v4",
     "actions/setup-python@v5",
     "actions/upload-artifact@v4",
+    "actions/upload-artifact@v5",
+    "actions/download-artifact@v4",
+    "actions/download-artifact@v5",
+    "actions/download-artifact@v6",
     "actions/github-script@v7",
     "astral-sh/setup-uv@v3",
+    "astral-sh/setup-uv@v5",
+    "astral-sh/setup-uv@v6",
     "azure/login@v2",
 )
 

{agentops_accelerator-0.5.0 → agentops_accelerator-0.5.2}/tests/unit/test_evaluators.py

@@ -201,6 +201,56 @@ class TestSelectEvaluators:
         with pytest.raises(ValueError, match="unknown evaluator"):
             select_evaluators(_PROMPT_AGENT, _shape(), overrides=["NotAnEvaluator"])
 
+    def test_override_mapping_patches_explicit_override(self) -> None:
+        # Grey-box: point GroundednessEvaluator at the live captured context.
+        result = select_evaluators(
+            _HTTP_AGENT,
+            _shape(),
+            overrides=["GroundednessEvaluator"],
+            override_mappings={
+                "GroundednessEvaluator": {"context": "$response.context"}
+            },
+        )
+        preset = [p for p in result if p.name == "GroundednessEvaluator"][0]
+        assert preset.input_mapping["context"] == "$response.context"
+        # Unpatched keys keep their preset defaults.
+        assert preset.input_mapping["response"] == "$prediction"
+
+    def test_override_mapping_patches_auto_selected_rag_preset(self) -> None:
+        # No explicit overrides: RAG presets are auto-selected from the
+        # context-bearing dataset, and the mapping still applies.
+        result = select_evaluators(
+            _HTTP_AGENT,
+            _shape(context=True),
+            override_mappings={
+                "RetrievalEvaluator": {"context": "$response.context"}
+            },
+        )
+        preset = [p for p in result if p.name == "RetrievalEvaluator"][0]
+        assert preset.input_mapping["context"] == "$response.context"
+
+    def test_override_mapping_does_not_mutate_catalog_preset(self) -> None:
+        select_evaluators(
+            _HTTP_AGENT,
+            _shape(),
+            overrides=["GroundednessEvaluator"],
+            override_mappings={
+                "GroundednessEvaluator": {"context": "$response.context"}
+            },
+        )
+        # The shared CATALOG preset must be untouched (dataclasses.replace
+        # returns a copy).
+        assert CATALOG["GroundednessEvaluator"].input_mapping["context"] == "$context"
+
+    def test_override_mapping_none_is_noop(self) -> None:
+        result = select_evaluators(
+            _HTTP_AGENT,
+            _shape(context=True),
+            override_mappings=None,
+        )
+        preset = [p for p in result if p.name == "GroundednessEvaluator"][0]
+        assert preset.input_mapping["context"] == "$context"
+
 
 # ---------------------------------------------------------------------------
 # merge_thresholds

agentops_accelerator-0.5.2/tests/unit/test_step_summary.py

@@ -0,0 +1,72 @@
+"""Unit tests for the GitHub Actions step-summary helpers."""
+
+from __future__ import annotations
+
+from agentops.core import step_summary
+
+
+def test_is_active_reflects_env(monkeypatch, tmp_path):
+    monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False)
+    assert step_summary.is_active() is False
+
+    summary = tmp_path / "summary.md"
+    monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+    assert step_summary.is_active() is True
+
+
+def test_append_step_summary_noop_without_env(monkeypatch):
+    monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False)
+    assert step_summary.append_step_summary("# hello") is False
+
+
+def test_append_step_summary_writes_and_appends(monkeypatch, tmp_path):
+    summary = tmp_path / "summary.md"
+    monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+
+    assert step_summary.append_step_summary("# first") is True
+    assert step_summary.append_step_summary("# second\n") is True
+
+    content = summary.read_text(encoding="utf-8")
+    assert "# first" in content
+    assert "# second" in content
+    # Each block ends with its own newline plus a separating blank line.
+    assert content.index("# first") < content.index("# second")
+
+
+def test_append_report_file_appends_contents(monkeypatch, tmp_path):
+    summary = tmp_path / "summary.md"
+    monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+    report = tmp_path / "report.md"
+    report.write_text("# AgentOps Evaluation Report\n\nPASS\n", encoding="utf-8")
+
+    assert step_summary.append_report_file(report) is True
+    content = summary.read_text(encoding="utf-8")
+    assert "AgentOps Evaluation Report" in content
+    assert "PASS" in content
+
+
+def test_append_report_file_with_heading(monkeypatch, tmp_path):
+    summary = tmp_path / "summary.md"
+    monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+    report = tmp_path / "report.md"
+    report.write_text("body\n", encoding="utf-8")
+
+    assert step_summary.append_report_file(report, heading="## Eval") is True
+    content = summary.read_text(encoding="utf-8")
+    assert "## Eval" in content
+    assert "body" in content
+
+
+def test_append_report_file_missing_file_is_false(monkeypatch, tmp_path):
+    summary = tmp_path / "summary.md"
+    monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+
+    assert step_summary.append_report_file(tmp_path / "missing.md") is False
+    assert summary.exists() is False or summary.read_text(encoding="utf-8") == ""
+
+
+def test_append_report_file_noop_without_env(monkeypatch, tmp_path):
+    monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False)
+    report = tmp_path / "report.md"
+    report.write_text("body\n", encoding="utf-8")
+    assert step_summary.append_report_file(report) is False