agentops-accelerator 0.8.2__tar.gz → 0.8.3__tar.gz

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/.claude-plugin/marketplace.json

@@ -13,7 +13,7 @@
       "name": "agentops-accelerator",
       "source": "../../plugins/agentops",
       "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.",
-      "version": "0.8.2",
+      "version": "0.8.3",
       "keywords": [
         "agentops",
         "evaluation",

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/.github/plugin/marketplace.json

@@ -13,7 +13,7 @@
       "name": "agentops-accelerator",
       "source": "../../plugins/agentops",
       "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.",
-      "version": "0.8.2",
+      "version": "0.8.3",
       "keywords": [
         "agentops",
         "evaluation",

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/.github/workflows/ci.yml

@@ -181,20 +181,24 @@ jobs:
 
       - name: Install from TestPyPI
         run: |
-          for i in 1 2 3 4 5; do
-            echo "Attempt $i: installing agentops-accelerator==${{ steps.version.outputs.version }}"
-            if pip install \
+          # TestPyPI serves its simple index through a CDN, so a freshly
+          # uploaded release is not immediately resolvable even after the
+          # upload returns 200 OK. Allow ~6 minutes for it to propagate.
+          ATTEMPTS=12
+          for i in $(seq 1 "$ATTEMPTS"); do
+            echo "Attempt $i/$ATTEMPTS: installing agentops-accelerator==${{ steps.version.outputs.version }}"
+            if pip install --no-cache-dir \
               "agentops-accelerator==${{ steps.version.outputs.version }}" \
               --index-url https://test.pypi.org/simple/ \
               --extra-index-url https://pypi.org/simple/; then
               exit 0
             fi
-            if [ "$i" -lt 5 ]; then
+            if [ "$i" -lt "$ATTEMPTS" ]; then
               echo "Not available yet, waiting 30s..."
               sleep 30
             fi
           done
-          echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after 5 attempts."
+          echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after $ATTEMPTS attempts (~6 min)."
           exit 1
 
       - name: Smoke test

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/.github/workflows/release.yml

@@ -124,20 +124,24 @@ jobs:
 
       - name: Install from TestPyPI
         run: |
-          for i in 1 2 3 4 5; do
-            echo "Attempt $i: installing agentops-accelerator==${{ steps.version.outputs.version }}"
-            if pip install \
+          # TestPyPI serves its simple index through a CDN, so a freshly
+          # uploaded release is not immediately resolvable even after the
+          # upload returns 200 OK. Allow ~6 minutes for it to propagate.
+          ATTEMPTS=12
+          for i in $(seq 1 "$ATTEMPTS"); do
+            echo "Attempt $i/$ATTEMPTS: installing agentops-accelerator==${{ steps.version.outputs.version }}"
+            if pip install --no-cache-dir \
               "agentops-accelerator==${{ steps.version.outputs.version }}" \
               --index-url https://test.pypi.org/simple/ \
               --extra-index-url https://pypi.org/simple/; then
               exit 0
             fi
-            if [ "$i" -lt 5 ]; then
+            if [ "$i" -lt "$ATTEMPTS" ]; then
               echo "Not available yet, waiting 30s..."
               sleep 30
             fi
           done
-          echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after 5 attempts."
+          echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after $ATTEMPTS attempts (~6 min)."
           exit 1
 
       - name: Smoke test — version and help

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/.github/workflows/staging.yml

@@ -98,20 +98,24 @@ jobs:
 
       - name: Install from TestPyPI
         run: |
-          for i in 1 2 3 4 5; do
-            echo "Attempt $i: installing agentops-accelerator==${{ steps.version.outputs.version }}"
-            if pip install \
+          # TestPyPI serves its simple index through a CDN, so a freshly
+          # uploaded release is not immediately resolvable even after the
+          # upload returns 200 OK. Allow ~6 minutes for it to propagate.
+          ATTEMPTS=12
+          for i in $(seq 1 "$ATTEMPTS"); do
+            echo "Attempt $i/$ATTEMPTS: installing agentops-accelerator==${{ steps.version.outputs.version }}"
+            if pip install --no-cache-dir \
               "agentops-accelerator==${{ steps.version.outputs.version }}" \
               --index-url https://test.pypi.org/simple/ \
               --extra-index-url https://pypi.org/simple/; then
               exit 0
             fi
-            if [ "$i" -lt 5 ]; then
+            if [ "$i" -lt "$ATTEMPTS" ]; then
               echo "Not available yet, waiting 30s..."
               sleep 30
             fi
           done
-          echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after 5 attempts."
+          echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after $ATTEMPTS attempts (~6 min)."
           exit 1
 
       - name: Smoke test — version and help

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/CHANGELOG.md

@@ -5,6 +5,49 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres
 
 ## [Unreleased]
 
+## [0.8.3] - 2026-08-07
+
+### Fixed
+- **`execution: cloud` now works with a Foundry hosted agent.** Setting
+  `execution: cloud` on a hosted agent endpoint failed with
+  `execution: cloud only supports Foundry prompt agents`, so hosted-agent users
+  had no way to run a server-side evaluation and see the results in the New
+  Foundry Evaluations panel. Their only option was `execution: local`, which
+  keeps everything on the client and publishes nothing.
+
+  This was an artificial restriction, not an API limit. The Foundry Evals API
+  identifies its target with `{"type": "azure_ai_agent", "name": ..., "version":
+  ...}`, and a hosted agent URL already carries both values in its path
+  (`/agents/<name>/versions/<version>`). The problem was that `classify_agent()`
+  never extracted them: it returned a `foundry_hosted` target with `name` and
+  `version` left as `None`, so the cloud path had nothing to send and rejected
+  the target up front.
+
+  `classify_agent()` now parses the agent name and version out of a hosted
+  endpoint URL, tolerating trailing path segments (`/responses`), query strings,
+  and fragments. The guards in `pipeline/cloud_runner.py` and
+  `pipeline/orchestrator.py` accept `foundry_hosted` alongside `foundry_prompt`.
+  A hosted URL with no `/versions/<version>` segment is still rejected, because
+  there is no version to pin the run to, but the error now names the missing
+  segment instead of rejecting the whole target kind.
+
+  `execution: local` remains the default for every target kind, so this change
+  adds an option without altering existing behavior.
+
+- **CI: `Install from TestPyPI` no longer fails on a healthy build.** After a
+  dev, staging, or release build uploaded a distribution to TestPyPI and
+  received `200 OK`, the verification step immediately tried to install it and
+  gave up after 5 attempts spaced 30 s apart (~2 min). TestPyPI serves its
+  simple index through a CDN, so propagation regularly takes longer than that,
+  and the job failed with
+  `No matching distribution found for agentops-accelerator==<version>` for a
+  package that had in fact been published successfully. The retry window is now
+  12 attempts (~6 min), and `pip install` runs with `--no-cache-dir` so a
+  negative index response from an earlier attempt is not reused within the same
+  job. Applied to `ci.yml` (`verify-dev`), `staging.yml`, and `release.yml` —
+  in `release.yml` this step gates publication to PyPI, so a false negative
+  there blocked the release entirely.
+
 ## [0.8.2] - 2026-08-07
 
 ### Fixed

{agentops_accelerator-0.8.2/src/agentops_accelerator.egg-info → agentops_accelerator-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentops-accelerator
-Version: 0.8.2
+Version: 0.8.3
 Summary: Release readiness gates and evidence for Microsoft Foundry agents
 License: MIT License
         

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/docs/how-it-works.md

@@ -335,7 +335,7 @@ That's a complete config. AgentOps:
 | `auth_header_env` | no | Env var name holding a Bearer token. |
 | `evaluators` | no | Escape-hatch list of evaluator names that overrides auto-selection. |
 | `publish` | no | Boolean. With `execution: local`, `true` uploads local metrics to Classic Foundry. With `execution: cloud`, publishing is implicit. |
-| `execution` | no | `local` (default) runs through AgentOps locally. `cloud` runs a Foundry prompt agent server-side through the OpenAI Evals API. |
+| `execution` | no | `local` (default) runs through AgentOps locally. `cloud` runs a Foundry agent server-side through the OpenAI Evals API — either a prompt agent (`name:version`) or a hosted agent URL containing `/agents/<name>/versions/<version>`. |
 | `project_endpoint` | no | Foundry project URL used by Foundry invocation and publishing. Falls back to `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`. |
 | `dataset_sync` | no | Cloud-evaluation dataset policy: `auto`, `foundry`, or `inline`. |
 
@@ -429,12 +429,27 @@ thresholds:
 
 ```yaml
 version: 1
-agent: my-rag:3                   # name:version is required for cloud mode
+agent: my-rag:3                   # prompt agent
 dataset: .agentops/data/qa.jsonl
 execution: cloud
 # project_endpoint: "https://<resource>.services.ai.azure.com/api/projects/<p>"
 ```
 
+A hosted agent works the same way, as long as the URL carries the agent
+name and version. That is the pair Foundry needs to resolve the target:
+
+```yaml
+version: 1
+agent: https://<resource>.services.ai.azure.com/api/projects/<p>/agents/helpdeskbot/versions/11
+dataset: .agentops/data/qa.jsonl
+protocol: responses
+execution: cloud
+```
+
+A hosted URL without a `/versions/<version>` segment cannot be cloud-executed,
+because there is no version to pin the run to. Add the version to the URL, or
+keep `execution: local` for that target.
+
 ## Datasets
 
 A dataset is a plain JSONL file. One row per line. No companion YAML.
@@ -567,7 +582,7 @@ link.
 |---|---|---|---|
 | `execution: local`, `publish: false` | AgentOps invokes target and evaluators locally | None; local artifacts only | Any target |
 | `execution: local`, `publish: true` | AgentOps local run, then metric upload | Classic Foundry Evaluations panel | Any target |
-| `execution: cloud` | Foundry runs agent + evaluators server-side through the OpenAI Evals API | New Foundry Evaluations panel; publish is implicit | Foundry Prompt Agent (`name:version`) |
+| `execution: cloud` | Foundry runs agent + evaluators server-side through the OpenAI Evals API | New Foundry Evaluations panel; publish is implicit | Foundry Prompt Agent (`name:version`) or Foundry Hosted Agent URL with `/agents/<name>/versions/<version>` |
 
 Foundry-visible modes:
 

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/docs/tutorial-end-to-end.md

@@ -444,6 +444,10 @@ Foundry through `agentops eval run`, so AgentOps can enforce thresholds and writ
 repo-side evidence. AgentOps keeps the local path for hosted endpoints, models,
 unsupported evaluator mappings, and fallback cases.
 
+The hosted-endpoint row above is the default, not a hard limit. A hosted agent
+URL that includes `/agents/<name>/versions/<version>` can also run cloud eval by
+setting `execution: cloud` in `agentops.yaml`.
+
 When the quality gate uses a task-specific rubric, keep it as an advanced
 Foundry / azd hardening step: first confirm the rubric evaluator exists in the
 Foundry project and that an azd run emits stable metric names for its scores.

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/docs/tutorial-hosted-agent-quickstart.md

@@ -586,9 +586,16 @@ Recommendation
   Copilot skills  installed - available for workflow adaptation handoff
 ```
 
-That is expected. Foundry prompt agents can use AgentOps cloud eval in Foundry;
-hosted endpoints use AgentOps local eval so the repo can invoke the endpoint,
-normalize results, apply thresholds, and keep a stable `results.json` contract.
+That is expected. The default for a hosted endpoint is AgentOps local eval, so
+the repo can invoke the endpoint, normalize results, apply thresholds, and keep
+a stable `results.json` contract.
+
+You can opt into server-side execution instead. Set `execution: cloud` in
+`agentops.yaml` and Foundry runs the agent and the evaluators, with the run
+appearing in the New Foundry Evaluations panel. This requires the hosted URL to
+include `/agents/<name>/versions/<version>`, since that pair is how Foundry
+identifies the target. The trade-off is that latency becomes Foundry-side
+rather than client-measured, and custom evaluators are skipped.
 
 ## 8. Run a local eval
 

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/plugins/agentops/package.json

@@ -2,7 +2,7 @@
   "name": "agentops-accelerator",
   "displayName": "AgentOps Accelerator — Skills for GitHub Copilot",
   "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.",
-  "version": "0.8.2",
+  "version": "0.8.3",
   "publisher": "AgentOpsAccelerator",
   "icon": "icon.png",
   "license": "MIT",

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/plugins/agentops/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "agentops-accelerator",
   "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.",
-  "version": "0.8.2",
+  "version": "0.8.3",
   "author": {
     "name": "AgentOps Accelerator",
     "url": "https://github.com/Azure/agentops"

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/src/agentops/core/agentops_config.py

@@ -26,6 +26,7 @@ kinds - ``foundry_prompt``, ``foundry_hosted``, ``http_json``, or
 
 from __future__ import annotations
 
+import re
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, List, Literal, Optional
@@ -64,7 +65,9 @@ TargetKind = Literal[
 #:   evaluators locally. Results are the canonical record.
 #: - ``cloud``: Foundry runs the agent and evaluators server-side via the
 #:   OpenAI Evals API. Use this when you want the run to appear in the
-#:   New Foundry Evaluations panel as the primary record.
+#:   New Foundry Evaluations panel as the primary record. Requires a Foundry
+#:   agent target: either ``name:version`` (prompt agent) or a hosted agent
+#:   endpoint URL containing ``/agents/<name>/versions/<version>``.
 #: - ``azd``: Azure Developer CLI runs ``azd ai agent eval``. This is a hard
 #:   dependency: if azd cannot run, AgentOps fails gracefully rather than
 #:   switching engines.
@@ -1017,7 +1020,9 @@ class AgentOpsConfig(BaseModel):
             "- local (default): AgentOps invokes the agent row-by-row locally.\n"
             "- cloud: Foundry runs the agent and evaluators server-side, and "
             "the run is implicitly published to the New Foundry Evaluations "
-            "panel (publish defaults to true).\n"
+            "panel (publish defaults to true). Requires a Foundry agent "
+            "target: 'name:version', or a hosted agent endpoint URL "
+            "containing '/agents/<name>/versions/<version>'.\n"
             "- azd: azd ai agent eval runs the Foundry-native evaluation. If "
             "azd cannot run, AgentOps fails gracefully and never switches "
             "engines implicitly.\n"
@@ -1202,8 +1207,12 @@ class TargetResolution:
     protocol: Optional[Protocol]
     raw: str
     #: For ``foundry_prompt``: the agent name (left of the colon).
+    #: For ``foundry_hosted``: parsed from the ``/agents/<name>/`` URL
+    #: segment when present (``None`` when the URL does not carry one).
     name: Optional[str] = None
     #: For ``foundry_prompt``: the version (right of the colon).
+    #: For ``foundry_hosted``: parsed from the ``/versions/<version>/``
+    #: URL segment when present.
     version: Optional[str] = None
     #: For ``foundry_hosted`` / ``http_json``: the target URL.
     url: Optional[str] = None
@@ -1230,6 +1239,38 @@ def _looks_like_foundry_url(url: str) -> bool:
     return any(domain in lowered for domain in foundry_domains)
 
 
+#: Matches the ``/agents/<name>/versions/<version>`` segment pair that Foundry
+#: hosted agent endpoints carry. Tolerates trailing path segments (e.g.
+#: ``/responses``), query strings, and fragments.
+_HOSTED_AGENT_REFERENCE_RE = re.compile(
+    r"/agents/(?P<name>[^/?#]+)/versions/(?P<version>[^/?#]+)",
+    re.IGNORECASE,
+)
+
+
+def _parse_hosted_agent_reference(url: str) -> tuple[Optional[str], Optional[str]]:
+    """Extract ``(name, version)`` from a Foundry hosted agent URL.
+
+    A hosted endpoint such as::
+
+        https://acct.services.ai.azure.com/api/projects/p/agents/bot/versions/11
+
+    references the very same Foundry agent that a ``foundry_prompt`` target
+    spells as ``bot:11``. Server-side evaluation (``execution: cloud``) needs
+    that ``name`` / ``version`` pair to build the ``azure_ai_agent`` target,
+    so parse it out instead of forcing users to duplicate the reference.
+
+    Returns ``(None, None)`` when the URL carries no such segment pair; the
+    caller decides whether that is fatal.
+    """
+    match = _HOSTED_AGENT_REFERENCE_RE.search(url)
+    if match is None:
+        return None, None
+    name = match.group("name").strip()
+    version = match.group("version").strip()
+    return (name or None), (version or None)
+
+
 def classify_agent(
     agent: str,
     protocol: Optional[Protocol] = None,
@@ -1276,11 +1317,14 @@ def classify_agent(
                     "Foundry hosted endpoints accept only protocol "
                     "'responses' or 'invocations'"
                 )
+            hosted_name, hosted_version = _parse_hosted_agent_reference(raw)
             return TargetResolution(
                 kind="foundry_hosted",
                 protocol=resolved_protocol,
                 raw=raw,
                 url=raw,
+                name=hosted_name,
+                version=hosted_version,
             )
 
         resolved_protocol = protocol or "http-json"

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/src/agentops/pipeline/cloud_runner.py

@@ -28,8 +28,10 @@ point of view.
 
 Limitations (documented in the YAML schema docstring as well):
 
-* Only ``foundry_prompt`` agents (``name:version``) are supported. HTTP
-  endpoints, local adapters, and direct model deployments are rejected.
+* Foundry agents only: a ``foundry_prompt`` target (``name:version``) or a
+  ``foundry_hosted`` endpoint URL carrying ``/agents/<name>/versions/<v>``.
+  Plain HTTP endpoints, local adapters, and direct model deployments are
+  rejected - Foundry has to be able to resolve the agent itself.
 * Only builtin evaluators that map cleanly onto ``azure_ai_evaluator``
   testing criteria are supported. Custom evaluators are skipped with a
   warning.
@@ -184,10 +186,11 @@ def run_on_foundry_cloud(
     """
     progress = progress or (lambda _msg: None)
 
-    if result.target.kind != "foundry_prompt":
+    if result.target.kind not in {"foundry_prompt", "foundry_hosted"}:
         raise ValueError(
-            "publish: foundry_cloud only supports Foundry agents declared "
-            "as 'name:version' (foundry_prompt targets). Got "
+            "execution: cloud only supports Foundry agents - either a "
+            "'name:version' prompt agent or a hosted agent endpoint URL "
+            "containing '/agents/<name>/versions/<version>'. Got "
             f"target.kind={result.target.kind!r}."
         )
     if not dataset_path.exists():
@@ -196,6 +199,15 @@ def run_on_foundry_cloud(
     agent_name = result.target.name
     agent_version = result.target.version
     if not agent_name or not agent_version:
+        if result.target.kind == "foundry_hosted":
+            raise ValueError(
+                "execution: cloud could not derive an agent name/version "
+                f"from the hosted endpoint {result.target.raw!r}. Foundry "
+                "runs the agent server-side and needs an explicit agent "
+                "reference. Use a URL that contains "
+                "'/agents/<name>/versions/<version>', or set "
+                "'agent: <name>:<version>' in agentops.yaml."
+            )
         raise ValueError(
             "Cloud publish requires a fully qualified 'name:version' agent "
             f"reference; got name={agent_name!r} version={agent_version!r}"

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/src/agentops/pipeline/orchestrator.py

@@ -272,10 +272,20 @@ def _run_evaluation_cloud(
         options.agent_override or config.agent,
         config.protocol,
     )
-    if target.kind != "foundry_prompt":
+    if target.kind not in {"foundry_prompt", "foundry_hosted"}:
+        raise ValueError(
+            "execution: cloud only supports Foundry agents - either a "
+            "'name:version' prompt agent or a hosted agent endpoint URL "
+            f"containing '/agents/<name>/versions/<version>'; got "
+            f"target.kind={target.kind!r}."
+        )
+    if not target.name or not target.version:
         raise ValueError(
-            "execution: cloud only supports Foundry prompt agents "
-            f"('name:version'); got target.kind={target.kind!r}."
+            "execution: cloud could not derive an agent name/version from "
+            f"{target.raw!r}. Foundry runs the agent server-side and needs "
+            "an explicit agent reference. Use a hosted endpoint URL that "
+            "contains '/agents/<name>/versions/<version>', or set "
+            "'agent: <name>:<version>' in agentops.yaml."
         )
 
     dataset_path = options.dataset_override or _resolve_dataset_path(config, options)

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3/src/agentops_accelerator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentops-accelerator
-Version: 0.8.2
+Version: 0.8.3
 Summary: Release readiness gates and evidence for Microsoft Foundry agents
 License: MIT License
         

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/tests/unit/test_agentops_config.py

@@ -55,6 +55,46 @@ class TestClassifyAgent:
         assert result.protocol == "responses"
         assert result.url == url
 
+    def test_foundry_hosted_parses_agent_name_and_version(self) -> None:
+        """A hosted endpoint references the same agent a prompt target
+        spells as 'name:version'; cloud execution needs that pair."""
+        url = (
+            "https://acct.services.ai.azure.com/api/projects/p"
+            "/agents/helpdeskbot/versions/11"
+        )
+        result = classify_agent(url)
+        assert result.kind == "foundry_hosted"
+        assert result.name == "helpdeskbot"
+        assert result.version == "11"
+        assert result.url == url
+
+    def test_foundry_hosted_parses_reference_with_trailing_segment(self) -> None:
+        url = (
+            "https://acct.services.ai.azure.com/api/projects/p"
+            "/agents/helpdeskbot/versions/11/responses"
+        )
+        result = classify_agent(url)
+        assert result.name == "helpdeskbot"
+        assert result.version == "11"
+
+    def test_foundry_hosted_parses_reference_with_query_string(self) -> None:
+        url = (
+            "https://acct.services.ai.azure.com/api/projects/p"
+            "/agents/helpdeskbot/versions/11?api-version=2025-05-01"
+        )
+        result = classify_agent(url)
+        assert result.name == "helpdeskbot"
+        assert result.version == "11"
+
+    def test_foundry_hosted_without_version_has_no_reference(self) -> None:
+        """No /versions/ segment means no resolvable agent reference. The
+        target still classifies (local execution works); only cloud cares."""
+        url = "https://my-project.services.ai.azure.com/agents/foo"
+        result = classify_agent(url)
+        assert result.kind == "foundry_hosted"
+        assert result.name is None
+        assert result.version is None
+
     def test_foundry_hosted_invocations(self) -> None:
         url = "https://my-project.services.ai.azure.com/agents/foo"
         result = classify_agent(url, protocol="invocations")

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/tests/unit/test_cloud_runner.py

@@ -185,7 +185,49 @@ def test_build_item_schema_empty_file_falls_back(tmp_path: Path):
 
 def test_publish_rejects_non_foundry_targets(dataset_file: Path):
     result = _make_result(kind="http_json")
-    with pytest.raises(ValueError, match="foundry_cloud only supports"):
+    with pytest.raises(ValueError, match="only supports Foundry agents"):
+        cloud_runner.run_on_foundry_cloud(
+            result,
+            dataset_path=dataset_file,
+            project_endpoint="https://x.example/api/projects/p",
+        )
+
+
+def test_publish_accepts_hosted_agent_with_resolved_reference(dataset_file: Path):
+    """A hosted agent endpoint carries the same name/version a prompt agent
+    spells as 'name:version', so cloud execution must accept it."""
+    result = _make_result(kind="foundry_hosted")
+    result.target = TargetInfo(
+        kind="foundry_hosted",
+        raw=(
+            "https://acct.services.ai.azure.com/api/projects/p"
+            "/agents/support-bot/versions/1"
+        ),
+        name="support-bot",
+        version="1",
+    )
+    # Guard passes; failure comes later from the absent azure SDK/network,
+    # never from the target-kind check itself.
+    with pytest.raises(Exception) as excinfo:  # noqa: PT011
+        cloud_runner.run_on_foundry_cloud(
+            result,
+            dataset_path=dataset_file,
+            project_endpoint="https://x.example/api/projects/p",
+        )
+    assert "only supports Foundry agents" not in str(excinfo.value)
+
+
+def test_publish_rejects_hosted_agent_without_version(dataset_file: Path):
+    """A hosted URL with no /versions/ segment cannot be resolved to an
+    azure_ai_agent target; the error must say how to fix it."""
+    result = _make_result(kind="foundry_hosted")
+    result.target = TargetInfo(
+        kind="foundry_hosted",
+        raw="https://acct.services.ai.azure.com/api/projects/p/agents/bot",
+        name=None,
+        version=None,
+    )
+    with pytest.raises(ValueError, match="could not derive an agent name"):
         cloud_runner.run_on_foundry_cloud(
             result,
             dataset_path=dataset_file,

{agentops_accelerator-0.8.2 → agentops_accelerator-0.8.3}/tests/unit/test_pipeline_publisher.py

@@ -368,13 +368,13 @@ def test_run_evaluation_cloud_requires_project_endpoint(tmp_path: Path, monkeypa
 
 
 def test_run_evaluation_cloud_rejects_non_foundry_target(tmp_path: Path):
-    """execution: cloud only works for Foundry prompt agents."""
+    """execution: cloud only works for Foundry agents."""
     from agentops.core.agentops_config import AgentOpsConfig
     from agentops.pipeline import orchestrator
 
-    # Construct config bypassing validators: only Foundry prompt agents
-    # can be cloud-executed, so build a model_direct target to exercise
-    # the orchestrator-level guard.
+    # Construct config bypassing validators: only Foundry agents can be
+    # cloud-executed, so build a model_direct target to exercise the
+    # orchestrator-level guard.
     dataset_path = tmp_path / "dataset.jsonl"
     dataset_path.write_text('{"input": "hi"}\n', encoding="utf-8")
 
@@ -392,5 +392,33 @@ def test_run_evaluation_cloud_rejects_non_foundry_target(tmp_path: Path):
         config_path=tmp_path / "agentops.yaml",
         output_dir=output_dir,
     )
-    with pytest.raises(ValueError, match="Foundry prompt agents"):
+    with pytest.raises(ValueError, match="only supports Foundry agents"):
+        orchestrator._run_evaluation_cloud(config, options=options)
+
+
+def test_run_evaluation_cloud_rejects_hosted_agent_without_version(tmp_path: Path):
+    """A hosted URL without /versions/ cannot name an azure_ai_agent target,
+    so the guard must reject it with an actionable message instead of
+    failing deep inside the Foundry SDK call."""
+    from agentops.core.agentops_config import AgentOpsConfig
+    from agentops.pipeline import orchestrator
+
+    dataset_path = tmp_path / "dataset.jsonl"
+    dataset_path.write_text('{"input": "hi"}\n', encoding="utf-8")
+
+    config = AgentOpsConfig(
+        version=1,
+        agent="https://acct.services.ai.azure.com/api/projects/p/agents/bot",
+        dataset=dataset_path,
+        execution="cloud",
+        project_endpoint="https://x.example/api/projects/p",
+    )
+    output_dir = tmp_path / "out"
+    output_dir.mkdir()
+
+    options = orchestrator.RunOptions(
+        config_path=tmp_path / "agentops.yaml",
+        output_dir=output_dir,
+    )
+    with pytest.raises(ValueError, match="could not derive an agent name"):
         orchestrator._run_evaluation_cloud(config, options=options)