applied-cli 0.6.1__tar.gz → 0.6.2__tar.gz

{applied_cli-0.6.1 → applied_cli-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: applied-cli
-Version: 0.6.1
+Version: 0.6.2
 Summary: CLI and shared client library for Applied Labs AI support agents
 Author: Applied Labs
 License-Expression: MIT

{applied_cli-0.6.1 → applied_cli-0.6.2}/applied_cli/cli.py

@@ -1640,6 +1640,38 @@ def benchmark_clone(
     typer.echo(result)
 
 
+@app.command("benchmark-delete")
+def benchmark_delete(
+    id: str = typer.Argument(..., help="Benchmark ID"),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Acknowledge and proceed with the cascade delete of scenarios/runs",
+    ),
+    detach_scenarios: bool = typer.Option(
+        False,
+        "--detach-scenarios",
+        help="Preserve scenarios by unlinking them before deleting the benchmark",
+    ),
+    shop_id: str = typer.Option(None, "--shop-id", help="Override shop ID"),
+    format: str = typer.Option(
+        "text", "--format", "-f", help="Output format: text or json"
+    ),
+) -> None:
+    """Delete a benchmark; refuses to silently wipe scenarios (see --force / --detach-scenarios)."""
+    client = get_client(shop_id=shop_id)
+    result = asyncio.run(
+        tools.benchmark_delete(
+            client,
+            benchmark_id=id,
+            force=force,
+            detach_scenarios=detach_scenarios,
+            output_format=format,
+        )
+    )
+    typer.echo(result)
+
+
 @app.command()
 def scenarios(
     benchmark_id: str = typer.Option(

{applied_cli-0.6.1 → applied_cli-0.6.2}/applied_cli/tools.py

@@ -5409,23 +5409,107 @@ async def benchmark_create(
 async def benchmark_delete(
     client: AppliedClient,
     benchmark_id: str,
+    *,
+    force: bool = False,
+    detach_scenarios: bool = False,
+    output_format: str = "text",
 ) -> str:
     """
-    Delete a benchmark.
+    Delete a benchmark, with a guardrail against silently wiping scenarios.
+
+    On the platform, deleting a benchmark cascades and permanently deletes every
+    scenario and scenario run beneath it. To prevent accidental data loss this
+    refuses to delete a benchmark that still has scenarios unless you either:
+
+    - pass detach_scenarios=True to first unlink the scenarios (they survive
+      under their agent, untagged) and then delete the now-empty benchmark, or
+    - pass force=True to acknowledge and proceed with the cascade delete.
+
+    An empty benchmark is deleted directly.
 
     Args:
         client: Authenticated AppliedClient
         benchmark_id: The benchmark UUID
+        force: Acknowledge and proceed with the cascade delete of scenarios/runs
+        detach_scenarios: Preserve scenarios by unlinking them before deleting
+        output_format: 'text' (default) or 'json'
 
     Returns:
-        Success message
+        Result message (or refusal with the impact that would occur).
     """
     try:
-        await client.delete_benchmark(benchmark_id)
+        scenarios = await client.list_scenarios(
+            benchmark_id=benchmark_id, fetch_all=True
+        )
     except AppliedAPIError as e:
         return _format_error(e)
 
-    return f"Benchmark {benchmark_id} deleted successfully."
+    scenario_count = len(scenarios)
+    run_count = sum(int(s.get("run_count") or 0) for s in scenarios)
+    result: dict[str, Any] = {
+        "benchmark_id": benchmark_id,
+        "scenario_count": scenario_count,
+        "run_count": run_count,
+        "deleted": False,
+        "refused": False,
+        "detached_scenarios": 0,
+    }
+
+    # Preserve scenarios: unlink each from this benchmark, then delete it empty.
+    if scenario_count and detach_scenarios:
+        detached = 0
+        try:
+            for scenario in scenarios:
+                detail = await client.get_scenario(str(scenario["id"]))
+                remaining = [
+                    str(b["id"])
+                    for b in (detail.get("benchmarks") or [])
+                    if b.get("id") and str(b["id"]) != benchmark_id
+                ]
+                await client.update_scenario(
+                    str(scenario["id"]), benchmark_ids=remaining
+                )
+                detached += 1
+            await client.delete_benchmark(benchmark_id)
+        except AppliedAPIError as e:
+            result["detached_scenarios"] = detached
+            result["message"] = _format_error(e)
+            return to_json(result) if output_format == "json" else result["message"]
+        result["deleted"] = True
+        result["detached_scenarios"] = detached
+        result["message"] = (
+            f"Detached {detached} scenario(s) and deleted empty benchmark "
+            f"{benchmark_id}. Scenarios preserved under their agent."
+        )
+        return to_json(result) if output_format == "json" else result["message"]
+
+    # Refuse to cascade-delete a non-empty benchmark unless explicitly forced.
+    if scenario_count and not force:
+        result["refused"] = True
+        result["message"] = (
+            f"Refusing to delete benchmark {benchmark_id}: it has "
+            f"{scenario_count} scenario(s) and {run_count} run(s) that would be "
+            f"PERMANENTLY deleted by the cascade. Re-run with detach_scenarios=True "
+            f"to preserve the scenarios, or force=True to delete them anyway."
+        )
+        return to_json(result) if output_format == "json" else result["message"]
+
+    try:
+        await client.delete_benchmark(benchmark_id)
+    except AppliedAPIError as e:
+        result["message"] = _format_error(e)
+        return to_json(result) if output_format == "json" else result["message"]
+
+    result["deleted"] = True
+    result["message"] = (
+        f"Benchmark {benchmark_id} deleted successfully"
+        + (
+            f" (cascaded {scenario_count} scenario(s), {run_count} run(s))."
+            if scenario_count
+            else "."
+        )
+    )
+    return to_json(result) if output_format == "json" else result["message"]
 
 
 def _scenario_input_conversation_id(scenario: dict) -> str | None:

{applied_cli-0.6.1 → applied_cli-0.6.2}/applied_cli/v2/scenarios.py

@@ -48,6 +48,8 @@ class BenchmarksCreateInput(StrictInput):
 
 class BenchmarksDeleteInput(StrictInput):
     benchmark_id: str
+    force: bool = False
+    detach_scenarios: bool = False
 
 
 class BenchmarksCloneInput(StrictInput):
@@ -494,14 +496,33 @@ async def benchmarks_delete_handler(
     client: AppliedClient,
     params: BenchmarksDeleteInput,
 ) -> ToolResult[Any]:
+    from applied_cli import tools as legacy_tools
+
+    raw = await legacy_tools.benchmark_delete(
+        client,
+        benchmark_id=params.benchmark_id,
+        force=params.force,
+        detach_scenarios=params.detach_scenarios,
+        output_format="json",
+    )
     try:
-        await client.delete_benchmark(params.benchmark_id)
-    except AppliedAPIError as exc:
-        return _api_error_result(exc)
+        data = json.loads(raw)
+    except (json.JSONDecodeError, TypeError):
+        return ToolResult(data={"message": raw}, summary=str(raw))
+
+    if data.get("refused"):
+        return ToolResult(
+            data=data,
+            summary=data.get("message", "Refused to delete a non-empty benchmark."),
+            next_actions=[
+                "Re-run with detach_scenarios=true to preserve the scenarios.",
+                "Or re-run with force=true to delete the scenarios and runs too.",
+            ],
+        )
 
     return ToolResult(
-        data={"benchmark_id": params.benchmark_id, "deleted": True},
-        summary=f"Deleted benchmark {params.benchmark_id}.",
+        data=data,
+        summary=data.get("message", f"Deleted benchmark {params.benchmark_id}."),
         next_actions=[
             "Use benchmarks_list to confirm the benchmark was removed.",
         ],
@@ -866,7 +887,11 @@ def scenario_specs() -> list[ToolSpec]:
         ToolSpec(
             name="benchmarks_delete",
             namespace="benchmarks",
-            description="Delete a benchmark and return a structured acknowledgement.",
+            description=(
+                "Delete a benchmark. Refuses to delete a benchmark that still "
+                "has scenarios (the platform cascade would permanently delete "
+                "them) unless detach_scenarios=true (preserve them) or force=true."
+            ),
             input_model=BenchmarksDeleteInput,
             output_model=None,
             handler=benchmarks_delete_handler,

{applied_cli-0.6.1 → applied_cli-0.6.2}/applied_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: applied-cli
-Version: 0.6.1
+Version: 0.6.2
 Summary: CLI and shared client library for Applied Labs AI support agents
 Author: Applied Labs
 License-Expression: MIT

{applied_cli-0.6.1 → applied_cli-0.6.2}/applied_cli.egg-info/SOURCES.txt

@@ -39,6 +39,7 @@ tests/test_agent_scoped_flows.py
 tests/test_audit_tools.py
 tests/test_auth_context.py
 tests/test_benchmark_clone.py
+tests/test_benchmark_delete_guardrail.py
 tests/test_benchmark_scenario_tools.py
 tests/test_cli.py
 tests/test_cli_v2.py

{applied_cli-0.6.1 → applied_cli-0.6.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "applied-cli"
-version = "0.6.1"
+version = "0.6.2"
 description = "CLI and shared client library for Applied Labs AI support agents"
 readme = "README.md"
 requires-python = ">=3.11"

applied_cli-0.6.2/tests/test_benchmark_delete_guardrail.py

@@ -0,0 +1,83 @@
+import json
+
+import pytest
+
+from applied_cli import tools
+
+
+class FakeDeleteClient:
+    def __init__(self, scenarios):
+        self._scenarios = scenarios
+        self.deleted = None
+        self.updates = []
+
+    async def list_scenarios(self, benchmark_id=None, fetch_all=True, **kwargs):
+        return list(self._scenarios)
+
+    async def get_scenario(self, scenario_id):
+        for s in self._scenarios:
+            if s["id"] == scenario_id:
+                return s
+        raise KeyError(scenario_id)
+
+    async def update_scenario(self, scenario_id, **updates):
+        self.updates.append({"id": scenario_id, **updates})
+        return {"id": scenario_id, **updates}
+
+    async def delete_benchmark(self, benchmark_id):
+        self.deleted = benchmark_id
+
+
+SCENARIOS = [
+    {"id": "s1", "run_count": 3, "benchmarks": [{"id": "bench-1"}]},
+    {"id": "s2", "run_count": 1, "benchmarks": [{"id": "bench-1"}, {"id": "bench-2"}]},
+]
+
+
+@pytest.mark.asyncio
+async def test_empty_benchmark_deletes_directly():
+    client = FakeDeleteClient(scenarios=[])
+    result = await tools.benchmark_delete(client, "bench-1", output_format="json")
+    data = json.loads(result)
+    assert data["deleted"] is True
+    assert data["refused"] is False
+    assert client.deleted == "bench-1"
+
+
+@pytest.mark.asyncio
+async def test_nonempty_benchmark_refuses_and_discloses_impact():
+    client = FakeDeleteClient(scenarios=SCENARIOS)
+    result = await tools.benchmark_delete(client, "bench-1")  # text, no force
+    assert "Refusing to delete" in result
+    assert "2 scenario(s)" in result
+    assert "4 run(s)" in result  # 3 + 1
+    assert client.deleted is None  # nothing wiped
+
+
+@pytest.mark.asyncio
+async def test_force_cascades_and_reports_counts():
+    client = FakeDeleteClient(scenarios=SCENARIOS)
+    result = await tools.benchmark_delete(
+        client, "bench-1", force=True, output_format="json"
+    )
+    data = json.loads(result)
+    assert data["deleted"] is True
+    assert data["scenario_count"] == 2
+    assert data["run_count"] == 4
+    assert client.deleted == "bench-1"
+
+
+@pytest.mark.asyncio
+async def test_detach_unlinks_then_deletes_empty_benchmark():
+    client = FakeDeleteClient(scenarios=SCENARIOS)
+    result = await tools.benchmark_delete(
+        client, "bench-1", detach_scenarios=True, output_format="json"
+    )
+    data = json.loads(result)
+    assert data["deleted"] is True
+    assert data["detached_scenarios"] == 2
+    # bench-1 removed from each scenario's links; bench-2 kept on s2.
+    by_id = {u["id"]: u["benchmark_ids"] for u in client.updates}
+    assert by_id["s1"] == []
+    assert by_id["s2"] == ["bench-2"]
+    assert client.deleted == "bench-1"

{applied_cli-0.6.1 → applied_cli-0.6.2}/tests/test_v2_scenarios.py

@@ -429,24 +429,46 @@ async def test_benchmarks_get_includes_structured_scenarios_without_legacy_wrapp
 
 
 @pytest.mark.asyncio
-async def test_benchmarks_delete_returns_structured_ack_without_legacy_wrapper(
-    monkeypatch,
-):
-    async def fail_legacy_wrapper(*args, **kwargs):
-        raise AssertionError("legacy benchmark_delete wrapper should not run")
-
-    monkeypatch.setattr("applied_cli.tools.benchmark_delete", fail_legacy_wrapper)
+async def test_benchmarks_delete_refuses_to_wipe_scenarios_without_force():
     client = FakeScenarioClient()
     spec = get_tool_catalog()["benchmarks_delete"]
 
     result = await spec.run(client, {"benchmark_id": "bench-1"})
 
     assert result.ok is True
-    assert result.summary == "Deleted benchmark bench-1."
-    assert result.data == {"benchmark_id": "bench-1", "deleted": True}
-    assert result.next_actions == [
-        "Use benchmarks_list to confirm the benchmark was removed.",
-    ]
+    assert result.data["refused"] is True
+    assert result.data["deleted"] is False
+    assert result.data["scenario_count"] >= 1
+    assert "Refusing to delete" in result.summary
+    # Guardrail must not have deleted the benchmark.
+    assert client.delete_benchmark_id is None
+
+
+@pytest.mark.asyncio
+async def test_benchmarks_delete_force_cascades():
+    client = FakeScenarioClient()
+    spec = get_tool_catalog()["benchmarks_delete"]
+
+    result = await spec.run(client, {"benchmark_id": "bench-1", "force": True})
+
+    assert result.data["deleted"] is True
+    assert client.delete_benchmark_id == "bench-1"
+
+
+@pytest.mark.asyncio
+async def test_benchmarks_delete_detach_preserves_scenarios():
+    client = FakeScenarioClient()
+    spec = get_tool_catalog()["benchmarks_delete"]
+
+    result = await spec.run(
+        client, {"benchmark_id": "bench-1", "detach_scenarios": True}
+    )
+
+    assert result.data["deleted"] is True
+    assert result.data["detached_scenarios"] >= 1
+    # Scenarios were unlinked (bench-1 removed from their benchmark list).
+    assert client.update_scenario_kwargs is not None
+    assert "bench-1" not in client.update_scenario_kwargs["updates"]["benchmark_ids"]
     assert client.delete_benchmark_id == "bench-1"