applied-cli 0.6.8__tar.gz → 0.6.10__tar.gz

{applied_cli-0.6.8 → applied_cli-0.6.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: applied-cli
-Version: 0.6.8
+Version: 0.6.10
 Summary: CLI and shared client library for Applied Labs AI support agents
 Author: Applied Labs
 License-Expression: MIT
@@ -97,6 +97,11 @@ applied benchmark-create --agent-id <agent_id> --name "Cancel Regression"
 applied scenario-create --input-conversation-id <conversation_id> --name "<name>" \
   --benchmark-id <benchmark_id>
 
+# Build a suite fast from several real conversations at once. Each scenario is
+# named from its conversation's title (or "<prefix> N" with --name-prefix).
+applied scenario-create-bulk --conversation-ids <id1>,<id2>,<id3> \
+  --benchmark-id <benchmark_id>
+
 # Port a suite to another agent (e.g. email -> chat). Cross-agent recreates the
 # scenarios under the destination agent; same-agent just tags them in.
 # Dry-run by default; add --apply to write.
@@ -116,6 +121,9 @@ applied scenario-bulk-cancel <job_id> --apply
 # Review pass/fail health (pass_status reflects the latest run per scenario)
 applied benchmark-results <benchmark_id> --format json
 
+# Portfolio go/no-go: pass rates across all of an agent's benchmarks at a glance
+applied benchmarks --agent-id <agent_id> --with-results --format json
+
 # Rate scenarios as you evaluate
 applied scenario-update <scenario_id> --pass-status pass --feedback "<note>"
 
@@ -167,6 +175,8 @@ conversations = await tools.conversation_query(
 | `benchmark_clone` | Copy all scenarios from one benchmark into another |
 | `benchmark_delete` | Delete a benchmark (guards against wiping scenarios) |
 | `benchmark_results` | Pass/fail/unrated tally and pass rate for a benchmark |
+| `benchmark_list` | List benchmarks (with per-benchmark pass rates via `with_results`) |
+| `scenario_create_bulk` | Build scenarios from several conversations at once |
 | `scenario_bulk_run` | Run scenarios (contact override + wait-to-completion) |
 | `scenario_bulk_cancel` | Cancel a stuck bulk run's queued/running scenario runs |
 

{applied_cli-0.6.8 → applied_cli-0.6.10}/README.md

@@ -71,6 +71,11 @@ applied benchmark-create --agent-id <agent_id> --name "Cancel Regression"
 applied scenario-create --input-conversation-id <conversation_id> --name "<name>" \
   --benchmark-id <benchmark_id>
 
+# Build a suite fast from several real conversations at once. Each scenario is
+# named from its conversation's title (or "<prefix> N" with --name-prefix).
+applied scenario-create-bulk --conversation-ids <id1>,<id2>,<id3> \
+  --benchmark-id <benchmark_id>
+
 # Port a suite to another agent (e.g. email -> chat). Cross-agent recreates the
 # scenarios under the destination agent; same-agent just tags them in.
 # Dry-run by default; add --apply to write.
@@ -90,6 +95,9 @@ applied scenario-bulk-cancel <job_id> --apply
 # Review pass/fail health (pass_status reflects the latest run per scenario)
 applied benchmark-results <benchmark_id> --format json
 
+# Portfolio go/no-go: pass rates across all of an agent's benchmarks at a glance
+applied benchmarks --agent-id <agent_id> --with-results --format json
+
 # Rate scenarios as you evaluate
 applied scenario-update <scenario_id> --pass-status pass --feedback "<note>"
 
@@ -141,6 +149,8 @@ conversations = await tools.conversation_query(
 | `benchmark_clone` | Copy all scenarios from one benchmark into another |
 | `benchmark_delete` | Delete a benchmark (guards against wiping scenarios) |
 | `benchmark_results` | Pass/fail/unrated tally and pass rate for a benchmark |
+| `benchmark_list` | List benchmarks (with per-benchmark pass rates via `with_results`) |
+| `scenario_create_bulk` | Build scenarios from several conversations at once |
 | `scenario_bulk_run` | Run scenarios (contact override + wait-to-completion) |
 | `scenario_bulk_cancel` | Cancel a stuck bulk run's queued/running scenario runs |
 

{applied_cli-0.6.8 → applied_cli-0.6.10}/applied_cli/cli.py

@@ -1785,6 +1785,46 @@ def scenario_create(
     typer.echo(result)
 
 
+@app.command("scenario-create-bulk")
+def scenario_create_bulk(
+    conversation_ids: str = typer.Option(
+        ..., "--conversation-ids", help="Comma-separated source conversation IDs"
+    ),
+    benchmark_id: str = typer.Option(
+        None, "--benchmark-id", help="Attach the scenarios to an existing benchmark"
+    ),
+    benchmark_name: str = typer.Option(
+        None, "--benchmark-name", help="Create or reuse a benchmark by name"
+    ),
+    agent_id: str = typer.Option(
+        None, "--agent-id", help="Required when --benchmark-name is used"
+    ),
+    name_prefix: str = typer.Option(
+        None,
+        "--name-prefix",
+        help="Name scenarios '<prefix> N' instead of deriving from conversation titles",
+    ),
+    shop_id: str = typer.Option(None, "--shop-id", help="Override shop ID"),
+    format: str = typer.Option(
+        "text", "--format", "-f", help="Output format: text or json"
+    ),
+) -> None:
+    """Create scenarios from several conversations at once (build a suite fast)."""
+    client = get_client(shop_id=shop_id)
+    result = asyncio.run(
+        tools.scenario_create_bulk(
+            client,
+            conversation_ids=_parse_csv_option(conversation_ids),
+            benchmark_id=benchmark_id,
+            benchmark_name=benchmark_name,
+            agent_id=agent_id,
+            name_prefix=name_prefix,
+            output_format=format,
+        )
+    )
+    typer.echo(result)
+
+
 @app.command("scenario-update")
 def scenario_update_cmd(
     id: str = typer.Argument(..., help="Scenario ID"),

{applied_cli-0.6.8 → applied_cli-0.6.10}/applied_cli/tools.py

@@ -5984,6 +5984,97 @@ async def scenario_create(
     return result
 
 
+async def scenario_create_bulk(
+    client: AppliedClient,
+    conversation_ids: list[str],
+    *,
+    benchmark_id: str | None = None,
+    benchmark_name: str | None = None,
+    agent_id: str | None = None,
+    name_prefix: str | None = None,
+    output_format: str = "text",
+) -> str:
+    """
+    Create scenarios from several conversations at once and attach them to a
+    benchmark — the fast way to build a regression suite from real conversations.
+
+    Each scenario's name is derived from its source conversation's title; pass
+    name_prefix to instead name them "<prefix> 1", "<prefix> 2", … (skips the
+    per-conversation title lookup). Names are de-duplicated server-side.
+
+    Args:
+        client: Authenticated AppliedClient
+        conversation_ids: Source conversation UUIDs (one scenario each)
+        benchmark_id: Attach the scenarios to this existing benchmark
+        benchmark_name: Create or reuse a benchmark by name (requires agent_id)
+        agent_id: Required when benchmark_name is used
+        name_prefix: Name scenarios "<prefix> N" instead of using conversation titles
+        output_format: 'text' (default) or 'json'
+
+    Returns:
+        Summary of created scenarios and any per-conversation errors.
+    """
+    if not conversation_ids:
+        return _format_argument_error("Pass at least one conversation id.")
+
+    created: list[dict[str, Any]] = []
+    errors: list[dict[str, Any]] = []
+    for index, conversation_id in enumerate(conversation_ids):
+        if name_prefix:
+            name = f"{name_prefix} {index + 1}"
+        else:
+            try:
+                conversation = await client.get_conversation(conversation_id)
+                name = conversation.get("title") or f"Scenario {str(conversation_id)[:8]}"
+            except AppliedAPIError:
+                name = f"Scenario {str(conversation_id)[:8]}"
+
+        try:
+            scenario = await client.create_scenario(
+                input_conversation_id=conversation_id,
+                name=name,
+                benchmark_id=benchmark_id,
+                benchmark_name=benchmark_name,
+                agent_id=agent_id,
+            )
+            created.append(
+                {
+                    "id": scenario.get("id"),
+                    "name": scenario.get("name"),
+                    "conversation_id": conversation_id,
+                }
+            )
+        except AppliedAPIError as e:
+            errors.append({"conversation_id": conversation_id, "error": str(e)})
+
+    summary = {
+        "requested": len(conversation_ids),
+        "created": len(created),
+        "failed": len(errors),
+        "scenarios": created,
+        "errors": errors,
+    }
+
+    if output_format == "json":
+        return to_json(summary)
+
+    lines = [
+        "# Bulk Scenario Create",
+        f"requested: {summary['requested']}",
+        f"created: {summary['created']}",
+        f"failed: {summary['failed']}",
+    ]
+    if created:
+        lines.append("\n# Created")
+        lines.extend(f"  - {s['name']} ({s['id']})" for s in created[:50])
+        if len(created) > 50:
+            lines.append(f"  ... and {len(created) - 50} more")
+    if errors:
+        lines.append(f"\n# Errors ({len(errors)})")
+        lines.extend(f"  - {e['conversation_id']}: {e['error']}" for e in errors[:20])
+    return "\n".join(lines)
+
+
 async def scenario_update(
     client: AppliedClient,
     scenario_id: str,

{applied_cli-0.6.8 → applied_cli-0.6.10}/applied_cli/v2/domains.py

@@ -112,6 +112,7 @@ DOMAIN_TOOL_RENAMES: dict[str, dict[str, str]] = {
         "scenario_list": "scenarios_list",
         "scenario_get": "scenarios_get",
         "scenario_create": "scenarios_create",
+        "scenario_create_bulk": "scenarios_create_bulk",
         "scenario_update": "scenarios_update",
         "scenario_delete": "scenarios_delete",
         "scenario_run_list": "scenarios_runs_list",

{applied_cli-0.6.8 → applied_cli-0.6.10}/applied_cli/v2/scenarios.py

@@ -98,6 +98,14 @@ class ScenariosCreateInput(StrictInput):
     agent_id: str | None = None
 
 
+class ScenariosCreateBulkInput(StrictInput):
+    conversation_ids: list[str]
+    benchmark_id: str | None = None
+    benchmark_name: str | None = None
+    agent_id: str | None = None
+    name_prefix: str | None = None
+
+
 class ScenariosUpdateInput(StrictInput):
     scenario_id: str
     name: str | None = None
@@ -651,6 +659,46 @@ async def scenarios_create_handler(
     )
 
 
+async def scenarios_create_bulk_handler(
+    client: AppliedClient,
+    params: ScenariosCreateBulkInput,
+) -> ToolResult[Any]:
+    from applied_cli import tools as legacy_tools
+
+    raw = await legacy_tools.scenario_create_bulk(
+        client,
+        conversation_ids=params.conversation_ids,
+        benchmark_id=params.benchmark_id,
+        benchmark_name=params.benchmark_name,
+        agent_id=params.agent_id,
+        name_prefix=params.name_prefix,
+        output_format="json",
+    )
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, TypeError):
+        return ToolResult(data={"message": raw}, summary=str(raw))
+
+    next_actions = []
+    if data.get("created"):
+        next_actions.append(
+            "Use scenarios_bulk_run with the benchmark_id to run the new scenarios."
+        )
+    return ToolResult(
+        data=data,
+        summary=(
+            f"Created {data.get('created', 0)}/{data.get('requested', 0)} "
+            f"scenarios ({data.get('failed', 0)} failed)."
+        ),
+        warnings=(
+            [f"{data['failed']} conversation(s) failed to convert."]
+            if data.get("failed")
+            else []
+        ),
+        next_actions=next_actions,
+    )
+
+
 async def scenarios_get_handler(
     client: AppliedClient,
     params: ScenariosGetInput,
@@ -1116,6 +1164,21 @@ def scenario_specs() -> list[ToolSpec]:
             read_write_mode="write",
             tags=["scenario_create", "native"],
         ),
+        ToolSpec(
+            name="scenarios_create_bulk",
+            namespace="scenarios",
+            description=(
+                "Create scenarios from several conversations at once and attach "
+                "them to a benchmark — build a regression suite from real "
+                "conversations. Names derive from each conversation's title "
+                "unless name_prefix is given."
+            ),
+            input_model=ScenariosCreateBulkInput,
+            output_model=None,
+            handler=scenarios_create_bulk_handler,
+            read_write_mode="write",
+            tags=["scenario_create_bulk", "native"],
+        ),
         ToolSpec(
             name="scenarios_update",
             namespace="scenarios",

{applied_cli-0.6.8 → applied_cli-0.6.10}/applied_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: applied-cli
-Version: 0.6.8
+Version: 0.6.10
 Summary: CLI and shared client library for Applied Labs AI support agents
 Author: Applied Labs
 License-Expression: MIT
@@ -97,6 +97,11 @@ applied benchmark-create --agent-id <agent_id> --name "Cancel Regression"
 applied scenario-create --input-conversation-id <conversation_id> --name "<name>" \
   --benchmark-id <benchmark_id>
 
+# Build a suite fast from several real conversations at once. Each scenario is
+# named from its conversation's title (or "<prefix> N" with --name-prefix).
+applied scenario-create-bulk --conversation-ids <id1>,<id2>,<id3> \
+  --benchmark-id <benchmark_id>
+
 # Port a suite to another agent (e.g. email -> chat). Cross-agent recreates the
 # scenarios under the destination agent; same-agent just tags them in.
 # Dry-run by default; add --apply to write.
@@ -116,6 +121,9 @@ applied scenario-bulk-cancel <job_id> --apply
 # Review pass/fail health (pass_status reflects the latest run per scenario)
 applied benchmark-results <benchmark_id> --format json
 
+# Portfolio go/no-go: pass rates across all of an agent's benchmarks at a glance
+applied benchmarks --agent-id <agent_id> --with-results --format json
+
 # Rate scenarios as you evaluate
 applied scenario-update <scenario_id> --pass-status pass --feedback "<note>"
 
@@ -167,6 +175,8 @@ conversations = await tools.conversation_query(
 | `benchmark_clone` | Copy all scenarios from one benchmark into another |
 | `benchmark_delete` | Delete a benchmark (guards against wiping scenarios) |
 | `benchmark_results` | Pass/fail/unrated tally and pass rate for a benchmark |
+| `benchmark_list` | List benchmarks (with per-benchmark pass rates via `with_results`) |
+| `scenario_create_bulk` | Build scenarios from several conversations at once |
 | `scenario_bulk_run` | Run scenarios (contact override + wait-to-completion) |
 | `scenario_bulk_cancel` | Cancel a stuck bulk run's queued/running scenario runs |
 

{applied_cli-0.6.8 → applied_cli-0.6.10}/applied_cli.egg-info/SOURCES.txt

@@ -54,6 +54,7 @@ tests/test_recovery.py
 tests/test_scenario_bulk_cancel.py
 tests/test_scenario_bulk_run_contact.py
 tests/test_scenario_bulk_run_wait.py
+tests/test_scenario_create_bulk.py
 tests/test_toolkit_contract.py
 tests/test_v2_agents.py
 tests/test_v2_articles.py

{applied_cli-0.6.8 → applied_cli-0.6.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "applied-cli"
-version = "0.6.8"
+version = "0.6.10"
 description = "CLI and shared client library for Applied Labs AI support agents"
 readme = "README.md"
 requires-python = ">=3.11"

applied_cli-0.6.10/tests/test_scenario_create_bulk.py

@@ -0,0 +1,112 @@
+import json
+
+import pytest
+
+from applied_cli import tools
+from applied_cli.client import AppliedAPIError
+
+
+class FakeBulkCreateClient:
+    def __init__(self, titles=None, fail_ids=None):
+        self._titles = titles or {}
+        self._fail_ids = set(fail_ids or [])
+        self.get_conversation_calls = []
+        self.created = []
+
+    async def get_conversation(self, conversation_id, *, shop_id=None):
+        self.get_conversation_calls.append(conversation_id)
+        return {"id": conversation_id, "title": self._titles.get(conversation_id)}
+
+    async def create_scenario(
+        self,
+        input_conversation_id,
+        name,
+        benchmark_id=None,
+        benchmark_name=None,
+        agent_id=None,
+    ):
+        if input_conversation_id in self._fail_ids:
+            raise AppliedAPIError("boom", status_code=400)
+        rec = {
+            "id": f"scn-{len(self.created) + 1}",
+            "name": name,
+            "input_conversation_id": input_conversation_id,
+            "benchmark_id": benchmark_id,
+        }
+        self.created.append(rec)
+        return rec
+
+
+@pytest.mark.asyncio
+async def test_bulk_create_names_from_conversation_titles():
+    client = FakeBulkCreateClient(
+        titles={"c1": "Cancel order BP123", "c2": "Refund request"}
+    )
+    out = await tools.scenario_create_bulk(
+        client, ["c1", "c2"], benchmark_id="b1", output_format="json"
+    )
+    data = json.loads(out)
+    assert data["created"] == 2
+    assert [s["name"] for s in client.created] == ["Cancel order BP123", "Refund request"]
+    assert all(s["benchmark_id"] == "b1" for s in client.created)
+    assert client.get_conversation_calls == ["c1", "c2"]
+
+
+@pytest.mark.asyncio
+async def test_name_prefix_skips_title_lookup():
+    client = FakeBulkCreateClient()
+    await tools.scenario_create_bulk(
+        client, ["c1", "c2", "c3"], name_prefix="DG Cancel", output_format="json"
+    )
+    assert [s["name"] for s in client.created] == [
+        "DG Cancel 1",
+        "DG Cancel 2",
+        "DG Cancel 3",
+    ]
+    # No per-conversation fetches when a prefix is supplied.
+    assert client.get_conversation_calls == []
+
+
+@pytest.mark.asyncio
+async def test_missing_title_falls_back_to_short_id():
+    client = FakeBulkCreateClient(titles={"abcdef12-0000": None})
+    await tools.scenario_create_bulk(
+        client, ["abcdef12-0000"], output_format="json"
+    )
+    assert client.created[0]["name"] == "Scenario abcdef12"
+
+
+@pytest.mark.asyncio
+async def test_partial_failures_are_reported_not_fatal():
+    client = FakeBulkCreateClient(
+        titles={"c1": "A", "c2": "B", "c3": "C"}, fail_ids=["c2"]
+    )
+    out = await tools.scenario_create_bulk(
+        client, ["c1", "c2", "c3"], output_format="json"
+    )
+    data = json.loads(out)
+    assert data["created"] == 2
+    assert data["failed"] == 1
+    assert data["errors"][0]["conversation_id"] == "c2"
+
+
+@pytest.mark.asyncio
+async def test_empty_list_is_an_argument_error():
+    client = FakeBulkCreateClient()
+    out = await tools.scenario_create_bulk(client, [], output_format="text")
+    assert "at least one conversation id" in out.lower()
+
+
+@pytest.mark.asyncio
+async def test_v2_scenarios_create_bulk_handler():
+    from applied_cli.v2.scenarios import (
+        ScenariosCreateBulkInput,
+        scenarios_create_bulk_handler,
+    )
+
+    client = FakeBulkCreateClient(titles={"c1": "A"})
+    result = await scenarios_create_bulk_handler(
+        client, ScenariosCreateBulkInput(conversation_ids=["c1"], benchmark_id="b1")
+    )
+    assert result.data["created"] == 1
+    assert "scenarios" in " ".join(result.next_actions).lower()