applied-cli 0.6.0__tar.gz → 0.6.1__tar.gz

{applied_cli-0.6.0 → applied_cli-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: applied-cli
-Version: 0.6.0
+Version: 0.6.1
 Summary: CLI and shared client library for Applied Labs AI support agents
 Author: Applied Labs
 License-Expression: MIT

{applied_cli-0.6.0 → applied_cli-0.6.1}/applied_cli/cli.py

@@ -42,6 +42,11 @@ app.add_typer(v2_app, name="v2")
 
 DEFAULT_BASE_URL = "https://api.appliedlabs.ai"
 DEFAULT_CLIENT_URL = "https://appliedlabs.ai"
+DEFAULT_RECOVERY_DIR_OPTION = typer.Option(
+    "/tmp/applied-labs-recovery-jun4-1915",
+    "--recovery-dir",
+    help="Directory containing exports/ and deleted_only/ recovery files",
+)
 AGENT_DEPLOY_AGENT_IDS_ARGUMENT = typer.Argument(
     ...,
     help="One or more Agent IDs to deploy",
@@ -1592,6 +1597,49 @@ def benchmark_create(
     typer.echo(result)
 
 
+@app.command("benchmark-clone")
+def benchmark_clone(
+    source_benchmark_id: str = typer.Argument(
+        ..., help="Benchmark to copy scenarios from"
+    ),
+    dest_benchmark_id: str = typer.Option(
+        None, "--dest-benchmark-id", help="Existing destination benchmark UUID"
+    ),
+    dest_benchmark_name: str = typer.Option(
+        None,
+        "--dest-benchmark-name",
+        help="Destination benchmark name (found or created under --target-agent-id)",
+    ),
+    target_agent_id: str = typer.Option(
+        None,
+        "--target-agent-id",
+        help="Agent for the destination benchmark when created by name "
+        "(e.g. port an email benchmark onto the chat agent)",
+    ),
+    apply: bool = typer.Option(
+        False, "--apply", help="Write changes (default is a dry-run plan)"
+    ),
+    shop_id: str = typer.Option(None, "--shop-id", help="Override shop ID"),
+    format: str = typer.Option(
+        "text", "--format", "-f", help="Output format: text or json"
+    ),
+) -> None:
+    """Copy all scenarios from one benchmark into another (e.g. email -> chat)."""
+    client = get_client(shop_id=shop_id)
+    result = asyncio.run(
+        tools.benchmark_clone(
+            client,
+            source_benchmark_id=source_benchmark_id,
+            dest_benchmark_id=dest_benchmark_id,
+            dest_benchmark_name=dest_benchmark_name,
+            target_agent_id=target_agent_id,
+            dry_run=not apply,
+            output_format=format,
+        )
+    )
+    typer.echo(result)
+
+
 @app.command()
 def scenarios(
     benchmark_id: str = typer.Option(
@@ -1772,6 +1820,59 @@ def scenario_run_update_cmd(
     typer.echo(result)
 
 
+@app.command("scenario-recover-catalog")
+def scenario_recover_catalog(
+    recovery_dir: Path = DEFAULT_RECOVERY_DIR_OPTION,
+    apply: bool = typer.Option(
+        False,
+        "--apply",
+        help="Create missing benchmarks/scenarios and restore rated runs",
+    ),
+    restore_ratings: bool = typer.Option(
+        True,
+        "--restore-ratings/--no-restore-ratings",
+        help="Restore rated scenario runs linked to recovered scenarios",
+    ),
+    max_rated_runs: int | None = typer.Option(
+        None,
+        "--max-rated-runs",
+        help="Cap rated run restore count for staged recovery",
+    ),
+    shop_id: str = typer.Option(None, "--shop-id", help="Override shop ID"),
+    format: str = typer.Option(
+        "json", "--format", "-f", help="Output format: json or text"
+    ),
+) -> None:
+    """Recover deleted benchmark/scenario catalog rows from local PITR exports."""
+    from applied_cli.recovery import recover_scenario_catalog
+
+    client = get_client(shop_id=shop_id)
+    result = asyncio.run(
+        recover_scenario_catalog(
+            client,
+            recovery_dir=recovery_dir,
+            dry_run=not apply,
+            restore_ratings=restore_ratings,
+            max_rated_runs=max_rated_runs,
+        )
+    )
+    if format == "text":
+        typer.echo(
+            "\n".join(
+                [
+                    f"dry_run: {result['dry_run']}",
+                    f"shop_id: {result['shop_id']}",
+                    f"restore_point_utc: {result['restore_point_utc']}",
+                    f"benchmarks: {result['benchmarks']}",
+                    f"scenarios: {result['scenarios']}",
+                    f"scenario_runs: {result['scenario_runs']}",
+                ]
+            )
+        )
+        return
+    typer.echo(json.dumps(result, indent=2, default=str))
+
+
 @app.command("scenario-bulk-run")
 def scenario_bulk_run(
     scenario_ids: str = typer.Option(

{applied_cli-0.6.0 → applied_cli-0.6.1}/applied_cli/client.py

@@ -2426,6 +2426,40 @@ class AppliedClient:
                 break
         return all_results
 
+    async def list_recovery_scenarios(
+        self,
+        *,
+        agent_id: str,
+        names: list[str],
+        limit: int = 50,
+    ) -> list[dict]:
+        """List scenarios for recovery by exact name and agent."""
+        by_id: dict[str, dict] = {}
+        for name in sorted({item for item in names if item}):
+            params: dict[str, Any] = {
+                "agent": agent_id,
+                "limit": limit,
+                "name": name,
+                "summary": "true",
+            }
+            page = 1
+            while True:
+                params["page"] = page
+                data = await self._request(
+                    "GET",
+                    "/v1/conversation-scenarios/",
+                    params=params,
+                )
+                for scenario in self._normalize_response(data):
+                    scenario_id = scenario.get("id")
+                    if scenario_id:
+                        by_id[str(scenario_id)] = scenario
+                if isinstance(data, dict) and data.get("next"):
+                    page += 1
+                else:
+                    break
+        return list(by_id.values())
+
     async def get_scenario(self, scenario_id: str) -> dict:
         """Get a single scenario by ID."""
         return await self._request("GET", f"/v1/conversation-scenarios/{scenario_id}/")
@@ -2469,6 +2503,22 @@ class AppliedClient:
             "PATCH", f"/v1/conversation-scenarios/{scenario_id}/", body=kwargs
         )
 
+    async def create_recovery_scenario(self, body: dict[str, Any]) -> dict:
+        """Create a scenario with raw recovery fields."""
+        return await self._request("POST", "/v1/conversation-scenarios/", body=body)
+
+    async def update_recovery_scenario(
+        self,
+        scenario_id: str,
+        body: dict[str, Any],
+    ) -> dict:
+        """Patch scenario benchmark links with raw recovery fields."""
+        return await self._request(
+            "PATCH",
+            f"/v1/conversation-scenarios/{scenario_id}/",
+            body=body,
+        )
+
     # -------------------------------------------------------------------------
     # Scenario Runs
     # -------------------------------------------------------------------------
@@ -2506,6 +2556,10 @@ class AppliedClient:
         """Update a scenario run's properties (ratings, feedback, etc.)."""
         return await self._request("PATCH", f"/v1/scenario-runs/{run_id}/", body=kwargs)
 
+    async def create_recovery_scenario_run(self, body: dict[str, Any]) -> dict:
+        """Create a scenario run linked to an existing output conversation."""
+        return await self._request("POST", "/v1/scenario-runs/", body=body)
+
     async def delete_scenario(self, scenario_id: str) -> None:
         """Delete a scenario."""
         await self._request("DELETE", f"/v1/conversation-scenarios/{scenario_id}/")

applied_cli-0.6.1/applied_cli/recovery.py

@@ -0,0 +1,400 @@
+"""Helpers for recovering scenario catalog rows from local PITR exports."""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class RecoveryDataset:
+    shop_id: str
+    restore_point_utc: str | None
+    benchmarks: list[dict[str, Any]]
+    scenarios: list[dict[str, Any]]
+    scenario_benchmark_links: list[dict[str, Any]]
+    scenario_runs: list[dict[str, Any]]
+
+    def links_by_scenario(self) -> dict[str, list[str]]:
+        links: dict[str, list[str]] = defaultdict(list)
+        for link in self.scenario_benchmark_links:
+            scenario_id = link.get("scenario_id")
+            benchmark_id = link.get("benchmark_id")
+            if scenario_id and benchmark_id:
+                links[str(scenario_id)].append(str(benchmark_id))
+        return links
+
+    def benchmark_ids_for_scenario(self, scenario: dict[str, Any]) -> list[str]:
+        scenario_id = str(scenario.get("id") or "")
+        ids: list[str] = []
+        primary = scenario.get("benchmark_id")
+        if primary:
+            ids.append(str(primary))
+        for benchmark_id in self.links_by_scenario().get(scenario_id, []):
+            if benchmark_id not in ids:
+                ids.append(benchmark_id)
+        return ids
+
+    def rated_runs(self) -> list[dict[str, Any]]:
+        return [run for run in self.scenario_runs if _is_rated_run(run)]
+
+    def rated_runs_by_scenario(self) -> dict[str, list[dict[str, Any]]]:
+        runs: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        for run in self.rated_runs():
+            scenario_id = run.get("scenario_id")
+            if scenario_id:
+                runs[str(scenario_id)].append(run)
+        return runs
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _read_ndjson(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open(encoding="utf-8") as handle:
+        for line in handle:
+            stripped = line.strip()
+            if stripped:
+                rows.append(json.loads(stripped))
+    return rows
+
+
+def load_recovery_dataset(recovery_dir: str | Path) -> RecoveryDataset:
+    root = Path(recovery_dir)
+    exports = root / "exports"
+    deleted_only = root / "deleted_only"
+    export_manifest = _read_json(exports / "manifest.json")
+    deleted_manifest = _read_json(deleted_only / "manifest.json")
+
+    return RecoveryDataset(
+        shop_id=str(deleted_manifest.get("shop_id") or export_manifest.get("shop_id")),
+        restore_point_utc=deleted_manifest.get("restore_point_utc")
+        or export_manifest.get("restore_point_utc"),
+        benchmarks=_read_ndjson(exports / "core_conversationbenchmark.ndjson"),
+        scenarios=_read_ndjson(deleted_only / "core_conversationscenario.ndjson"),
+        scenario_benchmark_links=_read_ndjson(
+            deleted_only / "core_conversationscenariobenchmark.ndjson"
+        ),
+        scenario_runs=_read_ndjson(deleted_only / "core_scenariorun.ndjson"),
+    )
+
+
+def _agent_id(row: dict[str, Any]) -> str | None:
+    if row.get("agent_id"):
+        return str(row["agent_id"])
+    agent = row.get("agent")
+    if isinstance(agent, dict) and agent.get("id"):
+        return str(agent["id"])
+    return None
+
+
+def _input_conversation_id(row: dict[str, Any]) -> str | None:
+    if row.get("input_conversation_id"):
+        return str(row["input_conversation_id"])
+    input_conversation = row.get("input_conversation")
+    if isinstance(input_conversation, dict) and input_conversation.get("id"):
+        return str(input_conversation["id"])
+    return None
+
+
+def _current_benchmark_ids(row: dict[str, Any]) -> list[str]:
+    ids: list[str] = []
+    primary = row.get("benchmark")
+    if isinstance(primary, dict) and primary.get("id"):
+        ids.append(str(primary["id"]))
+    for benchmark in row.get("benchmarks") or []:
+        if isinstance(benchmark, dict) and benchmark.get("id"):
+            benchmark_id = str(benchmark["id"])
+            if benchmark_id not in ids:
+                ids.append(benchmark_id)
+    return ids
+
+
+def _is_rated_run(run: dict[str, Any]) -> bool:
+    return bool(
+        run.get("pass_status") is not None
+        or run.get("csat_score") is not None
+        or run.get("reference_score") is not None
+        or (run.get("reference_notes") or "")
+        or (run.get("feedback") or "")
+        or run.get("is_escalated") is True
+    )
+
+
+def _scenario_run_body(run: dict[str, Any], scenario_id: str) -> dict[str, Any]:
+    body: dict[str, Any] = {"scenario": scenario_id}
+    if run.get("output_conversation_id"):
+        body["output_conversation_id"] = str(run["output_conversation_id"])
+    if run.get("pass_status") is not None:
+        body["pass_status"] = run.get("pass_status")
+    if run.get("csat_score") is not None:
+        body["csat_score"] = run.get("csat_score")
+    if run.get("reference_score") is not None:
+        body["reference_score"] = run.get("reference_score")
+    if run.get("reference_notes"):
+        body["reference_notes"] = run.get("reference_notes")
+    if run.get("feedback") is not None:
+        body["feedback"] = run.get("feedback")
+    if run.get("is_escalated") is True:
+        body["is_escalated"] = True
+    return body
+
+
+def _run_signature(row: dict[str, Any]) -> tuple[Any, ...]:
+    output_conversation = row.get("output_conversation")
+    output_conversation_id = row.get("output_conversation_id")
+    if isinstance(output_conversation, dict) and output_conversation.get("id"):
+        output_conversation_id = output_conversation["id"]
+    return (
+        str(output_conversation_id) if output_conversation_id else None,
+        row.get("pass_status"),
+        row.get("csat_score"),
+        row.get("reference_score"),
+        row.get("reference_notes") or "",
+        row.get("feedback") or "",
+        bool(row.get("is_escalated")),
+    )
+
+
+async def _list_current_benchmarks(
+    client: Any, agent_ids: list[str]
+) -> list[dict[str, Any]]:
+    by_id: dict[str, dict[str, Any]] = {}
+    for agent_id in agent_ids:
+        for benchmark in await client.list_benchmarks(agent_id=agent_id, limit=500):
+            benchmark_id = benchmark.get("id")
+            if benchmark_id:
+                by_id[str(benchmark_id)] = benchmark
+    return list(by_id.values())
+
+
+async def _list_current_scenarios(
+    client: Any, scenarios: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    by_id: dict[str, dict[str, Any]] = {}
+    names_by_agent: dict[str, list[str]] = defaultdict(list)
+    for scenario in scenarios:
+        agent_id = _agent_id(scenario)
+        name = scenario.get("name")
+        if agent_id and name:
+            names_by_agent[agent_id].append(str(name))
+
+    for agent_id, names in sorted(names_by_agent.items()):
+        if hasattr(client, "list_recovery_scenarios"):
+            rows = await client.list_recovery_scenarios(
+                agent_id=agent_id,
+                names=names,
+                limit=50,
+            )
+        else:
+            rows = await client.list_scenarios(
+                agent_id=agent_id,
+                limit=500,
+                fetch_all=True,
+            )
+        for scenario in rows:
+            scenario_id = scenario.get("id")
+            if scenario_id:
+                by_id[str(scenario_id)] = scenario
+    return list(by_id.values())
+
+
+async def recover_scenario_catalog(
+    client: Any,
+    *,
+    recovery_dir: str | Path,
+    dry_run: bool = True,
+    restore_ratings: bool = True,
+    max_rated_runs: int | None = None,
+) -> dict[str, Any]:
+    dataset = load_recovery_dataset(recovery_dir)
+    benchmark_agent_ids = sorted(
+        {agent_id for row in dataset.benchmarks if (agent_id := _agent_id(row))}
+    )
+
+    current_benchmarks = await _list_current_benchmarks(client, benchmark_agent_ids)
+    current_benchmark_by_key = {
+        (_agent_id(row), row.get("name")): row for row in current_benchmarks
+    }
+    benchmark_id_map: dict[str, str] = {}
+    benchmark_summary = {
+        "available": len(dataset.benchmarks),
+        "existing": 0,
+        "to_create": 0,
+        "created": 0,
+    }
+
+    for benchmark in dataset.benchmarks:
+        old_id = str(benchmark["id"])
+        key = (_agent_id(benchmark), benchmark.get("name"))
+        current = current_benchmark_by_key.get(key)
+        if current and current.get("id"):
+            benchmark_summary["existing"] += 1
+            benchmark_id_map[old_id] = str(current["id"])
+            continue
+
+        benchmark_summary["to_create"] += 1
+        if dry_run:
+            continue
+
+        created = await client.create_benchmark(
+            agent_id=str(benchmark["agent_id"]),
+            name=benchmark["name"],
+            description=benchmark.get("description") or "",
+        )
+        benchmark_summary["created"] += 1
+        benchmark_id_map[old_id] = str(created["id"])
+
+    current_scenarios = await _list_current_scenarios(client, dataset.scenarios)
+    current_scenario_by_exact = {
+        (_agent_id(row), row.get("name"), _input_conversation_id(row)): row
+        for row in current_scenarios
+    }
+    current_scenario_by_name = {
+        (_agent_id(row), row.get("name")): row for row in current_scenarios
+    }
+    scenario_id_map: dict[str, str] = {}
+    scenario_summary = {
+        "available": len(dataset.scenarios),
+        "existing": 0,
+        "to_create": 0,
+        "created": 0,
+        "to_update_links": 0,
+        "updated_links": 0,
+    }
+
+    for scenario in dataset.scenarios:
+        old_id = str(scenario["id"])
+        current = current_scenario_by_exact.get(
+            (
+                _agent_id(scenario),
+                scenario.get("name"),
+                _input_conversation_id(scenario),
+            )
+        ) or current_scenario_by_name.get((_agent_id(scenario), scenario.get("name")))
+
+        old_benchmark_ids = dataset.benchmark_ids_for_scenario(scenario)
+        current_benchmark_ids = [
+            benchmark_id_map[old_benchmark_id]
+            for old_benchmark_id in old_benchmark_ids
+            if old_benchmark_id in benchmark_id_map
+        ]
+        primary_benchmark_id = (
+            benchmark_id_map.get(str(scenario.get("benchmark_id")))
+            if scenario.get("benchmark_id")
+            else (current_benchmark_ids[0] if current_benchmark_ids else None)
+        )
+
+        if current and current.get("id"):
+            scenario_summary["existing"] += 1
+            current_id = str(current["id"])
+            scenario_id_map[old_id] = current_id
+            missing_links = [
+                benchmark_id
+                for benchmark_id in current_benchmark_ids
+                if benchmark_id not in _current_benchmark_ids(current)
+            ]
+            if missing_links:
+                scenario_summary["to_update_links"] += 1
+                if not dry_run:
+                    body = {"benchmark_ids": _current_benchmark_ids(current)}
+                    for benchmark_id in missing_links:
+                        if benchmark_id not in body["benchmark_ids"]:
+                            body["benchmark_ids"].append(benchmark_id)
+                    if primary_benchmark_id:
+                        body["benchmark_id"] = primary_benchmark_id
+                    await client.update_recovery_scenario(current_id, body)
+                    scenario_summary["updated_links"] += 1
+            continue
+
+        scenario_summary["to_create"] += 1
+        if dry_run:
+            continue
+
+        body: dict[str, Any] = {
+            "agent_id": str(scenario["agent_id"]),
+            "allow_remote_ticket_creation": bool(
+                scenario.get("allow_remote_ticket_creation")
+            ),
+            "benchmark_ids": current_benchmark_ids,
+            "context": scenario.get("context") or {},
+            "input_conversation_id": str(scenario["input_conversation_id"]),
+            "name": scenario["name"],
+        }
+        if primary_benchmark_id:
+            body["benchmark_id"] = primary_benchmark_id
+
+        created = await client.create_recovery_scenario(body)
+        scenario_summary["created"] += 1
+        scenario_id_map[old_id] = str(created["id"])
+
+    rated_runs_by_scenario = dataset.rated_runs_by_scenario()
+    rated_runs = dataset.rated_runs()
+    if max_rated_runs is not None:
+        rated_runs = rated_runs[:max_rated_runs]
+    allowed_old_run_ids = {str(run["id"]) for run in rated_runs if run.get("id")}
+    run_summary = {
+        "rated_available": len(dataset.rated_runs()),
+        "to_create": len(rated_runs) if restore_ratings else 0,
+        "created": 0,
+        "skipped_existing": 0,
+        "skipped_missing_scenario": 0,
+    }
+
+    if restore_ratings:
+        existing_run_signatures: dict[str, set[tuple[Any, ...]]] = {}
+        for scenario in dataset.scenarios:
+            old_scenario_id = str(scenario["id"])
+            current_scenario_id = scenario_id_map.get(old_scenario_id)
+            scenario_runs = [
+                run
+                for run in rated_runs_by_scenario.get(old_scenario_id, [])
+                if str(run.get("id")) in allowed_old_run_ids
+            ]
+            for run in scenario_runs:
+                if not current_scenario_id:
+                    run_summary["skipped_missing_scenario"] += 1
+                    continue
+
+                body = _scenario_run_body(run, current_scenario_id)
+                if dry_run:
+                    continue
+
+                if current_scenario_id not in existing_run_signatures:
+                    existing_runs = await client.list_scenario_runs(
+                        scenario_id=current_scenario_id,
+                        latest=False,
+                        limit=500,
+                    )
+                    existing_run_signatures[current_scenario_id] = {
+                        _run_signature(existing_run) for existing_run in existing_runs
+                    }
+
+                signature = _run_signature(body)
+                if signature in existing_run_signatures[current_scenario_id]:
+                    run_summary["skipped_existing"] += 1
+                    continue
+
+                await client.create_recovery_scenario_run(body)
+                existing_run_signatures[current_scenario_id].add(signature)
+                run_summary["created"] += 1
+
+    return {
+        "dry_run": dry_run,
+        "shop_id": dataset.shop_id,
+        "restore_point_utc": dataset.restore_point_utc,
+        "source": {
+            "benchmarks": len(dataset.benchmarks),
+            "scenarios": len(dataset.scenarios),
+            "scenario_benchmark_links": len(dataset.scenario_benchmark_links),
+            "scenario_runs": len(dataset.scenario_runs),
+        },
+        "benchmarks": benchmark_summary,
+        "scenarios": scenario_summary,
+        "scenario_runs": run_summary,
+    }