applied-cli 0.1.0__py3-none-any.whl

applied_cli/commands/coverage.py

@@ -0,0 +1,348 @@
+import json
+import uuid
+from typing import Any, Optional
+
+import typer
+
+from applied_cli.error_reporting import render_api_error
+from applied_cli.http import APIError, list_conversation_scenarios, list_scenario_runs
+from applied_cli.runtime import resolve_runtime
+
+app = typer.Typer(help="Summarize test coverage by intent/topic and flow paths.")
+
+
+def _validate_uuid(value: str, *, field_name: str) -> None:
+    try:
+        uuid.UUID(value)
+    except ValueError as exc:
+        raise typer.BadParameter(f"{field_name} must be a valid UUID.") from exc
+
+
+def _extract_name(value: Any) -> Optional[str]:
+    if isinstance(value, dict):
+        name = value.get("name")
+        if isinstance(name, str) and name.strip():
+            return name.strip()
+        return None
+    if isinstance(value, str) and value.strip():
+        return value.strip()
+    return None
+
+
+def _extract_conversation(run: dict[str, Any], scenario: dict[str, Any]) -> dict[str, Any]:
+    for key in ("output_conversation", "input_conversation"):
+        candidate = run.get(key)
+        if isinstance(candidate, dict):
+            return candidate
+    scenario_input = scenario.get("input_conversation")
+    if isinstance(scenario_input, dict):
+        return scenario_input
+    return {}
+
+
+def _extract_flow_ids_and_paths(conversation: dict[str, Any]) -> tuple[set[str], set[str]]:
+    flow_ids: set[str] = set()
+    flow_paths: set[str] = set()
+
+    metadata = conversation.get("metadata")
+    if isinstance(metadata, dict):
+        state = metadata.get("state")
+        if isinstance(state, dict):
+            state_flow_id = state.get("flow_id")
+            if isinstance(state_flow_id, str) and state_flow_id.strip():
+                flow_ids.add(state_flow_id.strip())
+
+            current_node = state.get("current_node")
+            if isinstance(current_node, dict):
+                node_name = current_node.get("name")
+                if isinstance(node_name, str) and node_name.strip():
+                    if isinstance(state_flow_id, str) and state_flow_id.strip():
+                        flow_paths.add(f"{state_flow_id.strip()}:{node_name.strip()}")
+                    else:
+                        flow_paths.add(node_name.strip())
+
+    messages = conversation.get("messages")
+    if not isinstance(messages, list):
+        return flow_ids, flow_paths
+
+    seq: list[str] = []
+    for item in messages:
+        if not isinstance(item, dict):
+            continue
+        flow_id_raw = item.get("flow_id")
+        node_raw = item.get("conversational_node_id")
+
+        flow_id = flow_id_raw.strip() if isinstance(flow_id_raw, str) else ""
+        node = node_raw.strip() if isinstance(node_raw, str) else ""
+
+        if flow_id:
+            flow_ids.add(flow_id)
+        if not node:
+            continue
+
+        token = f"{flow_id}:{node}" if flow_id else node
+        if not seq or seq[-1] != token:
+            seq.append(token)
+
+    if seq:
+        flow_paths.add(" -> ".join(seq))
+
+    return flow_ids, flow_paths
+
+
+def _build_coverage(
+    *,
+    scenarios: list[dict[str, Any]],
+    scenario_runs: dict[str, list[dict[str, Any]]],
+) -> dict[str, Any]:
+    intent_buckets: dict[str, dict[str, Any]] = {}
+    flow_buckets: dict[str, dict[str, Any]] = {}
+    flow_path_counts: dict[str, int] = {}
+
+    total_runs = 0
+    for scenario in scenarios:
+        scenario_id = str(scenario.get("id") or "")
+        runs = scenario_runs.get(scenario_id, [])
+        for run in runs:
+            total_runs += 1
+            conversation = _extract_conversation(run, scenario)
+            topic = _extract_name(conversation.get("label")) or "(none)"
+            intent = _extract_name(conversation.get("sublabel")) or "(none)"
+            pass_status = str(run.get("pass_status") or "").lower()
+            conv_id = str(conversation.get("id") or "")
+
+            intent_key = f"{topic}|||{intent}"
+            bucket = intent_buckets.setdefault(
+                intent_key,
+                {
+                    "topic": topic,
+                    "intent": intent,
+                    "runs": 0,
+                    "pass": 0,
+                    "fail": 0,
+                    "other": 0,
+                    "scenario_ids": set(),
+                    "conversation_ids": set(),
+                },
+            )
+            bucket["runs"] += 1
+            bucket["scenario_ids"].add(scenario_id)
+            if conv_id:
+                bucket["conversation_ids"].add(conv_id)
+            if pass_status == "pass":
+                bucket["pass"] += 1
+            elif pass_status == "fail":
+                bucket["fail"] += 1
+            else:
+                bucket["other"] += 1
+
+            flow_ids, flow_paths = _extract_flow_ids_and_paths(conversation)
+            if not flow_ids and not flow_paths:
+                flow_ids = {"(none)"}
+            for flow_id in flow_ids:
+                flow_bucket = flow_buckets.setdefault(
+                    flow_id,
+                    {
+                        "flow_id": flow_id,
+                        "runs": 0,
+                        "pass": 0,
+                        "fail": 0,
+                        "other": 0,
+                        "scenario_ids": set(),
+                        "conversation_ids": set(),
+                    },
+                )
+                flow_bucket["runs"] += 1
+                flow_bucket["scenario_ids"].add(scenario_id)
+                if conv_id:
+                    flow_bucket["conversation_ids"].add(conv_id)
+                if pass_status == "pass":
+                    flow_bucket["pass"] += 1
+                elif pass_status == "fail":
+                    flow_bucket["fail"] += 1
+                else:
+                    flow_bucket["other"] += 1
+
+            for path in flow_paths:
+                flow_path_counts[path] = flow_path_counts.get(path, 0) + 1
+
+    intents = []
+    for row in intent_buckets.values():
+        intents.append(
+            {
+                "topic": row["topic"],
+                "intent": row["intent"],
+                "runs": row["runs"],
+                "pass": row["pass"],
+                "fail": row["fail"],
+                "other": row["other"],
+                "scenario_count": len(row["scenario_ids"]),
+                "conversation_count": len(row["conversation_ids"]),
+            }
+        )
+
+    flows = []
+    for row in flow_buckets.values():
+        flows.append(
+            {
+                "flow_id": row["flow_id"],
+                "runs": row["runs"],
+                "pass": row["pass"],
+                "fail": row["fail"],
+                "other": row["other"],
+                "scenario_count": len(row["scenario_ids"]),
+                "conversation_count": len(row["conversation_ids"]),
+            }
+        )
+
+    flow_paths = [{"path": key, "runs": value} for key, value in flow_path_counts.items()]
+    flow_paths.sort(key=lambda x: (-x["runs"], x["path"]))
+
+    intents.sort(key=lambda x: (-x["runs"], x["topic"], x["intent"]))
+    flows.sort(key=lambda x: (-x["runs"], x["flow_id"]))
+
+    scenarios_with_runs = sum(1 for value in scenario_runs.values() if value)
+    return {
+        "summary": {
+            "scenario_count": len(scenarios),
+            "scenarios_with_runs": scenarios_with_runs,
+            "scenarios_without_runs": max(0, len(scenarios) - scenarios_with_runs),
+            "run_count": total_runs,
+            "intent_bucket_count": len(intents),
+            "flow_bucket_count": len(flows),
+            "flow_path_count": len(flow_paths),
+        },
+        "intents": intents,
+        "flows": flows,
+        "flow_paths": flow_paths,
+    }
+
+
+@app.command(
+    "summary",
+    help=(
+        "Summarize coverage by topic/intent and flow paths. Example: applied-cli test coverage "
+        "summary --agent-id <uuid> --benchmark-id <uuid> --all"
+    ),
+)
+def summary_cmd(
+    agent_id: str = typer.Option(..., "--agent-id", "--agent", help="Target agent UUID."),
+    benchmark_id: Optional[str] = typer.Option(
+        None, "--benchmark-id", "--benchmark", help="Optional benchmark UUID."
+    ),
+    latest: bool = typer.Option(
+        True, "--latest/--all", help="Use latest run per scenario or include all runs."
+    ),
+    include_none_flow: bool = typer.Option(
+        True,
+        "--include-none-flow/--no-include-none-flow",
+        help="Include '(none)' flow bucket for runs without flow metadata.",
+    ),
+    output_json: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    base_url: Optional[str] = typer.Option(None, help="Applied base URL."),
+    shop_id: Optional[str] = typer.Option(None, help="Target shop UUID."),
+    api_token: Optional[str] = typer.Option(None, help="Applied API token."),
+) -> None:
+    _validate_uuid(agent_id, field_name="agent-id")
+    if benchmark_id:
+        _validate_uuid(benchmark_id, field_name="benchmark-id")
+
+    try:
+        resolved_base_url, resolved_shop_id, resolved_token = resolve_runtime(
+            base_url=base_url,
+            shop_id=shop_id,
+            api_token=api_token,
+        )
+        scenarios = list_conversation_scenarios(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            agent_id=agent_id,
+            benchmark_id=benchmark_id,
+            limit=500,
+            ordering="-created_at",
+        )
+    except APIError as exc:
+        typer.echo(render_api_error(exc, action="list scenarios for coverage summary"), err=True)
+        raise typer.Exit(code=1) from exc
+
+    scenario_runs: dict[str, list[dict[str, Any]]] = {}
+    for scenario in scenarios:
+        scenario_id = str(scenario.get("id") or "")
+        if not scenario_id:
+            continue
+        try:
+            runs = list_scenario_runs(
+                base_url=resolved_base_url,
+                shop_id=resolved_shop_id,
+                api_token=resolved_token,
+                scenario_id=scenario_id,
+                latest_only=latest,
+            )
+        except APIError as exc:
+            typer.echo(
+                render_api_error(
+                    exc, action=f"list runs for coverage scenario {scenario_id}"
+                ),
+                err=True,
+            )
+            raise typer.Exit(code=1) from exc
+        scenario_runs[scenario_id] = runs
+
+    coverage = _build_coverage(scenarios=scenarios, scenario_runs=scenario_runs)
+    if not include_none_flow:
+        coverage["flows"] = [
+            row for row in coverage["flows"] if row.get("flow_id") != "(none)"
+        ]
+
+    result = {
+        "agent_id": agent_id,
+        "benchmark_id": benchmark_id,
+        "latest_only": latest,
+        **coverage,
+    }
+
+    if output_json:
+        typer.echo(json.dumps(result, indent=2, default=str))
+        return
+
+    summary = result["summary"]
+    typer.echo(
+        "summary="
+        + ", ".join(
+            [
+                f"scenarios={summary['scenario_count']}",
+                f"scenarios_with_runs={summary['scenarios_with_runs']}",
+                f"scenarios_without_runs={summary['scenarios_without_runs']}",
+                f"runs={summary['run_count']}",
+                f"intent_buckets={summary['intent_bucket_count']}",
+                f"flow_buckets={summary['flow_bucket_count']}",
+                f"flow_paths={summary['flow_path_count']}",
+            ]
+        )
+    )
+
+    typer.echo("\nIntent Coverage:")
+    if not result["intents"]:
+        typer.echo("- no runs")
+    else:
+        for row in result["intents"]:
+            typer.echo(
+                f"- topic={row['topic']} | intent={row['intent']} | runs={row['runs']} | pass={row['pass']} | fail={row['fail']} | other={row['other']}"
+            )
+
+    typer.echo("\nFlow Coverage:")
+    if not result["flows"]:
+        typer.echo("- no runs")
+    else:
+        for row in result["flows"]:
+            typer.echo(
+                f"- flow_id={row['flow_id']} | runs={row['runs']} | pass={row['pass']} | fail={row['fail']} | other={row['other']}"
+            )
+
+    typer.echo("\nFlow Paths:")
+    if not result["flow_paths"]:
+        typer.echo("- no flow paths observed")
+    else:
+        for row in result["flow_paths"]:
+            typer.echo(f"- runs={row['runs']} | path={row['path']}")