agentcore-cli 0.31.1__py3-none-any.whl

agentcore/cli/commands/run.py

@@ -0,0 +1,808 @@
+"""`agentcore run eval|batch-evaluation|insights|recommendation|ab-test` (port of commands/run/command.tsx).
+
+`run ingest` belongs to the knowledge-base area (cli/commands/run_ingest.py).
+"""
+
+import json
+from typing import Any
+
+import typer
+
+from agentcore.cli.commands._jobs_common import fail_early, print_json, red, require_project
+from agentcore.cli.errors import get_error_message
+from agentcore.cli.groups import group
+from agentcore.cli.run_command import run_cli_command
+from agentcore.cli.variadic import variadic
+from agentcore.lib import ConfigIO, ValidationError, serialize_result
+from agentcore.operations.eval.run_eval import handle_run_eval
+from agentcore.operations.jobs import create_job_engine, run_dataset_phase1, wait_for_terminal
+from agentcore.operations.jobs.ab_test.format import print_ab_test_detail
+from agentcore.operations.jobs.recommendation.format import failure_text
+from agentcore.operations.jobs.shared.format import parse_int, to_fixed, to_locale_string
+from agentcore.schema import is_valid_kms_key_arn
+
+RECOMMENDATION_TYPE_MAP = {
+    "system-prompt": "SYSTEM_PROMPT_RECOMMENDATION",
+    "tool-description": "TOOL_DESCRIPTION_RECOMMENDATION",
+}
+_DATASET_VERSION_ERROR = "--dataset-version requires --dataset (the version selects a version of that dataset)"
+_ENDPOINT_HELP = "Runtime endpoint name (e.g. PROMPT_V1). Defaults to AGENTCORE_RUNTIME_ENDPOINT env var, then DEFAULT"
+_KMS_HELP_ERROR = (
+    "--kms-key must be a valid KMS key ARN "
+    "(e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)"
+)
+
+run_app = group("run")
+
+
+def _job_json(record: dict[str, Any], extra: dict[str, Any] | None = None) -> None:
+    print_json(serialize_result({"success": True, **record, **(extra or {})}))
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# run eval
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+def format_run_output(value: dict[str, Any]) -> None:
+    run = value["run"]
+    lookback = "" if run.get("source") == "dataset" else f" | Lookback: {run.get('lookbackDays')}d"
+    dataset = run.get("dataset")
+    dataset_str = (
+        f" | Dataset: {dataset['id']}@{dataset['version']}" if run.get("source") == "dataset" and dataset else ""
+    )
+    typer.echo(
+        f"\nAgent: {run['agent']} | {to_locale_string(run['timestamp'])} | Sessions: {run['sessionCount']}"
+        f"{lookback}{dataset_str}"
+    )
+    refs = run.get("referenceInputs")
+    if refs:
+        parts = []
+        if refs.get("assertions"):
+            parts.append(f"{len(refs['assertions'])} assertion(s)")
+        if refs.get("expectedResponse"):
+            parts.append("expected response")
+        if refs.get("expectedTrajectory"):
+            parts.append(f"{len(refs['expectedTrajectory'])} trajectory step(s)")
+        if parts:
+            typer.echo(f"Reference inputs: {', '.join(parts)}")
+    typer.echo("")
+    for r in run["results"]:
+        errors = sum(1 for s in r["sessionScores"] if s.get("errorMessage"))
+        suffix = f" ({errors} errors)" if errors else ""
+        typer.echo(f"  {r['evaluator']}: {to_fixed(r['aggregateScore'], 2)}{suffix}")
+    if value.get("filePath"):
+        typer.echo(f"\nResults saved to: {value['filePath']}")
+
+
+@run_app.command(
+    "eval",
+    help="Run on-demand evaluation of runtime traces. Use --runtime-arn to evaluate runtimes outside the project.",
+    cls=variadic("-e", "--evaluator", "--evaluator-arn", "-A", "--assertion"),
+)
+def run_eval(
+    runtime: str | None = typer.Option(None, "-r", "--runtime", help="Runtime name from project config"),
+    runtime_arn: str | None = typer.Option(None, "--runtime-arn", help="Runtime ARN — run outside a project directory"),
+    evaluator: list[str] | None = typer.Option(
+        None, "-e", "--evaluator", help="Evaluator name(s) — project evaluators or Builtin.* IDs"
+    ),
+    evaluator_arn: list[str] | None = typer.Option(
+        None, "--evaluator-arn", help="Evaluator ARN(s) — use with --runtime-arn for standalone mode"
+    ),
+    region: str | None = typer.Option(
+        None, "--region", help="AWS region (required with --runtime-arn, auto-detected otherwise)"
+    ),
+    session_id: str | None = typer.Option(None, "-s", "--session-id", help="Evaluate a specific session only"),
+    trace_id: str | None = typer.Option(None, "-t", "--trace-id", help="Evaluate a specific trace only"),
+    endpoint: str | None = typer.Option(None, "--endpoint", help=_ENDPOINT_HELP),
+    days: str = typer.Option("7", "--days", help="Lookback window in days"),
+    assertion: list[str] | None = typer.Option(
+        None, "-A", "--assertion", help="Ground truth assertion the agent response must satisfy (repeatable)"
+    ),
+    expected_trajectory: str | None = typer.Option(
+        None, "--expected-trajectory", help="Ground truth: expected tool call names in order (comma-separated)"
+    ),
+    expected_response: str | None = typer.Option(
+        None, "--expected-response", help="Ground truth: expected agent response text to compare against"
+    ),
+    output: str | None = typer.Option(None, "--output", help="Custom output file path for results"),
+    dataset: str | None = typer.Option(
+        None, "--dataset", help="Dataset name — invoke agent with dataset scenarios instead of historical traces"
+    ),
+    dataset_version: str | None = typer.Option(
+        None, "--dataset-version", help="Dataset version to use (omit for local file, or N/DRAFT)"
+    ),
+    as_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    if not (runtime_arn and evaluator_arn):
+        require_project()
+    if not evaluator and not evaluator_arn:
+        fail_early(as_json, "At least one --evaluator or --evaluator-arn is required")
+    if dataset_version and not dataset:
+        fail_early(as_json, _DATASET_VERSION_ERROR)
+
+    options = {
+        "agent": runtime,
+        "agentArn": runtime_arn,
+        "evaluator": evaluator or [],
+        "evaluatorArn": evaluator_arn or None,
+        "region": region,
+        "sessionId": session_id,
+        "traceId": trace_id,
+        "endpoint": endpoint,
+        "assertions": assertion or None,
+        "expectedTrajectory": [s.strip() for s in expected_trajectory.split(",")] if expected_trajectory else None,
+        "expectedResponse": expected_response,
+        "days": parse_int(days),
+        "output": output,
+        "dataset": dataset,
+        "datasetVersion": dataset_version,
+        "onProgress": None if as_json else (lambda _phase, message: typer.echo(message)),
+    }
+    try:
+        result = handle_run_eval(options)
+        if as_json:
+            print_json(serialize_result(result))
+        elif result.success:
+            format_run_output(result.value)
+        else:
+            red(str(result.error))
+        raise typer.Exit(0 if result.success else 1)
+    except typer.Exit:
+        raise
+    except Exception as exc:  # noqa: BLE001
+        if as_json:
+            print_json({"success": False, "error": get_error_message(exc)})
+        else:
+            red(f"Error: {get_error_message(exc)}")
+        raise typer.Exit(1) from None
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# run batch-evaluation
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+def print_batch_eval_result(record: dict[str, Any]) -> None:
+    results = record.get("evaluationResults")
+    if results:
+        parts = []
+        if results.get("totalNumberOfSessions") is not None:
+            parts.append(f"{results['totalNumberOfSessions']} sessions")
+        if results.get("numberOfSessionsCompleted") is not None:
+            parts.append(f"{results['numberOfSessionsCompleted']} completed")
+        if results.get("numberOfSessionsFailed"):
+            parts.append(f"{results['numberOfSessionsFailed']} failed")
+        if parts:
+            typer.echo(f"Sessions: {', '.join(parts)}")
+    summaries = (results or {}).get("evaluatorSummaries")
+    if summaries:
+        typer.echo("\nResults:")
+        for s in summaries:
+            avg = (s.get("statistics") or {}).get("averageScore")
+            fail = f" ({s['totalFailed']} failed)" if s.get("totalFailed") else ""
+            count = f" [{s['totalEvaluated']} evaluated]" if s.get("totalEvaluated") is not None else ""
+            typer.echo(f"  {s.get('evaluatorId')}: {to_fixed(avg, 2) if avg is not None else 'N/A'} avg{fail}{count}")
+    elif record.get("results"):
+        typer.echo("\nResults:")
+        groups: dict[str, list[dict[str, Any]]] = {}
+        for r in record["results"]:
+            groups.setdefault(r.get("evaluatorId"), []).append(r)
+        for evaluator_id, entries in groups.items():
+            scores = [r["score"] for r in entries if not r.get("error") and r.get("score") is not None]
+            avg = sum(scores) / len(scores) if scores else 0
+            errors = sum(1 for r in entries if r.get("error"))
+            typer.echo(f"  {evaluator_id}: {to_fixed(avg, 2)} avg{f' ({errors} errors)' if errors else ''}")
+
+
+@run_app.command(
+    "batch-evaluation",
+    help="Run evaluators in batch across all agent sessions in CloudWatch",
+    cls=variadic("-e", "--evaluator", "--evaluator-arn", "-s", "--session-ids"),
+)
+def run_batch_evaluation(
+    runtime: str | None = typer.Option(
+        None, "-r", "--runtime", help="Runtime name from project config [non-interactive]"
+    ),
+    evaluator: list[str] | None = typer.Option(
+        None, "-e", "--evaluator", help="Evaluator name(s) — Builtin.* IDs [non-interactive]"
+    ),
+    evaluator_arn: list[str] | None = typer.Option(
+        None, "--evaluator-arn", help="Evaluator ARN(s) — use instead of -e when referencing evaluators by ARN"
+    ),
+    name: str | None = typer.Option(
+        None, "-n", "--name", help="Name for the batch evaluation (auto-generated if omitted)"
+    ),
+    lookback_days: str | None = typer.Option(
+        None, "-d", "--lookback-days", help="Lookback window in days (filters sessions by time range)"
+    ),
+    session_ids: list[str] | None = typer.Option(None, "-s", "--session-ids", help="Specific session IDs to evaluate"),
+    ground_truth: str | None = typer.Option(
+        None,
+        "-g",
+        "--ground-truth",
+        help="JSON file with session metadata and ground truth (assertions, expected trajectory, turns)",
+    ),
+    region: str | None = typer.Option(None, "--region", help="AWS region (auto-detected if omitted)"),
+    endpoint: str | None = typer.Option(None, "--endpoint", help=_ENDPOINT_HELP),
+    dataset: str | None = typer.Option(
+        None, "--dataset", help="Dataset name — invoke agent with dataset scenarios before batch evaluation"
+    ),
+    dataset_version: str | None = typer.Option(
+        None, "--dataset-version", help="Dataset version to use (omit for local file, or N/DRAFT)"
+    ),
+    kms_key: str | None = typer.Option(None, "--kms-key", help="KMS key ARN for encrypting batch evaluation results"),
+    wait: bool = typer.Option(False, "--wait", help="Block until the batch evaluation reaches a terminal state"),
+    as_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    require_project()
+    if not runtime and not as_json:
+        from agentcore.cli import interactive
+        from agentcore.cli.commands._run_wizards import run_batch_eval_wizard
+
+        interactive.require_tty()
+        run_batch_eval_wizard()
+        raise typer.Exit(0)
+    if not runtime or (not evaluator and not evaluator_arn):
+        fail_early(
+            as_json, "--runtime and at least one --evaluator or --evaluator-arn are required in non-interactive mode"
+        )
+    if dataset_version and not dataset:
+        fail_early(as_json, _DATASET_VERSION_ERROR)
+
+    def log(message: str) -> None:
+        if not as_json:
+            typer.echo(message)
+
+    def run() -> None:
+        engine = create_job_engine(ConfigIO())
+        session_metadata = None
+        if ground_truth:
+            with open(ground_truth, encoding="utf-8") as fh:
+                data = json.load(fh)
+            parsed = (
+                data if isinstance(data, list) else (data.get("sessionMetadata") if isinstance(data, dict) else None)
+            )
+            if not isinstance(parsed, list):
+                raise Exception(
+                    "Ground truth file must be a JSON array of session metadata entries, or an object with a "
+                    '"sessionMetadata" key'
+                )
+            session_metadata = parsed
+        days = parse_int(lookback_days) if lookback_days else None
+        if lookback_days and (days is None or days < 1 or days > 90):
+            raise Exception("--lookback-days must be between 1 and 90")
+        ids = list(session_ids) if session_ids else None
+        dataset_info = {"id": dataset, "version": dataset_version or "LOCAL"} if dataset else None
+        if dataset:
+            phase1 = run_dataset_phase1(
+                agent=runtime or "",
+                dataset_name=dataset,
+                dataset_version=dataset_version,
+                endpoint=endpoint,
+                on_progress=lambda _p, m: log(m),
+            )
+            if not phase1.success:
+                raise phase1.error
+            ids = [*(ids or []), *phase1.value["sessionIds"]]
+            session_metadata = [*(session_metadata or []), *phase1.value["sessionMetadata"]]
+        start = engine.start(
+            "batch-evaluation",
+            {
+                "agent": runtime,
+                "evaluators": [*(evaluator or []), *(evaluator_arn or [])],
+                "name": name,
+                "region": region,
+                "endpoint": endpoint,
+                "sessionIds": ids,
+                "lookbackDays": days or None,
+                "sessionMetadata": session_metadata,
+                "source": "dataset" if dataset else "traces",
+                "dataset": dataset_info,
+                "kmsKeyArn": kms_key,
+                "onProgress": None if as_json else (lambda _s, m: typer.echo(m)),
+            },
+        )
+        if not start.success:
+            raise start.error
+        record = start.value
+        if wait:
+            final = wait_for_terminal(engine, "batch-evaluation", record["id"], on_tick=lambda s: log(f"Status: {s}"))
+            if final:
+                record = final
+        if as_json:
+            _job_json(record)
+        else:
+            typer.echo(f"\n✓ Batch evaluation started: {record['id']} ({record['status']})")
+            print_batch_eval_result(record)
+            if not wait:
+                typer.echo(f"\nNext: agentcore view batch-evaluation {record['id']}")
+            typer.echo("")
+
+    run_cli_command(as_json, run)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# run insights
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+def print_insights_result(record: dict[str, Any]) -> None:
+    categories = (record.get("failureAnalysisResult") or {}).get("failureCategories")
+    summaries = (record.get("evaluationResults") or {}).get("evaluatorSummaries")
+    if categories:
+        typer.echo("\nFailure Analysis:")
+        for cat in categories:
+            typer.echo(
+                f"  {cat.get('failureCategoryName') or 'Unknown'}: {cat.get('failureCategoryDescription') or ''}"
+            )
+            for rc in cat.get("rootCauses") or []:
+                typer.echo(f"    - {rc.get('rootCauseCategory') or ''}: {rc.get('rootCauseDescription') or ''}")
+                if rc.get("recommendation"):
+                    typer.echo(f"      Recommendation: {rc['recommendation']}")
+    elif summaries:
+        typer.echo("\nEvaluation Results:")
+        for s in summaries:
+            avg = (s.get("statistics") or {}).get("averageScore")
+            typer.echo(f"  {s.get('evaluatorId')}: {to_fixed(avg, 2) if avg is not None else 'N/A'}")
+
+
+@run_app.command(
+    "insights",
+    help="[preview] Run failure analysis across agent sessions",
+    cls=variadic("--insights", "-e", "--evaluator", "-s", "--session-ids"),
+)
+def run_insights(
+    runtime: str | None = typer.Option(None, "-r", "--runtime", help="Runtime name from project config"),
+    insights: list[str] | None = typer.Option(
+        None, "--insights", help="Insight type(s) (default: Builtin.Insight.FailureAnalysis)"
+    ),
+    evaluator: list[str] | None = typer.Option(
+        None, "-e", "--evaluator", help="Evaluator(s) to include (needed for chaining into recommendations)"
+    ),
+    online_eval_config_arn: str | None = typer.Option(
+        None, "--online-eval-config-arn", help="Use an existing OnlineEvaluationConfig as session source"
+    ),
+    lookback_days: str | None = typer.Option(
+        None, "-d", "--lookback-days", help="Lookback window in days (default: 7)"
+    ),
+    start_time: str | None = typer.Option(None, "--start-time", help="Session filter start time"),
+    end_time: str | None = typer.Option(None, "--end-time", help="Session filter end time"),
+    session_ids: list[str] | None = typer.Option(None, "-s", "--session-ids", help="Limit to specific session IDs"),
+    name: str | None = typer.Option(None, "-n", "--name", help="Job name (auto-generated if omitted)"),
+    kms_key: str | None = typer.Option(None, "--kms-key", help="KMS key ARN for encrypting insights job results"),
+    wait: bool = typer.Option(False, "--wait", help="Block until the job reaches a terminal state"),
+    region: str | None = typer.Option(None, "--region", help="AWS region (auto-detected if omitted)"),
+    endpoint: str | None = typer.Option(None, "--endpoint", help="Runtime endpoint name (e.g. PROMPT_V1)"),
+    as_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    require_project()
+
+    def log(message: str) -> None:
+        if not as_json:
+            typer.echo(message)
+
+    def run() -> None:
+        if name is not None and name.strip() == "":
+            raise ValidationError("--name must not be empty. Omit the flag to auto-generate a name.")
+        if kms_key and not is_valid_kms_key_arn(kms_key):
+            raise ValidationError(_KMS_HELP_ERROR)
+        engine = create_job_engine(ConfigIO())
+        days = parse_int(lookback_days) if lookback_days else None
+        if lookback_days and (days is None or days < 1 or days > 90):
+            raise ValidationError("--lookback-days must be a positive integer between 1 and 90.")
+        start = engine.start(
+            "insights",
+            {
+                "agent": runtime,
+                "insights": list(insights) if insights else ["Builtin.Insight.FailureAnalysis"],
+                "evaluators": list(evaluator) if evaluator else None,
+                "onlineEvalConfigArn": online_eval_config_arn,
+                "lookbackDays": days or None,
+                "startTime": start_time,
+                "endTime": end_time,
+                "sessionIds": list(session_ids) if session_ids else None,
+                "name": name,
+                "kmsKeyArn": kms_key,
+                "region": region,
+                "endpoint": endpoint,
+                "onProgress": None if as_json else (lambda _s, m: typer.echo(m)),
+            },
+        )
+        if not start.success:
+            raise start.error
+        record = start.value
+        if wait:
+            final = wait_for_terminal(engine, "insights", record["id"], on_tick=lambda s: log(f"Status: {s}"))
+            if final:
+                record = final
+        if as_json:
+            _job_json(record)
+        else:
+            typer.echo(f"\n✓ Insights job started: {record['id']} ({record['status']})")
+            print_insights_result(record)
+            if not wait:
+                typer.echo(f"\nNext: agentcore insights results --id {record['id']}")
+            typer.echo("")
+
+    run_cli_command(as_json, run)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# run recommendation
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+def print_recommendation_result(record: dict[str, Any]) -> None:
+    result = record.get("result") or {}
+    sys_result = result.get("systemPromptRecommendationResult") or {}
+    tool_result = result.get("toolDescriptionRecommendationResult") or {}
+    if sys_result.get("recommendedSystemPrompt"):
+        if sys_result.get("explanation"):
+            typer.echo("\n--- Explanation ---")
+            typer.echo(sys_result["explanation"])
+        typer.echo("\n+++ Recommended System Prompt +++")
+        typer.echo(sys_result["recommendedSystemPrompt"])
+    elif tool_result.get("tools"):
+        for t in tool_result["tools"]:
+            typer.echo(f"\nTool: {t.get('toolName')}")
+            if t.get("explanation"):
+                typer.echo(f"Explanation: {t['explanation']}")
+            typer.echo(f"Recommended: {t.get('recommendedToolDescription')}")
+    elif record.get("status") == "FAILED":
+        typer.echo(f"\nError: {failure_text(record)}")
+    if record.get("syncedVersionId"):
+        typer.echo(f"\nNew config bundle version {record['syncedVersionId']} applied to agentcore.json.")
+
+
+@run_app.command(
+    "recommendation",
+    help="Optimize a system prompt or tool descriptions using agent traces as signal",
+    cls=variadic("--tool-desc-json-path", "--tools", "-s", "--session-id"),
+)
+def run_recommendation(
+    type_: str | None = typer.Option(
+        None, "-t", "--type", help="What to optimize: system-prompt or tool-description (default: system-prompt)"
+    ),
+    runtime: str | None = typer.Option(None, "-r", "--runtime", help="Runtime name from project config"),
+    evaluator: str | None = typer.Option(
+        None,
+        "-e",
+        "--evaluator",
+        help="Evaluator name (exactly one). Required for system-prompt unless the trace source is a batch or online "
+        "evaluation, which inherits its evaluator",
+    ),
+    prompt_file: str | None = typer.Option(None, "--prompt-file", help="Load the current system prompt from a file"),
+    inline: str | None = typer.Option(
+        None, "--inline", help="Provide the current system prompt or tool descriptions inline"
+    ),
+    bundle_name: str | None = typer.Option(
+        None, "--bundle-name", help="Read current content from a deployed config bundle"
+    ),
+    bundle_version: str | None = typer.Option(
+        None, "--bundle-version", help="Config bundle version (used with --bundle-name)"
+    ),
+    system_prompt_json_path: str | None = typer.Option(
+        None,
+        "--system-prompt-json-path",
+        help='Field name under "configuration" in the bundle (e.g. "systemPrompt"). The CLI resolves it to the '
+        "full path automatically. Do not use bracket notation — use dot notation only.",
+    ),
+    tool_desc_json_path: list[str] | None = typer.Option(
+        None,
+        "--tool-desc-json-path",
+        help="Tool name:field pairs for tool descriptions in a config bundle (e.g. --tool-desc-json-path "
+        '"search:searchDesc"). The CLI resolves each to the full path automatically.',
+    ),
+    tools: list[str] | None = typer.Option(
+        None,
+        "--tools",
+        help='Tool name:description pairs (repeatable, e.g. --tools "search:Searches the web" --tools "calc:Does '
+        'math")',
+    ),
+    spans_file: str | None = typer.Option(
+        None, "--spans-file", help="JSON file with OTEL session spans (use instead of CloudWatch traces)"
+    ),
+    lookback: str = typer.Option(
+        "7",
+        "--lookback",
+        help="How far back to collect traces (days). Also bounds the online-evaluation reuse window: startTime = "
+        "now − lookback, endTime = now",
+    ),
+    session_id: list[str] | None = typer.Option(
+        None, "-s", "--session-id", help="Limit trace collection to specific session IDs"
+    ),
+    run_name: str | None = typer.Option(None, "-n", "--run", help="Run name prefix for the recommendation"),
+    region: str | None = typer.Option(None, "--region", help="AWS region"),
+    kms_key: str | None = typer.Option(None, "--kms-key", help="KMS key ARN for encrypting recommendation results"),
+    from_insights: str | None = typer.Option(
+        None, "--from-insights", help="Use a local insights run as trace source (resolves batch eval ARN)"
+    ),
+    batch_evaluation_arn: str | None = typer.Option(
+        None, "--batch-evaluation-arn", help="Use a batch evaluation ARN directly as trace source"
+    ),
+    online_evaluation_arn: str | None = typer.Option(
+        None, "--online-evaluation-arn", help="Use an online evaluation config ARN as trace source"
+    ),
+    wait: bool = typer.Option(False, "--wait", help="Block until the recommendation reaches a terminal state"),
+    as_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    require_project()
+    source_flags = [
+        ("--batch-evaluation-arn", bool(batch_evaluation_arn)),
+        ("--from-insights", bool(from_insights)),
+        ("--online-evaluation-arn", bool(online_evaluation_arn)),
+        ("--spans-file", bool(spans_file)),
+        ("--session-id", bool(session_id)),
+    ]
+    has_source = any(is_set for _, is_set in source_flags)
+    if not runtime and not has_source and not as_json:
+        from agentcore.cli import interactive
+        from agentcore.cli.commands._run_wizards import run_recommendation_wizard
+
+        interactive.require_tty()
+        run_recommendation_wizard()
+        raise typer.Exit(0)
+
+    set_flags = [n for n, is_set in source_flags if is_set]
+    if len(set_flags) > 1:
+        fail_early(
+            as_json,
+            f"Only one trace source may be specified. Got: {', '.join(set_flags)}. Use exactly one of "
+            "--batch-evaluation-arn, --from-insights, --online-evaluation-arn, --spans-file, or --session-id.",
+        )
+    type_key = type_ if type_ is not None else "system-prompt"
+    rec_type = RECOMMENDATION_TYPE_MAP.get(type_key)
+    if not rec_type:
+        fail_early(as_json, f'Invalid --type "{type_key}". Must be one of: {", ".join(RECOMMENDATION_TYPE_MAP)}')
+    is_batch = bool(from_insights or batch_evaluation_arn)
+    is_online = bool(online_evaluation_arn)
+    no_agent = is_batch or is_online
+    if not runtime and not no_agent:
+        fail_early(
+            as_json,
+            "--runtime is required (unless --from-insights, --batch-evaluation-arn, or --online-evaluation-arn is "
+            "provided)",
+        )
+    if rec_type == "SYSTEM_PROMPT_RECOMMENDATION" and not evaluator and not no_agent:
+        fail_early(
+            as_json,
+            "--evaluator is required for system-prompt recommendations (unless using --from-insights, "
+            "--batch-evaluation-arn, or --online-evaluation-arn)",
+        )
+    if prompt_file:
+        input_source = "file"
+    elif inline:
+        input_source = "inline"
+    elif bundle_name:
+        input_source = "config-bundle"
+    else:
+        input_source = "inline"
+    if is_batch:
+        trace_source = "batch-evaluation"
+    elif is_online:
+        trace_source = "online-evaluation"
+    elif spans_file:
+        trace_source = "spans-file"
+    elif session_id:
+        trace_source = "sessions"
+    else:
+        trace_source = "cloudwatch"
+    pairs = []
+    for pair in tool_desc_json_path or []:
+        idx = pair.find(":")
+        if idx > 0:
+            pairs.append({"toolName": pair[:idx], "toolDescriptionJsonPath": pair[idx + 1 :]})
+
+    def run() -> None:
+        engine = create_job_engine(ConfigIO())
+        start = engine.start(
+            "recommendation",
+            {
+                "type": rec_type,
+                "agent": runtime or "",
+                "evaluators": [evaluator] if evaluator else [],
+                "promptFile": prompt_file,
+                "inlineContent": inline,
+                "bundleName": bundle_name,
+                "bundleVersion": bundle_version,
+                "systemPromptJsonPath": system_prompt_json_path,
+                "toolDescJsonPaths": pairs or None,
+                "tools": list(tools) if tools else None,
+                "lookbackDays": parse_int(lookback),
+                "sessionIds": list(session_id) if session_id else None,
+                "spansFile": spans_file,
+                "fromInsights": from_insights,
+                "batchEvaluationArn": batch_evaluation_arn,
+                "onlineEvaluationArn": online_evaluation_arn,
+                "recommendationName": run_name,
+                "region": region,
+                "kmsKeyArn": kms_key,
+                "inputSource": input_source,
+                "traceSource": trace_source,
+                "onProgress": None if as_json else (lambda _s, m: typer.echo(m)),
+            },
+        )
+        if not start.success:
+            raise start.error
+        record = start.value
+        if wait:
+
+            def tick(status: str) -> None:
+                if not as_json:
+                    typer.echo(f"Status: {status}")
+
+            final = wait_for_terminal(engine, "recommendation", record["id"], on_tick=tick)
+            if final:
+                record = final
+        if as_json:
+            _job_json(record)
+        else:
+            typer.echo(f"\n✓ Recommendation started: {record['id']} ({record['status']})")
+            print_recommendation_result(record)
+            if not wait:
+                suffix = (
+                    " — the new config bundle will be applied to agentcore.json automatically."
+                    if input_source == "config-bundle"
+                    else ""
+                )
+                typer.echo(f"\nNext: agentcore view recommendation {record['id']}{suffix}")
+            typer.echo("")
+
+    run_cli_command(as_json, run)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# run ab-test
+# ──────────────────────────────────────────────────────────────────────────────
+
+_AB_TEST_EPILOG = """\b
+Config-bundle mode example:
+  agentcore run ab-test -n MyTest -g MyGateway \\
+    --control-bundle PromptV1 --control-version LATEST \\
+    --treatment-bundle PromptV2 --treatment-version LATEST \\
+    --online-eval QualityEval
+
+\b
+Target-based mode example:
+  agentcore run ab-test -n MyTest -g MyGateway --mode target-based \\
+    --control-target prod-target --treatment-target staging-target \\
+    --control-online-eval ProdEval --treatment-online-eval StagingEval
+"""
+
+
+def weight_error(control_raw: str, treatment_raw: str) -> tuple[int | None, int | None, str | None]:
+    cw, tw = parse_int(control_raw), parse_int(treatment_raw)
+    if cw is None or cw < 0 or cw > 100:
+        return cw, tw, f'Invalid --control-weight "{control_raw}". Must be an integer between 0 and 100.'
+    if tw is None or tw < 0 or tw > 100:
+        return cw, tw, f'Invalid --treatment-weight "{treatment_raw}". Must be an integer between 0 and 100.'
+    if cw + tw != 100:
+        return cw, tw, f"Variant weights must sum to 100 (got {cw} + {tw} = {cw + tw})."
+    return cw, tw, None
+
+
+def start_ab_test(opts: dict[str, Any], *, wait: bool, as_json: bool) -> None:
+    engine = create_job_engine(ConfigIO())
+    start = engine.start("ab-test", opts)
+    if not start.success:
+        raise start.error
+    record = start.value
+    if wait:
+
+        def tick(status: str) -> None:
+            if not as_json:
+                typer.echo(f"Status: {status}")
+
+        final = wait_for_terminal(engine, "ab-test", record["id"], on_tick=tick)
+        if final:
+            record = final
+    if as_json:
+        _job_json(record)
+    else:
+        typer.echo(f"\n✓ A/B test started: {record['id']} ({record['status']})")
+        print_ab_test_detail(record)
+        if not wait:
+            typer.echo(f"\nNext: agentcore view ab-test {record['id']}")
+        typer.echo("")
+
+
+@run_app.command(
+    "ab-test", help="Start an A/B test comparing two config-bundle or gateway-target variants", epilog=_AB_TEST_EPILOG
+)
+def run_ab_test(
+    name: str | None = typer.Option(None, "-n", "--name", help="Name for the A/B test [non-interactive]"),
+    gateway: str | None = typer.Option(
+        None, "-g", "--gateway", help="Gateway name (must be deployed) [non-interactive]"
+    ),
+    mode: str = typer.Option(
+        "config-bundle", "-m", "--mode", help="config-bundle | target-based (default: config-bundle)"
+    ),
+    description: str | None = typer.Option(None, "--description", help="Description"),
+    runtime: str | None = typer.Option(None, "-r", "--runtime", help="Runtime name (recorded as the agent)"),
+    control_weight: str = typer.Option("50", "--control-weight", help="Control traffic weight 0-100 (default: 50)"),
+    treatment_weight: str = typer.Option(
+        "50", "--treatment-weight", help="Treatment traffic weight 0-100 (default: 50)"
+    ),
+    role_arn: str | None = typer.Option(None, "--role-arn", help="Execution role ARN (auto-created if omitted)"),
+    disable_on_create: bool = typer.Option(
+        False, "--disable-on-create", help="Create without starting (default: enabled)"
+    ),
+    region: str | None = typer.Option(None, "--region", help="AWS region (auto-detected if omitted)"),
+    wait: bool = typer.Option(False, "--wait", help="Block until terminal state"),
+    as_json: bool = typer.Option(False, "--json", help="Output as JSON"),
+    control_bundle: str | None = typer.Option(
+        None, "--control-bundle", help="[config-bundle] Control bundle name or ARN"
+    ),
+    control_version: str | None = typer.Option(
+        None, "--control-version", help="[config-bundle] Control bundle version (or LATEST)"
+    ),
+    treatment_bundle: str | None = typer.Option(
+        None, "--treatment-bundle", help="[config-bundle] Treatment bundle name or ARN"
+    ),
+    treatment_version: str | None = typer.Option(
+        None, "--treatment-version", help="[config-bundle] Treatment bundle version (or LATEST)"
+    ),
+    online_eval: str | None = typer.Option(
+        None, "--online-eval", help="[config-bundle] Shared online eval config name or ARN"
+    ),
+    control_target: str | None = typer.Option(
+        None, "--control-target", help="[target-based] Control gateway-target name"
+    ),
+    treatment_target: str | None = typer.Option(
+        None, "--treatment-target", help="[target-based] Treatment gateway-target name"
+    ),
+    control_online_eval: str | None = typer.Option(
+        None, "--control-online-eval", help="[target-based] Online eval for control endpoint (required)"
+    ),
+    treatment_online_eval: str | None = typer.Option(
+        None, "--treatment-online-eval", help="[target-based] Online eval for treatment endpoint (required)"
+    ),
+    gateway_filter: str | None = typer.Option(
+        None,
+        "--gateway-filter",
+        help='Single gateway target path pattern to scope the test (e.g. "/orders/*"). Applies to both modes. '
+        "Optional; omit for no gateway filter.",
+    ),
+) -> None:
+    require_project()
+    if not name and not as_json:
+        from agentcore.cli import interactive
+        from agentcore.cli.commands._run_wizards import run_ab_test_wizard
+
+        interactive.require_tty()
+        run_ab_test_wizard()
+        raise typer.Exit(0)
+    if not name or not gateway:
+        fail_early(as_json, "--name and --gateway are required in non-interactive mode")
+    if mode not in ("config-bundle", "target-based"):
+        fail_early(as_json, f'Invalid --mode "{mode}". Must be one of: config-bundle, target-based')
+    cw, tw, error = weight_error(control_weight, treatment_weight)
+    if error:
+        fail_early(as_json, error)
+
+    opts = {
+        "name": name,
+        "mode": mode,
+        "description": description,
+        "gateway": gateway,
+        "agent": runtime,
+        "controlBundle": control_bundle,
+        "controlVersion": control_version,
+        "treatmentBundle": treatment_bundle,
+        "treatmentVersion": treatment_version,
+        "onlineEval": online_eval,
+        "runtime": runtime,
+        "controlTarget": control_target,
+        "treatmentTarget": treatment_target,
+        "controlOnlineEval": control_online_eval,
+        "treatmentOnlineEval": treatment_online_eval,
+        "gatewayFilter": gateway_filter,
+        "controlWeight": cw,
+        "treatmentWeight": tw,
+        "enableOnCreate": not disable_on_create,
+        "region": region,
+        "roleArn": role_arn,
+        "onProgress": None if as_json else (lambda _s, m: typer.echo(m)),
+    }
+    run_cli_command(as_json, lambda: start_ab_test(opts, wait=wait, as_json=as_json))