applied-cli 0.1.0__py3-none-any.whl

applied_cli/commands/fix.py

@@ -0,0 +1,1204 @@
+"""
+Benchmark fix workflow commands for AI agents.
+
+This module provides commands optimized for AI agents to fix failing test scenarios:
+1. `fix context` - Returns all context needed to understand and fix failing scenarios
+2. `fix test` - Replays a scenario's input message to validate a fix
+3. `fix batch` - Batch test multiple scenarios with retries and parallelism
+4. `fix status` - Track progress between source and target benchmarks
+"""
+
+import json
+import random
+import time
+import uuid as uuid_module
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import httpx
+import typer
+
+from applied_cli.commands._parsers import validate_uuid
+from applied_cli.commands.responses import RESPONSE_SEMANTICS
+from applied_cli.error_reporting import render_api_error
+from applied_cli.http import (
+    APIError,
+    create_conversation_benchmark,
+    create_conversation_scenario,
+    get_agent,
+    get_conversation,
+    get_conversation_benchmark,
+    get_conversation_scenario,
+    list_conversation_scenarios,
+    list_responses,
+    patch_conversation_scenario,
+)
+from applied_cli.runtime import resolve_runtime
+
+app = typer.Typer(
+    help=(
+        "Fix failing benchmark scenarios. Optimized for AI agent workflows.\n\n"
+        "Get all context needed to fix failures:\n"
+        "  applied-cli test fix context --benchmark-id <uuid>\n\n"
+        "Test a fix by replaying a scenario:\n"
+        "  applied-cli test fix test --scenario-id <uuid> --benchmark-id <uuid>\n\n"
+        "Batch test all failing scenarios:\n"
+        "  applied-cli test fix batch --source <uuid> --target <uuid>\n\n"
+        "Track progress between benchmarks:\n"
+        "  applied-cli test fix status --source <uuid> --target <uuid>"
+    )
+)
+
+
+# Default test contact names for auto-generation
+_TEST_FIRST_NAMES = ["Alex", "Jordan", "Taylor", "Morgan", "Casey", "Riley", "Quinn", "Avery"]
+_TEST_LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Davis", "Miller", "Wilson", "Moore"]
+
+
+def _generate_test_contact() -> tuple[str, str]:
+    """Generate a random test contact name and email."""
+    first = random.choice(_TEST_FIRST_NAMES)
+    last = random.choice(_TEST_LAST_NAMES)
+    name = f"{first} {last}"
+    email = f"{first.lower()}.{last.lower()}.{random.randint(100, 999)}@test.example.com"
+    return name, email
+
+
+@dataclass
+class TestResult:
+    """Result of testing a single scenario."""
+
+    original_scenario_id: str
+    success: bool
+    new_scenario_id: str | None = None
+    conversation_id: str | None = None
+    resolution: str | None = None
+    error: str | None = None
+    retries_used: int = 0
+
+
+def _extract_user_messages(conversation: dict[str, Any]) -> list[dict[str, str]]:
+    """Extract user messages from a conversation for replay."""
+    messages = conversation.get("messages") or []
+    user_messages = []
+    for msg in messages:
+        if msg.get("role") == "user":
+            user_messages.append({
+                "content": msg.get("content") or msg.get("text") or "",
+                "role": "user",
+            })
+    return user_messages
+
+
+def _get_failing_scenarios(
+    *,
+    base_url: str,
+    shop_id: str,
+    api_token: str,
+    benchmark_id: str,
+    limit: int = 200,
+) -> list[dict[str, Any]]:
+    """Fetch all failing scenarios from a benchmark."""
+    all_scenarios = list_conversation_scenarios(
+        base_url=base_url,
+        shop_id=shop_id,
+        api_token=api_token,
+        benchmark_id=benchmark_id,
+        limit=limit,
+        ordering="-created_at",
+    )
+    return [s for s in all_scenarios if s.get("pass_status") == "fail"]
+
+
+def _summarize_scenario(scenario: dict[str, Any]) -> dict[str, Any]:
+    """Create a concise summary of a scenario for AI consumption."""
+    input_conv = scenario.get("input_conversation") or {}
+    user_messages = _extract_user_messages(input_conv)
+
+    return {
+        "scenario_id": scenario.get("id"),
+        "name": scenario.get("name"),
+        "pass_status": scenario.get("pass_status"),
+        "feedback": scenario.get("feedback") or "",
+        "label": (input_conv.get("label") or {}).get("name"),
+        "sublabel": (input_conv.get("sublabel") or {}).get("name"),
+        "user_messages": user_messages,
+        "run_count": scenario.get("run_count", 0),
+    }
+
+
+def _summarize_response(response: dict[str, Any]) -> dict[str, Any]:
+    """Create a concise summary of a response rule."""
+    return {
+        "response_id": response.get("id"),
+        "type": response.get("type"),
+        "question": response.get("question"),
+        "answer": response.get("answer"),
+        "guardrail": response.get("guardrail") or "",
+        "active": response.get("active", True),
+    }
+
+
+def _create_test_conversation(
+    client: httpx.Client,
+    *,
+    base_url: str,
+    shop_id: str,
+    api_token: str,
+    agent_id: str,
+    channel: str,
+    contact_name: str | None = None,
+    contact_email: str | None = None,
+) -> str:
+    """Create a test conversation for replay."""
+    # Parse contact name into first/last
+    first_name = ""
+    last_name = ""
+    if contact_name:
+        parts = contact_name.strip().split(" ", 1)
+        first_name = parts[0]
+        last_name = parts[1] if len(parts) > 1 else ""
+
+    context: dict[str, object] = {
+        "channel": channel,
+        "firstName": first_name,
+        "lastName": last_name,
+        "contact": {
+            "contextFields": {
+                "channel": channel,
+            },
+        },
+    }
+    if contact_email:
+        context["email"] = contact_email
+        context["contact"]["email"] = contact_email  # type: ignore[index]
+
+    payload: dict[str, object] = {
+        "agent_id": agent_id,
+        "is_test": True,
+        "metadata": {
+            "isTest": True,
+            "source": "applied-cli-fix",
+            "context": context,
+        },
+    }
+    if channel == "email":
+        payload["type"] = "email"
+
+    headers = {
+        "Authorization": f"Bearer {api_token}",
+        "X-Shop-Id": shop_id,
+        "Content-Type": "application/json",
+    }
+
+    response = client.post(
+        f"{base_url}/v1/c/",
+        json=payload,
+        headers=headers,
+        timeout=10.0,
+    )
+
+    if response.status_code >= 400:
+        raise APIError(
+            f"Failed to create test conversation ({response.status_code})",
+            status_code=response.status_code,
+            code="CONVERSATION_CREATE_FAILED",
+        )
+
+    data = response.json()
+    return str(data.get("id"))
+
+
+def _send_message_and_get_response(
+    client: httpx.Client,
+    *,
+    base_url: str,
+    shop_id: str,
+    api_token: str,
+    agent_id: str,
+    conversation_id: str,
+    message: str,
+    timeout: float = 60.0,
+) -> str:
+    """Send a message and get the agent's response."""
+    headers = {
+        "Authorization": f"Bearer {api_token}",
+        "X-Shop-Id": shop_id,
+        "Content-Type": "application/json",
+    }
+
+    transcript = [
+        {
+            "id": str(uuid_module.uuid4()),
+            "role": "user",
+            "content": message,
+            "text": message,
+            "format": "TEXT",
+            "entity": {"type": "user"},
+        }
+    ]
+
+    payload = {
+        "conversation_id": conversation_id,
+        "context": "EVALUATE",
+        "transcript": transcript,
+        "metadata": {
+            "source": "applied-cli-fix",
+            "isTest": True,
+        },
+        "draft": False,
+    }
+
+    # Use non-streaming completion for simplicity
+    response = client.post(
+        f"{base_url}/v1/agents/{agent_id}/complete/",
+        headers=headers,
+        json=payload,
+        timeout=timeout,
+    )
+
+    if response.status_code >= 400:
+        raise APIError(
+            f"Completion failed ({response.status_code})",
+            status_code=response.status_code,
+            code="COMPLETION_FAILED",
+        )
+
+    # Parse streamed response to extract content
+    generated_text = ""
+    for line in response.text.split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            data = json.loads(line)
+            content = data.get("content")
+            if isinstance(content, str):
+                generated_text += content
+        except json.JSONDecodeError:
+            continue
+
+    return generated_text
+
+
+def _get_conversation_resolution(
+    *,
+    base_url: str,
+    shop_id: str,
+    api_token: str,
+    conversation_id: str,
+) -> str | None:
+    """Get the resolution status of a conversation."""
+    try:
+        conv = get_conversation(
+            base_url=base_url,
+            shop_id=shop_id,
+            api_token=api_token,
+            conversation_id=conversation_id,
+        )
+        return conv.get("resolution")
+    except Exception:
+        return None
+
+
+def _run_single_test(
+    *,
+    base_url: str,
+    shop_id: str,
+    api_token: str,
+    original_scenario: dict[str, Any],
+    target_benchmark_id: str,
+    contact_name: str | None = None,
+    contact_email: str | None = None,
+    auto_pass: bool = False,
+    timeout: float = 90.0,
+    retries: int = 3,
+    expect_escalation: bool | None = None,
+    quiet: bool = False,
+) -> TestResult:
+    """
+    Run a single test with retry logic.
+
+    Returns a TestResult with success/failure status.
+    """
+    scenario_id = original_scenario.get("id", "")
+    agent_data = original_scenario.get("agent") or {}
+    agent_id = agent_data.get("id")
+
+    if not agent_id:
+        return TestResult(
+            original_scenario_id=scenario_id,
+            success=False,
+            error="Scenario has no associated agent",
+        )
+
+    modality = (agent_data.get("modality") or "chat").lower()
+    channel = "email" if modality == "email" else "chat"
+
+    # Extract user messages
+    input_conv = original_scenario.get("input_conversation") or {}
+    user_messages = _extract_user_messages(input_conv)
+
+    if not user_messages:
+        return TestResult(
+            original_scenario_id=scenario_id,
+            success=False,
+            error="Scenario has no user messages to replay",
+        )
+
+    test_message = user_messages[0]["content"]
+
+    # Generate contact info if not provided
+    if not contact_name or not contact_email:
+        gen_name, gen_email = _generate_test_contact()
+        contact_name = contact_name or gen_name
+        contact_email = contact_email or gen_email
+
+    last_error: str | None = None
+    retries_used = 0
+
+    for attempt in range(retries + 1):
+        retries_used = attempt
+        try:
+            with httpx.Client() as client:
+                # Create test conversation
+                conversation_id = _create_test_conversation(
+                    client,
+                    base_url=base_url,
+                    shop_id=shop_id,
+                    api_token=api_token,
+                    agent_id=agent_id,
+                    channel=channel,
+                    contact_name=contact_name,
+                    contact_email=contact_email,
+                )
+
+                # Send message and get response
+                _send_message_and_get_response(
+                    client,
+                    base_url=base_url,
+                    shop_id=shop_id,
+                    api_token=api_token,
+                    agent_id=agent_id,
+                    conversation_id=conversation_id,
+                    message=test_message,
+                    timeout=timeout,
+                )
+
+            # Brief delay for processing
+            time.sleep(2)
+
+            # Get conversation resolution
+            resolution = _get_conversation_resolution(
+                base_url=base_url,
+                shop_id=shop_id,
+                api_token=api_token,
+                conversation_id=conversation_id,
+            )
+
+            # Validate expectations if set
+            if expect_escalation is not None:
+                is_escalated = resolution == "escalated"
+                if expect_escalation and not is_escalated:
+                    return TestResult(
+                        original_scenario_id=scenario_id,
+                        success=False,
+                        conversation_id=conversation_id,
+                        resolution=resolution,
+                        error="Expected escalation but agent responded",
+                        retries_used=retries_used,
+                    )
+                elif not expect_escalation and is_escalated:
+                    return TestResult(
+                        original_scenario_id=scenario_id,
+                        success=False,
+                        conversation_id=conversation_id,
+                        resolution=resolution,
+                        error="Expected response but agent escalated",
+                        retries_used=retries_used,
+                    )
+
+            # Create new scenario in target benchmark
+            new_scenario_name = f"[Fix Test] {original_scenario.get('name', 'Unnamed')}"
+            new_scenario = create_conversation_scenario(
+                base_url=base_url,
+                shop_id=shop_id,
+                api_token=api_token,
+                agent_id=agent_id,
+                benchmark_id=target_benchmark_id,
+                name=new_scenario_name,
+                input_conversation_id=conversation_id,
+            )
+
+            new_scenario_id = new_scenario.get("id")
+
+            # Auto-mark as pass if requested
+            if auto_pass and new_scenario_id:
+                patch_conversation_scenario(
+                    base_url=base_url,
+                    shop_id=shop_id,
+                    api_token=api_token,
+                    scenario_id=new_scenario_id,
+                    payload={"pass_status": "pass"},
+                )
+
+            return TestResult(
+                original_scenario_id=scenario_id,
+                success=True,
+                new_scenario_id=new_scenario_id,
+                conversation_id=conversation_id,
+                resolution=resolution,
+                retries_used=retries_used,
+            )
+
+        except httpx.TimeoutException as exc:
+            last_error = f"Timeout after {timeout}s"
+            if attempt < retries:
+                if not quiet:
+                    typer.echo(f"  Retry {attempt + 1}/{retries} after timeout...")
+                time.sleep(2 ** attempt)  # Exponential backoff
+                continue
+            break
+
+        except httpx.HTTPError as exc:
+            last_error = f"Network error: {exc}"
+            if attempt < retries:
+                if not quiet:
+                    typer.echo(f"  Retry {attempt + 1}/{retries} after error...")
+                time.sleep(2 ** attempt)
+                continue
+            break
+
+        except APIError as exc:
+            last_error = f"API error: {exc}"
+            # Don't retry API errors (likely not transient)
+            break
+
+        except Exception as exc:
+            last_error = f"Unexpected error: {exc}"
+            break
+
+    return TestResult(
+        original_scenario_id=scenario_id,
+        success=False,
+        error=last_error,
+        retries_used=retries_used,
+    )
+
+
+@app.command(
+    "context",
+    help=(
+        "Get all context needed to fix failing scenarios in a benchmark.\n\n"
+        "Returns:\n"
+        "- All failing scenarios with feedback and user messages\n"
+        "- Agent responses (knowledge base)\n"
+        "- Agent guardrails\n"
+        "- Response type semantics\n"
+        "- Instructions for making fixes\n\n"
+        "Example: applied-cli test fix context --benchmark-id <uuid>"
+    ),
+)
+def fix_context(
+    benchmark_id: str = typer.Option(
+        ..., "--benchmark-id", "--benchmark", "--id", help="Benchmark UUID."
+    ),
+    include_passing: bool = typer.Option(
+        False, "--include-passing", help="Include passing scenarios for reference."
+    ),
+    output_json: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    base_url: Optional[str] = typer.Option(None, help="Applied base URL."),
+    shop_id: Optional[str] = typer.Option(None, help="Target shop UUID."),
+    api_token: Optional[str] = typer.Option(None, help="Applied API token."),
+) -> None:
+    validate_uuid(benchmark_id, field_name="benchmark-id")
+
+    try:
+        resolved_base_url, resolved_shop_id, resolved_token = resolve_runtime(
+            base_url=base_url, shop_id=shop_id, api_token=api_token
+        )
+
+        # Get benchmark details
+        benchmark = get_conversation_benchmark(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=benchmark_id,
+        )
+
+        # Get agent from benchmark
+        agent_data = benchmark.get("agent") or {}
+        agent_id = agent_data.get("id")
+        if not agent_id:
+            raise typer.BadParameter("Benchmark has no associated agent.")
+
+        # Get full agent details including guardrails
+        agent = get_agent(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            agent_id=agent_id,
+        )
+
+        # Get all scenarios
+        all_scenarios = list_conversation_scenarios(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=benchmark_id,
+            limit=500,
+            ordering="-created_at",
+        )
+
+        # For failing scenarios, we need full details including input_conversation
+        failing_scenarios = []
+        passing_scenarios = []
+        for scenario in all_scenarios:
+            if scenario.get("pass_status") == "fail":
+                # Get full scenario details
+                full_scenario = get_conversation_scenario(
+                    base_url=resolved_base_url,
+                    shop_id=resolved_shop_id,
+                    api_token=resolved_token,
+                    scenario_id=scenario.get("id"),
+                )
+                failing_scenarios.append(_summarize_scenario(full_scenario))
+            elif include_passing and scenario.get("pass_status") == "pass":
+                passing_scenarios.append({
+                    "scenario_id": scenario.get("id"),
+                    "name": scenario.get("name"),
+                    "pass_status": "pass",
+                })
+
+        # Get agent responses (knowledge base)
+        responses = list_responses(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            agent_id=agent_id,
+            active=True,
+            limit=500,
+        )
+
+    except APIError as exc:
+        typer.echo(render_api_error(exc, action="get fix context"), err=True)
+        raise typer.Exit(code=1) from exc
+
+    # Build the context output
+    context: dict[str, Any] = {
+        "benchmark": {
+            "id": benchmark_id,
+            "name": benchmark.get("name"),
+            "scenario_count": benchmark.get("scenario_count"),
+        },
+        "agent": {
+            "id": agent_id,
+            "name": agent.get("name"),
+            "modality": agent.get("modality"),
+            "guardrail": agent.get("guardrail") or "",
+        },
+        "summary": {
+            "total_scenarios": len(all_scenarios),
+            "failing_count": len(failing_scenarios),
+            "passing_count": len([s for s in all_scenarios if s.get("pass_status") == "pass"]),
+            "unrated_count": len([s for s in all_scenarios if s.get("pass_status") is None]),
+        },
+        "failing_scenarios": failing_scenarios,
+        "responses": [_summarize_response(r) for r in responses],
+        "response_type_semantics": RESPONSE_SEMANTICS,
+        "instructions": {
+            "overview": (
+                "To fix failing scenarios, analyze each failure's feedback and user messages, "
+                "then update responses or agent guardrails accordingly."
+            ),
+            "response_types": {
+                "escalation": "Use for messages that should be escalated to humans. Set question to trigger criteria.",
+                "exact": "Use for verbatim templated responses. Answer is returned exactly as written.",
+                "qa": "Use for knowledge-grounded answers. Answer is used as context for generation.",
+                "context": "Use for background knowledge that informs responses broadly.",
+            },
+            "commands": {
+                "add_knowledge": "applied-cli knowledge upsert --agent-id <agent_id> --type <type> --question '<question>' --answer '<answer>' --yes",
+                "update_knowledge": "applied-cli knowledge update --response-id <response_id> --answer '<new_answer>' --yes",
+                "update_guardrail": "applied-cli agent update --agent-id <agent_id> --guardrail '<guardrail_text>' --yes",
+                "test_fix": "applied-cli test fix test --scenario-id <scenario_id> --benchmark-id <new_benchmark_id>",
+                "create_benchmark": "applied-cli benchmarks create --agent-id <agent_id> --name 'Fix Validation'",
+            },
+        },
+    }
+
+    if include_passing:
+        context["passing_scenarios"] = passing_scenarios
+
+    if output_json:
+        typer.echo(json.dumps(context, indent=2, default=str))
+    else:
+        typer.echo(f"Benchmark: {context['benchmark']['name']} ({benchmark_id})")
+        typer.echo(f"Agent: {context['agent']['name']} ({agent_id})")
+        typer.echo(f"Modality: {context['agent']['modality']}")
+        typer.echo("")
+        typer.echo(f"Summary: {context['summary']['failing_count']} failing, {context['summary']['passing_count']} passing, {context['summary']['unrated_count']} unrated")
+        typer.echo("")
+
+        if failing_scenarios:
+            typer.echo("=== FAILING SCENARIOS ===")
+            for scenario in failing_scenarios:
+                typer.echo(f"\n--- {scenario['name']} ---")
+                typer.echo(f"ID: {scenario['scenario_id']}")
+                typer.echo(f"Label: {scenario['label']} / {scenario['sublabel']}")
+                typer.echo(f"Feedback: {scenario['feedback'] or '(none)'}")
+                typer.echo("User messages:")
+                for msg in scenario['user_messages']:
+                    content = msg['content'][:200] + "..." if len(msg['content']) > 200 else msg['content']
+                    typer.echo(f"  - {content}")
+
+        typer.echo("\n=== AGENT RESPONSES ===")
+        typer.echo(f"Total: {len(responses)} active responses")
+        for r in responses[:10]:  # Show first 10
+            typer.echo(f"  [{r.get('type')}] {str(r.get('question') or '')[:60]}")
+        if len(responses) > 10:
+            typer.echo(f"  ... and {len(responses) - 10} more")
+
+        typer.echo("\n=== AGENT GUARDRAIL ===")
+        guardrail = context['agent']['guardrail']
+        if guardrail:
+            typer.echo(guardrail[:500] + "..." if len(guardrail) > 500 else guardrail)
+        else:
+            typer.echo("(no guardrail set)")
+
+        typer.echo("\nUse --json for full machine-readable output.")
+
+
+@app.command(
+    "test",
+    help=(
+        "Test a fix by replaying a scenario's input message.\n\n"
+        "This command:\n"
+        "1. Gets the original scenario's input message\n"
+        "2. Sends it to the agent to get a new response\n"
+        "3. Creates a new scenario in the target benchmark\n"
+        "4. Returns the scenario ID for rating\n\n"
+        "Example: applied-cli test fix test --scenario-id <uuid> --benchmark-id <uuid>"
+    ),
+)
+def fix_test(
+    scenario_id: str = typer.Option(
+        ..., "--scenario-id", "--scenario", help="Original scenario UUID to replay."
+    ),
+    benchmark_id: str = typer.Option(
+        ..., "--benchmark-id", "--benchmark", help="Target benchmark UUID for the new scenario."
+    ),
+    contact_name: Optional[str] = typer.Option(
+        None, "--contact-name", "--name", help="Contact name for the test. Auto-generated if not provided."
+    ),
+    contact_email: Optional[str] = typer.Option(
+        None, "--contact-email", "--email", help="Contact email for the test. Auto-generated if not provided."
+    ),
+    auto_pass: bool = typer.Option(
+        False, "--auto-pass", help="Automatically mark the new scenario as pass."
+    ),
+    expect_escalation: Optional[bool] = typer.Option(
+        None, "--expect-escalation/--expect-response",
+        help="Validate escalation behavior. --expect-escalation fails if agent responds, --expect-response fails if agent escalates."
+    ),
+    timeout: int = typer.Option(
+        90, "--timeout", "-t", help="Timeout in seconds for agent response."
+    ),
+    retries: int = typer.Option(
+        2, "--retry", "-r", help="Number of retries on timeout/network errors."
+    ),
+    feedback: Optional[str] = typer.Option(
+        None, "--feedback", help="Feedback to add to the new scenario."
+    ),
+    quiet: bool = typer.Option(
+        False, "--quiet", "-q", help="Minimal output (only show result)."
+    ),
+    output_json: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    base_url: Optional[str] = typer.Option(None, help="Applied base URL."),
+    shop_id: Optional[str] = typer.Option(None, help="Target shop UUID."),
+    api_token: Optional[str] = typer.Option(None, help="Applied API token."),
+) -> None:
+    validate_uuid(scenario_id, field_name="scenario-id")
+    validate_uuid(benchmark_id, field_name="benchmark-id")
+
+    try:
+        resolved_base_url, resolved_shop_id, resolved_token = resolve_runtime(
+            base_url=base_url, shop_id=shop_id, api_token=api_token
+        )
+
+        # Get the original scenario
+        original_scenario = get_conversation_scenario(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            scenario_id=scenario_id,
+        )
+
+        if not quiet:
+            input_conv = original_scenario.get("input_conversation") or {}
+            user_messages = _extract_user_messages(input_conv)
+            test_message = user_messages[0]["content"] if user_messages else ""
+            typer.echo(f"Replaying scenario: {original_scenario.get('name')}")
+            typer.echo(f"Message: {test_message[:100]}...")
+            typer.echo("")
+            typer.echo("Sending message to agent...")
+
+    except APIError as exc:
+        typer.echo(render_api_error(exc, action="get original scenario"), err=True)
+        raise typer.Exit(code=1) from exc
+
+    # Run the test with retry logic
+    test_result = _run_single_test(
+        base_url=resolved_base_url,
+        shop_id=resolved_shop_id,
+        api_token=resolved_token,
+        original_scenario=original_scenario,
+        target_benchmark_id=benchmark_id,
+        contact_name=contact_name,
+        contact_email=contact_email,
+        auto_pass=auto_pass,
+        timeout=float(timeout),
+        retries=retries,
+        expect_escalation=expect_escalation,
+        quiet=quiet,
+    )
+
+    # Handle feedback update if provided
+    if test_result.success and feedback and test_result.new_scenario_id:
+        try:
+            patch_conversation_scenario(
+                base_url=resolved_base_url,
+                shop_id=resolved_shop_id,
+                api_token=resolved_token,
+                scenario_id=test_result.new_scenario_id,
+                payload={"feedback": feedback},
+            )
+        except APIError:
+            pass  # Non-critical
+
+    result = {
+        "result": "success" if test_result.success else "failed",
+        "original_scenario_id": scenario_id,
+        "new_scenario_id": test_result.new_scenario_id,
+        "benchmark_id": benchmark_id,
+        "conversation_id": test_result.conversation_id,
+        "resolution": test_result.resolution,
+        "pass_status": "pass" if auto_pass and test_result.success else None,
+        "retries_used": test_result.retries_used,
+        "error": test_result.error,
+    }
+
+    if output_json:
+        typer.echo(json.dumps(result, indent=2, default=str))
+    elif test_result.success:
+        if quiet:
+            typer.echo(f"✓ {test_result.new_scenario_id}")
+        else:
+            typer.echo(f"\nCreated new scenario: {test_result.new_scenario_id}")
+            typer.echo(f"In benchmark: {benchmark_id}")
+            typer.echo(f"Resolution: {test_result.resolution or 'answered'}")
+            if auto_pass:
+                typer.echo("Status: pass (auto-marked)")
+            else:
+                typer.echo("Status: unrated")
+                typer.echo(f"\nTo rate: applied-cli scenarios update --scenario-id {test_result.new_scenario_id} --pass-status pass")
+    else:
+        if quiet:
+            typer.echo(f"✗ {scenario_id}: {test_result.error}")
+        else:
+            typer.echo(f"\nTest failed: {test_result.error}", err=True)
+            if test_result.retries_used > 0:
+                typer.echo(f"Retries used: {test_result.retries_used}", err=True)
+        raise typer.Exit(code=1)
+
+
+@app.command(
+    "summary",
+    help=(
+        "Get a quick summary of a benchmark's pass/fail status.\n\n"
+        "Example: applied-cli test fix summary --benchmark-id <uuid>"
+    ),
+)
+def fix_summary(
+    benchmark_id: str = typer.Option(
+        ..., "--benchmark-id", "--benchmark", "--id", help="Benchmark UUID."
+    ),
+    output_json: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    base_url: Optional[str] = typer.Option(None, help="Applied base URL."),
+    shop_id: Optional[str] = typer.Option(None, help="Target shop UUID."),
+    api_token: Optional[str] = typer.Option(None, help="Applied API token."),
+) -> None:
+    validate_uuid(benchmark_id, field_name="benchmark-id")
+
+    try:
+        resolved_base_url, resolved_shop_id, resolved_token = resolve_runtime(
+            base_url=base_url, shop_id=shop_id, api_token=api_token
+        )
+
+        benchmark = get_conversation_benchmark(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=benchmark_id,
+        )
+
+        all_scenarios = list_conversation_scenarios(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=benchmark_id,
+            limit=500,
+        )
+
+    except APIError as exc:
+        typer.echo(render_api_error(exc, action="get benchmark summary"), err=True)
+        raise typer.Exit(code=1) from exc
+
+    passing = [s for s in all_scenarios if s.get("pass_status") == "pass"]
+    failing = [s for s in all_scenarios if s.get("pass_status") == "fail"]
+    unrated = [s for s in all_scenarios if s.get("pass_status") is None]
+
+    summary = {
+        "benchmark_id": benchmark_id,
+        "benchmark_name": benchmark.get("name"),
+        "total": len(all_scenarios),
+        "passing": len(passing),
+        "failing": len(failing),
+        "unrated": len(unrated),
+        "pass_rate": f"{len(passing) / len(all_scenarios) * 100:.1f}%" if all_scenarios else "N/A",
+        "failing_scenario_ids": [s.get("id") for s in failing],
+    }
+
+    if output_json:
+        typer.echo(json.dumps(summary, indent=2, default=str))
+    else:
+        typer.echo(f"Benchmark: {summary['benchmark_name']}")
+        typer.echo(f"Total: {summary['total']} scenarios")
+        typer.echo(f"Passing: {summary['passing']} ({summary['pass_rate']})")
+        typer.echo(f"Failing: {summary['failing']}")
+        typer.echo(f"Unrated: {summary['unrated']}")
+        if failing:
+            typer.echo(f"\nFailing scenario IDs:")
+            for sid in summary['failing_scenario_ids']:
+                typer.echo(f"  - {sid}")
+
+
+@app.command(
+    "batch",
+    help=(
+        "Batch test all failing scenarios from a source benchmark.\n\n"
+        "This command:\n"
+        "1. Fetches all failing scenarios from the source benchmark\n"
+        "2. Tests each one with retries and parallelism\n"
+        "3. Creates new scenarios in the target benchmark\n"
+        "4. Reports overall progress\n\n"
+        "Example: applied-cli test fix batch --source <uuid> --target <uuid> --auto-pass"
+    ),
+)
+def fix_batch(
+    source_benchmark_id: str = typer.Option(
+        ..., "--source", "--source-benchmark", help="Source benchmark UUID with failing scenarios."
+    ),
+    target_benchmark_id: str = typer.Option(
+        ..., "--target", "--target-benchmark", help="Target benchmark UUID for new scenarios."
+    ),
+    pass_status_filter: str = typer.Option(
+        "fail", "--pass-status", help="Filter scenarios by pass status: fail, pass, unrated, or all."
+    ),
+    auto_pass: bool = typer.Option(
+        False, "--auto-pass", help="Automatically mark successful tests as pass."
+    ),
+    timeout: int = typer.Option(
+        90, "--timeout", "-t", help="Timeout in seconds per test."
+    ),
+    retries: int = typer.Option(
+        2, "--retry", "-r", help="Number of retries per test."
+    ),
+    parallel: int = typer.Option(
+        1, "--parallel", "-p", help="Number of parallel tests (1-10)."
+    ),
+    limit: int = typer.Option(
+        0, "--limit", "-l", help="Max scenarios to test (0 = all)."
+    ),
+    continue_on_error: bool = typer.Option(
+        True, "--continue-on-error/--stop-on-error",
+        help="Continue testing even if some scenarios fail."
+    ),
+    output_json: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    base_url: Optional[str] = typer.Option(None, help="Applied base URL."),
+    shop_id: Optional[str] = typer.Option(None, help="Target shop UUID."),
+    api_token: Optional[str] = typer.Option(None, help="Applied API token."),
+) -> None:
+    validate_uuid(source_benchmark_id, field_name="source-benchmark")
+    validate_uuid(target_benchmark_id, field_name="target-benchmark")
+
+    # Clamp parallel workers
+    parallel = max(1, min(10, parallel))
+
+    try:
+        resolved_base_url, resolved_shop_id, resolved_token = resolve_runtime(
+            base_url=base_url, shop_id=shop_id, api_token=api_token
+        )
+
+        # Get source scenarios
+        all_scenarios = list_conversation_scenarios(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=source_benchmark_id,
+            limit=500,
+        )
+
+        # Filter by pass status
+        if pass_status_filter == "all":
+            scenarios_to_test = all_scenarios
+        else:
+            filter_value = None if pass_status_filter == "unrated" else pass_status_filter
+            scenarios_to_test = [s for s in all_scenarios if s.get("pass_status") == filter_value]
+
+        # Apply limit
+        if limit > 0:
+            scenarios_to_test = scenarios_to_test[:limit]
+
+        if not scenarios_to_test:
+            typer.echo(f"No scenarios found with pass_status={pass_status_filter}")
+            raise typer.Exit(code=0)
+
+        # Fetch full scenario details for each
+        full_scenarios = []
+        for scenario in scenarios_to_test:
+            full = get_conversation_scenario(
+                base_url=resolved_base_url,
+                shop_id=resolved_shop_id,
+                api_token=resolved_token,
+                scenario_id=scenario.get("id"),
+            )
+            full_scenarios.append(full)
+
+    except APIError as exc:
+        typer.echo(render_api_error(exc, action="get scenarios"), err=True)
+        raise typer.Exit(code=1) from exc
+
+    typer.echo(f"Testing {len(full_scenarios)} scenarios...")
+    typer.echo(f"  Source: {source_benchmark_id}")
+    typer.echo(f"  Target: {target_benchmark_id}")
+    typer.echo(f"  Parallel: {parallel}, Retries: {retries}, Timeout: {timeout}s")
+    typer.echo("")
+
+    results: list[TestResult] = []
+    success_count = 0
+    fail_count = 0
+
+    def run_test(scenario: dict[str, Any]) -> TestResult:
+        return _run_single_test(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            original_scenario=scenario,
+            target_benchmark_id=target_benchmark_id,
+            auto_pass=auto_pass,
+            timeout=float(timeout),
+            retries=retries,
+            quiet=True,
+        )
+
+    if parallel == 1:
+        # Sequential execution with progress
+        for i, scenario in enumerate(full_scenarios):
+            name = scenario.get("name", "Unnamed")[:40]
+            typer.echo(f"[{i+1}/{len(full_scenarios)}] {name}...", nl=False)
+
+            result = run_test(scenario)
+            results.append(result)
+
+            if result.success:
+                success_count += 1
+                typer.echo(f" ✓ {result.resolution or 'answered'}")
+            else:
+                fail_count += 1
+                typer.echo(f" ✗ {result.error}")
+                if not continue_on_error:
+                    break
+    else:
+        # Parallel execution
+        with ThreadPoolExecutor(max_workers=parallel) as executor:
+            future_to_scenario = {
+                executor.submit(run_test, s): s for s in full_scenarios
+            }
+
+            for i, future in enumerate(as_completed(future_to_scenario)):
+                scenario = future_to_scenario[future]
+                name = scenario.get("name", "Unnamed")[:40]
+
+                try:
+                    result = future.result()
+                    results.append(result)
+
+                    if result.success:
+                        success_count += 1
+                        typer.echo(f"[{i+1}/{len(full_scenarios)}] ✓ {name}")
+                    else:
+                        fail_count += 1
+                        typer.echo(f"[{i+1}/{len(full_scenarios)}] ✗ {name}: {result.error}")
+                except Exception as exc:
+                    fail_count += 1
+                    results.append(TestResult(
+                        original_scenario_id=scenario.get("id", ""),
+                        success=False,
+                        error=str(exc),
+                    ))
+                    typer.echo(f"[{i+1}/{len(full_scenarios)}] ✗ {name}: {exc}")
+
+    typer.echo("")
+    typer.echo(f"=== Results ===")
+    typer.echo(f"Tested: {len(results)}")
+    typer.echo(f"Success: {success_count}")
+    typer.echo(f"Failed: {fail_count}")
+
+    if output_json:
+        summary = {
+            "source_benchmark_id": source_benchmark_id,
+            "target_benchmark_id": target_benchmark_id,
+            "total_tested": len(results),
+            "success": success_count,
+            "failed": fail_count,
+            "results": [
+                {
+                    "original_id": r.original_scenario_id,
+                    "new_id": r.new_scenario_id,
+                    "success": r.success,
+                    "resolution": r.resolution,
+                    "error": r.error,
+                }
+                for r in results
+            ],
+        }
+        typer.echo(json.dumps(summary, indent=2, default=str))
+
+    if fail_count > 0 and not continue_on_error:
+        raise typer.Exit(code=1)
+
+
+@app.command(
+    "status",
+    help=(
+        "Track fix progress between source and target benchmarks.\n\n"
+        "Shows which scenarios from the source have been tested in the target.\n\n"
+        "Example: applied-cli test fix status --source <uuid> --target <uuid>"
+    ),
+)
+def fix_status(
+    source_benchmark_id: str = typer.Option(
+        ..., "--source", "--source-benchmark", help="Source benchmark UUID with original scenarios."
+    ),
+    target_benchmark_id: str = typer.Option(
+        ..., "--target", "--target-benchmark", help="Target benchmark UUID with test results."
+    ),
+    output_json: bool = typer.Option(False, "--json", help="Emit JSON output."),
+    base_url: Optional[str] = typer.Option(None, help="Applied base URL."),
+    shop_id: Optional[str] = typer.Option(None, help="Target shop UUID."),
+    api_token: Optional[str] = typer.Option(None, help="Applied API token."),
+) -> None:
+    validate_uuid(source_benchmark_id, field_name="source-benchmark")
+    validate_uuid(target_benchmark_id, field_name="target-benchmark")
+
+    try:
+        resolved_base_url, resolved_shop_id, resolved_token = resolve_runtime(
+            base_url=base_url, shop_id=shop_id, api_token=api_token
+        )
+
+        # Get both benchmarks
+        source_benchmark = get_conversation_benchmark(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=source_benchmark_id,
+        )
+
+        target_benchmark = get_conversation_benchmark(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=target_benchmark_id,
+        )
+
+        # Get scenarios from both
+        source_scenarios = list_conversation_scenarios(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=source_benchmark_id,
+            limit=500,
+        )
+
+        target_scenarios = list_conversation_scenarios(
+            base_url=resolved_base_url,
+            shop_id=resolved_shop_id,
+            api_token=resolved_token,
+            benchmark_id=target_benchmark_id,
+            limit=500,
+        )
+
+    except APIError as exc:
+        typer.echo(render_api_error(exc, action="get benchmark status"), err=True)
+        raise typer.Exit(code=1) from exc
+
+    # Analyze source
+    source_failing = [s for s in source_scenarios if s.get("pass_status") == "fail"]
+    source_passing = [s for s in source_scenarios if s.get("pass_status") == "pass"]
+    source_unrated = [s for s in source_scenarios if s.get("pass_status") is None]
+
+    # Analyze target
+    target_passing = [s for s in target_scenarios if s.get("pass_status") == "pass"]
+    target_failing = [s for s in target_scenarios if s.get("pass_status") == "fail"]
+    target_unrated = [s for s in target_scenarios if s.get("pass_status") is None]
+
+    # Calculate progress
+    source_fail_count = len(source_failing)
+    target_tested = len(target_scenarios)
+    target_pass_count = len(target_passing)
+
+    progress_pct = (target_tested / source_fail_count * 100) if source_fail_count > 0 else 0
+    pass_rate = (target_pass_count / target_tested * 100) if target_tested > 0 else 0
+
+    status = {
+        "source": {
+            "benchmark_id": source_benchmark_id,
+            "name": source_benchmark.get("name"),
+            "total": len(source_scenarios),
+            "failing": source_fail_count,
+            "passing": len(source_passing),
+            "unrated": len(source_unrated),
+        },
+        "target": {
+            "benchmark_id": target_benchmark_id,
+            "name": target_benchmark.get("name"),
+            "total": target_tested,
+            "passing": target_pass_count,
+            "failing": len(target_failing),
+            "unrated": len(target_unrated),
+        },
+        "progress": {
+            "tested": target_tested,
+            "of_source_failing": source_fail_count,
+            "progress_pct": round(progress_pct, 1),
+            "pass_rate": round(pass_rate, 1),
+            "remaining": max(0, source_fail_count - target_tested),
+        },
+    }
+
+    if output_json:
+        typer.echo(json.dumps(status, indent=2, default=str))
+    else:
+        typer.echo(f"=== Fix Progress ===")
+        typer.echo("")
+        typer.echo(f"Source: {status['source']['name']}")
+        typer.echo(f"  Total: {status['source']['total']}, Failing: {status['source']['failing']}")
+        typer.echo("")
+        typer.echo(f"Target: {status['target']['name']}")
+        typer.echo(f"  Tested: {status['target']['total']}, Passing: {status['target']['passing']}")
+        typer.echo("")
+        typer.echo(f"Progress: {status['progress']['tested']}/{status['progress']['of_source_failing']} ({status['progress']['progress_pct']}%)")
+        typer.echo(f"Pass Rate: {status['progress']['pass_rate']}%")
+        typer.echo(f"Remaining: {status['progress']['remaining']} scenarios")