agentcore-cli 0.31.1__py3-none-any.whl

agentcore/operations/jobs/ab_test/resolve.py

@@ -0,0 +1,221 @@
+"""AB-test resolution helpers (port of operations/jobs/ab-test/resolve.ts).
+
+IAM role create/reuse/delete, and ARN resolution for gateway / config-bundle / online-eval
+references against the merged deployed-resources dict (camelCase).
+"""
+
+import hashlib
+import json
+import re
+import time
+from collections.abc import Callable
+from typing import Any
+
+from agentcore.aws._aws_errors import aws_error_name
+from agentcore.aws.account import create_aws_client
+from agentcore.aws.iam import validate_iam_role_trust_policy
+from agentcore.aws.partition import arn_prefix
+
+AB_TEST_ROLE_POLICY_NAME = "ABTestExecutionPolicy"
+
+# IAM policy propagation wait after creating/updating the role (ms).
+IAM_PROPAGATION_DELAY_MS = 15_000
+
+
+def generate_role_name(project_name: str, test_name: str) -> str:
+    """AgentCore-{ProjectName}-ABTest{TestName}-{Hash} (deterministic, max 64 chars)."""
+    digest = hashlib.sha256(f"{project_name}:{test_name}".encode()).hexdigest()[:8]
+    base = f"AgentCore-{project_name}-ABTest{test_name}"
+    return f"{base[:55]}-{digest}"
+
+
+def role_name_from_arn(role_arn: str) -> str:
+    return role_arn.split("/")[-1]
+
+
+def _trust_policy(region: str, account_id: str) -> dict[str, Any]:
+    return {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": {"Service": "bedrock-agentcore.amazonaws.com"},
+                "Action": "sts:AssumeRole",
+                "Condition": {
+                    "StringEquals": {"aws:SourceAccount": account_id},
+                    "ArnLike": {"aws:SourceArn": f"{arn_prefix(region)}:bedrock-agentcore:*:{account_id}:ab-test/*"},
+                },
+            }
+        ],
+    }
+
+
+def _execution_policy(region: str, account_id: str) -> dict[str, Any]:
+    prefix = arn_prefix(region)
+    return {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "AgentCoreResources",
+                "Effect": "Allow",
+                "Action": [
+                    "bedrock-agentcore:GetGateway",
+                    "bedrock-agentcore:GetGatewayTarget",
+                    "bedrock-agentcore:ListGatewayTargets",
+                    "bedrock-agentcore:CreateGatewayRule",
+                    "bedrock-agentcore:UpdateGatewayRule",
+                    "bedrock-agentcore:GetGatewayRule",
+                    "bedrock-agentcore:DeleteGatewayRule",
+                    "bedrock-agentcore:ListGatewayRules",
+                    "bedrock-agentcore:GetOnlineEvaluationConfig",
+                    "bedrock-agentcore:GetEvaluator",
+                    "bedrock-agentcore:GetConfigurationBundle",
+                    "bedrock-agentcore:GetConfigurationBundleVersion",
+                    "bedrock-agentcore:ListConfigurationBundleVersions",
+                ],
+                "Resource": f"{prefix}:bedrock-agentcore:*:{account_id}:*",
+                "Condition": {"StringEquals": {"aws:ResourceAccount": account_id}},
+            },
+            {"Sid": "CloudWatchLogsDescribe", "Effect": "Allow", "Action": ["logs:DescribeLogGroups"], "Resource": "*"},
+            {
+                "Sid": "CloudWatchLogs",
+                "Effect": "Allow",
+                "Action": [
+                    "logs:DescribeIndexPolicies",
+                    "logs:PutIndexPolicy",
+                    "logs:StartQuery",
+                    "logs:GetQueryResults",
+                    "logs:StopQuery",
+                    "logs:FilterLogEvents",
+                    "logs:GetLogEvents",
+                ],
+                "Resource": [
+                    f"{prefix}:logs:*:{account_id}:log-group:/aws/bedrock-agentcore/evaluations/*",
+                    f"{prefix}:logs:*:{account_id}:log-group:/aws/bedrock-agentcore/runtimes/*",
+                    f"{prefix}:logs:*:{account_id}:log-group:aws/spans",
+                    f"{prefix}:logs:*:{account_id}:log-group:aws/spans:*",
+                ],
+            },
+        ],
+    }
+
+
+def _compact(value: Any) -> str:
+    return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
+
+
+def get_or_create_ab_test_role(
+    *,
+    region: str,
+    project_name: str,
+    test_name: str,
+    gateway_arn: str,
+    propagation_delay_ms: int | None = None,
+    sleep: Callable[[float], None] = time.sleep,
+) -> str:
+    """Create (or reuse) the AB-test execution role + inline policy, then wait for IAM propagation."""
+    iam = create_aws_client("iam", region)
+    parts = gateway_arn.split(":")
+    account_id = parts[4] if len(parts) > 4 else "*"
+    role_name = generate_role_name(project_name, test_name)
+    trust = _trust_policy(region, account_id)
+    try:
+        created = iam.create_role(
+            RoleName=role_name,
+            AssumeRolePolicyDocument=_compact(trust),
+            Description=f"Auto-created execution role for AgentCore AB test: {test_name}",
+            Tags=[
+                {"Key": "agentcore:created-by", "Value": "agentcore-cli"},
+                {"Key": "agentcore:project-name", "Value": project_name},
+                {"Key": "agentcore:ab-test-name", "Value": test_name},
+            ],
+        )
+        role_arn = (created.get("Role") or {}).get("Arn") or ""
+        if not role_arn:
+            raise Exception(f'IAM CreateRole succeeded but returned no role ARN for "{role_name}"')
+    except Exception as exc:
+        if aws_error_name(exc) not in ("EntityAlreadyExistsException", "EntityAlreadyExists"):
+            raise
+        existing = iam.get_role(RoleName=role_name)
+        role_arn = (existing.get("Role") or {}).get("Arn") or ""
+        if not role_arn:
+            raise Exception(f'Role "{role_name}" already exists but ARN could not be retrieved') from None
+        validate_iam_role_trust_policy(
+            (existing.get("Role") or {}).get("AssumeRolePolicyDocument"),
+            trust,
+            role_name,
+            "Delete the conflicting role or provide a customer-managed role with --role-arn.",
+        )
+
+    iam.put_role_policy(
+        RoleName=role_name,
+        PolicyName=AB_TEST_ROLE_POLICY_NAME,
+        PolicyDocument=_compact(_execution_policy(region, account_id)),
+    )
+    delay = propagation_delay_ms if propagation_delay_ms is not None else IAM_PROPAGATION_DELAY_MS
+    sleep(delay / 1000)
+    return role_arn
+
+
+def delete_ab_test_role(region: str, role_arn: str) -> None:
+    """Best-effort role cleanup: delete the inline policy then the role."""
+    iam = create_aws_client("iam", region)
+    role_name = role_name_from_arn(role_arn)
+    try:
+        iam.delete_role_policy(RoleName=role_name, PolicyName=AB_TEST_ROLE_POLICY_NAME)
+    except Exception:  # noqa: BLE001, S110
+        pass
+    try:
+        iam.delete_role(RoleName=role_name)
+    except Exception:  # noqa: BLE001, S110
+        pass
+
+
+def resolve_gateway_arn(ref: str, deployed: dict[str, Any] | None) -> str | None:
+    if ref.startswith("arn:"):
+        return ref
+    m = re.fullmatch(r"\{\{gateway:(.+)\}\}", ref)
+    name = m.group(1) if m else ref
+    deployed = deployed or {}
+    mcp_gw = ((deployed.get("mcp") or {}).get("gateways") or {}).get(name) if name else None
+    if mcp_gw:
+        return mcp_gw.get("gatewayArn")
+    http_gw = (deployed.get("gateways") or {}).get(name) if name else None
+    if http_gw:
+        return http_gw.get("gatewayArn")
+    return None
+
+
+def resolve_config_bundle_arn(ref: str, deployed: dict[str, Any] | None) -> str | None:
+    if ref.startswith("arn:"):
+        return ref
+    bundle = ((deployed or {}).get("configBundles") or {}).get(ref)
+    return bundle.get("bundleArn") if bundle else None
+
+
+def resolve_config_bundle_version(bundle_ref: str, version_ref: str, deployed: dict[str, Any] | None) -> str | None:
+    if version_ref != "LATEST":
+        return version_ref
+    if bundle_ref.startswith("arn:"):
+        return None
+    bundle = ((deployed or {}).get("configBundles") or {}).get(bundle_ref)
+    return bundle.get("versionId") if bundle else None
+
+
+def resolve_online_eval_arn(ref: str, deployed: dict[str, Any] | None) -> str | None:
+    if ref.startswith("arn:"):
+        return ref
+    config = ((deployed or {}).get("onlineEvalConfigs") or {}).get(ref)
+    return config.get("onlineEvaluationConfigArn") if config else None
+
+
+def resolve_runtime_target_names(gateway_name: str | None, runtime_name: str | None, project: dict[str, Any]) -> list:
+    """Every httpRuntime target on the gateway that fronts the runtime, in spec order."""
+    if not gateway_name or not runtime_name:
+        return []
+    gateway = next((g for g in project.get("agentCoreGateways") or [] if g.get("name") == gateway_name), None)
+    return [
+        t["name"]
+        for t in (gateway or {}).get("targets") or []
+        if t.get("targetType") == "httpRuntime" and (t.get("httpRuntime") or {}).get("runtime") == runtime_name
+    ]

agentcore/operations/jobs/batch_evaluation/build_source.py

@@ -0,0 +1,68 @@
+"""Batch-evaluation start-time helpers (port of operations/jobs/batch-evaluation/build-source.ts)."""
+
+import re
+import time
+from typing import Any
+
+from agentcore.aws.cloudwatch import resolve_endpoint_name, runtime_log_group
+from agentcore.lib import ValidationError
+from agentcore.operations.jobs.shared.constants import BATCH_EVAL_NAME_REGEX
+from agentcore.operations.jobs.shared.format import iso_from_ms, iso_now
+
+
+def resolve_batch_evaluator_ids(deployed_state: dict[str, Any], agent: str, evaluators: list[str]) -> list[str]:
+    """Evaluator references -> the SHORT ids the batch API expects (Builtin.*, ARNs, deployed names)."""
+    target_resources = next(
+        (
+            (t or {}).get("resources")
+            for t in (deployed_state.get("targets") or {}).values()
+            if (((t or {}).get("resources") or {}).get("runtimes") or {}).get(agent) is not None
+        ),
+        None,
+    )
+    out: list[str] = []
+    for name in evaluators:
+        short = name.split("evaluator/")[-1] if "evaluator/" in name else name
+        if short.startswith("Builtin."):
+            out.append(short)
+            continue
+        deployed = ((target_resources or {}).get("evaluators") or {}).get(short)
+        out.append(deployed["evaluatorId"] if deployed and deployed.get("evaluatorId") else short)
+    return out
+
+
+def build_cloud_watch_source(project_name: str, agent: str, runtime_id: str, endpoint: str | None) -> dict[str, str]:
+    """CloudWatch service name + log group for the agent's runtime traces."""
+    endpoint_name = resolve_endpoint_name(endpoint)
+    return {
+        "serviceName": f"{project_name}_{agent}.{endpoint_name}",
+        "logGroupName": runtime_log_group(runtime_id, endpoint),
+    }
+
+
+def resolve_batch_eval_name(name: str | None, project_name: str, agent: str) -> str:
+    """Validate an explicit name or auto-generate one."""
+    if name:
+        if not BATCH_EVAL_NAME_REGEX.match(name):
+            raise ValidationError(
+                "Batch evaluation name must start with a letter and contain only letters, digits, and "
+                f'underscores (max 48 chars). Got: "{name}"'
+            )
+        return name
+    return re.sub(r"[^a-zA-Z0-9_]", "_", f"{project_name}_{agent}_{int(time.time() * 1000)}")[:48]
+
+
+def build_cloud_watch_filter_config(session_ids: list[str] | None, lookback_days: int | None) -> dict | None:
+    """sessionIds take precedence over a lookback timeRange; None when neither is given."""
+    effective = list(dict.fromkeys(session_ids or []))
+    if effective:
+        return {"sessionIds": effective}
+    if lookback_days:
+        now_ms = time.time() * 1000
+        return {
+            "timeRange": {
+                "startTime": iso_from_ms(now_ms - lookback_days * 24 * 60 * 60 * 1000),
+                "endTime": iso_now(),
+            }
+        }
+    return None

agentcore/operations/jobs/batch_evaluation/dataset_phase1.py

@@ -0,0 +1,95 @@
+"""Dataset Phase-1 for batch evaluation (port of operations/jobs/batch-evaluation/dataset-phase1.ts).
+
+Invokes the agent against every dataset scenario, waits for CloudWatch ingestion, and returns the
+sessionIds + ground-truth sessionMetadata to hand to ``engine.start('batch-evaluation', ...)``.
+"""
+
+import time
+from collections.abc import Callable
+from typing import Any
+
+from agentcore.lib import ConfigIO
+from agentcore.lib.errors.types import to_error
+from agentcore.lib.result import Err, Ok
+from agentcore.operations.eval.shared.dataset_session_provider import run_dataset_scenarios
+
+# Delay before submitting batch eval to allow CloudWatch span ingestion. Matches the SDK default.
+BATCH_INGESTION_DELAY_MS = 180_000
+
+
+def _resolve_agent_context(**kwargs: Any) -> Any:
+    from agentcore.operations.invoke.resolve_agent_context import resolve_agent_context
+
+    return resolve_agent_context(**kwargs)
+
+
+def _ground_truth(scenario: dict[str, Any] | None) -> dict[str, Any] | None:
+    if scenario is None:
+        return None
+    inline: dict[str, Any] = {}
+    if scenario.get("assertions"):
+        inline["assertions"] = [{"text": a} for a in scenario["assertions"]]
+    if scenario.get("expected_trajectory"):
+        inline["expectedTrajectory"] = {"toolNames": scenario["expected_trajectory"]}
+    turns = scenario.get("turns") or []
+    if any(t.get("expectedResponse") for t in turns):
+        inline["turns"] = [
+            {
+                "input": {"prompt": t.get("input")},
+                **({"expectedResponse": {"text": t["expectedResponse"]}} if t.get("expectedResponse") else {}),
+            }
+            for t in turns
+        ]
+    return {"inline": inline}
+
+
+def run_dataset_phase1(
+    *,
+    agent: str,
+    dataset_name: str,
+    dataset_version: str | None = None,
+    endpoint: str | None = None,
+    config_io: ConfigIO | None = None,
+    on_progress: Callable[[str, str], None] | None = None,
+    ingestion_delay_ms: int | None = None,
+    sleep: Callable[[float], None] = time.sleep,
+) -> Ok | Err:
+    """Ok({"sessionIds": [...], "sessionMetadata": [...]}) or Err(error)."""
+    config_io = config_io or ConfigIO()
+    progress = on_progress or (lambda _p, _m: None)
+    try:
+        agent_context = _resolve_agent_context(
+            project=config_io.read_project_spec(),
+            deployed_state=config_io.read_deployed_state(),
+            aws_targets=config_io.resolve_aws_deployment_targets(),
+            agent_name=agent,
+            endpoint=endpoint,
+        )
+        progress("invoking", f'Invoking agent with dataset "{dataset_name}"...')
+        dataset_result = run_dataset_scenarios(
+            agent_context=agent_context,
+            dataset_name=dataset_name,
+            version=dataset_version,
+            config_base_dir=config_io.get_config_root(),
+            on_progress=progress,
+        )
+        successful = [r for r in dataset_result["scenarioResults"] if r["status"] == "success"]
+        if not successful:
+            return Err(Exception("All scenarios failed during invocation. No sessions to evaluate."))
+
+        session_ids = [r["sessionId"] for r in successful]
+        session_metadata = []
+        for r in successful:
+            scenario = next((s for s in dataset_result["scenarios"] if s["scenario_id"] == r["scenarioId"]), None)
+            entry: dict[str, Any] = {"sessionId": r["sessionId"], "testScenarioId": r["scenarioId"]}
+            gt = _ground_truth(scenario)
+            if gt is not None:
+                entry["groundTruth"] = gt
+            session_metadata.append(entry)
+
+        progress("invoking", f"✓ {len(successful)} sessions ready for batch evaluation")
+        progress("ingesting", "Waiting 180s for CloudWatch span ingestion...")
+        sleep((ingestion_delay_ms if ingestion_delay_ms is not None else BATCH_INGESTION_DELAY_MS) / 1000)
+        return Ok({"sessionIds": session_ids, "sessionMetadata": session_metadata})
+    except Exception as exc:  # noqa: BLE001
+        return Err(to_error(exc))

agentcore/operations/jobs/batch_evaluation/format.py

@@ -0,0 +1,58 @@
+"""Presentation helpers for batch-evaluation jobs (port of operations/jobs/batch-evaluation/format.ts)."""
+
+from typing import Any
+
+import typer
+
+from agentcore.operations.jobs.shared.format import format_job_date, to_fixed
+
+
+def print_batch_evaluation_history(records: list[dict[str, Any]]) -> None:
+    if not records:
+        typer.echo("No batch evaluation jobs found. Run `agentcore run batch-evaluation` to create one.")
+        return
+    typer.echo(f"\n{'Date'.ljust(22)} {'Status'.ljust(22)} {'Evaluators'.ljust(28)} ID")
+    typer.echo("─" * 100)
+    for r in records:
+        typer.echo(
+            f"{format_job_date(r.get('createdAt')).ljust(22)} {str(r.get('status')).ljust(22)} "
+            f"{', '.join(r.get('evaluators') or []).ljust(28)} {r['id']}"
+        )
+    typer.echo("")
+
+
+def summary_line(s: dict[str, Any]) -> str:
+    avg = (s.get("statistics") or {}).get("averageScore")
+    failed = f" ({s['totalFailed']} failed)" if s.get("totalFailed") else ""
+    return f"  {s.get('evaluatorId')}: {to_fixed(avg, 2) if avg is not None else 'N/A'}{failed}"
+
+
+def print_batch_evaluation_detail(record: dict[str, Any]) -> None:
+    typer.echo(f"\nBatch evaluation: {record['id']}")
+    typer.echo(f"Name: {record.get('name')}")
+    typer.echo(f"Status: {record.get('status')}")
+    typer.echo(f"Agent: {record.get('agent')}")
+    typer.echo(f"Evaluators: {', '.join(record.get('evaluators') or [])}")
+    typer.echo(f"Started: {format_job_date(record.get('createdAt'))}")
+    if record.get("completedAt"):
+        typer.echo(f"Completed: {format_job_date(record['completedAt'])}")
+    if record.get("source"):
+        typer.echo(f"Source: {record['source']}")
+
+    summaries = (record.get("evaluationResults") or {}).get("evaluatorSummaries")
+    if summaries:
+        typer.echo("\nResults:")
+        for s in summaries:
+            typer.echo(summary_line(s))
+    elif record.get("results"):
+        typer.echo("\nResults:")
+        for r in record["results"]:
+            score = r.get("score")
+            typer.echo(
+                f"  {r.get('evaluatorId')}: {to_fixed(score, 2) if score is not None else (r.get('label') or 'N/A')}"
+            )
+    else:
+        typer.echo("\nResults not yet available.")
+    if record.get("logFilePath"):
+        typer.echo(f"\nLog: {record['logFilePath']}")
+    typer.echo("")

agentcore/operations/jobs/batch_evaluation/handler.py

@@ -0,0 +1,203 @@
+"""Batch-evaluation job handler (port of operations/jobs/batch-evaluation/handler.ts).
+
+- create():  resolve agent + evaluators, build the CloudWatch data source, ONE StartBatchEvaluation.
+- refresh(): GET latest status (404 -> NOT_FOUND); on terminal status fetch per-session scores
+             from the CloudWatch output log once (``resultsFetched``).
+- stop():    StopBatchEvaluation.   archive(): DeleteBatchEvaluation.
+"""
+
+import json
+from typing import Any
+
+from agentcore.aws import agentcore_batch_evaluation as api
+from agentcore.aws.account import create_aws_client
+from agentcore.lib import ConfigIO, JobNotFoundError, ResourceNotFoundError
+from agentcore.lib.errors.types import to_error
+from agentcore.lib.result import Err, Ok
+from agentcore.lib.utils.json_format import to_json_string
+from agentcore.logging.exec_logger import ExecLogger
+from agentcore.operations.jobs.batch_evaluation.build_source import (
+    build_cloud_watch_filter_config,
+    build_cloud_watch_source,
+    resolve_batch_eval_name,
+    resolve_batch_evaluator_ids,
+)
+from agentcore.operations.jobs.shared.constants import NOT_FOUND_STATUS
+from agentcore.operations.jobs.shared.format import iso_now
+from agentcore.operations.jobs.shared.region import region_of, resolve_job_region
+from agentcore.operations.jobs.shared.resolve_agent_state import deployed_dict, resolve_agent_state
+from agentcore.operations.jobs.shared.storage import drop_undefined
+
+_TERMINAL = ("COMPLETED", "COMPLETED_WITH_ERRORS", "FAILED", "STOPPED", "CANCELLED")
+
+
+def new_logger(command: str) -> ExecLogger | None:
+    try:
+        return ExecLogger(command)
+    except Exception:  # noqa: BLE001 - logger creation is non-fatal
+        return None
+
+
+def fetch_results_from_cloud_watch(region: str, log_group_name: str, log_stream_name: str) -> list[dict[str, Any]]:
+    """Per-session evaluation scores from the batch's CloudWatch output log stream."""
+    client = create_aws_client("logs", region)
+    response = client.get_log_events(logGroupName=log_group_name, logStreamName=log_stream_name, startFromHead=True)
+    results: list[dict[str, Any]] = []
+    for event in response.get("events") or []:
+        message = event.get("message")
+        if not message:
+            continue
+        try:
+            parsed = json.loads(message)
+        except ValueError:
+            continue
+        if not isinstance(parsed, dict):
+            continue
+        attrs = parsed.get("attributes") or {}
+        evaluator_id = attrs.get("gen_ai.evaluation.name")
+        if not evaluator_id:
+            continue
+        results.append(
+            drop_undefined(
+                {
+                    "evaluatorId": evaluator_id,
+                    "score": attrs.get("gen_ai.evaluation.score.value"),
+                    "label": attrs.get("gen_ai.evaluation.score.label"),
+                    "explanation": attrs.get("gen_ai.evaluation.explanation"),
+                }
+            )
+        )
+    return results
+
+
+def create(opts: dict[str, Any], config_io: ConfigIO) -> Ok | Err:
+    logger = new_logger("batch-evaluate")
+    try:
+        if logger:
+            logger.start_step("Load project config")
+        project = config_io.read_project_spec()
+        deployed_state = deployed_dict(config_io.read_deployed_state())
+        aws_targets = config_io.resolve_aws_deployment_targets()
+        region = resolve_job_region(opts.get("region"), aws_targets)
+        if logger:
+            logger.end_step("success")
+            logger.start_step("Resolve agent")
+        agent = opts["agent"]
+        agent_state = resolve_agent_state(deployed_state, agent)
+        if not agent_state:
+            err = ResourceNotFoundError(f'Agent "{agent}" not deployed. Run `agentcore deploy` first.')
+            if logger:
+                logger.end_step("error", err.message)
+                logger.finalize(False)
+            return Err(err)
+        source = build_cloud_watch_source(project.name, agent, agent_state["runtimeId"], opts.get("endpoint"))
+        if logger:
+            logger.log(f"Service name: {source['serviceName']}")
+            logger.log(f"Log group: {source['logGroupName']}")
+            logger.end_step("success")
+
+        eval_name = resolve_batch_eval_name(opts.get("name"), project.name, agent)
+        resolved = resolve_batch_evaluator_ids(deployed_state, agent, opts.get("evaluators") or [])
+
+        session_metadata = opts.get("sessionMetadata")
+        metadata_ids = [m.get("sessionId") for m in (session_metadata or []) if m.get("sessionId")]
+        effective_ids = list(dict.fromkeys([*(opts.get("sessionIds") or []), *metadata_ids]))
+        filter_config = build_cloud_watch_filter_config(effective_ids, opts.get("lookbackDays"))
+
+        if logger:
+            logger.start_step("Start batch evaluation")
+        on_progress = opts.get("onProgress")
+        if on_progress:
+            on_progress("starting", f'Starting batch evaluation "{eval_name}"...')
+        cloud_watch: dict[str, Any] = {
+            "serviceNames": [source["serviceName"]],
+            "logGroupNames": [source["logGroupName"]],
+        }
+        if filter_config:
+            cloud_watch["filterConfig"] = filter_config
+        start = api.start_batch_evaluation(
+            region=region,
+            name=eval_name,
+            evaluators=[{"evaluatorId": i} for i in resolved],
+            data_source_config={"cloudWatchLogs": cloud_watch},
+            evaluation_metadata={"sessionMetadata": session_metadata} if session_metadata else None,
+            kms_key_arn=opts.get("kmsKeyArn") or None,
+            client_token=api.generate_client_token(),
+        )
+        if logger:
+            logger.log(f"Response: {to_json_string(start)}")
+            logger.end_step("success")
+        if on_progress:
+            on_progress("started", f"Batch evaluation created: {start['batchEvaluationId']} ({start['status']})")
+        if logger:
+            logger.finalize(True)
+
+        record: dict[str, Any] = {
+            "type": "batch-evaluation",
+            "id": start["batchEvaluationId"],
+            "arn": start["batchEvaluationArn"],
+            "status": start["status"],
+            "createdAt": start.get("createdAt") or iso_now(),
+            "agent": agent,
+            "logFilePath": logger.log_file_path if logger else None,
+            "name": eval_name,
+            "evaluators": resolved,
+            "source": opts.get("source"),
+            "dataset": opts.get("dataset"),
+        }
+        if opts.get("kmsKeyArn"):
+            record["kmsKeyArn"] = opts["kmsKeyArn"]
+        return Ok(drop_undefined(record))
+    except Exception as exc:  # noqa: BLE001
+        if logger:
+            logger.finalize(False)
+        return Err(to_error(exc))
+
+
+def refresh(record: dict[str, Any]) -> Ok | Err:
+    region = region_of(record["arn"])
+    try:
+        response = api.get_batch_evaluation(region=region, batch_evaluation_id=record["id"])
+    except JobNotFoundError:
+        return Ok({**record, "status": NOT_FOUND_STATUS, "resultsFetched": True})
+    except Exception as exc:  # noqa: BLE001
+        return Err(to_error(exc))
+
+    updated = {
+        **record,
+        "status": response["status"],
+        "completedAt": response.get("updatedAt") or record.get("completedAt"),
+        "evaluationResults": response.get("evaluationResults") or record.get("evaluationResults"),
+        "kmsKeyArn": response.get("kmsKeyArn") or record.get("kmsKeyArn"),
+    }
+    terminal = response["status"] in _TERMINAL
+    cw = (response.get("outputConfig") or {}).get("cloudWatchConfig")
+    if terminal and not record.get("resultsFetched") and cw:
+        try:
+            results = fetch_results_from_cloud_watch(region, cw["logGroupName"], cw["logStreamName"])
+            if results or not record.get("results"):
+                updated["results"] = results
+            updated["resultsFetched"] = True
+        except Exception:  # noqa: BLE001, S110 - leave resultsFetched false so the next read retries
+            pass
+    elif terminal and not cw:
+        updated["resultsFetched"] = True
+    return Ok(drop_undefined(updated))
+
+
+def stop(record: dict[str, Any]) -> Ok | Err:
+    try:
+        api.stop_batch_evaluation(region=region_of(record["arn"]), batch_evaluation_id=record["id"])
+        return Ok(None)
+    except Exception as exc:  # noqa: BLE001
+        return Err(to_error(exc))
+
+
+def archive(record: dict[str, Any]) -> Ok | Err:
+    try:
+        api.delete_batch_evaluation(region=region_of(record["arn"]), batch_evaluation_id=record["id"])
+        return Ok(None)
+    except JobNotFoundError:
+        return Ok(None)
+    except Exception as exc:  # noqa: BLE001
+        return Err(to_error(exc))