agentcore-cli 0.31.1__py3-none-any.whl

agentcore/operations/eval/run_eval.py

@@ -0,0 +1,354 @@
+"""On-demand evaluation (port of operations/eval/run-eval.ts).
+
+Project mode resolves the runtime/evaluators from agentcore.json + deployed state; ARN mode works
+from a runtime ARN without a project. Dataset mode invokes the agent with dataset scenarios first.
+"""
+
+import os
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+from agentcore.aws import agentcore_control as control
+from agentcore.aws.account import create_aws_client
+from agentcore.aws.cloudwatch import runtime_log_group
+from agentcore.lib import ConfigIO, ResourceNotFoundError, ValidationError
+from agentcore.lib.result import Err, Ok
+from agentcore.lib.schemas.io.config_io import to_jsonable
+from agentcore.operations import resolve_agent as _resolve_agent
+from agentcore.operations.eval.shared import span_collector
+from agentcore.operations.eval.shared.dataset_session_provider import run_dataset_scenarios_and_collect_spans
+from agentcore.operations.eval.shared.evaluator_runner import run_evaluators_over_sessions
+from agentcore.operations.eval.storage import generate_filename, run_json, save_eval_run
+from agentcore.operations.jobs.shared.format import iso_now
+
+BUILTIN_EVALUATOR_LEVELS: dict[str, str] = {
+    "Builtin.GoalSuccessRate": "SESSION",
+    "Builtin.Correctness": "TRACE",
+    "Builtin.Faithfulness": "TRACE",
+    "Builtin.Helpfulness": "TRACE",
+    "Builtin.ResponseRelevance": "TRACE",
+    "Builtin.Conciseness": "TRACE",
+    "Builtin.Coherence": "TRACE",
+    "Builtin.InstructionFollowing": "TRACE",
+    "Builtin.Refusal": "TRACE",
+    "Builtin.ToolSelectionAccuracy": "TOOL_CALL",
+    "Builtin.SkillSelectionAccuracy": "TOOL_CALL",
+    "Builtin.SkillInstructionFollowing": "TOOL_CALL",
+}
+
+MAX_DISCOVERED_SESSIONS = 50
+
+
+def _resolve_evaluator_arns(arns: list[str]) -> list[str]:
+    out = []
+    for value in arns:
+        m = re.search(r"evaluator/(.+)$", value)
+        out.append(m.group(1) if m else value)
+    return out
+
+
+def _resolve_from_arn(options: dict[str, Any]) -> dict[str, Any]:
+    arn = options["agentArn"]
+    parts = arn.split(":")
+    if len(parts) < 6:
+        return {"success": False, "error": f"Invalid agent runtime ARN: {arn}"}
+    region = options.get("region") or parts[3]
+    if not region:
+        return {"success": False, "error": "Could not determine region from ARN. Use --region to specify."}
+    m = re.search(r"runtime/(.+)$", ":".join(parts[5:]))
+    if not m:
+        return {"success": False, "error": f"Could not extract runtime ID from ARN: {arn}"}
+    runtime_id = m.group(1)
+    ids: list[str] = []
+    labels: list[str] = []
+    for name in options.get("evaluator") or []:
+        if not name.startswith("Builtin."):
+            return {
+                "success": False,
+                "error": f'Custom evaluator "{name}" cannot be resolved in ARN mode. Use --evaluator-arn with an '
+                "evaluator ARN or ID, or use Builtin.* evaluators.",
+            }
+        ids.append(name)
+        labels.append(name)
+    if options.get("evaluatorArn"):
+        ids += _resolve_evaluator_arns(options["evaluatorArn"])
+        labels += options["evaluatorArn"]
+    if not ids:
+        return {
+            "success": False,
+            "error": "No evaluators specified. Use -e/--evaluator with Builtin.* or --evaluator-arn.",
+        }
+    return {
+        "success": True,
+        "ctx": {
+            "agentLabel": runtime_id,
+            "region": region,
+            "runtimeId": runtime_id,
+            "runtimeLogGroup": runtime_log_group(runtime_id, options.get("endpoint")),
+            "evaluatorIds": ids,
+            "evaluatorLabels": labels,
+        },
+    }
+
+
+def _resolve_from_project(context: Any, options: dict[str, Any]) -> dict[str, Any]:
+    resolved = _resolve_agent.resolve_agent(context, options.get("agent"))
+    if not resolved.success:
+        return {"success": False, "error": resolved.error}
+    agent = resolved.agent
+    deployed = to_jsonable(context.deployed_state)
+    resources = ((deployed.get("targets") or {}).get(agent.target_name) or {}).get("resources") or {}
+    ids: list[str] = []
+    labels: list[str] = []
+    for name in options.get("evaluator") or []:
+        if name.startswith("Builtin."):
+            ids.append(name)
+            labels.append(name)
+            continue
+        state = (resources.get("evaluators") or {}).get(name)
+        if not state:
+            return {"success": False, "error": f'Evaluator "{name}" not found in deployed state. Has it been deployed?'}
+        ids.append(state["evaluatorId"])
+        labels.append(name)
+    if options.get("evaluatorArn"):
+        ids += _resolve_evaluator_arns(options["evaluatorArn"])
+        labels += options["evaluatorArn"]
+    if not ids:
+        return {"success": False, "error": "No evaluators specified. Use -e/--evaluator or --evaluator-arn."}
+    return {
+        "success": True,
+        "ctx": {
+            "agentLabel": agent.agent_name,
+            "region": agent.region,
+            "runtimeId": agent.runtime_id,
+            "runtimeLogGroup": runtime_log_group(agent.runtime_id, options.get("endpoint")),
+            "evaluatorIds": ids,
+            "evaluatorLabels": labels,
+        },
+    }
+
+
+def resolve_evaluator_levels(evaluator_ids: list[str], region: str) -> dict[str, str]:
+    levels: dict[str, str] = {}
+    for evaluator_id in evaluator_ids:
+        if evaluator_id in BUILTIN_EVALUATOR_LEVELS:
+            levels[evaluator_id] = BUILTIN_EVALUATOR_LEVELS[evaluator_id]
+        elif evaluator_id.startswith("Builtin."):
+            levels[evaluator_id] = "SESSION"
+        else:
+            try:
+                levels[evaluator_id] = (
+                    control.get_evaluator(region=region, evaluator_id=evaluator_id).get("level") or "SESSION"
+                )
+            except Exception:  # noqa: BLE001
+                levels[evaluator_id] = "SESSION"
+    return levels
+
+
+def discover_sessions(*, runtime_id: str, region: str, lookback_days: int) -> list[dict[str, Any]]:
+    """Lightweight session discovery: [{sessionId, spanCount, firstSeen}]."""
+    end_ms = int(time.time() * 1000)
+    start_ms = end_ms - lookback_days * 24 * 60 * 60 * 1000
+    client = create_aws_client("logs", region)
+    query = f"""fields attributes.session.id as sessionId
+     | parse resource.attributes.cloud.resource_id "runtime/*/" as parsedAgentId
+     | filter parsedAgentId = '{span_collector.sanitize_query_value(runtime_id)}'
+     | filter ispresent(kind)
+     | stats count(*) as spanCount, min(@timestamp) as firstSeen by sessionId
+     | sort firstSeen desc
+     | limit {MAX_DISCOVERED_SESSIONS}"""
+    args = (query, start_ms // 1000, end_ms // 1000)
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        a = pool.submit(span_collector.execute_query_graceful, client, span_collector.SPANS_LOG_GROUP, *args)
+        b = pool.submit(span_collector.execute_query_graceful, client, runtime_log_group(runtime_id), *args)
+        rows = a.result() + b.result()
+    sessions = []
+    for row in rows:
+        values = {f.get("field"): f.get("value") for f in row}
+        session_id = values.get("sessionId")
+        try:
+            span_count = int(values.get("spanCount") or "0")
+        except ValueError:
+            span_count = 0
+        if session_id and session_id != "unknown":
+            sessions.append(
+                {"sessionId": session_id, "spanCount": span_count, "firstSeen": values.get("firstSeen") or ""}
+            )
+    return sessions
+
+
+def handle_run_eval(options: dict[str, Any]) -> Ok | Err:
+    """Ok({"run": EvalRunResult, "filePath": str}). Options use upstream RunEvalOptions keys."""
+    if options.get("agentArn"):
+        resolution = _resolve_from_arn(options)
+    else:
+        resolution = _resolve_from_project(_resolve_agent.load_deployed_project_config(), options)
+    if not resolution["success"]:
+        return Err(ResourceNotFoundError(resolution["error"]))
+    ctx = resolution["ctx"]
+
+    if options.get("dataset"):
+        from agentcore.operations.invoke.resolve_agent_context import resolve_agent_context
+
+        config_io = ConfigIO()
+        agent_context = resolve_agent_context(
+            project=config_io.read_project_spec(),
+            deployed_state=config_io.read_deployed_state(),
+            aws_targets=config_io.read_aws_deployment_targets(),
+            agent_name=options.get("agent"),
+            endpoint=options.get("endpoint"),
+        )
+
+        def query_spans(region: str, log_group: str, session_id: str) -> list[Any]:
+            result = span_collector.fetch_session_spans(
+                runtime_id=agent_context.runtime_id,
+                runtime_log_group=log_group,
+                region=region,
+                lookback_days=1,
+                session_id=session_id,
+            )
+            return result[0]["spans"] if result else []
+
+        dataset_result = run_dataset_scenarios_and_collect_spans(
+            agent_context=agent_context,
+            dataset_name=options["dataset"],
+            version=options.get("datasetVersion"),
+            config_base_dir=config_io.get_config_root(),
+            query_spans=query_spans,
+            on_progress=options.get("onProgress"),
+        )
+        if not dataset_result["sessions"]:
+            return Err(
+                ResourceNotFoundError("No spans collected from dataset scenarios. All sessions may have timed out.")
+            )
+        levels = resolve_evaluator_levels(ctx["evaluatorIds"], ctx["region"])
+        refs: dict[str, list[dict[str, Any]]] = {}
+        for ref in dataset_result["referenceInputs"]:
+            refs.setdefault(ref["context"]["spanContext"]["sessionId"], []).append(ref)
+        scenario_by_session = {r["sessionId"]: r["scenarioId"] for r in dataset_result["scenarioResults"]}
+        sessions = [{**s, "scenarioId": scenario_by_session.get(s["sessionId"])} for s in dataset_result["sessions"]]
+        results = run_evaluators_over_sessions(
+            region=ctx["region"],
+            evaluator_ids=ctx["evaluatorIds"],
+            evaluator_labels=ctx["evaluatorLabels"],
+            evaluator_levels=levels,
+            sessions=sessions,
+            ref_inputs_by_session=refs,
+        )
+        run = {
+            "timestamp": iso_now(),
+            "agent": ctx["agentLabel"],
+            "evaluators": ctx["evaluatorLabels"],
+            "sessionCount": len(sessions),
+            "results": results,
+            "source": "dataset",
+            "datasetName": options["dataset"],
+            "dataset": {"id": options["dataset"], "version": options.get("datasetVersion") or "LOCAL"},
+        }
+        # Upstream returns --output as the path here without writing it.
+        file_path = options.get("output") or save_eval_run(run)
+        return Ok({"run": run, "filePath": file_path})
+
+    days = options["days"]
+    sessions = span_collector.fetch_session_spans(
+        runtime_id=ctx["runtimeId"],
+        runtime_log_group=ctx["runtimeLogGroup"],
+        region=ctx["region"],
+        lookback_days=days,
+        session_id=options.get("sessionId"),
+        trace_id=options.get("traceId"),
+    )
+    if options.get("sessionIds"):
+        selected = set(options["sessionIds"])
+        sessions = [s for s in sessions if s["sessionId"] in selected]
+    if not sessions:
+        return Err(
+            ResourceNotFoundError(
+                f'No session spans found for agent "{ctx["agentLabel"]}" in the last {days} day(s). '
+                "Has the agent been invoked?"
+            )
+        )
+    levels = resolve_evaluator_levels(ctx["evaluatorIds"], ctx["region"])
+
+    assertions = options.get("assertions") or []
+    trajectory = options.get("expectedTrajectory") or []
+    expected_response = options.get("expectedResponse")
+    has_refs = bool(assertions) or bool(trajectory) or bool(expected_response)
+    if has_refs and len(sessions) != 1:
+        return Err(
+            ValidationError(
+                "Ground truth flags (-A, --expected-trajectory, --expected-response) require exactly one session. "
+                "Use -s/--session-id to target a single session."
+            )
+        )
+    reference_inputs: list[dict[str, Any]] | None = None
+    if has_refs:
+        first = sessions[0]
+        refs_list: list[dict[str, Any]] = []
+        session_ref: dict[str, Any] = {"context": {"spanContext": {"sessionId": first["sessionId"]}}}
+        if trajectory:
+            session_ref["expectedTrajectory"] = {"toolNames": trajectory}
+        if assertions:
+            session_ref["assertions"] = [{"text": a} for a in assertions]
+        if trajectory or assertions:
+            refs_list.append(session_ref)
+        if expected_response:
+            trace_id = options.get("traceId")
+            if not trace_id:
+                ids = span_collector.extract_trace_ids(first["spans"])
+                trace_id = ids[-1] if ids else None
+            if not trace_id:
+                return Err(
+                    ValidationError(
+                        "Expected response provided but no trace IDs found in session spans. "
+                        "Use -t/--trace-id to specify."
+                    )
+                )
+            refs_list.append(
+                {
+                    "context": {"spanContext": {"sessionId": first["sessionId"], "traceId": trace_id}},
+                    "expectedResponse": {"text": expected_response},
+                }
+            )
+        reference_inputs = refs_list or None
+
+    results = run_evaluators_over_sessions(
+        region=ctx["region"],
+        evaluator_ids=ctx["evaluatorIds"],
+        evaluator_labels=ctx["evaluatorLabels"],
+        evaluator_levels=levels,
+        sessions=sessions,
+        ref_inputs_by_session={sessions[0]["sessionId"]: reference_inputs} if reference_inputs else None,
+    )
+    timestamp = iso_now()
+    run: dict[str, Any] = {
+        "timestamp": timestamp,
+        "agent": ctx["agentLabel"],
+        "evaluators": ctx["evaluatorLabels"],
+        "lookbackDays": days,
+        "sessionCount": len(sessions),
+        "results": results,
+    }
+    if has_refs:
+        ref_summary: dict[str, Any] = {}
+        if assertions:
+            ref_summary["assertions"] = assertions
+        if trajectory:
+            ref_summary["expectedTrajectory"] = trajectory
+        if expected_response:
+            ref_summary["expectedResponse"] = expected_response
+        run["referenceInputs"] = ref_summary
+
+    if options.get("output"):
+        file_path = options["output"]
+        with open(file_path, "w", encoding="utf-8", newline="") as fh:
+            fh.write(run_json(run))
+    elif options.get("agentArn"):
+        file_path = os.path.join(os.getcwd(), f"{generate_filename(timestamp)}.json")
+        with open(file_path, "w", encoding="utf-8", newline="") as fh:
+            fh.write(run_json(run))
+    else:
+        file_path = save_eval_run(run)
+    return Ok({"run": run, "filePath": file_path})

agentcore/operations/eval/shared/dataset_loader.py

@@ -0,0 +1,68 @@
+"""Load and validate dataset scenarios for evaluation (port of operations/eval/shared/dataset-loader.ts).
+
+- Local file (no version): reads ``config.managed.location`` directly.
+- Version mode (N or DRAFT): downloads from the service via the pre-signed URL.
+"""
+
+import json
+import os
+from typing import Any
+
+from agentcore.aws import agentcore_datasets as api
+from agentcore.lib import ConfigIO
+from agentcore.lib.schemas.io.config_io import to_jsonable
+from agentcore.operations.dataset.resolve_dataset import resolve_dataset
+
+
+def load_dataset_scenarios(*, dataset_name: str, config_base_dir: str, version: str | None = None) -> list[dict]:
+    resolved = resolve_dataset(dataset_name)
+    project = to_jsonable(ConfigIO().read_project_spec())
+    spec = next((d for d in project.get("datasets") or [] if d.get("name") == dataset_name), None)
+    if spec and spec.get("schemaType") == "AGENTCORE_EVALUATION_SIMULATED_V1":
+        raise Exception(
+            "Simulated scenarios (actor profiles) are not supported yet. Use predefined turns or wait for Phase 4."
+        )
+    if not version:
+        with open(os.path.join(config_base_dir, resolved.location), encoding="utf-8") as fh:
+            content = fh.read()
+    else:
+        info = api.get_dataset(
+            region=resolved.region, dataset_id=resolved.dataset_id, version=None if version == "DRAFT" else version
+        )
+        if not info.get("downloadUrl"):
+            raise Exception(
+                "Dataset has no download URL available. The dataset may not be ready yet. Please try again later."
+            )
+        content = api.download_dataset(info["downloadUrl"], mode="buffer")
+    return parse_and_validate(content)
+
+
+def parse_and_validate(content: str) -> list[dict[str, Any]]:
+    """JSONL -> validated predefined scenarios {scenario_id, turns, assertions?, expected_trajectory?}."""
+    lines = [line for line in content.split("\n") if line.strip()]
+    if not lines:
+        raise Exception("Dataset has no examples. Add scenarios to your dataset file first.")
+    scenarios = []
+    for index, line in enumerate(lines):
+        try:
+            obj = json.loads(line)
+        except ValueError as exc:
+            snippet = line[:120] + "..." if len(line) > 120 else line
+            raise Exception(f"Invalid JSON at line {index + 1}: {exc}\n  {snippet}") from None
+        if not isinstance(obj, dict):
+            obj = {}
+        if not obj.get("scenario_id") or not isinstance(obj.get("scenario_id"), str):
+            raise Exception(f'Line {index + 1}: missing required field "scenario_id"')
+        turns = obj.get("turns")
+        if not turns or not isinstance(turns, list):
+            raise Exception(f'Line {index + 1}: "turns" must be a non-empty array')
+        for i, turn in enumerate(turns):
+            if not isinstance(turn, dict) or not turn.get("input") or not isinstance(turn.get("input"), str):
+                raise Exception(f'Line {index + 1}, turn {i + 1}: each turn must have a string "input" field')
+        scenario: dict[str, Any] = {"scenario_id": obj["scenario_id"], "turns": turns}
+        if obj.get("assertions") is not None:
+            scenario["assertions"] = obj["assertions"]
+        if obj.get("expected_trajectory") is not None:
+            scenario["expected_trajectory"] = obj["expected_trajectory"]
+        scenarios.append(scenario)
+    return scenarios

agentcore/operations/eval/shared/dataset_session_provider.py

@@ -0,0 +1,122 @@
+"""Dataset scenario orchestration (port of operations/eval/shared/dataset-session-provider.ts).
+
+- run_dataset_scenarios: load + invoke (used by batch eval).
+- run_dataset_scenarios_and_collect_spans: + span collection + ground truth (on-demand eval).
+"""
+
+from collections.abc import Callable
+from typing import Any
+
+from agentcore.aws.cloudwatch import runtime_log_group
+from agentcore.lib import ConfigIO
+from agentcore.operations.eval.shared import span_collector
+from agentcore.operations.eval.shared.dataset_loader import load_dataset_scenarios
+from agentcore.operations.eval.shared.scenario_executor import execute_scenarios
+
+
+def build_reference_inputs(*, scenario: dict[str, Any], session_id: str, trace_ids: list[str]) -> list[dict]:
+    """Session-level assertions/trajectory + per-trace expectedResponse (turn i -> traceIds[i])."""
+    inputs: list[dict[str, Any]] = []
+    assertions = scenario.get("assertions") or []
+    trajectory = scenario.get("expected_trajectory") or []
+    if assertions or trajectory:
+        entry: dict[str, Any] = {"context": {"spanContext": {"sessionId": session_id}}}
+        if assertions:
+            entry["assertions"] = [{"text": a} for a in assertions]
+        if trajectory:
+            entry["expectedTrajectory"] = {"toolNames": trajectory}
+        inputs.append(entry)
+    for i, turn in enumerate(scenario.get("turns") or []):
+        if not turn.get("expectedResponse"):
+            continue
+        if i >= len(trace_ids):
+            break
+        inputs.append(
+            {
+                "context": {"spanContext": {"sessionId": session_id, "traceId": trace_ids[i]}},
+                "expectedResponse": {"text": turn["expectedResponse"]},
+            }
+        )
+    return inputs
+
+
+def run_dataset_scenarios(
+    *,
+    agent_context: Any,
+    dataset_name: str,
+    version: str | None = None,
+    config_base_dir: str | None = None,
+    on_progress: Callable[[str, str], None] | None = None,
+) -> dict[str, Any]:
+    """{"scenarioResults", "scenarios"}; raises if every scenario fails."""
+    progress = on_progress or (lambda _p, _m: None)
+    progress("load", f'Loading dataset "{dataset_name}"...')
+    base = config_base_dir or ConfigIO().get_config_root()
+    scenarios = load_dataset_scenarios(dataset_name=dataset_name, version=version, config_base_dir=base)
+    progress("load", f"Loaded {len(scenarios)} scenarios")
+
+    progress("invoke", f"Invoking agent with {len(scenarios)} scenarios...")
+
+    def on_scenario(completed: int, total: int, current: dict[str, Any]) -> None:
+        mark = "✓" if current["status"] == "success" else "✗"
+        progress("invoke", f"[{completed}/{total}] {current['scenarioId']}: {mark}")
+
+    results = execute_scenarios(scenarios=scenarios, agent_context=agent_context, on_progress=on_scenario)
+    ok = [r for r in results if r["status"] == "success"]
+    failed = len(results) - len(ok)
+    progress("invoke", f"✓ {len(ok)}/{len(results)} scenarios invoked{f' ({failed} failed)' if failed else ''}")
+    if not ok:
+        raise Exception("All scenarios failed during invocation. No sessions to evaluate.")
+    return {"scenarioResults": results, "scenarios": scenarios}
+
+
+def run_dataset_scenarios_and_collect_spans(
+    *,
+    agent_context: Any,
+    dataset_name: str,
+    query_spans: Callable[[str, str, str], list[Any]],
+    version: str | None = None,
+    config_base_dir: str | None = None,
+    on_progress: Callable[[str, str], None] | None = None,
+) -> dict[str, Any]:
+    """{"sessions", "referenceInputs", "scenarioResults", "scenarios"}."""
+    progress = on_progress or (lambda _p, _m: None)
+    ran = run_dataset_scenarios(
+        agent_context=agent_context,
+        dataset_name=dataset_name,
+        version=version,
+        config_base_dir=config_base_dir,
+        on_progress=on_progress,
+    )
+    successful = [r for r in ran["scenarioResults"] if r["status"] == "success"]
+    log_group = runtime_log_group(agent_context.runtime_id, agent_context.endpoint)
+    session_ids = [r["sessionId"] for r in successful]
+
+    progress("collect", "Waiting for span ingestion (15s)...")
+    collected = span_collector.collect_spans(
+        session_ids=session_ids,
+        region=agent_context.region,
+        log_group=log_group,
+        query_spans=query_spans,
+        on_progress=lambda c, t, *_: progress("collect", f"Collecting spans... ({c}/{t} sessions)"),
+    )
+    if collected["timedOut"]:
+        progress("collect", f"⚠ {len(collected['timedOut'])} sessions timed out waiting for spans")
+    progress("collect", f"✓ {len(collected['spans'])}/{len(session_ids)} sessions collected")
+
+    sessions = []
+    references: list[dict[str, Any]] = []
+    for result in successful:
+        spans = collected["spans"].get(result["sessionId"])
+        if not spans:
+            continue
+        sessions.append({"sessionId": result["sessionId"], "spans": spans})
+        scenario = next((s for s in ran["scenarios"] if s["scenario_id"] == result["scenarioId"]), None)
+        if scenario is None:
+            continue
+        references.extend(
+            build_reference_inputs(
+                scenario=scenario, session_id=result["sessionId"], trace_ids=span_collector.extract_trace_ids(spans)
+            )
+        )
+    return {**ran, "sessions": sessions, "referenceInputs": references}

agentcore/operations/eval/shared/evaluator_runner.py

@@ -0,0 +1,89 @@
+"""Shared evaluator-loop runner (port of operations/eval/shared/evaluator-runner.ts).
+
+TRACE/TOOL_CALL/SESSION routing, batching target ids into chunks of 10 (Evaluate API limit),
+per-session reference inputs and score aggregation.
+"""
+
+from typing import Any
+
+from agentcore.aws import agentcore as agentcore_api
+from agentcore.operations.eval.shared.span_collector import extract_tool_call_span_ids, extract_trace_ids
+
+BATCH_SIZE = 10
+
+
+def _batch_target_ids(trace_ids: list[str] | None, span_ids: list[str] | None) -> list[dict[str, Any]]:
+    if trace_ids:
+        return [{"traceIds": trace_ids[i : i + BATCH_SIZE]} for i in range(0, len(trace_ids), BATCH_SIZE)]
+    if span_ids:
+        return [{"spanIds": span_ids[i : i + BATCH_SIZE]} for i in range(0, len(span_ids), BATCH_SIZE)]
+    return [{}]
+
+
+def _resolve_targets(level: str, spans: list[Any]) -> dict[str, Any] | None:
+    if level == "TRACE":
+        ids = extract_trace_ids(spans)
+        return {"traceIds": ids} if ids else None
+    if level == "TOOL_CALL":
+        ids = extract_tool_call_span_ids(spans)
+        return {"spanIds": ids} if ids else None
+    return {}
+
+
+def run_evaluators_over_sessions(
+    *,
+    region: str,
+    evaluator_ids: list[str],
+    evaluator_labels: list[str],
+    evaluator_levels: dict[str, str],
+    sessions: list[dict[str, Any]],
+    ref_inputs_by_session: dict[str, list[dict[str, Any]]] | None = None,
+) -> list[dict[str, Any]]:
+    """[{evaluator, aggregateScore, sessionScores, tokenUsage}] per evaluator."""
+    results = []
+    for i, evaluator_id in enumerate(evaluator_ids):
+        name = evaluator_labels[i] if i < len(evaluator_labels) else evaluator_id
+        level = evaluator_levels.get(evaluator_id, "SESSION")
+        scores: list[dict[str, Any]] = []
+        input_tokens = output_tokens = total_tokens = 0
+        for session in sessions:
+            targets = _resolve_targets(level, session["spans"])
+            if targets is None:
+                continue
+            for batch in _batch_target_ids(targets.get("traceIds"), targets.get("spanIds")):
+                response = agentcore_api.evaluate(
+                    region=region,
+                    evaluator_id=evaluator_id,
+                    session_spans=session["spans"],
+                    target_trace_ids=batch.get("traceIds"),
+                    target_span_ids=batch.get("spanIds"),
+                    evaluation_reference_inputs=(ref_inputs_by_session or {}).get(session["sessionId"]),
+                )
+                for r in response["evaluationResults"]:
+                    ctx = r.get("context") or {}
+                    score: dict[str, Any] = {
+                        "sessionId": ctx.get("sessionId") or session["sessionId"],
+                        "scenarioId": session.get("scenarioId"),
+                        "traceId": ctx.get("traceId"),
+                        "spanId": ctx.get("spanId"),
+                        "value": r.get("value") if r.get("value") is not None else 0,
+                        "label": r.get("label"),
+                        "explanation": r.get("explanation"),
+                        "errorMessage": r.get("errorMessage"),
+                    }
+                    scores.append({k: v for k, v in score.items() if v is not None})
+                    usage = r.get("tokenUsage") or {}
+                    input_tokens += usage.get("inputTokens") or 0
+                    output_tokens += usage.get("outputTokens") or 0
+                    total_tokens += usage.get("totalTokens") or 0
+        valid = [s for s in scores if not s.get("errorMessage")]
+        aggregate = sum(s["value"] for s in valid) / len(valid) if valid else 0
+        results.append(
+            {
+                "evaluator": name,
+                "aggregateScore": aggregate,
+                "sessionScores": scores,
+                "tokenUsage": {"inputTokens": input_tokens, "outputTokens": output_tokens, "totalTokens": total_tokens},
+            }
+        )
+    return results