agentcore-cli 0.31.1__py3-none-any.whl

agentcore/operations/eval/shared/scenario_executor.py

@@ -0,0 +1,71 @@
+"""Execute dataset scenarios against a deployed agent (port of operations/eval/shared/scenario-executor.ts).
+
+Turns run sequentially in one session; up to 5 scenarios run concurrently; a turn failure halts
+(and fails) that scenario.
+"""
+
+import threading
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+from agentcore.aws import agentcore as agentcore_api
+from agentcore.operations.session import generate_session_id
+
+MAX_CONCURRENT = 5
+
+
+def _execute_single(scenario: dict[str, Any], ctx: Any) -> dict[str, Any]:
+    session_id = generate_session_id()
+    try:
+        for turn in scenario["turns"]:
+            agentcore_api.invoke_agent_runtime(
+                agentcore_api.InvokeAgentRuntimeOptions(
+                    region=ctx.region,
+                    runtime_arn=ctx.runtime_arn,
+                    payload=turn["input"],
+                    session_id=session_id,
+                    bearer_token=ctx.bearer_token,
+                    baggage=ctx.baggage,
+                    endpoint=ctx.endpoint,
+                )
+            )
+        return {
+            "scenarioId": scenario["scenario_id"],
+            "sessionId": session_id,
+            "turnCount": len(scenario["turns"]),
+            "status": "success",
+        }
+    except Exception as exc:  # noqa: BLE001
+        return {
+            "scenarioId": scenario["scenario_id"],
+            "sessionId": session_id,
+            "turnCount": len(scenario["turns"]),
+            "status": "failed",
+            "error": str(exc),
+        }
+
+
+def execute_scenarios(
+    *,
+    scenarios: list[dict[str, Any]],
+    agent_context: Any,
+    on_progress: Callable[[int, int, dict[str, Any]], None] | None = None,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any] | None] = [None] * len(scenarios)
+    lock = threading.Lock()
+    completed = 0
+
+    def run(i: int) -> None:
+        nonlocal completed
+        result = _execute_single(scenarios[i], agent_context)
+        with lock:
+            results[i] = result
+            completed += 1
+            if on_progress:
+                on_progress(completed, len(scenarios), result)
+
+    if scenarios:
+        with ThreadPoolExecutor(max_workers=min(MAX_CONCURRENT, len(scenarios))) as pool:
+            list(pool.map(run, range(len(scenarios))))
+    return [r for r in results if r is not None]

agentcore/operations/eval/shared/span_collector.py

@@ -0,0 +1,255 @@
+"""Collect spans from CloudWatch (port of operations/eval/shared/span-collector.ts)."""
+
+import json
+import time
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+from agentcore.aws._aws_errors import aws_error_name
+from agentcore.aws.account import create_aws_client
+
+# Default delay before first span query (CloudWatch ingestion buffer). Matches the SDK's 180s.
+SPAN_INGESTION_DELAY_MS = 180_000
+SPAN_POLL_TIMEOUT_MS = 60_000
+SPAN_POLL_INTERVAL_MS = 5_000
+
+SPANS_LOG_GROUP = "aws/spans"
+
+SUPPORTED_SCOPES = frozenset(
+    {
+        "strands.telemetry.tracer",
+        "opentelemetry.instrumentation.langchain",
+        "openinference.instrumentation.langchain",
+    }
+)
+
+ResultRow = list[dict[str, str]]
+
+# Injectable for tests.
+sleep: Callable[[float], None] = time.sleep
+
+
+def _is_permanent_error(err: BaseException) -> bool:
+    msg = str(err)
+    return "AccessDenied" in msg or "InvalidParameter" in msg
+
+
+def _poll_one_session(
+    session_id: str, query_spans: Callable[[str, str, str], list[Any]], region: str, log_group: str, timeout_ms: int
+) -> list[Any] | None:
+    deadline = time.time() * 1000 + timeout_ms
+    while time.time() * 1000 < deadline:
+        try:
+            spans = query_spans(region, log_group, session_id)
+            if spans:
+                return spans
+        except Exception as exc:
+            if _is_permanent_error(exc):
+                raise Exception(f"CloudWatch query failed: {exc}") from exc
+        sleep(SPAN_POLL_INTERVAL_MS / 1000)
+    return None
+
+
+def collect_spans(
+    *,
+    session_ids: list[str],
+    region: str,
+    log_group: str,
+    query_spans: Callable[[str, str, str], list[Any]],
+    on_progress: Callable[..., None] | None = None,
+) -> dict[str, Any]:
+    """{"spans": {sessionId: [...]}, "timedOut": [...]} after the ingestion delay."""
+    progress = on_progress or (lambda *_a: None)
+    progress(0, len(session_ids), f"Waiting for span ingestion ({SPAN_INGESTION_DELAY_MS // 1000}s)...")
+    sleep(SPAN_INGESTION_DELAY_MS / 1000)
+
+    collected_count = 0
+
+    def poll(session_id: str) -> tuple[str, list[Any] | None]:
+        nonlocal collected_count
+        spans = _poll_one_session(session_id, query_spans, region, log_group, SPAN_POLL_TIMEOUT_MS)
+        if spans:
+            collected_count += 1
+            progress(collected_count, len(session_ids))
+        return session_id, spans
+
+    collected: dict[str, list[Any]] = {}
+    timed_out: list[str] = []
+    if not session_ids:
+        return {"spans": collected, "timedOut": timed_out}
+    with ThreadPoolExecutor(max_workers=min(8, len(session_ids))) as pool:
+        futures = [pool.submit(poll, s) for s in session_ids]
+        for future in futures:
+            try:
+                sid, spans = future.result()
+            except Exception:  # noqa: BLE001 - rejected sessions are treated as timed out
+                timed_out.append("unknown")
+                continue
+            if spans:
+                collected[sid] = spans
+            else:
+                timed_out.append(sid)
+    return {"spans": collected, "timedOut": timed_out}
+
+
+def extract_trace_ids(spans: list[Any]) -> list[str]:
+    """Unique traceIds in appearance order."""
+    seen: dict[str, None] = {}
+    for span in spans:
+        trace_id = span.get("traceId") if isinstance(span, dict) else None
+        if trace_id and trace_id not in seen:
+            seen[trace_id] = None
+    return list(seen)
+
+
+def extract_tool_call_span_ids(spans: list[Any]) -> list[str]:
+    out = []
+    for span in spans:
+        if not isinstance(span, dict) or not span.get("spanId"):
+            continue
+        attrs = span.get("attributes") or {}
+        tool = attrs.get("gen_ai.tool.name")
+        if tool is None:
+            tool = attrs.get("tool.name")
+        if tool:
+            out.append(span["spanId"])
+    return out
+
+
+def sanitize_query_value(value: str) -> str:
+    return value.replace("'", "")
+
+
+def execute_query(client: Any, log_group_name: str, query_string: str, start_sec: int, end_sec: int) -> list[ResultRow]:
+    """Run a Logs Insights query and wait (max 60s) for the results."""
+    start = client.start_query(
+        logGroupName=log_group_name, startTime=start_sec, endTime=end_sec, queryString=query_string
+    )
+    query_id = start.get("queryId")
+    if not query_id:
+        raise Exception("Failed to start CloudWatch Logs Insights query")
+    for _ in range(60):
+        sleep(1)
+        results = client.get_query_results(queryId=query_id)
+        status = results.get("status") or "Unknown"
+        if status in ("Failed", "Cancelled"):
+            raise Exception(f"CloudWatch query {status.lower()}")
+        if status == "Complete":
+            return results.get("results") or []
+    raise Exception("CloudWatch query timed out after 60 seconds")
+
+
+def execute_query_graceful(client: Any, log_group_name: str, query: str, start_sec: int, end_sec: int) -> list:
+    """``execute_query`` returning [] when the log group does not exist."""
+    try:
+        return execute_query(client, log_group_name, query, start_sec, end_sec)
+    except Exception as exc:
+        if aws_error_name(exc) == "ResourceNotFoundException" or "does not exist" in str(exc):
+            return []
+        raise
+
+
+def _field(row: ResultRow, name: str) -> str | None:
+    return next((f.get("value") for f in row if f.get("field") == name), None)
+
+
+def _extract_messages(rows: list[ResultRow]) -> list[dict[str, Any]]:
+    docs = []
+    for row in rows:
+        value = _field(row, "@message")
+        if value:
+            try:
+                doc = json.loads(value)
+            except ValueError:
+                continue
+            if isinstance(doc, dict):
+                docs.append(doc)
+    return docs
+
+
+def _is_relevant_for_eval(doc: dict[str, Any]) -> bool:
+    scope = doc.get("scope")
+    if isinstance(scope, dict) and scope.get("name") in SUPPORTED_SCOPES:
+        return True
+    body = doc.get("body")
+    return isinstance(body, dict) and ("input" in body or "output" in body)
+
+
+def fetch_session_spans(
+    *,
+    runtime_id: str,
+    runtime_log_group: str,
+    region: str,
+    lookback_days: float,
+    session_id: str | None = None,
+    trace_id: str | None = None,
+) -> list[dict[str, Any]]:
+    """OTel spans from ``aws/spans`` + runtime logs, grouped by session: [{sessionId, spans}]."""
+    end_ms = int(time.time() * 1000)
+    start_ms = end_ms - int(lookback_days * 24 * 60 * 60 * 1000)
+    start_sec, end_sec = start_ms // 1000, end_ms // 1000
+    client = create_aws_client("logs", region)
+
+    span_query = f"""fields @message, attributes.session.id as sessionId, traceId
+     | parse resource.attributes.cloud.resource_id "runtime/*/" as parsedAgentId
+     | filter parsedAgentId = '{sanitize_query_value(runtime_id)}'
+     | filter ispresent(scope.name) and ispresent(kind)"""
+    if session_id:
+        span_query += f"\n     | filter attributes.session.id = '{sanitize_query_value(session_id)}'"
+    if trace_id:
+        span_query += f"\n     | filter traceId = '{sanitize_query_value(trace_id)}'"
+    span_query += "\n     | sort startTimeUnixNano asc\n     | limit 10000"
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        shared_f = pool.submit(execute_query_graceful, client, SPANS_LOG_GROUP, span_query, start_sec, end_sec)
+        runtime_f = pool.submit(execute_query_graceful, client, runtime_log_group, span_query, start_sec, end_sec)
+        all_rows = shared_f.result() + runtime_f.result()
+
+    session_map: dict[str, list[Any]] = {}
+    trace_ids: dict[str, None] = {}
+    for row in all_rows:
+        message = _field(row, "@message")
+        if not message:
+            continue
+        try:
+            doc = json.loads(message)
+        except ValueError:
+            continue
+        sid = _field(row, "sessionId") or "unknown"
+        session_map.setdefault(sid, []).append(doc)
+        tid = _field(row, "traceId")
+        if tid:
+            trace_ids[tid] = None
+
+    if not session_map:
+        return []
+
+    if trace_ids:
+        trace_filter = ", ".join(f"'{sanitize_query_value(t)}'" for t in trace_ids)
+        try:
+            log_rows = execute_query(
+                client,
+                runtime_log_group,
+                f"""fields @message, traceId
+         | filter traceId in [{trace_filter}]
+         | sort @timestamp asc
+         | limit 10000""",
+                start_sec,
+                end_sec,
+            )
+        except Exception:  # noqa: BLE001 - runtime log group may not exist yet
+            log_rows = []
+        trace_to_session: dict[str, str] = {}
+        for row in all_rows:
+            tid, sid = _field(row, "traceId"), _field(row, "sessionId")
+            if tid and sid:
+                trace_to_session[tid] = sid
+        for log_doc in _extract_messages(log_rows):
+            if not _is_relevant_for_eval(log_doc):
+                continue
+            tid = log_doc.get("traceId")
+            sid = trace_to_session.get(tid, "unknown") if tid else "unknown"
+            session_map.setdefault(sid, []).append(log_doc)
+
+    return [{"sessionId": sid, "spans": docs} for sid, docs in session_map.items() if docs]

agentcore/operations/eval/storage.py

@@ -0,0 +1,67 @@
+"""Local eval-run history (port of operations/eval/storage.ts): ``<configRoot>/.cli/eval-results/eval_*.json``."""
+
+import json
+import os
+from typing import Any
+
+from agentcore.lib import find_config_root
+from agentcore.lib.utils.json_format import to_json_string
+from agentcore.operations.jobs.shared.format import parse_js_date
+
+EVAL_RESULTS_DIR = "eval-results"
+
+
+def _results_dir() -> str:
+    config_root = find_config_root()
+    if not config_root:
+        raise Exception("No agentcore project found. Run `agentcore create` first.")
+    return os.path.join(config_root, ".cli", EVAL_RESULTS_DIR)
+
+
+def generate_filename(timestamp: str) -> str:
+    d = parse_js_date(timestamp)
+    if d is None:
+        return "eval_NaN-NaN-NaN_NaN-NaN-NaN"
+    from datetime import UTC
+
+    d = d.astimezone(UTC)
+    return f"eval_{d.year}-{d.month:02d}-{d.day:02d}_{d.hour:02d}-{d.minute:02d}-{d.second:02d}"
+
+
+def run_json(run: dict[str, Any]) -> str:
+    return to_json_string(run, 2)
+
+
+def save_eval_run(run: dict[str, Any]) -> str:
+    directory = _results_dir()
+    os.makedirs(directory, exist_ok=True)
+    file_path = os.path.join(directory, f"{generate_filename(run['timestamp'])}.json")
+    with open(file_path, "w", encoding="utf-8", newline="") as fh:
+        fh.write(run_json(run))
+    return file_path
+
+
+def load_eval_run(filename: str) -> dict[str, Any]:
+    directory = _results_dir()
+    name = filename if filename.endswith(".json") else f"{filename}.json"
+    file_path = os.path.join(directory, name)
+    if not os.path.exists(file_path):
+        raise Exception(f'Eval run "{filename}" not found at {file_path}')
+    with open(file_path, encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+def list_eval_runs() -> list[dict[str, Any]]:
+    directory = _results_dir()
+    if not os.path.isdir(directory):
+        return []
+    files = sorted((f for f in os.listdir(directory) if f.startswith("eval_") and f.endswith(".json")), reverse=True)
+    runs = []
+    for f in files:
+        with open(os.path.join(directory, f), encoding="utf-8") as fh:
+            runs.append(json.load(fh))
+    return runs
+
+
+def get_results_path() -> str:
+    return _results_dir()

agentcore/operations/feedback/__init__.py

@@ -0,0 +1,21 @@
+"""Feedback submission to AWS's public feedback service (port of src/cli/operations/feedback)."""
+
+from agentcore.operations.feedback.aperture_client import ApertureError
+from agentcore.operations.feedback.constants import CONSENT_TEXT
+from agentcore.operations.feedback.submit_feedback import (
+    FEEDBACK_MESSAGE_MAX_LENGTH,
+    FeedbackValidationError,
+    submit_feedback,
+    validate_feedback_message,
+    validate_screenshot_path,
+)
+
+__all__ = [
+    "CONSENT_TEXT",
+    "FEEDBACK_MESSAGE_MAX_LENGTH",
+    "ApertureError",
+    "FeedbackValidationError",
+    "submit_feedback",
+    "validate_feedback_message",
+    "validate_screenshot_path",
+]

agentcore/operations/feedback/aperture_client.py

@@ -0,0 +1,98 @@
+"""HTTP client for AWS's public feedback service (port of aperture-client.ts).
+
+``_send`` is the single network seam (tests replace it); it never raises on HTTP error statuses.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import Any
+
+from agentcore.operations.feedback.constants import APERTURE_INGESTION_URL, APERTURE_PRESIGNED_URL_ENDPOINT
+
+_TIMEOUT_SECONDS = 60
+
+
+class ApertureError(Exception):
+    def __init__(self, message: str, status: int | None = None, body: str | None = None):
+        super().__init__(message)
+        self.name = "ApertureError"
+        self.status = status
+        self.body = body
+
+
+def _send(method: str, url: str, headers: dict[str, str], body: bytes) -> tuple[int, bytes]:
+    request = urllib.request.Request(url, data=body, method=method, headers=headers)
+    try:
+        with urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) as response:  # noqa: S310 - fixed https URLs
+            return response.status, response.read()
+    except urllib.error.HTTPError as exc:
+        try:
+            data = exc.read()
+        except Exception:  # noqa: BLE001
+            data = b""
+        return exc.code, data
+
+
+def _text(body: bytes) -> str:
+    return body.decode("utf-8", errors="replace")
+
+
+def fetch_presigned_url(request: dict[str, Any], user_agent: str) -> str:
+    """The S3 presigned URL for a screenshot upload (returned as a plain-text body)."""
+    status, body = _send(
+        "POST",
+        APERTURE_PRESIGNED_URL_ENDPOINT,
+        {"content-type": "application/json", "user-agent": user_agent},
+        json.dumps(request, separators=(",", ":")).encode(),
+    )
+    if not 200 <= status < 300:
+        raise ApertureError(f"Failed to fetch screenshot upload URL (HTTP {status}).", status, _text(body))
+    return _text(body).strip()
+
+
+def upload_file_to_s3(presigned_url: str, data: bytes, content_type: str, sha256_base64: str, user_agent: str) -> None:
+    status, body = _send(
+        "PUT",
+        presigned_url,
+        {
+            "content-type": content_type,
+            "x-amz-checksum-algorithm": "SHA256",
+            "x-amz-checksum-sha256": sha256_base64,
+            "x-amz-tagging": "scanstatus=NOT_SCANNED",
+            "user-agent": user_agent,
+        },
+        data,
+    )
+    if not 200 <= status < 300:
+        raise ApertureError(f"Failed to upload screenshot (HTTP {status}).", status, _text(body))
+
+
+def submit_form(payload: dict[str, Any], user_agent: str) -> dict[str, Any]:
+    status, body = _send(
+        "POST",
+        APERTURE_INGESTION_URL,
+        {"content-type": "application/json", "user-agent": user_agent},
+        json.dumps(payload, separators=(",", ":")).encode(),
+    )
+    if not 200 <= status < 300:
+        text = _text(body)
+        raise ApertureError(_map_status_to_message(status, text), status, text)
+    return json.loads(body)
+
+
+def _map_status_to_message(status: int, body: str) -> str:
+    if status == 400:
+        return f"Feedback service rejected the submission (HTTP 400). {body or 'Form payload may be malformed.'}"
+    if status == 412:
+        return "Feedback service is missing required headers (HTTP 412)."
+    if status == 417:
+        return "Feedback service rejected the request content type (HTTP 417)."
+    if status == 500:
+        return "Feedback service returned an internal error (HTTP 500). Please try again later."
+    return f"Feedback service returned HTTP {status}."
+
+
+__all__ = ["ApertureError", "fetch_presigned_url", "submit_form", "upload_file_to_s3"]

agentcore/operations/feedback/build_payload.py

@@ -0,0 +1,97 @@
+"""Feedback form payload (port of build-payload.ts).
+
+Deliberate deviation: this client identifies itself honestly as the Python port — the CLI version is
+sent as ``<version>-python`` and the runtime as ``python <version>`` instead of Node. The product
+identifiers (``agentcore-cli`` / ``AgentCoreCLI``) are unchanged.
+"""
+
+from __future__ import annotations
+
+import platform
+import sys
+from typing import Any, Literal
+
+from agentcore import __version__
+from agentcore.operations.feedback.constants import (
+    APERTURE_FORM_CATEGORY,
+    APERTURE_FORM_NAME,
+    APERTURE_FORM_VERSION,
+    APERTURE_LOCALE,
+    FEEDBACK_ATTACHMENT_QUESTION,
+    FEEDBACK_MESSAGE_QUESTION,
+    FEEDBACK_REFERENCE,
+    METADATA_KEY_CLI_VERSION,
+    METADATA_KEY_OS,
+)
+
+FeedbackMode = Literal["cli", "tui"]
+
+CLI_VERSION = f"{__version__}-python"
+
+
+def _platform() -> str:
+    """Node's ``process.platform`` naming (darwin / linux / win32)."""
+    return sys.platform
+
+
+def _python_version() -> str:
+    return platform.python_version()
+
+
+def build_os_descriptor() -> str:
+    return f"{_platform()} {platform.release()}"
+
+
+def build_location_descriptor(cli_version: str, mode: FeedbackMode) -> str:
+    return f"agentcore-cli@{cli_version} ({_platform()}; python {_python_version()}; {mode})"
+
+
+def build_user_agent(cli_version: str) -> str:
+    return f"AgentCoreCLI/{cli_version} ({_platform()} {platform.release()}; python/{_python_version()})"
+
+
+def build_feedback_payload(
+    message: str,
+    screenshot_reference: str | None = None,
+    mode: FeedbackMode | None = None,
+    cli_version: str | None = None,
+    os_descriptor: str | None = None,
+) -> dict[str, Any]:
+    version = cli_version or CLI_VERSION
+    responses: list[dict[str, Any]] = [
+        {
+            "question": FEEDBACK_MESSAGE_QUESTION,
+            "pii": False,
+            "response": {"responseType": "textArea", "responseValue": message},
+        }
+    ]
+    if screenshot_reference:
+        responses.append(
+            {
+                "question": FEEDBACK_ATTACHMENT_QUESTION,
+                "pii": True,
+                "response": {"responseType": "fileUpload", "responseValue": [screenshot_reference]},
+            }
+        )
+    return {
+        "category": APERTURE_FORM_CATEGORY,
+        "name": APERTURE_FORM_NAME,
+        "version": APERTURE_FORM_VERSION,
+        "locale": APERTURE_LOCALE,
+        "reference": FEEDBACK_REFERENCE,
+        "location": build_location_descriptor(version, mode or "cli"),
+        "customerResponses": responses,
+        "metadataList": [
+            {"key": METADATA_KEY_CLI_VERSION, "value": version},
+            {"key": METADATA_KEY_OS, "value": os_descriptor or build_os_descriptor()},
+        ],
+    }
+
+
+__all__ = [
+    "CLI_VERSION",
+    "build_feedback_payload",
+    "build_location_descriptor",
+    "build_os_descriptor",
+    "build_user_agent",
+]

agentcore/operations/feedback/constants.py

@@ -0,0 +1,26 @@
+"""Port of src/cli/operations/feedback/constants.ts."""
+
+APERTURE_FORM_CATEGORY = "AgentCore"
+APERTURE_FORM_NAME = "CLI"
+APERTURE_FORM_VERSION = "0.1.0"
+APERTURE_LOCALE = "en_US"
+
+APERTURE_INGESTION_URL = "https://ingestion.aperture-public-api.feedback.console.aws.dev/form"
+APERTURE_PRESIGNED_URL_ENDPOINT = "https://presignedurl.aperture-public-api.feedback.console.aws.dev/presignedurl"
+
+MAX_SCREENSHOT_BYTES = 100 * 1024 * 1024
+ALLOWED_SCREENSHOT_EXTENSIONS = (".png", ".jpg", ".jpeg")
+
+FEEDBACK_MESSAGE_QUESTION = "What feedback do you have for the AgentCore CLI"
+FEEDBACK_ATTACHMENT_QUESTION = "Attachments"
+METADATA_KEY_CLI_VERSION = "cli-version"
+METADATA_KEY_OS = "os"
+
+FEEDBACK_REFERENCE = "agentcore-cli"
+
+CONSENT_TEXT = (
+    "All feedback submissions, including any uploaded text and images, are subject "
+    "to the AWS Customer Agreement (https://aws.amazon.com/agreement/). By submitting "
+    'feedback, you agree that your submissions constitute "Suggestions" as defined '
+    "in the AWS Customer Agreement."
+)