tribunal-kit 9.2.3 → 9.2.4

package/.agent/skills/hf-cloud-sagemaker-production-defaults/scripts/_common.py

@@ -0,0 +1,148 @@
+"""Shared helpers for the deployment scripts (deploy.py, deploy_async.py).
+
+Anything that should look identical between real-time and async deployments
+lives here — tag schema, name format, env-var parsing, model creation,
+endpoint waiting. Mode-specific logic (endpoint config shape, autoscaling
+policies, alarm metrics) stays in the calling script.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from datetime import datetime, timezone
+from typing import Any
+
+from botocore.exceptions import ClientError
+
+
+# Shared tagging convention. CreatedBy lets us find every resource these
+# scripts have ever created via:
+#   aws resourcegroupstaggingapi get-resources \
+#     --tag-filters Key=CreatedBy,Values=agentic-deploy-skills
+CREATED_BY_TAG_VALUE = "agentic-deploy-skills"
+
+
+def log(prefix: str, msg: str) -> None:
+    """Stream a log line to stderr with a per-script prefix."""
+    print(f"[{prefix}] {msg}", file=sys.stderr, flush=True)
+
+
+def parse_env(env_args: list[str]) -> dict[str, str]:
+    """Parse --env KEY=VALUE flags from argparse into a dict."""
+    env: dict[str, str] = {}
+    for item in env_args:
+        if "=" not in item:
+            raise SystemExit(f"--env must be KEY=VALUE, got: {item}")
+        k, v = item.split("=", 1)
+        env[k] = v
+    return env
+
+
+def make_endpoint_name(model_name: str, override: str | None) -> str:
+    """Generate a SageMaker-legal endpoint name.
+
+    Default format: <model-name>-<YYYYMMDD-HHMM>. SageMaker names are limited
+    to 63 chars and must be DNS-friendly (lowercase, hyphens, no underscores).
+    """
+    if override:
+        return override
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M")
+    base = model_name.replace("_", "-").lower()
+    return f"{base}-{stamp}"[:63]
+
+
+def build_tags(
+    *,
+    project: str,
+    caller_arn: str,
+    environment: str,
+    model_s3_uri: str | None = None,
+    extra: dict[str, str] | None = None,
+) -> list[dict[str, str]]:
+    """Build the shared tag set applied to every resource we create.
+
+    `extra` lets the caller add deployment-mode-specific tags (e.g.
+    `{"InferenceMode": "async"}`) without forking the schema.
+    """
+    owner = caller_arn.split("/")[-1] if "/" in caller_arn else caller_arn
+    tags = [
+        {"Key": "Project", "Value": project},
+        {"Key": "Owner", "Value": owner},
+        {"Key": "Environment", "Value": environment},
+        {"Key": "CreatedBy", "Value": CREATED_BY_TAG_VALUE},
+    ]
+    if model_s3_uri:
+        tags.append({"Key": "ModelArtifact", "Value": model_s3_uri})
+    if extra:
+        for k, v in extra.items():
+            tags.append({"Key": k, "Value": v})
+    return tags
+
+
+def create_model(
+    sm: Any,
+    *,
+    model_name: str,
+    image_uri: str,
+    role_arn: str,
+    model_s3_uri: str | None,
+    env: dict[str, str],
+    tags: list[dict[str, str]],
+    log_prefix: str = "deploy",
+) -> str:
+    """Create a SageMaker Model. Idempotent — reuses on AlreadyExists.
+
+    Returns the model name on success. The model definition itself is mode-
+    independent: same call, same parameters, whether the eventual endpoint
+    is real-time or async.
+    """
+    log(log_prefix, f"Creating model: {model_name}")
+    primary_container: dict[str, Any] = {"Image": image_uri}
+    if env:
+        primary_container["Environment"] = env
+    if model_s3_uri:
+        primary_container["ModelDataUrl"] = model_s3_uri
+
+    try:
+        sm.create_model(
+            ModelName=model_name,
+            PrimaryContainer=primary_container,
+            ExecutionRoleArn=role_arn,
+            Tags=tags,
+        )
+    except ClientError as e:
+        if "Cannot create already existing model" in str(e):
+            log(log_prefix, f"Model {model_name} already exists — reusing")
+        else:
+            raise
+    return model_name
+
+
+def wait_for_endpoint(
+    sm: Any,
+    endpoint_name: str,
+    timeout_minutes: int = 30,
+    log_prefix: str = "deploy",
+) -> None:
+    """Poll DescribeEndpoint until InService, or raise on Failed/timeout."""
+    log(log_prefix, f"Waiting for {endpoint_name} to reach InService (up to {timeout_minutes} min)...")
+    start = time.time()
+    deadline = start + (timeout_minutes * 60)
+
+    while time.time() < deadline:
+        resp = sm.describe_endpoint(EndpointName=endpoint_name)
+        status = resp["EndpointStatus"]
+        elapsed = int(time.time() - start)
+
+        if status == "InService":
+            log(log_prefix, f"InService after {elapsed}s")
+            return
+        if status == "Failed":
+            reason = resp.get("FailureReason", "(no reason given)")
+            raise RuntimeError(f"Endpoint creation failed after {elapsed}s: {reason}")
+
+        log(log_prefix, f"  status={status} elapsed={elapsed}s")
+        time.sleep(30)
+
+    raise TimeoutError(f"Endpoint did not reach InService within {timeout_minutes} minutes")

package/.agent/skills/hf-cloud-sagemaker-production-defaults/scripts/deploy.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python
+"""Create a SageMaker real-time endpoint with production defaults.
+
+Minimal usage:
+    python deploy.py --model-name <name> --image-uri <uri> \
+        --inference-ami-version <ami> --role-arn <arn> \
+        --instance-type ml.g5.xlarge --region <region>
+
+See the SKILL.md for full flags and the recommended chained-with-resolver pattern.
+"""
+
+import argparse
+import json
+import sys
+import time
+from typing import Any
+
+import boto3
+from botocore.exceptions import ClientError
+
+from _common import (
+    build_tags,
+    create_model,
+    log as _log,
+    make_endpoint_name,
+    parse_env,
+    wait_for_endpoint,
+)
+
+
+# Defaults — change here, not at call sites.
+# Reasoning lives in references/deployment-template.md.
+DEFAULTS = {
+    "initial_instance_count": 1,
+    "min_capacity": 1,
+    "max_capacity": 4,
+    "target_invocations_per_instance": 20,
+    "scale_in_cooldown_seconds": 300,
+    "scale_out_cooldown_seconds": 60,
+    "data_capture_sampling_percent": 100,
+    "alarm_latency_threshold_ms": 30_000,
+    "alarm_5xx_threshold_count": 5,
+    "alarm_overhead_threshold_ms": 2_000,
+    "alarm_evaluation_periods": 1,
+    "alarm_period_seconds": 300,
+    "environment_tag": "dev",
+}
+
+
+def log(msg: str) -> None:
+    _log("deploy", msg)
+
+
+def create_endpoint_config(
+    sm: Any, *, config_name: str, model_name: str, instance_type: str,
+    initial_instance_count: int, inference_ami_version: str | None,
+    data_capture_enabled: bool, data_capture_s3_uri: str | None,
+    tags: list[dict],
+) -> str:
+    log(f"Creating endpoint config: {config_name}")
+
+    production_variant: dict[str, Any] = {
+        "VariantName": "AllTraffic",
+        "ModelName": model_name,
+        "InstanceType": instance_type,
+        "InitialInstanceCount": initial_instance_count,
+        "InitialVariantWeight": 1.0,
+    }
+
+    # InferenceAmiVersion required for vLLM DLC with CUDA 13+. Without it the
+    # container dies on startup with no logs. See hf-cloud-serving-image-selection skill.
+    if inference_ami_version:
+        production_variant["InferenceAmiVersion"] = inference_ami_version
+        log(f"  InferenceAmiVersion set to: {inference_ami_version}")
+
+    kwargs: dict[str, Any] = {
+        "EndpointConfigName": config_name,
+        "ProductionVariants": [production_variant],
+        "Tags": tags,
+    }
+
+    if data_capture_enabled:
+        if not data_capture_s3_uri:
+            raise ValueError("data_capture_enabled but no data_capture_s3_uri provided")
+        kwargs["DataCaptureConfig"] = {
+            "EnableCapture": True,
+            "InitialSamplingPercentage": DEFAULTS["data_capture_sampling_percent"],
+            "DestinationS3Uri": data_capture_s3_uri,
+            "CaptureOptions": [{"CaptureMode": "Input"}, {"CaptureMode": "Output"}],
+            "CaptureContentTypeHeader": {"JsonContentTypes": ["application/json"]},
+        }
+
+    try:
+        sm.create_endpoint_config(**kwargs)
+    except ClientError as e:
+        if "Cannot create already existing endpoint configuration" in str(e):
+            log(f"Endpoint config {config_name} already exists — reusing")
+        else:
+            raise
+    return config_name
+
+
+def create_endpoint(sm: Any, *, endpoint_name: str, config_name: str, tags: list[dict]) -> None:
+    log(f"Creating endpoint: {endpoint_name}")
+    sm.create_endpoint(
+        EndpointName=endpoint_name,
+        EndpointConfigName=config_name,
+        Tags=tags,
+    )
+
+
+def register_autoscaling(
+    *, endpoint_name: str, variant_name: str, min_capacity: int, max_capacity: int,
+    target_invocations: int, scale_in_cooldown: int, scale_out_cooldown: int, region: str,
+) -> None:
+    log(f"Registering autoscaling: min={min_capacity} max={max_capacity} target={target_invocations}/min")
+    appscaling = boto3.client("application-autoscaling", region_name=region)
+    resource_id = f"endpoint/{endpoint_name}/variant/{variant_name}"
+
+    appscaling.register_scalable_target(
+        ServiceNamespace="sagemaker",
+        ResourceId=resource_id,
+        ScalableDimension="sagemaker:variant:DesiredInstanceCount",
+        MinCapacity=min_capacity,
+        MaxCapacity=max_capacity,
+    )
+    appscaling.put_scaling_policy(
+        PolicyName=f"{endpoint_name}-target-tracking",
+        ServiceNamespace="sagemaker",
+        ResourceId=resource_id,
+        ScalableDimension="sagemaker:variant:DesiredInstanceCount",
+        PolicyType="TargetTrackingScaling",
+        TargetTrackingScalingPolicyConfiguration={
+            "TargetValue": float(target_invocations),
+            "PredefinedMetricSpecification": {
+                "PredefinedMetricType": "SageMakerVariantInvocationsPerInstance",
+            },
+            "ScaleInCooldown": scale_in_cooldown,
+            "ScaleOutCooldown": scale_out_cooldown,
+        },
+    )
+
+
+def create_alarms(*, endpoint_name: str, variant_name: str, sns_topic_arn: str | None, region: str) -> None:
+    log(f"Creating CloudWatch alarms for {endpoint_name}")
+    cw = boto3.client("cloudwatch", region_name=region)
+    actions = [sns_topic_arn] if sns_topic_arn else []
+    common_dims = [
+        {"Name": "EndpointName", "Value": endpoint_name},
+        {"Name": "VariantName", "Value": variant_name},
+    ]
+
+    alarms = [
+        {
+            "AlarmName": f"{endpoint_name}-ModelLatencyP99",
+            "MetricName": "ModelLatency",
+            "ExtendedStatistic": "p99",
+            "Threshold": DEFAULTS["alarm_latency_threshold_ms"] * 1000,  # microseconds
+            "ComparisonOperator": "GreaterThanThreshold",
+            "AlarmDescription": "Model inference latency p99 > 30s",
+        },
+        {
+            "AlarmName": f"{endpoint_name}-Invocation5XXErrors",
+            "MetricName": "Invocation5XXErrors",
+            "Statistic": "Sum",
+            "Threshold": DEFAULTS["alarm_5xx_threshold_count"],
+            "ComparisonOperator": "GreaterThanThreshold",
+            "AlarmDescription": "5XX errors > 5 in 5min",
+        },
+        {
+            "AlarmName": f"{endpoint_name}-OverheadLatencyP99",
+            "MetricName": "OverheadLatency",
+            "ExtendedStatistic": "p99",
+            "Threshold": DEFAULTS["alarm_overhead_threshold_ms"] * 1000,
+            "ComparisonOperator": "GreaterThanThreshold",
+            "AlarmDescription": "Platform overhead latency p99 > 2s",
+        },
+    ]
+
+    for spec in alarms:
+        params = {
+            "AlarmName": spec["AlarmName"],
+            "AlarmDescription": spec["AlarmDescription"],
+            "MetricName": spec["MetricName"],
+            "Namespace": "AWS/SageMaker",
+            "Dimensions": common_dims,
+            "Period": DEFAULTS["alarm_period_seconds"],
+            "EvaluationPeriods": DEFAULTS["alarm_evaluation_periods"],
+            "Threshold": spec["Threshold"],
+            "ComparisonOperator": spec["ComparisonOperator"],
+            "TreatMissingData": "notBreaching",
+            "AlarmActions": actions,
+        }
+        if "Statistic" in spec:
+            params["Statistic"] = spec["Statistic"]
+        if "ExtendedStatistic" in spec:
+            params["ExtendedStatistic"] = spec["ExtendedStatistic"]
+        cw.put_metric_alarm(**params)
+
+    if not sns_topic_arn:
+        log("WARNING: no --sns-alarm-topic — alarms exist but won't notify anyone.")
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    # Required
+    p.add_argument("--model-name", required=True)
+    p.add_argument("--image-uri", required=True, help="From hf-cloud-serving-image-selection")
+    p.add_argument("--role-arn", required=True, help="From hf-cloud-sagemaker-iam-preflight")
+    p.add_argument("--instance-type", required=True, help="e.g. ml.g5.xlarge")
+    p.add_argument("--region", required=True, help="From hf-cloud-aws-context-discovery")
+
+    # Conditional
+    p.add_argument("--model-s3-uri", default=None,
+                   help="S3 model artifact extracted to /opt/ml/model (preferred); omit only for Hub-at-runtime")
+    p.add_argument("--env", action="append", default=[], help="KEY=VALUE; repeatable")
+    p.add_argument(
+        "--inference-ami-version", default=None,
+        help=(
+            "REQUIRED for vLLM DLC with CUDA 13+ (e.g. al2-ami-sagemaker-inference-gpu-3-1). "
+            "Without this, container dies on startup with no logs. "
+            "See hf-cloud-serving-image-selection's 'vLLM AMI requirement' table to map a tag to the AMI version."
+        ),
+    )
+
+    # Naming
+    p.add_argument("--endpoint-name", default=None, help="Default: <model-name>-<timestamp>")
+    p.add_argument("--project", default=None, help="Tag value (default: model name)")
+    p.add_argument("--environment", default=DEFAULTS["environment_tag"])
+
+    # Capacity / scaling
+    p.add_argument("--initial-instance-count", type=int, default=DEFAULTS["initial_instance_count"])
+    p.add_argument("--min-capacity", type=int, default=DEFAULTS["min_capacity"])
+    p.add_argument("--max-capacity", type=int, default=DEFAULTS["max_capacity"])
+    p.add_argument("--target-invocations-per-instance", type=int, default=DEFAULTS["target_invocations_per_instance"])
+    p.add_argument("--no-autoscaling", action="store_true", help="NOT RECOMMENDED")
+
+    # Data capture (off by default)
+    p.add_argument("--enable-data-capture", action="store_true", help="Log requests/responses to S3")
+    p.add_argument("--data-capture-s3-uri", default=None)
+
+    # Alarms
+    p.add_argument("--sns-alarm-topic", default=None, help="SNS topic ARN for alarm notifications")
+    p.add_argument("--no-alarms", action="store_true")
+
+    args = p.parse_args()
+
+    env_dict = parse_env(args.env)
+    endpoint_name = make_endpoint_name(args.model_name, args.endpoint_name)
+    config_name = f"{endpoint_name}-config"
+
+    sts = boto3.client("sts", region_name=args.region)
+    sm = boto3.client("sagemaker", region_name=args.region)
+    caller_arn = sts.get_caller_identity()["Arn"]
+    account_id = sts.get_caller_identity()["Account"]
+
+    if args.enable_data_capture and not args.data_capture_s3_uri:
+        args.data_capture_s3_uri = f"s3://sagemaker-{args.region}-{account_id}/{endpoint_name}/data-capture/"
+        log(f"Data capture URI defaulted to: {args.data_capture_s3_uri}")
+
+    tags = build_tags(
+        project=args.project or args.model_name,
+        caller_arn=caller_arn,
+        environment=args.environment,
+        model_s3_uri=args.model_s3_uri,
+    )
+
+    create_model(
+        sm, model_name=args.model_name, image_uri=args.image_uri,
+        role_arn=args.role_arn, model_s3_uri=args.model_s3_uri,
+        env=env_dict, tags=tags, log_prefix="deploy",
+    )
+    create_endpoint_config(
+        sm, config_name=config_name, model_name=args.model_name,
+        instance_type=args.instance_type, initial_instance_count=args.initial_instance_count,
+        inference_ami_version=args.inference_ami_version,
+        data_capture_enabled=args.enable_data_capture,
+        data_capture_s3_uri=args.data_capture_s3_uri, tags=tags,
+    )
+    create_endpoint(sm, endpoint_name=endpoint_name, config_name=config_name, tags=tags)
+    wait_for_endpoint(sm, endpoint_name, log_prefix="deploy")
+
+    if not args.no_autoscaling:
+        register_autoscaling(
+            endpoint_name=endpoint_name, variant_name="AllTraffic",
+            min_capacity=args.min_capacity, max_capacity=args.max_capacity,
+            target_invocations=args.target_invocations_per_instance,
+            scale_in_cooldown=DEFAULTS["scale_in_cooldown_seconds"],
+            scale_out_cooldown=DEFAULTS["scale_out_cooldown_seconds"],
+            region=args.region,
+        )
+    else:
+        log("WARNING: autoscaling skipped. Endpoint won't scale with traffic.")
+
+    if not args.no_alarms:
+        create_alarms(
+            endpoint_name=endpoint_name, variant_name="AllTraffic",
+            sns_topic_arn=args.sns_alarm_topic, region=args.region,
+        )
+
+    # Summary
+    log("")
+    log(f"Deployment complete: {endpoint_name}")
+    log(f"  Instance:        {args.instance_type}")
+    log(f"  Autoscaling:     {'OFF' if args.no_autoscaling else f'{args.min_capacity}-{args.max_capacity} instances'}")
+    log(f"  Data capture:    {args.data_capture_s3_uri if args.enable_data_capture else 'OFF (pass --enable-data-capture)'}")
+    log("")
+    log(f"Test:     python3 invoke_endpoint.py --endpoint-name {endpoint_name} \\")
+    log(f"            --payload '{{\"prompt\": \"hello\"}}' --region {args.region}")
+    log("          (BOM-safe + cross-platform; use 'python' on Windows)")
+    log(f"Teardown: python3 teardown.py {endpoint_name} {args.region}")
+
+    # Machine-readable summary for downstream scripting
+    print(json.dumps({
+        "endpoint_name": endpoint_name,
+        "endpoint_config_name": config_name,
+        "model_name": args.model_name,
+        "region": args.region,
+        "instance_type": args.instance_type,
+    }))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())