agentcore-cli 0.31.1__py3-none-any.whl

agentcore/primitives/evaluator.py

@@ -0,0 +1,909 @@
+"""Custom evaluator add/remove (port of src/cli/primitives/EvaluatorPrimitive.ts).
+
+Also hosts the evaluator wizard constants from tui/screens/evaluator/types.ts (rating scale
+presets, level placeholders, model options) since the flags and the prompts share them.
+"""
+
+import json
+import math
+import os
+import re
+import shutil
+from dataclasses import dataclass
+from typing import Any
+
+import typer
+from pydantic import TypeAdapter
+from pydantic import ValidationError as PydanticValidationError
+
+from agentcore.cli import interactive
+from agentcore.cli.errors import get_error_message
+from agentcore.cli.naming import generate_unique_name
+from agentcore.cli.run_command import run_cli_command
+from agentcore.lib import ConflictError, ResourceNotFoundError, find_config_root, serialize_result
+from agentcore.lib.errors.types import to_error
+from agentcore.lib.result import Err, Ok
+from agentcore.lib.schemas.io.config_io import to_jsonable
+from agentcore.operations.jobs.shared.format import parse_int
+from agentcore.primitives.base import BasePrimitive, RemovableResource, RemovalPreview, SchemaChange, emit_json
+from agentcore.schema import EVALUATION_LEVELS, EvaluatorModelId, EvaluatorName, is_valid_kms_key_arn
+from agentcore.schema.schemas.primitives.evaluator import is_valid_bedrock_model_id
+from agentcore.templates import render_code_based_evaluator_template, render_third_party_evaluator_template
+
+# ============================================================================
+# Third-party library registry
+# ============================================================================
+
+THIRD_PARTY_EVALUATOR_LIBRARIES: dict[str, dict[str, Any]] = {
+    "deepeval": {
+        "templateDir": "deepeval-lambda",
+        "defaultTimeoutSeconds": 300,
+        "warnings": [
+            {
+                "metrics": {
+                    "FaithfulnessMetric",
+                    "HallucinationMetric",
+                    "ContextualRelevancyMetric",
+                    "ContextualPrecisionMetric",
+                    "ContextualRecallMetric",
+                },
+                "message": "requires retrieval_context from tool-role messages. "
+                "If your agent has no tool calls, the evaluator will return MISSING_REQUIRED_FIELD at runtime.",
+            },
+            {
+                "metrics": {"ContextualPrecisionMetric", "ContextualRecallMetric"},
+                "message": "requires expected_output via evaluationReferenceInputs. "
+                "Caller must provide referenceInputs when invoking the Evaluate API.",
+            },
+        ],
+    },
+    "autoevals": {
+        "templateDir": "autoevals-lambda",
+        "defaultTimeoutSeconds": 60,
+        "warnings": [
+            {
+                "metrics": {"Factuality", "ClosedQA"},
+                "message": "requires expected_output via evaluationReferenceInputs. "
+                "Caller must provide referenceInputs when invoking the Evaluate API.",
+            },
+            {
+                "metrics": {"SQL"},
+                "message": "requires expected_output (reference SQL) via evaluationReferenceInputs. "
+                "Caller must provide referenceInputs when invoking the Evaluate API.",
+            },
+        ],
+    },
+}
+SUPPORTED_LIBRARIES = list(THIRD_PARTY_EVALUATOR_LIBRARIES)
+MODEL_PROVIDERS = ("openai", "bedrock")
+
+DEFAULT_CODE_ENTRYPOINT = "lambda_function.handler"
+DEFAULT_CODE_TIMEOUT = 60
+_KWARG_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_LAMBDA_ARN = re.compile(r"^arn:aws[a-z-]*:lambda:[a-z0-9-]+:\d{12}:function:.+$")
+KMS_ARN_ERROR = (
+    "--kms-key-arn must be a valid KMS key ARN "
+    "(e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)"
+)
+
+# ============================================================================
+# Wizard constants (tui/screens/evaluator/types.ts)
+# ============================================================================
+
+EVALUATOR_TYPE_OPTIONS = (
+    ("code-based", "Code-based (custom Lambda function)", "Write custom Python code for evaluation logic"),
+    ("llm-as-a-judge", "LLM-as-a-Judge (model-based)", "Use a foundation model to evaluate quality"),
+)
+CODE_BASED_TYPE_OPTIONS = (
+    ("managed", "Create new (CLI scaffolds and deploys the Lambda for you)", "CLI scaffolds code and deploys Lambda"),
+    ("external", "Existing Lambda ARN (bring your own)", "Use an existing Lambda function"),
+)
+EVALUATION_LEVEL_OPTIONS = (
+    ("SESSION", "Session", "Overall quality across an entire conversation"),
+    ("TRACE", "Trace", "Per-turn accuracy of individual agent responses"),
+    ("TOOL_CALL", "Tool Call", "Correctness of individual tool selections and usage"),
+)
+DEFAULT_MODEL = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+DEFAULT_OPENAI_MODEL = "openai.gpt-5.4"
+CUSTOM_MODEL_ID = "__custom__"
+EVALUATOR_MODEL_PROVIDER_OPTIONS = (
+    ("Bedrock", "Amazon Bedrock", "Use the Bedrock Converse API"),
+    ("OpenResponses", "OpenAI on Bedrock", "Use the OpenResponses API through Bedrock Mantle"),
+)
+BEDROCK_EVALUATOR_MODEL_OPTIONS = (
+    ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", "Claude Sonnet 4.5", "Recommended — balanced speed and accuracy"),
+    (
+        "global.anthropic.claude-opus-4-5-20251101-v1:0",
+        "Claude Opus 4.5",
+        "Most capable — best for complex evaluations",
+    ),
+    ("us.anthropic.claude-haiku-4-5-20251001-v1:0", "Claude Haiku 4.5", "Fastest — good for high-volume evaluations"),
+    ("us.amazon.nova-pro-v1:0", "Amazon Nova Pro", "Amazon foundation model — strong reasoning"),
+    ("us.amazon.nova-lite-v1:0", "Amazon Nova Lite", "Amazon foundation model — fast and cost-effective"),
+    (CUSTOM_MODEL_ID, "Other", "Enter a custom Bedrock model ID or ARN"),
+)
+OPENAI_EVALUATOR_MODEL_OPTIONS = (
+    ("openai.gpt-5.4", "GPT-5.4", "Recommended — high-quality reasoning and evaluation"),
+    ("openai.gpt-5.5", "GPT-5.5", "Most capable OpenAI model"),
+    ("openai.gpt-oss-120b", "GPT OSS 120B", "Open-weight model for complex evaluations"),
+    ("openai.gpt-oss-20b", "GPT OSS 20B", "Open-weight model for faster evaluations"),
+    (CUSTOM_MODEL_ID, "Other", "Enter a custom OpenAI model ID"),
+)
+
+LEVEL_PLACEHOLDERS: dict[str, list[str]] = {
+    "SESSION": ["context", "available_tools"],
+    "TRACE": ["context", "assistant_turn"],
+    "TOOL_CALL": [
+        "available_tools",
+        "context",
+        "tool_turn",
+        "available_skills",
+        "invoked_skill",
+        "skill_content",
+        "user_message",
+    ],
+}
+REFERENCE_INPUT_PLACEHOLDERS: dict[str, list[str]] = {
+    "SESSION": ["assertions", "expected_tool_trajectory", "actual_tool_trajectory"],
+    "TRACE": ["expected_response"],
+    "TOOL_CALL": [],
+}
+PLACEHOLDER_DESCRIPTIONS: dict[str, str] = {
+    "context": "full conversation history (user + assistant messages)",
+    "assistant_turn": "the specific assistant response being evaluated",
+    "available_tools": "list of tools the agent can call",
+    "tool_turn": "the specific tool call and its result",
+    "assertions": "caller-provided assertions the agent should satisfy",
+    "expected_tool_trajectory": "caller-provided expected sequence of tool calls",
+    "actual_tool_trajectory": "actual sequence of tool calls from the session",
+    "expected_response": "caller-provided expected agent response",
+    "available_skills": "skills offered to the agent (name + description)",
+    "invoked_skill": "the skill the agent selected for this span",
+    "skill_content": "the selected skill's SKILL.md instructions",
+    "user_message": "the user request in the turn that triggered the skill invocation",
+}
+DEFAULT_INSTRUCTIONS: dict[str, str] = {
+    "SESSION": "Evaluate the agent session. Context: {context}. Available tools: {available_tools}. The agent must "
+    "satisfy: {assertions}. Expected trajectory: {expected_tool_trajectory}. Actual trajectory: "
+    "{actual_tool_trajectory}. Rate the overall quality.",
+    "TRACE": "Evaluate the agent trace. Context: {context}. Assistant turn: {assistant_turn}. Expected response: "
+    "{expected_response}. Rate the quality of this response.",
+    "TOOL_CALL": "Evaluate the tool call. Context: {context}. Tool turn: {tool_turn}. Rate the quality of this tool "
+    "usage.",
+}
+CUSTOM_RATING_SCALE_ID = "__custom__"
+RATING_SCALE_TYPE_OPTIONS = (
+    ("numerical", "Numerical", "Scored values (e.g. 1–5)"),
+    ("categorical", "Categorical", "Named labels (e.g. Pass/Fail)"),
+)
+RATING_SCALE_PRESETS: list[dict[str, Any]] = [
+    {
+        "id": "1-5-quality",
+        "title": "1–5 Quality",
+        "description": "Numerical · Poor(1), Fair(2), Good(3), Very Good(4), Excellent(5)",
+        "ratingScale": {
+            "numerical": [
+                {"value": 1, "label": "Poor", "definition": "Fails to meet expectations"},
+                {"value": 2, "label": "Fair", "definition": "Partially meets expectations"},
+                {"value": 3, "label": "Good", "definition": "Meets expectations"},
+                {"value": 4, "label": "Very Good", "definition": "Exceeds expectations"},
+                {"value": 5, "label": "Excellent", "definition": "Far exceeds expectations"},
+            ]
+        },
+    },
+    {
+        "id": "1-3-simple",
+        "title": "1–3 Simple",
+        "description": "Numerical · Low(1), Medium(2), High(3)",
+        "ratingScale": {
+            "numerical": [
+                {"value": 1, "label": "Low", "definition": "Below acceptable quality"},
+                {"value": 2, "label": "Medium", "definition": "Acceptable quality"},
+                {"value": 3, "label": "High", "definition": "Above acceptable quality"},
+            ]
+        },
+    },
+    {
+        "id": "pass-fail",
+        "title": "Pass / Fail",
+        "description": "Categorical · binary pass or fail assessment",
+        "ratingScale": {
+            "categorical": [
+                {"label": "Pass", "definition": "Meets the evaluation criteria"},
+                {"label": "Fail", "definition": "Does not meet the evaluation criteria"},
+            ]
+        },
+    },
+    {
+        "id": "good-neutral-bad",
+        "title": "Good / Neutral / Bad",
+        "description": "Categorical · three-tier quality assessment",
+        "ratingScale": {
+            "categorical": [
+                {"label": "Good", "definition": "Positive outcome, meets or exceeds criteria"},
+                {"label": "Neutral", "definition": "Acceptable but unremarkable outcome"},
+                {"label": "Bad", "definition": "Negative outcome, fails to meet criteria"},
+            ]
+        },
+    },
+]
+PRESET_IDS = [p["id"] for p in RATING_SCALE_PRESETS]
+
+
+def validate_instruction_placeholders(instructions: str, level: str) -> str | None:
+    """Error message unless the instructions contain a level placeholder."""
+    placeholders = LEVEL_PLACEHOLDERS[level]
+    if any(f"{{{p}}}" in instructions for p in placeholders):
+        return None
+    return f"Instructions must contain at least one placeholder: {', '.join(f'{{{p}}}' for p in placeholders)}"
+
+
+def _js_number(raw: str) -> float | int | None:
+    """``Number(raw)``; None for NaN."""
+    text = raw.strip()
+    if text == "":
+        return 0
+    try:
+        value = float(int(text, 0)) if re.fullmatch(r"0[xXoObB][0-9a-fA-F]+", text) else float(text)
+    except ValueError:
+        return None
+    if math.isnan(value):
+        return None
+    return int(value) if value.is_integer() else value
+
+
+def parse_custom_rating_scale(text: str, scale_type: str) -> tuple[bool, Any]:
+    """(True, ratingScale) or (False, error) from "1:Poor:Def, 5:Great:Def" / "Pass:Def, Fail:Def"."""
+    entries = [e.strip() for e in text.split(",") if e.strip()]
+    if len(entries) < 2:
+        return False, "At least 2 entries required (comma-separated)"
+    if scale_type == "numerical":
+        numerical = []
+        for entry in entries:
+            first = entry.find(":")
+            second = entry.find(":", first + 1) if first >= 0 else -1
+            if first < 0 or second < 0:
+                return False, f'Invalid entry "{entry}". Format: value:label:definition'
+            raw = entry[:first].strip()
+            value = _js_number(raw)
+            if value is None:
+                return False, f'"{raw}" is not a valid number in "{entry}"'
+            numerical.append(
+                {"value": value, "label": entry[first + 1 : second].strip(), "definition": entry[second + 1 :].strip()}
+            )
+        return True, {"numerical": numerical}
+    categorical = []
+    for entry in entries:
+        first = entry.find(":")
+        if first < 0:
+            return False, f'Invalid entry "{entry}". Format: label:definition'
+        categorical.append({"label": entry[:first].strip(), "definition": entry[first + 1 :].strip()})
+    return True, {"categorical": categorical}
+
+
+def json_to_python_value(value: Any) -> str:
+    if value is None:
+        return "None"
+    if value is True:
+        return "True"
+    if value is False:
+        return "False"
+    if isinstance(value, (int, float)):
+        return str(int(value)) if isinstance(value, float) and value.is_integer() else str(value)
+    if isinstance(value, str):
+        return json.dumps(value, ensure_ascii=False)
+    if isinstance(value, list):
+        return f"[{', '.join(json_to_python_value(v) for v in value)}]"
+    if isinstance(value, dict):
+        return (
+            "{"
+            + ", ".join(f"{json.dumps(k, ensure_ascii=False)}: {json_to_python_value(v)}" for k, v in value.items())
+            + "}"
+        )
+    return str(value)
+
+
+def json_to_kwargs(text: str) -> str:
+    obj = json.loads(text)
+    if not isinstance(obj, dict):
+        raise TypeError("Expected a JSON object of keyword arguments")
+    parts = []
+    for key, value in obj.items():
+        if not _KWARG_NAME.match(key):
+            raise Exception(f'Invalid Python kwarg name "{key}"')
+        parts.append(f"{key}={json_to_python_value(value)}")
+    return ", ".join(parts)
+
+
+def get_warnings_for_metric(library_config: dict[str, Any], metric_class: str) -> list[str]:
+    return [f"⚠️ {metric_class} {w['message']}" for w in library_config["warnings"] if metric_class in w["metrics"]]
+
+
+def _parse_schema(schema: Any, value: Any) -> tuple[bool, Any]:
+    try:
+        return True, TypeAdapter(schema).validate_python(value)
+    except PydanticValidationError as exc:
+        errors = exc.errors()
+        return False, errors[0]["msg"] if errors else str(exc)
+
+
+@dataclass
+class ThirdPartyLibraryOptions:
+    library: str
+    metric_class: str
+    metric_params: str | None = None
+    model_provider: str | None = None
+    model: str | None = None
+
+
+@dataclass
+class AddEvaluatorOptions:
+    name: str
+    level: str
+    config: dict[str, Any]
+    description: str | None = None
+    kms_key_arn: str | None = None
+    third_party: ThirdPartyLibraryOptions | None = None
+
+
+def build_code_based_config(name: str, lambda_arn: str | None = None, timeout: str | None = None) -> dict:
+    if lambda_arn:
+        return {"codeBased": {"external": {"lambdaArn": lambda_arn}}}
+    return {
+        "codeBased": {
+            "managed": {
+                "codeLocation": f"app/{name}/",
+                "entrypoint": DEFAULT_CODE_ENTRYPOINT,
+                "timeoutSeconds": parse_int(timeout) if timeout else DEFAULT_CODE_TIMEOUT,
+                "additionalPolicies": ["execution-role-policy.json"],
+            }
+        }
+    }
+
+
+def build_third_party_config(name: str, library_config: dict[str, Any], timeout: str | None = None) -> dict:
+    return {
+        "codeBased": {
+            "managed": {
+                "codeLocation": f"app/{name}/",
+                "entrypoint": DEFAULT_CODE_ENTRYPOINT,
+                "timeoutSeconds": parse_int(timeout) if timeout else library_config["defaultTimeoutSeconds"],
+                "additionalPolicies": ["execution-role-policy.json"],
+            }
+        }
+    }
+
+
+class _Fail(Exception):
+    pass
+
+
+class EvaluatorPrimitive(BasePrimitive):
+    kind = "evaluator"
+    label = "Evaluator"
+    article = "an"
+
+    def _project(self) -> dict[str, Any]:
+        return to_jsonable(self.read_project_spec())
+
+    def _project_root(self) -> str:
+        return os.path.dirname(find_config_root() or "")
+
+    def add(self, options: AddEvaluatorOptions) -> Ok | Err:
+        try:
+            evaluator: dict[str, Any] = {"name": options.name, "level": options.level}
+            if options.description:
+                evaluator["description"] = options.description
+            evaluator["config"] = options.config
+            if options.kms_key_arn:
+                evaluator["kmsKeyArn"] = options.kms_key_arn
+
+            def mutate(project: Any) -> Any:
+                data = to_jsonable(project)
+                self.check_duplicate([RemovableResource(e["name"]) for e in data["evaluators"]], options.name)
+                data["evaluators"].append(evaluator)
+                return data
+
+            self.update_project_spec(mutate)
+
+            managed = (options.config.get("codeBased") or {}).get("managed")
+            if managed:
+                target_dir = os.path.join(self._project_root(), managed["codeLocation"])
+                if options.third_party:
+                    tp = options.third_party
+                    data: dict[str, Any] = {
+                        "Name": options.name,
+                        "EvaluatorClass": tp.metric_class,
+                        "EvaluatorParams": tp.metric_params or "",
+                    }
+                    if tp.model_provider == "bedrock":
+                        data["ModelProviderBedrock"] = True
+                    if tp.model:
+                        data["Model"] = tp.model
+                    render_third_party_evaluator_template(
+                        THIRD_PARTY_EVALUATOR_LIBRARIES[tp.library]["templateDir"], data, target_dir
+                    )
+                else:
+                    render_code_based_evaluator_template(options.name, target_dir)
+                return Ok({"evaluatorName": options.name, "codePath": managed["codeLocation"]})
+            return Ok({"evaluatorName": options.name})
+        except Exception as exc:  # noqa: BLE001
+            return Err(to_error(exc))
+
+    def remove(self, name: str) -> Ok | Err:
+        try:
+            project = self._project()
+            index = next((i for i, e in enumerate(project["evaluators"]) if e["name"] == name), -1)
+            if index == -1:
+                return Err(ResourceNotFoundError(f'Evaluator "{name}" not found.'))
+            refs = [c["name"] for c in project["onlineEvalConfigs"] if name in (c.get("evaluators") or [])]
+            if refs:
+                return Err(
+                    ConflictError(
+                        f'Evaluator "{name}" is referenced by online eval config(s): {", ".join(refs)}. '
+                        "Remove those references first."
+                    )
+                )
+            managed = ((project["evaluators"][index].get("config") or {}).get("codeBased") or {}).get("managed")
+            if managed:
+                code_dir = os.path.join(self._project_root(), managed["codeLocation"])
+                if os.path.exists(code_dir):
+                    shutil.rmtree(code_dir, ignore_errors=True)
+            project["evaluators"].pop(index)
+            self.write_project_spec(project)
+            return Ok(None)
+        except Exception as exc:  # noqa: BLE001
+            return Err(to_error(exc))
+
+    def preview_remove(self, name: str) -> RemovalPreview:
+        project = self._project()
+        evaluator = next((e for e in project["evaluators"] if e["name"] == name), None)
+        if evaluator is None:
+            raise Exception(f'Evaluator "{name}" not found.')
+        summary = [f"Removing evaluator: {name}"]
+        directories: list[str] = []
+        refs = [c["name"] for c in project["onlineEvalConfigs"] if name in (c.get("evaluators") or [])]
+        if refs:
+            summary.append(
+                f"Blocked: Referenced by online eval config(s): {', '.join(refs)}. Remove those references first."
+            )
+        managed = ((evaluator.get("config") or {}).get("codeBased") or {}).get("managed")
+        if managed:
+            location = managed["codeLocation"]
+            if os.path.exists(os.path.join(self._project_root(), location)):
+                directories.append(location)
+                summary.append(f"Will delete directory: {location}")
+        after = {**project, "evaluators": [e for e in project["evaluators"] if e["name"] != name]}
+        return RemovalPreview(summary, directories, [SchemaChange("agentcore/agentcore.json", project, after)])
+
+    def get_removable(self) -> list[RemovableResource]:
+        try:
+            return [RemovableResource(e["name"]) for e in self._project()["evaluators"]]
+        except Exception:  # noqa: BLE001
+            return []
+
+    def get_all_names(self) -> list[str]:
+        return [r.name for r in self.get_removable()]
+
+    # --- CLI ----------------------------------------------------------------------------
+
+    def register_commands(self, add_app: typer.Typer, remove_app: typer.Typer) -> None:
+        @add_app.command("evaluator", help="Add a custom evaluator to the project")
+        def add_evaluator(
+            name: str | None = typer.Option(None, "--name", help="Evaluator name"),
+            level: str | None = typer.Option(None, "--level", help="Evaluation level: SESSION, TRACE, TOOL_CALL"),
+            type_: str | None = typer.Option(
+                None, "--type", help="Evaluator type: llm-as-a-judge (default) or code-based"
+            ),
+            model: str | None = typer.Option(
+                None, "--model", help="[LLM] Bedrock inference profile ID or OpenResponses model ID for LLM-as-a-Judge"
+            ),
+            model_provider: str | None = typer.Option(
+                None, "--model-provider", help="[LLM] Model provider: Bedrock (default) or OpenResponses"
+            ),
+            instructions: str | None = typer.Option(
+                None,
+                "--instructions",
+                help="[LLM] Evaluation prompt instructions (must include level-appropriate placeholders, e.g. {context})",
+            ),
+            rating_scale: str | None = typer.Option(
+                None,
+                "--rating-scale",
+                help=f"[LLM] Rating scale preset: {', '.join(PRESET_IDS)} (default: 1-5-quality)",
+            ),
+            lambda_arn: str | None = typer.Option(
+                None, "--lambda-arn", help="[Code-based] Existing Lambda function ARN (external)"
+            ),
+            timeout: str | None = typer.Option(
+                None, "--timeout", help="[Code-based] Lambda timeout in seconds, 1-300 (default: 60)"
+            ),
+            template_json: str | None = typer.Option(
+                None,
+                "--3p-template-json",
+                help='[Code-based] Inline JSON with 3P library config: {"library", "metric", "modelProvider", '
+                '"model", "params"}',
+            ),
+            template_json_file: str | None = typer.Option(
+                None,
+                "--3p-template-json-file",
+                help="[Code-based] Path to JSON file with 3P library config (same format as --3p-template-json)",
+            ),
+            config: str | None = typer.Option(
+                None,
+                "--config",
+                help="Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) "
+                "[non-interactive]",
+            ),
+            kms_key_arn: str | None = typer.Option(
+                None, "--kms-key-arn", help="KMS key ARN for evaluator encryption (optional)"
+            ),
+            as_json: bool = typer.Option(False, "--json", help="Output as JSON [non-interactive]"),
+        ) -> None:
+            if not find_config_root():
+                typer.echo("No agentcore project found. Run `agentcore create` first.", err=True)
+                raise typer.Exit(1)
+            if not (name or as_json):
+                try:
+                    interactive.require_tty()
+                    self.add_interactive()
+                except typer.Exit:
+                    raise
+                except Exception as exc:  # noqa: BLE001
+                    typer.echo(get_error_message(exc), err=True)
+                    raise typer.Exit(1) from None
+                raise typer.Exit(0)
+
+            run_cli_command(
+                as_json,
+                lambda: self._add_from_flags(
+                    name=name,
+                    level=level,
+                    eval_type=type_,
+                    model=model,
+                    model_provider=model_provider,
+                    instructions=instructions,
+                    rating_scale=rating_scale,
+                    lambda_arn=lambda_arn,
+                    timeout=timeout,
+                    template_json=template_json,
+                    template_json_file=template_json_file,
+                    config=config,
+                    kms_key_arn=kms_key_arn,
+                    as_json=as_json,
+                ),
+            )
+
+        self.register_remove_subcommand(remove_app)
+
+    def _add_from_flags(self, **o: Any) -> None:
+        def fail(message: str) -> None:
+            raise Exception(message)
+
+        name, level = o["name"], o["level"]
+        if not name or not level:
+            fail("--name and --level are required in non-interactive mode")
+        if level not in EVALUATION_LEVELS:
+            fail(f'Invalid --level "{level}". Must be one of: SESSION, TRACE, TOOL_CALL')
+        if o["template_json"] and o["template_json_file"]:
+            fail("--3p-template-json and --3p-template-json-file cannot be used together")
+        template_text = None
+        if o["template_json"]:
+            template_text = o["template_json"]
+        elif o["template_json_file"]:
+            if not os.path.exists(o["template_json_file"]):
+                fail(f"--3p-template-json-file not found: {o['template_json_file']}")
+            with open(o["template_json_file"], encoding="utf-8") as fh:
+                template_text = fh.read()
+
+        library = metric = tp_provider = tp_model = tp_params = None
+        if template_text:
+            try:
+                template = json.loads(template_text)
+            except ValueError:
+                raise Exception("--3p-template-json must be valid JSON") from None
+            if not isinstance(template, dict):
+                template = {}
+            if not template.get("library") or not isinstance(template.get("library"), str):
+                fail('--3p-template-json must include "library" (e.g. "deepeval" or "autoevals")')
+            if not template.get("metric") or not isinstance(template.get("metric"), str):
+                fail('--3p-template-json must include "metric" (e.g. "AnswerRelevancyMetric")')
+            library = str(template["library"])
+            if library not in THIRD_PARTY_EVALUATOR_LIBRARIES:
+                fail(f'Invalid library "{library}". Supported: {", ".join(SUPPORTED_LIBRARIES)}')
+            metric = template["metric"]
+            if template.get("modelProvider"):
+                raw_provider = str(template["modelProvider"])
+                if raw_provider.lower() not in MODEL_PROVIDERS:
+                    fail(f'Invalid modelProvider "{raw_provider}". Supported: {", ".join(MODEL_PROVIDERS)}')
+                tp_provider = raw_provider.lower()
+            if template.get("model"):
+                tp_model = template["model"]
+            if template.get("params") and isinstance(template["params"], (dict, list)):
+                try:
+                    params = template["params"]
+                    items = params.items() if isinstance(params, dict) else enumerate(params)
+                    parts = []
+                    for k, v in items:
+                        if not _KWARG_NAME.match(str(k)):
+                            raise Exception(f'Invalid Python kwarg name "{k}"')
+                        parts.append(f"{k}={json_to_python_value(v)}")
+                    tp_params = ", ".join(parts)
+                except Exception as exc:  # noqa: BLE001
+                    fail(f"Invalid params in --3p-template-json: {get_error_message(exc)}")
+            if not tp_provider and tp_model:
+                tp_provider = "bedrock"
+            if tp_provider == "bedrock" and not tp_model:
+                fail(
+                    '--3p-template-json requires "model" when modelProvider is bedrock. '
+                    "Pass a Bedrock inference profile ID (e.g. us.anthropic.claude-sonnet-4-20250514-v1:0)"
+                )
+            if tp_provider == "openai":
+                typer.echo(
+                    "\n  ⚠️  OpenAI model provider selected. You must set OPENAI_API_KEY as a Lambda "
+                    "environment variable after deployment.\n",
+                    err=True,
+                )
+
+        timeout = o["timeout"]
+        if timeout:
+            value = parse_int(timeout)
+            if value is None or value < 1 or value > 300:
+                fail("--timeout must be an integer between 1 and 300")
+
+        eval_type = o["eval_type"] or ("code-based" if library else "llm-as-a-judge")
+        if eval_type not in ("llm-as-a-judge", "code-based"):
+            fail(f'Invalid --type "{eval_type}". Must be one of: llm-as-a-judge, code-based')
+        if eval_type != "code-based":
+            if o["lambda_arn"]:
+                fail("--lambda-arn requires --type code-based")
+            if timeout:
+                fail("--timeout requires --type code-based")
+            if library:
+                fail("--3p-template-json requires --type code-based")
+        if eval_type == "code-based":
+            if o["model"]:
+                fail("--model cannot be used with --type code-based")
+            if o["model_provider"]:
+                fail("--model-provider cannot be used with --type code-based")
+            if o["instructions"]:
+                fail("--instructions cannot be used with --type code-based")
+            if o["rating_scale"]:
+                fail("--rating-scale cannot be used with --type code-based")
+        if o["config"] and o["model_provider"]:
+            fail(
+                "--model-provider cannot be used with --config; set config.llmAsAJudge.modelProvider in the config file"
+            )
+
+        third_party = None
+        if library:
+            config = build_third_party_config(name, THIRD_PARTY_EVALUATOR_LIBRARIES[library], timeout)
+            third_party = ThirdPartyLibraryOptions(library, metric, tp_params, tp_provider, tp_model)  # type: ignore[arg-type]
+        elif o["config"]:
+            with open(o["config"], encoding="utf-8") as fh:
+                config = json.load(fh)
+        elif eval_type == "code-based":
+            config = build_code_based_config(name, o["lambda_arn"], timeout)
+        else:
+            if not o["model"]:
+                fail("Either --config or --model is required for LLM-as-a-Judge evaluators")
+            ok, model_id = _parse_schema(EvaluatorModelId, o["model"])
+            if not ok:
+                fail(model_id or "Invalid --model")
+            provider = o["model_provider"] if o["model_provider"] is not None else "Bedrock"
+            if provider not in ("Bedrock", "OpenResponses"):
+                fail(f'Invalid --model-provider "{o["model_provider"]}". Must be one of: Bedrock, OpenResponses')
+            instructions = o["instructions"]
+            if not instructions:
+                placeholders = ", ".join(f"{{{p}}}" for p in LEVEL_PLACEHOLDERS[level])
+                fail(
+                    "--instructions is required in non-interactive mode (or use --config). "
+                    f"Must include at least one placeholder for {level}: {placeholders}"
+                )
+            problem = validate_instruction_placeholders(instructions, level)
+            if problem:
+                fail(problem)
+            scale_input = o["rating_scale"] if o["rating_scale"] is not None else "1-5-quality"
+            preset = next((p for p in RATING_SCALE_PRESETS if p["id"] == scale_input), None)
+            if preset:
+                scale = preset["ratingScale"]
+            else:
+                numerical = bool(re.match(r"^\d", scale_input.strip()))
+                ok, scale = parse_custom_rating_scale(scale_input, "numerical" if numerical else "categorical")
+                if not ok:
+                    fail(
+                        f'Invalid --rating-scale "{scale_input}". Use a preset ({", ".join(PRESET_IDS)}) '
+                        'or custom format: "1:Label:Definition, 2:Label:Definition" (numerical) '
+                        'or "Label:Definition, Label:Definition" (categorical)'
+                    )
+            llm: dict[str, Any] = {}
+            if provider == "OpenResponses":
+                llm["modelProvider"] = provider
+            llm.update({"model": model_id, "instructions": instructions, "ratingScale": scale})
+            config = {"llmAsAJudge": llm}
+
+        if o["kms_key_arn"] and not is_valid_kms_key_arn(o["kms_key_arn"]):
+            fail(KMS_ARN_ERROR)
+
+        result = self.add(
+            AddEvaluatorOptions(
+                name=name, level=level, config=config, kms_key_arn=o["kms_key_arn"], third_party=third_party
+            )
+        )
+        if not result.success:
+            raise result.error
+        if o["as_json"]:
+            emit_json(serialize_result(result))
+            return
+        value = result.value
+        if value.get("codePath"):
+            typer.echo(f"Created evaluator '{value['evaluatorName']}'")
+            typer.echo(f"  Code: {value['codePath']}lambda_function.py")
+            typer.echo(f"  IAM:  {value['codePath']}execution-role-policy.json")
+            typer.echo("\n  Next: Edit lambda_function.py with your evaluation logic, then run `agentcore deploy`")
+        else:
+            typer.echo(f"Added evaluator '{value['evaluatorName']}'")
+        if third_party:
+            for warning in get_warnings_for_metric(
+                THIRD_PARTY_EVALUATOR_LIBRARIES[third_party.library], third_party.metric_class
+            ):
+                typer.echo(f"\n  {warning}", err=True)
+            if third_party.model_provider == "openai":
+                typer.echo(
+                    "\n  ⚠️  OpenAI model provider selected. You must set OPENAI_API_KEY as a Lambda "
+                    "environment variable for the evaluator to call the LLM judge.",
+                    err=True,
+                )
+
+    # --- interactive (tui/screens/evaluator/AddEvaluatorScreen) ---------------------------
+
+    def add_interactive(self) -> None:
+        existing = self.get_all_names()
+        choice = interactive.Choice
+        eval_type = interactive.select(
+            "What type of evaluator would you like to create?",
+            [choice(t, i, d) for i, t, d in EVALUATOR_TYPE_OPTIONS],
+        )
+        code_type = "managed"
+        if eval_type == "code-based":
+            code_type = interactive.select(
+                "How would you like to provide the Lambda?", [choice(t, i, d) for i, t, d in CODE_BASED_TYPE_OPTIONS]
+            )
+
+        def validate_name(value: str) -> str | None:
+            ok, msg = _parse_schema(EvaluatorName, value)
+            if not ok:
+                return msg
+            return "Evaluator name already exists" if value in existing else None
+
+        name = interactive.text(
+            "Evaluator name", default=generate_unique_name("MyEvaluator", existing), validate=validate_name
+        )
+        level = interactive.select(
+            "What level should this evaluator operate at?", [choice(t, i, d) for i, t, d in EVALUATION_LEVEL_OPTIONS]
+        )
+        rows: list[tuple[str, str]] = [("Type", eval_type), ("Name", name), ("Level", level)]
+
+        if eval_type == "llm-as-a-judge":
+            provider = interactive.select(
+                "Select model provider", [choice(t, i, d) for i, t, d in EVALUATOR_MODEL_PROVIDER_OPTIONS]
+            )
+            options = OPENAI_EVALUATOR_MODEL_OPTIONS if provider == "OpenResponses" else BEDROCK_EVALUATOR_MODEL_OPTIONS
+            model = interactive.select("Select model", [choice(t, i, d) for i, t, d in options])
+            if model == CUSTOM_MODEL_ID:
+                if provider == "OpenResponses":
+                    model = interactive.text(
+                        "OpenResponses model ID",
+                        validate=lambda v: (
+                            None
+                            if _parse_schema(EvaluatorModelId, v)[0]
+                            else "A valid OpenResponses model ID is required"
+                        ),
+                    )
+                else:
+                    model = interactive.text(
+                        "Bedrock model ID",
+                        validate=lambda v: (
+                            None
+                            if is_valid_bedrock_model_id(v)
+                            else "Must be a valid Bedrock model ID (e.g. us.anthropic.claude-sonnet-4-5-20250929-v1:0) "
+                            "or model ARN"
+                        ),
+                    )
+            typer.echo("Evaluation instructions")
+            typer.echo("Available placeholders:")
+            for p in LEVEL_PLACEHOLDERS[level]:
+                typer.echo(f"  {{{p}}} — {PLACEHOLDER_DESCRIPTIONS.get(p, p)}")
+            if REFERENCE_INPUT_PLACEHOLDERS[level]:
+                typer.echo("Reference inputs — provided by caller at eval time, may be empty:")
+                for p in REFERENCE_INPUT_PLACEHOLDERS[level]:
+                    typer.echo(f"  {{{p}}} — {PLACEHOLDER_DESCRIPTIONS.get(p, p)}")
+            instructions = interactive.text(
+                "Instructions",
+                default=DEFAULT_INSTRUCTIONS[level],
+                validate=lambda v: validate_instruction_placeholders(v, level),
+            )
+            scale_id = interactive.select(
+                "Rating scale",
+                [choice(p["title"], p["id"], p["description"]) for p in RATING_SCALE_PRESETS]
+                + [choice("Custom", CUSTOM_RATING_SCALE_ID, "Define your own rating scale")],
+            )
+            if scale_id == CUSTOM_RATING_SCALE_ID:
+                scale_type = interactive.select(
+                    "Scale type", [choice(t, i, d) for i, t, d in RATING_SCALE_TYPE_OPTIONS]
+                )
+                typer.echo("Define rating scale entries")
+                typer.echo(
+                    "Format: value:label:definition, ... (e.g. 1:Poor:Fails, 3:Good:Meets, 5:Excellent:Exceeds)"
+                    if scale_type == "numerical"
+                    else "Format: label:definition, ... (e.g. Pass:Meets criteria, Fail:Does not meet)"
+                )
+                scale_text = interactive.text(
+                    "Entries",
+                    validate=lambda v: (
+                        None
+                        if parse_custom_rating_scale(v, scale_type)[0]
+                        else parse_custom_rating_scale(v, scale_type)[1]
+                    ),
+                )
+                scale = parse_custom_rating_scale(scale_text, scale_type)[1]
+            else:
+                scale = next(p["ratingScale"] for p in RATING_SCALE_PRESETS if p["id"] == scale_id)
+            llm: dict[str, Any] = {}
+            if provider == "OpenResponses":
+                llm["modelProvider"] = provider
+            llm.update({"model": model, "instructions": instructions, "ratingScale": scale})
+            config: dict[str, Any] = {"llmAsAJudge": llm}
+            rows += [("Provider", provider), ("Model", model), ("Prompt", instructions)]
+        elif code_type == "external":
+            arn = interactive.text(
+                "Lambda function ARN",
+                validate=lambda v: None if _LAMBDA_ARN.match(v) else "Must be a valid Lambda function ARN",
+            )
+            config = build_code_based_config(name, arn)
+            rows.append(("Lambda", arn))
+        else:
+
+            def validate_timeout(v: str) -> str | None:
+                num = parse_int(v)
+                if num is None:
+                    return "Must be a number"
+                return None if 1 <= num <= 300 else "Must be between 1 and 300"
+
+            timeout = interactive.text(
+                "Lambda timeout in seconds (1-300)", default=str(DEFAULT_CODE_TIMEOUT), validate=validate_timeout
+            )
+            config = build_code_based_config(name, None, timeout)
+            rows.append(("Timeout", f"{parse_int(timeout)}s"))
+
+        kms = interactive.text(
+            "KMS key ARN for encryption (optional, press Enter to skip)",
+            validate=lambda v: (
+                None
+                if v == "" or is_valid_kms_key_arn(v)
+                else "Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)"
+            ),
+        )
+        if kms:
+            rows.append(("KMS Key", kms))
+        for label, value in rows:
+            typer.echo(f"  {label}: {value}")
+        if not interactive.confirm("Add this evaluator?", default=True):
+            typer.echo("Cancelled.")
+            return
+        result = self.add(AddEvaluatorOptions(name=name, level=level, config=config, kms_key_arn=kms or None))
+        if not result.success:
+            typer.echo("Failed to add evaluator", err=True)
+            typer.echo(get_error_message(result.error), err=True)
+            raise typer.Exit(1)
+        typer.echo(f"Added evaluator: {name}")
+        code_path = result.value.get("codePath")
+        if code_path:
+            typer.echo(f'Created evaluator "{name}"')
+            typer.echo(f"  Code: {code_path}lambda_function.py")
+            typer.echo(f"  IAM:  {code_path}execution-role-policy.json")
+            typer.echo("\n  Next: Edit lambda_function.py with your evaluation logic, then run `agentcore deploy`")
+        else:
+            typer.echo("Evaluator added to project in `agentcore/agentcore.json`. Deploy with `agentcore deploy`.")
+
+
+PRIMITIVE = EvaluatorPrimitive()