agentcore-cli 0.31.1__py3-none-any.whl

agentcore/schema/schemas/primitives/evaluator.py

@@ -0,0 +1,171 @@
+"""Custom evaluators (port of ``primitives/evaluator.ts``)."""
+
+from __future__ import annotations
+
+from typing import Annotated, Final, Literal, get_args
+
+from pydantic import Field, model_validator
+
+from ..._zod import ZodModel, js_regex, raise_issues, zint, zstr
+
+EvaluationLevel = Literal["SESSION", "TRACE", "TOOL_CALL"]
+EVALUATION_LEVELS: Final[tuple[EvaluationLevel, ...]] = get_args(EvaluationLevel)
+
+EvaluatorName = zstr(
+    min=1,
+    max=48,
+    min_msg="Name is required",
+    pattern=r"^[a-zA-Z][a-zA-Z0-9_]{0,47}$",
+    pattern_msg="Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)",
+)
+
+# ============================================================================
+# Rating Scale
+# ============================================================================
+
+
+class NumericalRating(ZodModel):
+    value: zint()
+    label: zstr(min=1)
+    definition: zstr(min=1)
+
+
+class CategoricalRating(ZodModel):
+    label: zstr(min=1)
+    definition: zstr(min=1)
+
+
+class RatingScale(ZodModel):
+    numerical: list[NumericalRating] | None = None
+    categorical: list[CategoricalRating] | None = None
+
+    @model_validator(mode="after")
+    def _exactly_one(self) -> RatingScale:
+        # Boolean([]) is true in JS, so presence (not non-emptiness) is what counts.
+        if (self.numerical is not None) == (self.categorical is not None):
+            raise_issues([((), "Rating scale must have either numerical or categorical, not both")], self)
+        return self
+
+
+# ============================================================================
+# LLM-as-a-Judge Config
+# ============================================================================
+
+EvaluatorModelProvider = Literal["Bedrock", "OpenResponses"]
+EVALUATOR_MODEL_PROVIDERS: Final[tuple[EvaluatorModelProvider, ...]] = get_args(EvaluatorModelProvider)
+
+_BEDROCK_MODEL_ID_PATTERN = js_regex(r"^[a-z][a-z0-9-]*\.[a-zA-Z0-9._-]+(:[0-9]+)?$")
+_BEDROCK_ARN_PATTERN = js_regex(r"^arn:aws[a-z-]*:bedrock:[a-z0-9-]+:\d{12}:(inference-profile|foundation-model)\/.+$")
+
+
+def is_valid_bedrock_model_id(value: str) -> bool:
+    return bool(_BEDROCK_MODEL_ID_PATTERN.search(value) or _BEDROCK_ARN_PATTERN.search(value))
+
+
+EvaluatorModelId = zstr(
+    trim=True,
+    min=1,
+    max=2048,
+    min_msg="Model ID is required",
+    max_msg="Model ID must be 2048 characters or fewer",
+    pattern=r"^[\x21-\x7e]+$",
+    pattern_msg="Model ID must contain only printable ASCII characters without spaces",
+)
+
+# Retained for compatibility with existing imports.
+BedrockModelId = EvaluatorModelId
+
+
+class LlmAsAJudgeConfig(ZodModel):
+    model_provider: EvaluatorModelProvider | None = None
+    model: EvaluatorModelId
+    instructions: zstr(min=1, min_msg="Evaluation instructions are required")
+    rating_scale: RatingScale
+
+
+# ============================================================================
+# Code-Based Evaluator Config
+# ============================================================================
+
+
+class ManagedCodeBasedConfig(ZodModel):
+    code_location: zstr(min=1)
+    entrypoint: zstr(min=1) = "lambda_function.handler"
+    timeout_seconds: zint(ge=1, le=300) = 60
+    additional_policies: list[zstr(min=1)] | None = None
+
+
+_LAMBDA_ARN_PATTERN = js_regex(r"^arn:aws[a-z-]*:lambda:[a-z0-9-]+:\d{12}:function:.+$")
+
+
+class ExternalCodeBasedConfig(ZodModel):
+    lambda_arn: zstr(min=1, pattern=_LAMBDA_ARN_PATTERN, pattern_msg="Must be a valid Lambda function ARN")
+
+
+class CodeBasedConfig(ZodModel):
+    managed: ManagedCodeBasedConfig | None = None
+    external: ExternalCodeBasedConfig | None = None
+
+    @model_validator(mode="after")
+    def _exactly_one(self) -> CodeBasedConfig:
+        if (self.managed is not None) == (self.external is not None):
+            raise_issues([((), "Code-based config must have either managed or external, not both")], self)
+        return self
+
+
+# ============================================================================
+# Evaluator Config
+# ============================================================================
+
+
+class EvaluatorConfig(ZodModel):
+    llm_as_a_judge: Annotated[LlmAsAJudgeConfig | None, Field(alias="llmAsAJudge")] = None
+    code_based: CodeBasedConfig | None = None
+
+    @model_validator(mode="after")
+    def _exactly_one(self) -> EvaluatorConfig:
+        if (self.llm_as_a_judge is not None) == (self.code_based is not None):
+            raise_issues([((), "Config must have either llmAsAJudge or codeBased, not both")], self)
+        return self
+
+
+# ============================================================================
+# KMS Key ARN Validation
+# ============================================================================
+
+# KMS key ARNs accepted by the AgentCore Evaluation service (alias ARNs are not supported).
+KMS_KEY_ARN_PATTERN: Final = js_regex(r"^arn:[^:]+:kms:[a-zA-Z0-9-]*:[0-9]{12}:key\/[a-zA-Z0-9-]{36}$")
+
+KmsKeyArn = zstr(
+    pattern=KMS_KEY_ARN_PATTERN,
+    pattern_msg=(
+        "Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)"
+    ),
+)
+
+
+def is_valid_kms_key_arn(value: str) -> bool:
+    return KMS_KEY_ARN_PATTERN.search(value) is not None
+
+
+__all__ = [
+    "EVALUATION_LEVELS",
+    "EVALUATOR_MODEL_PROVIDERS",
+    "KMS_KEY_ARN_PATTERN",
+    "BedrockModelId",
+    "CategoricalRating",
+    "CodeBasedConfig",
+    "EvaluationLevel",
+    "EvaluatorConfig",
+    "EvaluatorModelId",
+    "EvaluatorModelProvider",
+    "EvaluatorName",
+    "ExternalCodeBasedConfig",
+    "KmsKeyArn",
+    "LlmAsAJudgeConfig",
+    "ManagedCodeBasedConfig",
+    "NumericalRating",
+    "RatingScale",
+    "is_valid_bedrock_model_id",
+    "is_valid_kms_key_arn",
+]