azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_azure/_models.py

@@ -0,0 +1,227 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# pylint: disable=too-many-instance-attributes
+# pylint: disable=too-many-locals
+# pylint: disable=line-too-long
+
+from typing import Dict, List, NamedTuple, Optional, Union
+from msrest.serialization import Model
+from azure.core.credentials import AzureSasCredential, TokenCredential
+
+
+class BlobStoreInfo(NamedTuple):
+    name: str
+    account_name: str
+    endpoint: str
+    container_name: str
+    credential: Optional[Union[AzureSasCredential, TokenCredential, str]]
+
+
+class WorkspaceHubConfig(Model):
+    """WorkspaceHub's configuration object."""
+
+    _attribute_map = {
+        "additional_workspace_storage_accounts": {"key": "additionalWorkspaceStorageAccounts", "type": "[str]"},
+        "default_workspace_resource_group": {"key": "defaultWorkspaceResourceGroup", "type": "str"},
+    }
+
+    def __init__(
+        self,
+        *,
+        additional_workspace_storage_accounts: Optional[List[str]] = None,
+        default_workspace_resource_group: Optional[str] = None,
+        **kwargs
+    ):
+        super(WorkspaceHubConfig, self).__init__(**kwargs)
+        self.additional_workspace_storage_accounts = additional_workspace_storage_accounts
+        self.default_workspace_resource_group = default_workspace_resource_group
+
+
+class Workspace(Model):
+    """An object that represents a machine learning workspace.
+
+    Variables are only populated by the server, and will be ignored when sending a request."""
+
+    _validation = {
+        "id": {"readonly": True},
+        "name": {"readonly": True},
+        "type": {"readonly": True},
+        #'system_data': {'readonly': True},
+        "agents_endpoint_uri": {"readonly": True},
+        "ml_flow_tracking_uri": {"readonly": True},
+        #'notebook_info': {'readonly': True},
+        # "private_endpoint_connections": {"readonly": True},
+        #'private_link_count': {'readonly': True},
+        "provisioning_state": {"readonly": True},
+        "service_provisioned_resource_group": {"readonly": True},
+        "storage_hns_enabled": {"readonly": True},
+        "tenant_id": {"readonly": True},
+        "workspace_id": {"readonly": True},
+    }
+
+    _attribute_map = {
+        "id": {"key": "id", "type": "str"},
+        "name": {"key": "name", "type": "str"},
+        "type": {"key": "type", "type": "str"},
+        #'system_data': {'key': 'systemData', 'type': 'SystemData'},
+        #'identity': {'key': 'identity', 'type': 'ManagedServiceIdentity'},
+        "kind": {"key": "kind", "type": "str"},
+        "location": {"key": "location", "type": "str"},
+        #'sku': {'key': 'sku', 'type': 'Sku'},
+        "tags": {"key": "tags", "type": "{str}"},
+        "agents_endpoint_uri": {"key": "properties.agentsEndpointUri", "type": "str"},
+        "allow_public_access_when_behind_vnet": {"key": "properties.allowPublicAccessWhenBehindVnet", "type": "bool"},
+        "allow_role_assignment_on_rg": {"key": "properties.allowRoleAssignmentOnRG", "type": "bool"},
+        "application_insights": {"key": "properties.applicationInsights", "type": "str"},
+        "associated_workspaces": {"key": "properties.associatedWorkspaces", "type": "[str]"},
+        "container_registries": {"key": "properties.containerRegistries", "type": "[str]"},
+        "container_registry": {"key": "properties.containerRegistry", "type": "str"},
+        "description": {"key": "properties.description", "type": "str"},
+        "discovery_url": {"key": "properties.discoveryUrl", "type": "str"},
+        "enable_data_isolation": {"key": "properties.enableDataIsolation", "type": "bool"},
+        "enable_service_side_cmk_encryption": {"key": "properties.enableServiceSideCMKEncryption", "type": "bool"},
+        "enable_simplified_cmk": {"key": "properties.enableSimplifiedCmk", "type": "bool"},
+        "enable_software_bill_of_materials": {"key": "properties.enableSoftwareBillOfMaterials", "type": "bool"},
+        #'encryption': {'key': 'properties.encryption', 'type': 'EncryptionProperty'},
+        "existing_workspaces": {"key": "properties.existingWorkspaces", "type": "[str]"},
+        #'feature_store_settings': {'key': 'properties.featureStoreSettings', 'type': 'FeatureStoreSettings'},
+        "friendly_name": {"key": "properties.friendlyName", "type": "str"},
+        "hbi_workspace": {"key": "properties.hbiWorkspace", "type": "bool"},
+        "hub_resource_id": {"key": "properties.hubResourceId", "type": "str"},
+        "image_build_compute": {"key": "properties.imageBuildCompute", "type": "str"},
+        "ip_allowlist": {"key": "properties.ipAllowlist", "type": "[str]"},
+        "key_vault": {"key": "properties.keyVault", "type": "str"},
+        "key_vaults": {"key": "properties.keyVaults", "type": "[str]"},
+        #'managed_network': {'key': 'properties.managedNetwork', 'type': 'ManagedNetworkSettings'},
+        "ml_flow_tracking_uri": {"key": "properties.mlFlowTrackingUri", "type": "str"},
+        #'network_acls': {'key': 'properties.networkAcls', 'type': 'NetworkAcls'},
+        #'notebook_info': {'key': 'properties.notebookInfo', 'type': 'NotebookResourceInfo'},
+        "primary_user_assigned_identity": {"key": "properties.primaryUserAssignedIdentity", "type": "str"},
+        # "private_endpoint_connections": {
+        #    "key": "properties.privateEndpointConnections",
+        #    "type": "[PrivateEndpointConnection]",
+        # },
+        "private_link_count": {"key": "properties.privateLinkCount", "type": "int"},
+        "provision_network_now": {"key": "properties.provisionNetworkNow", "type": "bool"},
+        "provisioning_state": {"key": "properties.provisioningState", "type": "str"},
+        #'public_network_access': {'key': 'properties.publicNetworkAccess', 'type': 'str'},
+        #'serverless_compute_settings': {'key': 'properties.serverlessComputeSettings', 'type': 'ServerlessComputeSettings'},
+        #'service_managed_resources_settings': {'key': 'properties.serviceManagedResourcesSettings', 'type': 'ServiceManagedResourcesSettings'},
+        "service_provisioned_resource_group": {"key": "properties.serviceProvisionedResourceGroup", "type": "str"},
+        #'shared_private_link_resources': {'key': 'properties.sharedPrivateLinkResources', 'type': '[SharedPrivateLinkResource]'},
+        "soft_delete_retention_in_days": {"key": "properties.softDeleteRetentionInDays", "type": "int"},
+        "storage_account": {"key": "properties.storageAccount", "type": "str"},
+        "storage_accounts": {"key": "properties.storageAccounts", "type": "[str]"},
+        "storage_hns_enabled": {"key": "properties.storageHnsEnabled", "type": "bool"},
+        #'system_datastores_auth_mode': {'key': 'properties.systemDatastoresAuthMode', 'type': 'str'},
+        "tenant_id": {"key": "properties.tenantId", "type": "str"},
+        "v1_legacy_mode": {"key": "properties.v1LegacyMode", "type": "bool"},
+        "workspace_hub_config": {"key": "properties.workspaceHubConfig", "type": "WorkspaceHubConfig"},
+        "workspace_id": {"key": "properties.workspaceId", "type": "str"},
+    }
+
+    def __init__(
+        self,
+        *,
+        # system_data: Optional[SystemData] = None,
+        # identity: Optional["ManagedServiceIdentity"] = None,
+        kind: Optional[str] = None,
+        location: Optional[str] = None,
+        # sku: Optional["Sku"] = None,
+        tags: Optional[Dict[str, str]] = None,
+        allow_public_access_when_behind_vnet: Optional[bool] = None,
+        allow_role_assignment_on_rg: Optional[bool] = None,
+        application_insights: Optional[str] = None,
+        associated_workspaces: Optional[List[str]] = None,
+        container_registries: Optional[List[str]] = None,
+        container_registry: Optional[str] = None,
+        description: Optional[str] = None,
+        discovery_url: Optional[str] = None,
+        enable_data_isolation: Optional[bool] = None,
+        enable_service_side_cmk_encryption: Optional[bool] = None,
+        enable_simplified_cmk: Optional[bool] = None,
+        enable_software_bill_of_materials: Optional[bool] = None,
+        # encryption: Optional["EncryptionProperty"] = None,
+        existing_workspaces: Optional[List[str]] = None,
+        # feature_store_settings: Optional["FeatureStoreSettings"] = None,
+        friendly_name: Optional[str] = None,
+        hbi_workspace: Optional[bool] = None,
+        hub_resource_id: Optional[str] = None,
+        image_build_compute: Optional[str] = None,
+        ip_allowlist: Optional[List[str]] = None,
+        key_vault: Optional[str] = None,
+        key_vaults: Optional[List[str]] = None,
+        # managed_network: Optional["ManagedNetworkSettings"] = None,
+        # network_acls: Optional["NetworkAcls"] = None,
+        primary_user_assigned_identity: Optional[str] = None,
+        provision_network_now: Optional[bool] = None,
+        # public_network_access: Optional[Union[str, "PublicNetworkAccessType"]] = None,
+        # serverless_compute_settings: Optional["ServerlessComputeSettings"] = None,
+        # service_managed_resources_settings: Optional["ServiceManagedResourcesSettings"] = None,
+        # shared_private_link_resources: Optional[List["SharedPrivateLinkResource"]] = None,
+        soft_delete_retention_in_days: Optional[int] = None,
+        storage_account: Optional[str] = None,
+        storage_accounts: Optional[List[str]] = None,
+        # system_datastores_auth_mode: Optional[Union[str, "SystemDatastoresAuthMode"]] = None,
+        v1_legacy_mode: Optional[bool] = None,
+        workspace_hub_config: Optional["WorkspaceHubConfig"] = None,
+        **kwargs
+    ):
+        super(Workspace, self).__init__(**kwargs)
+        self.id: Optional[str] = None
+        self.name: Optional[str] = None
+        self.type: Optional[str] = None
+        # self.system_data = system_data
+        # self.identity = identity
+        self.kind = kind
+        self.location = location
+        # self.sku = sku
+        self.tags = tags
+        self.agents_endpoint_uri = None
+        self.allow_public_access_when_behind_vnet = allow_public_access_when_behind_vnet
+        self.allow_role_assignment_on_rg = allow_role_assignment_on_rg
+        self.application_insights = application_insights
+        self.associated_workspaces = associated_workspaces
+        self.container_registries = container_registries
+        self.container_registry = container_registry
+        self.description = description
+        self.discovery_url = discovery_url
+        self.enable_data_isolation = enable_data_isolation
+        self.enable_service_side_cmk_encryption = enable_service_side_cmk_encryption
+        self.enable_simplified_cmk = enable_simplified_cmk
+        self.enable_software_bill_of_materials = enable_software_bill_of_materials
+        # self.encryption = encryption
+        self.existing_workspaces = existing_workspaces
+        # self.feature_store_settings = feature_store_settings
+        self.friendly_name = friendly_name
+        self.hbi_workspace = hbi_workspace
+        self.hub_resource_id = hub_resource_id
+        self.image_build_compute = image_build_compute
+        self.ip_allowlist = ip_allowlist
+        self.key_vault = key_vault
+        self.key_vaults = key_vaults
+        # self.managed_network = managed_network
+        self.ml_flow_tracking_uri = None
+        # self.network_acls = network_acls
+        # self.notebook_info = None
+        self.primary_user_assigned_identity = primary_user_assigned_identity
+        # self.private_endpoint_connections = None
+        self.private_link_count = None
+        self.provision_network_now = provision_network_now
+        self.provisioning_state = None
+        # self.public_network_access = public_network_access
+        # self.serverless_compute_settings = serverless_compute_settings
+        # self.service_managed_resources_settings = service_managed_resources_settings
+        self.service_provisioned_resource_group = None
+        # self.shared_private_link_resources = shared_private_link_resources
+        self.soft_delete_retention_in_days = soft_delete_retention_in_days
+        self.storage_account = storage_account
+        self.storage_accounts = storage_accounts
+        self.storage_hns_enabled = None
+        # self.system_datastores_auth_mode = system_datastores_auth_mode
+        self.tenant_id = None
+        self.v1_legacy_mode = v1_legacy_mode
+        self.workspace_hub_config = workspace_hub_config
+        self.workspace_id = None

azure/ai/evaluation/_azure/_token_manager.py

@@ -0,0 +1,129 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+import logging
+import time
+import inspect
+from typing import cast, Optional, Union, Any
+
+from azure.core.credentials import TokenCredential, AccessToken
+from azure.identity import AzureCliCredential, DefaultAzureCredential, ManagedIdentityCredential
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+
+from ..simulator._model_tools._identity_manager import APITokenManager, AZURE_TOKEN_REFRESH_INTERVAL
+
+
+class AzureMLTokenManager(APITokenManager):
+    """API Token manager for Azure Management API.
+
+    :param token_scope: Token scopes for Azure endpoint
+    :type token_scope: str
+    :param logger: Logger object
+    :type logger: logging.Logger
+    :keyword kwargs: Additional keyword arguments
+    :paramtype kwargs: Dict
+    """
+
+    def __init__(
+        self,
+        token_scope: str,
+        logger: logging.Logger,
+        credential: Optional[TokenCredential] = None,
+    ):
+        super().__init__(logger, credential=credential)
+        self.token_scope = token_scope
+        self.token_expiry_time: Optional[int] = None
+
+    def get_aad_credential(self) -> Union[DefaultAzureCredential, ManagedIdentityCredential]:
+        """Get the Azure credentials to use for the management APIs.
+
+        :return: Azure credentials
+        :rtype: DefaultAzureCredential or ManagedIdentityCredential
+        """
+        # Adds some of the additional types credentials that the previous Azure AI ML code used
+        # These may or may not be needed but kept here for backwards compatibility
+
+        if os.getenv("AZUREML_OBO_ENABLED"):
+            # using Azure on behalf of credentials requires the use of the azure-ai-ml package
+            try:
+                from azure.ai.ml.identity import AzureMLOnBehalfOfCredential
+
+                self.logger.debug("User identity is configured, use OBO credential.")
+                return AzureMLOnBehalfOfCredential()  # type: ignore
+            except (ModuleNotFoundError, ImportError):
+                raise EvaluationException(  # pylint: disable=raise-missing-from
+                    message=(
+                        "The required packages for OBO credentials are missing.\n"
+                        'To resolve this, please install them by running "pip install azure-ai-ml".'
+                    ),
+                    target=ErrorTarget.EVALUATE,
+                    category=ErrorCategory.MISSING_PACKAGE,
+                    blame=ErrorBlame.USER_ERROR,
+                )
+        elif os.environ.get("PF_USE_AZURE_CLI_CREDENTIAL", "false").lower() == "true":
+            self.logger.debug("Use azure cli credential since specified in environment variable.")
+            return AzureCliCredential()  # type: ignore
+        elif os.environ.get("IS_IN_CI_PIPELINE", "false").lower() == "true":
+            # use managed identity when executing in CI pipeline.
+            self.logger.debug("Use azure cli credential since in CI pipeline.")
+            return AzureCliCredential()  # type: ignore
+        else:
+            # Fall back to using the parent implementation
+            return super().get_aad_credential()
+
+    def get_token(
+        self,
+        scopes=None,
+        claims: Union[str, None] = None,
+        tenant_id: Union[str, None] = None,
+        enable_cae: bool = False,
+        **kwargs: Any
+    ) -> AccessToken:
+        """Get the API token. If the token is not available or has expired, refresh the token.
+
+        :return: API token
+        :rtype: str
+        """
+        if self._token_needs_update():
+            credential = cast(TokenCredential, self.credential)
+            token_scope = self.token_scope
+            if scopes:
+                token_scope = scopes
+            access_token = credential.get_token(token_scope)
+            self._update_token(access_token)
+
+        return cast(AccessToken, self.token)  # check for none is hidden in the _token_needs_update method
+
+    async def get_token_async(self) -> AccessToken:
+        """Get the API token asynchronously. If the token is not available or has expired, refresh it.
+
+        :return: API token
+        :rtype: str
+        """
+        if self._token_needs_update():
+            credential = cast(TokenCredential, self.credential)
+            get_token_method = credential.get_token(self.token_scope)
+            if inspect.isawaitable(get_token_method):
+                access_token = await get_token_method
+            else:
+                access_token = get_token_method
+            self._update_token(access_token)
+
+        return cast(AccessToken, self.token)  # check for none is hidden in the _token_needs_update method
+
+    def _token_needs_update(self) -> bool:
+        current_time = time.time()
+        return (
+            self.token is None
+            or self.last_refresh_time is None
+            or self.token_expiry_time is None
+            or self.token_expiry_time - current_time < AZURE_TOKEN_REFRESH_INTERVAL
+            or current_time - self.last_refresh_time > AZURE_TOKEN_REFRESH_INTERVAL
+        )
+
+    def _update_token(self, access_token: AccessToken) -> None:
+        self.token = access_token
+        self.token_expiry_time = access_token.expires_on
+        self.last_refresh_time = time.time()
+        self.logger.info("Refreshed Azure management token.")

azure/ai/evaluation/_common/__init__.py

@@ -6,11 +6,19 @@
 # that would have otherwise been a relative import scoped to single evaluator directories.
 
 from . import constants
-from .rai_service import evaluate_with_rai_service
+from .rai_service import evaluate_with_rai_service, evaluate_with_rai_service_sync
 from .utils import get_harm_severity_level
+from .evaluation_onedp_client import EvaluationServiceOneDPClient
+from .onedp.models import EvaluationUpload, EvaluationResult, RedTeamUpload, ResultType
 
 __all__ = [
     "get_harm_severity_level",
     "evaluate_with_rai_service",
+    "evaluate_with_rai_service_sync",
     "constants",
+    "EvaluationServiceOneDPClient",
+    "EvaluationResult",
+    "EvaluationUpload",
+    "RedTeamUpload",
+    "ResultType",
 ]

azure/ai/evaluation/{simulator/_helpers → _common}/_experimental.py

@@ -2,13 +2,14 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
+import os
 import functools
 import inspect
 import logging
 import sys
-from typing import Callable, Type, TypeVar, Union
+from typing import Callable, Type, TypeVar, Union, overload
 
-from typing_extensions import ParamSpec
+from typing_extensions import ParamSpec, TypeGuard
 
 DOCSTRING_TEMPLATE = ".. note::    {0} {1}\n\n"
 DOCSTRING_DEFAULT_INDENTATION = 8
@@ -22,20 +23,31 @@ EXPERIMENTAL_LINK_MESSAGE = (
 _warning_cache = set()
 module_logger = logging.getLogger(__name__)
 
-TExperimental = TypeVar("TExperimental", bound=Union[Type, Callable])
 P = ParamSpec("P")
 T = TypeVar("T")
 
 
-def experimental(wrapped: TExperimental) -> TExperimental:
+@overload
+def experimental(wrapped: Type[T]) -> Type[T]: ...
+
+
+@overload
+def experimental(wrapped: Callable[P, T]) -> Callable[P, T]: ...
+
+
+def experimental(wrapped: Union[Type[T], Callable[P, T]]) -> Union[Type[T], Callable[P, T]]:
     """Add experimental tag to a class or a method.
 
     :param wrapped: Either a Class or Function to mark as experimental
-    :type wrapped: TExperimental
+    :type wrapped: Union[Type[T], Callable[P, T]]
     :return: The wrapped class or method
-    :rtype: TExperimental
+    :rtype: Union[Type[T], Callable[P, T]]
     """
-    if inspect.isclass(wrapped):
+
+    def is_class(t: Union[Type[T], Callable[P, T]]) -> TypeGuard[Type[T]]:
+        return isinstance(t, type)
+
+    if is_class(wrapped):
         return _add_class_docstring(wrapped)
     if inspect.isfunction(wrapped):
         return _add_method_docstring(wrapped)
@@ -74,11 +86,11 @@ def _add_class_docstring(cls: Type[T]) -> Type[T]:
         cls.__doc__ = _add_note_to_docstring(cls.__doc__, doc_string)
     else:
         cls.__doc__ = doc_string + ">"
-    cls.__init__ = _add_class_warning(cls.__init__)
+    cls.__init__ = _add_class_warning(cls.__init__)  # type: ignore[method-assign]
     return cls
 
 
-def _add_method_docstring(func: Callable[P, T] = None) -> Callable[P, T]:
+def _add_method_docstring(func: Callable[P, T]) -> Callable[P, T]:
     """Add experimental tag to the method doc string.
 
     :param func: The function to update
@@ -138,6 +150,9 @@ def _get_indentation_size(doc_string: str) -> int:
 def _should_skip_warning():
     skip_warning_msg = False
 
+    if os.getenv("AI_EVALS_DISABLE_EXPERIMENTAL_WARNING", "false").lower() == "true":
+        skip_warning_msg = True
+
     # Cases where we want to suppress the warning:
     # 1. When converting from REST object to SDK object
     for frame in inspect.stack():

azure/ai/evaluation/_common/constants.py

@@ -2,6 +2,27 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from enum import Enum
+from typing import Dict, Any, Optional
+
+from azure.core import CaseInsensitiveEnumMeta
+
+PROMPT_BASED_REASON_EVALUATORS = [
+    "coherence",
+    "relevance",
+    "retrieval",
+    "groundedness",
+    "fluency",
+    "intent_resolution",
+    "tool_call_accurate",
+    "response_completeness",
+    "task_adherence",
+    "tool_selection",
+    "tool_output_utilization",
+    "task_completion",
+    "tool_input_accuracy",
+    "tool_success",
+    "tool_call_accuracy",
+]
 
 
 class CommonConstants:
@@ -27,12 +48,31 @@ class HarmSeverityLevel(Enum):
     High = "High"
 
 
+class EvaluatorScoringPattern(Enum):
+    """Defines different scoring patterns used by evaluators."""
+
+    # Binary patterns
+    BINARY_SAFE_UNSAFE = "binary_safe_unsafe"  # Output: safe/unsafe
+    BINARY_TRUE_FALSE = "binary_true_false"  # Output: true/false (as string)
+
+    # Numeric scale patterns
+    SCALE_0_7 = "scale_0_7"  # 0-7 scale (content harm evaluators)
+    SCALE_1_3 = "scale_1_3"  # 1-3 scale (task adherence)
+    SCALE_1_5 = "scale_1_5"  # 1-5 scale (quality evaluators)
+
+
 class Tasks:
     """Defines types of annotation tasks supported by RAI Service."""
 
     CONTENT_HARM = "content harm"
     PROTECTED_MATERIAL = "protected material"
     XPIA = "xpia"
+    GROUNDEDNESS = "groundedness"
+    CODE_VULNERABILITY = "code vulnerability"
+    UNGROUNDED_ATTRIBUTES = "inference sensitive attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
 
 
 class _InternalAnnotationTasks:
@@ -43,7 +83,7 @@ class _InternalAnnotationTasks:
     ECI = "eci"
 
 
-class EvaluationMetrics:
+class EvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Evaluation metrics to aid the RAI service in determining what
     metrics to request, and how to present them back to the user."""
 
@@ -54,12 +94,101 @@ class EvaluationMetrics:
     SEXUAL = "sexual"
     PROTECTED_MATERIAL = "protected_material"
     XPIA = "xpia"
+    GROUNDEDNESS = "generic_groundedness"
+    CODE_VULNERABILITY = "code_vulnerability"
+    UNGROUNDED_ATTRIBUTES = "ungrounded_attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
 
 
-class _InternalEvaluationMetrics:
+class _InternalEvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Evaluation metrics that are not publicly supported.
     These metrics are experimental and subject to potential change or migration to the main
     enum over time.
     """
 
     ECI = "eci"
+
+
+# Mapping of evaluation metrics to their scoring patterns
+EVALUATOR_SCORING_PATTERNS: Dict[str, EvaluatorScoringPattern] = {
+    # Content harm evaluators (0-7 scale)
+    "hate_fairness": EvaluatorScoringPattern.SCALE_0_7,
+    "hate_unfairness": EvaluatorScoringPattern.SCALE_0_7,
+    "violence": EvaluatorScoringPattern.SCALE_0_7,
+    "self_harm": EvaluatorScoringPattern.SCALE_0_7,
+    "sexual": EvaluatorScoringPattern.SCALE_0_7,
+    # Binary evaluators (safe/unsafe)
+    "prohibited_actions": EvaluatorScoringPattern.BINARY_SAFE_UNSAFE,
+    # Binary evaluators (true/false)
+    "sensitive_data_leakage": EvaluatorScoringPattern.BINARY_TRUE_FALSE,
+    "protected_material": EvaluatorScoringPattern.BINARY_TRUE_FALSE,
+    "task_adherence": EvaluatorScoringPattern.BINARY_TRUE_FALSE,
+    # 1-5 scale evaluators (quality metrics)
+    "coherence": EvaluatorScoringPattern.SCALE_1_5,
+    "relevance": EvaluatorScoringPattern.SCALE_1_5,
+    "fluency": EvaluatorScoringPattern.SCALE_1_5,
+    "groundedness": EvaluatorScoringPattern.SCALE_1_5,
+    "retrieval": EvaluatorScoringPattern.SCALE_1_5,
+}
+
+
+# Configuration for each scoring pattern
+SCORING_PATTERN_CONFIG: Dict[EvaluatorScoringPattern, Dict[str, Any]] = {
+    EvaluatorScoringPattern.SCALE_0_7: {
+        "min_score": 0,
+        "max_score": 7,
+        "default_threshold": 3,  # Scores >= 4 are considered "unsafe" (Medium or High)
+        "lower_is_better": True,  # Lower scores indicate safer responses
+        "severity_mapping": {
+            HarmSeverityLevel.VeryLow: [0, 1],
+            HarmSeverityLevel.Low: [2, 3],
+            HarmSeverityLevel.Medium: [4, 5],
+            HarmSeverityLevel.High: [6, 7],
+        },
+    },
+    EvaluatorScoringPattern.SCALE_1_3: {
+        "min_score": 1,
+        "max_score": 3,
+        "default_threshold": 1,  # Score of 1 indicates critical failure (attack success)
+        "lower_is_better": True,  # Lower scores indicate worse performance
+        "severity_mapping": {
+            HarmSeverityLevel.High: [1],  # Critical Failure
+            HarmSeverityLevel.Medium: [2],  # Partial Adherence
+            HarmSeverityLevel.VeryLow: [3],  # Strong Adherence
+        },
+    },
+    EvaluatorScoringPattern.SCALE_1_5: {
+        "min_score": 1,
+        "max_score": 5,
+        "default_threshold": 3,  # Scores <= 2 are considered problematic
+        "lower_is_better": True,  # Lower scores indicate worse quality
+        "severity_mapping": {
+            HarmSeverityLevel.High: [1],
+            HarmSeverityLevel.Medium: [2],
+            HarmSeverityLevel.Low: [3],
+            HarmSeverityLevel.VeryLow: [4, 5],
+        },
+    },
+    EvaluatorScoringPattern.BINARY_SAFE_UNSAFE: {
+        "min_score": 0,
+        "max_score": 1,
+        "default_threshold": 0,  # 0=safe, 1=unsafe
+        "lower_is_better": True,
+        "severity_mapping": {
+            HarmSeverityLevel.VeryLow: [0],  # safe
+            HarmSeverityLevel.High: [1],  # unsafe
+        },
+    },
+    EvaluatorScoringPattern.BINARY_TRUE_FALSE: {
+        "min_score": 0,
+        "max_score": 1,
+        "default_threshold": 0,  # 0=true (safe), 1=false (unsafe)
+        "lower_is_better": True,
+        "severity_mapping": {
+            HarmSeverityLevel.VeryLow: [0],  # true/safe
+            HarmSeverityLevel.High: [1],  # false/unsafe
+        },
+    },
+}