azure-ai-evaluation 1.8.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_code_vulnerability/_code_vulnerability.py

@@ -8,15 +8,16 @@ from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
+
 # cspell:ignore ssrf, vuln
 @experimental
 class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
     """
-    Evaluates code vulnerability for a given query and response for a single-turn evaluation only, 
+    Evaluates code vulnerability for a given query and response for a single-turn evaluation only,
     where query represents the user query or code before the completion, and response represents the code recommended by the assistant.
 
     The code vulnerability evaluation checks for vulnerabilities in the following coding languages:
-    
+
     - Python
     - Java
     - C++
@@ -26,7 +27,7 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
     - SQL
 
     The code vulnerability evaluation identifies the following vulnerabilities:
-    
+
     - path-injection
     - sql-injection
     - code-injection
@@ -49,9 +50,9 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param kwargs: Additional arguments to pass to the evaluator.
     :type kwargs: Any
 
@@ -63,13 +64,13 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call CodeVulnerabilityEvaluator with a query and response using azure.ai.evaluation.AzureAIProject.
-        
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START code_vulnerability_evaluator]
             :end-before: [END code_vulnerability_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call CodeVulnerabilityEvaluator using Azure AI Project URL in following format 
+            :caption: Initialize and call CodeVulnerabilityEvaluator using Azure AI Project URL in following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     .. note::
@@ -78,19 +79,26 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         for the code vulnerability will be "code_vulnerability_label".
     """
 
-    id = "code_vulnerability"
+    id = "azureai://built-in/evaluators/code_vulnerability"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(
         self,
         credential,
         azure_ai_project,
+        **kwargs,
     ):
+        # Set default for evaluate_query if not provided
+        if "evaluate_query" not in kwargs:
+            kwargs["evaluate_query"] = True
+
         super().__init__(
             eval_metric=EvaluationMetrics.CODE_VULNERABILITY,
             azure_ai_project=azure_ai_project,
             credential=credential,
+            **kwargs,
         )
 
     @overload
@@ -99,7 +107,7 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         *,
         query: str,
         response: str,
-    ) -> Dict[str, Union[str, float]]: 
+    ) -> Dict[str, Union[str, float]]:
         """Evaluate a given query/response pair for code vulnerability
 
         :keyword query: The query to be evaluated.
@@ -116,7 +124,7 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         *args,
         **kwargs,
     ):
-        """Evaluate code vulnerability. Accepts query and response for a single-turn evaluation only. 
+        """Evaluate code vulnerability. Accepts query and response for a single-turn evaluation only.
 
         :keyword query: The query to be evaluated.
         :paramtype query: Optional[str]
@@ -124,5 +132,5 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         :paramtype response: Optional[str]
         :rtype: Dict[str, Union[str, bool]]
         """
-        
+
         return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -32,17 +32,17 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call CoherenceEvaluator using azure.ai.evaluation.AzureAIProject
-        
+
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START coherence_evaluator]
             :end-before: [END coherence_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call CoherenceEvaluator using Azure AI Project URL in following format 
+            :caption: Initialize and call CoherenceEvaluator using Azure AI Project URL in following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
-    
+
     .. admonition:: Example with Threshold:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py
@@ -62,7 +62,7 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE = "coherence.prompty"
     _RESULT_KEY = "coherence"
 
-    id = "azureml://registries/azureml/models/Coherence-Evaluator/versions/4"
+    id = "azureai://built-in/evaluators/coherence"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
@@ -76,7 +76,7 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
             threshold=threshold,
-            _higher_is_better=self._higher_is_better
+            _higher_is_better=self._higher_is_better,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -4,14 +4,34 @@
 
 import inspect
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Generic, List, TypedDict, TypeVar, Union, cast, final, Optional
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Generic,
+    List,
+    TypedDict,
+    TypeVar,
+    Union,
+    cast,
+    final,
+    Optional,
+)
 
 from azure.ai.evaluation._legacy._adapters.utils import async_run_allowing_running_loop
 from typing_extensions import ParamSpec, TypeAlias, get_overloads
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._exceptions import (
+    ErrorBlame,
+    ErrorCategory,
+    ErrorTarget,
+    EvaluationException,
+)
 from azure.ai.evaluation._common.utils import remove_optional_singletons
-from azure.ai.evaluation._constants import _AggregationType, EVALUATION_PASS_FAIL_MAPPING
+from azure.ai.evaluation._constants import (
+    _AggregationType,
+    EVALUATION_PASS_FAIL_MAPPING,
+)
 from azure.ai.evaluation._model_configurations import Conversation
 from azure.ai.evaluation._common._experimental import experimental
 
@@ -176,7 +196,9 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
             singletons.extend([p for p in params if p != "self"])
         return singletons
 
-    def _derive_conversation_converter(self) -> Callable[[Dict], List[DerivedEvalInput]]:
+    def _derive_conversation_converter(
+        self,
+    ) -> Callable[[Dict], List[DerivedEvalInput]]:
         """Produce the function that will be used to convert conversations to a list of evaluable inputs.
         This uses the inputs derived from the _derive_singleton_inputs function to determine which
         aspects of a conversation ought to be extracted.
@@ -235,7 +257,9 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
 
         return converter
 
-    def _derive_multi_modal_conversation_converter(self) -> Callable[[Dict], List[Dict[str, Any]]]:
+    def _derive_multi_modal_conversation_converter(
+        self,
+    ) -> Callable[[Dict], List[Dict[str, Any]]]:
         """Produce the function that will be used to convert multi-modal conversations to a list of evaluable inputs.
         This uses the inputs derived from the _derive_singleton_inputs function to determine which
         aspects of a conversation ought to be extracted.
@@ -288,7 +312,7 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
 
         return multi_modal_converter
 
-    def _convert_kwargs_to_eval_input(self, **kwargs) -> Union[List[Dict], List[DerivedEvalInput]]:
+    def _convert_kwargs_to_eval_input(self, **kwargs) -> Union[List[Dict], List[DerivedEvalInput], Dict[str, Any]]:
         """Convert an arbitrary input into a list of inputs for evaluators.
         It is assumed that evaluators generally make use of their inputs in one of two ways.
         Either they receive a collection of keyname inputs that are all single values
@@ -416,12 +440,12 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
                         threshold_key = f"{base_key}_threshold"
                         result[threshold_key] = self._threshold
                         if self._higher_is_better:
-                            if int(score_value) >= self._threshold:
+                            if float(score_value) >= self._threshold:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[True]
                             else:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[False]
                         else:
-                            if int(score_value) <= self._threshold:
+                            if float(score_value) <= self._threshold:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[True]
                             else:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[False]
@@ -498,9 +522,19 @@ class AsyncEvaluatorBase:
     # Since we want this to be relatively call-agnostic, we just account for every input that any children
     # are known to throw at this, mash them into kwargs, and then pass them into the real call.
     async def __call__(
-        self, *, query=None, response=None, context=None, conversation=None, ground_truth=None,
-            tool_calls=None, tool_definitions=None, messages=None, retrieval_ground_truth=None,
-            retrieved_documents=None,**kwargs
+        self,
+        *,
+        query=None,
+        response=None,
+        context=None,
+        conversation=None,
+        ground_truth=None,
+        tool_calls=None,
+        tool_definitions=None,
+        messages=None,
+        retrieval_ground_truth=None,
+        retrieved_documents=None,
+        **kwargs,
     ):
         if conversation is not None:
             kwargs["conversation"] = conversation

azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -20,9 +20,14 @@ from ..._common.utils import construct_prompty_model_config, validate_model_conf
 from . import EvaluatorBase
 
 try:
-    from ..._user_agent import USER_AGENT
+    from ..._user_agent import UserAgentSingleton
 except ImportError:
-    USER_AGENT = "None"
+
+    class UserAgentSingleton:
+        @property
+        def value(self) -> str:
+            return "None"
+
 
 T = TypeVar("T")
 
@@ -50,8 +55,17 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
     _LLM_CALL_TIMEOUT = 600
     _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
-    def __init__(self, *, result_key: str, prompty_file: str, model_config: dict, eval_last_turn: bool = False,
-                 threshold: int = 3, _higher_is_better: bool = False, **kwargs) -> None:
+    def __init__(
+        self,
+        *,
+        result_key: str,
+        prompty_file: str,
+        model_config: dict,
+        eval_last_turn: bool = False,
+        threshold: int = 3,
+        _higher_is_better: bool = False,
+        **kwargs,
+    ) -> None:
         self._result_key = result_key
         self._is_reasoning_model = kwargs.get("is_reasoning_model", False)
         self._prompty_file = prompty_file
@@ -60,15 +74,16 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
         super().__init__(eval_last_turn=eval_last_turn, threshold=threshold, _higher_is_better=_higher_is_better)
 
         subclass_name = self.__class__.__name__
-        user_agent = f"{USER_AGENT} (type=evaluator subtype={subclass_name})"
+        user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={subclass_name})"
         prompty_model_config = construct_prompty_model_config(
             validate_model_config(model_config),
             self._DEFAULT_OPEN_API_VERSION,
             user_agent,
         )
 
-        self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config,
-                                       is_reasoning_model=self._is_reasoning_model)
+        self._flow = AsyncPrompty.load(
+            source=self._prompty_file, model=prompty_model_config, is_reasoning_model=self._is_reasoning_model
+        )
 
     # __call__ not overridden here because child classes have such varied signatures that there's no point
     # defining a default here.
@@ -132,7 +147,7 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
                 score = float(match.group())
                 binary_result = self._get_binary_result(score)
             return {
-                self._result_key: float(score), 
+                self._result_key: float(score),
                 f"gpt_{self._result_key}": float(score),
                 f"{self._result_key}_result": binary_result,
                 f"{self._result_key}_threshold": self._threshold,
@@ -140,7 +155,7 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
 
         binary_result = self._get_binary_result(score)
         return {
-            self._result_key: float(score), 
+            self._result_key: float(score),
             f"gpt_{self._result_key}": float(score),
             f"{self._result_key}_result": binary_result,
             f"{self._result_key}_threshold": self._threshold,

azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py

@@ -36,14 +36,17 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]):
         aggregated. Per-turn results are still be available in the output via the "evaluation_per_turn" key
         when this occurs. Default is False, resulting full conversation evaluation and aggregation.
     :type eval_last_turn: bool
-    :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation
-        to produce a single result.
+    :param conversation_aggregation_type: The type of aggregation to perform on the per-turn results of a conversation        to produce a single result.
         Default is ~azure.ai.evaluation._AggregationType.MEAN.
     :type conversation_aggregation_type: ~azure.ai.evaluation._AggregationType
     :param threshold: The threshold for the evaluation. Default is 3.
     :type threshold: Optional[int]
     :param _higher_is_better: If True, higher scores are better. Default is True.
     :type _higher_is_better: Optional[bool]
+    :param evaluate_query: If True, the query will be included in the evaluation data when evaluating
+        query-response pairs. If False, only the response will be evaluated. Default is False.
+        Can be passed as a keyword argument.
+    :type evaluate_query: bool
     """
 
     @override
@@ -56,12 +59,21 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]):
         conversation_aggregation_type: _AggregationType = _AggregationType.MEAN,
         threshold: int = 3,
         _higher_is_better: Optional[bool] = False,
+        **kwargs,
     ):
-        super().__init__(eval_last_turn=eval_last_turn, conversation_aggregation_type=conversation_aggregation_type, threshold=threshold, _higher_is_better=_higher_is_better)
+        super().__init__(
+            eval_last_turn=eval_last_turn,
+            conversation_aggregation_type=conversation_aggregation_type,
+            threshold=threshold,
+            _higher_is_better=_higher_is_better,
+        )
         self._eval_metric = eval_metric
         self._azure_ai_project = validate_azure_ai_project(azure_ai_project)
         self._credential = credential
         self._threshold = threshold
+
+        # Handle evaluate_query parameter from kwargs
+        self._evaluate_query = kwargs.get("evaluate_query", False)
         self._higher_is_better = _higher_is_better
 
     @override
@@ -96,7 +108,7 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]):
         :return: The evaluation result.
         :rtype: Dict
         """
-        if "query" in eval_input and "response" in eval_input:
+        if "response" in eval_input:
             return await self._evaluate_query_response(eval_input)
 
         conversation = eval_input.get("conversation", None)
@@ -126,17 +138,20 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]):
         return result
 
     async def _evaluate_query_response(self, eval_input: Dict) -> Dict[str, T]:
-        query = str(eval_input.get("query", None))
-        response = str(eval_input.get("response", None))
-        if query is None or response is None:
+        query = eval_input.get("query", None)
+        response = eval_input.get("response", None)
+        if response is None:
             raise EvaluationException(
                 message="Not implemented",
                 internal_message=(
-                    "Reached query/response evaluation without supplying query or response."
+                    "Reached query/response evaluation without supplying response."
                     + " This should have failed earlier."
                 ),
             )
-        input_data = {"query": query, "response": response}
+        input_data = {"response": str(response)}
+
+        if query is not None and self._evaluate_query:
+            input_data["query"] = str(query)
 
         if "context" in self._singleton_inputs:
             context = eval_input.get("context", None)

azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py

@@ -22,9 +22,9 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param violence_threshold: The threshold for violence evaluation. Default is 3.
     :type violence_threshold: int
     :param sexual_threshold: The threshold for sexual content evaluation. Default is 3.
@@ -33,6 +33,8 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
     :type self_harm_threshold: int
     :param hate_unfairness_threshold: The threshold for hate/unfairness evaluation. Default is 3.
     :type hate_unfairness_threshold: int
+    :param evaluate_query: Whether to also evaluate the query in addition to the response. Default is False.
+    :type evaluate_query: bool
     :param kwargs: Additional arguments to pass to the evaluator.
     :type kwargs: Any
     :return: A function that evaluates content-safety metrics for "question-answering" scenario.
@@ -45,19 +47,19 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call ContentSafetyEvaluator using azure.ai.evaluation.AzureAIProject.
-    
+
     .. admonition:: Example using Azure AI Project URL:
-    
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START content_safety_evaluator]
             :end-before: [END content_safety_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call ContentSafetyEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call ContentSafetyEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
-    
+
     .. admonition:: Example with Threshold:
-    
+
         .. literalinclude:: ../samples/evaluation_samples_threshold.py
             :start-after: [START threshold_content_safety_evaluator]
             :end-before: [END threshold_content_safety_evaluator]
@@ -66,19 +68,20 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
             :caption: Initialize with threshold and call a ContentSafetyEvaluator with a query and response.
     """
 
-    id = "content_safety"
+    id = "azureai://built-in/evaluators/content_safety"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     def __init__(
-        self, 
-        credential, 
+        self,
+        credential,
         azure_ai_project,
-        *, 
+        *,
         violence_threshold: int = 3,
         sexual_threshold: int = 3,
         self_harm_threshold: int = 3,
         hate_unfairness_threshold: int = 3,
-        **kwargs
+        **kwargs,
     ):
         # Type checking
         for name, value in [
@@ -89,12 +92,19 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
         ]:
             if not isinstance(value, int):
                 raise TypeError(f"{name} must be an int, got {type(value)}")
-        
+
+        # Extract evaluate_query from kwargs if present
+        evaluate_query_kwargs = {}
+        if "evaluate_query" in kwargs:
+            evaluate_query_kwargs["evaluate_query"] = kwargs["evaluate_query"]
+
         evaluators = [
-            ViolenceEvaluator(credential, azure_ai_project, threshold=violence_threshold),
-            SexualEvaluator(credential, azure_ai_project, threshold=sexual_threshold),
-            SelfHarmEvaluator(credential, azure_ai_project, threshold=self_harm_threshold),
-            HateUnfairnessEvaluator(credential, azure_ai_project, threshold=hate_unfairness_threshold),
+            ViolenceEvaluator(credential, azure_ai_project, threshold=violence_threshold, **evaluate_query_kwargs),
+            SexualEvaluator(credential, azure_ai_project, threshold=sexual_threshold, **evaluate_query_kwargs),
+            SelfHarmEvaluator(credential, azure_ai_project, threshold=self_harm_threshold, **evaluate_query_kwargs),
+            HateUnfairnessEvaluator(
+                credential, azure_ai_project, threshold=hate_unfairness_threshold, **evaluate_query_kwargs
+            ),
         ]
         super().__init__(evaluators=evaluators, **kwargs)
 

azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py

@@ -45,9 +45,9 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param threshold: The threshold for the HateUnfairness evaluator. Default is 3.
     :type threshold: int
 
@@ -59,17 +59,17 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call HateUnfairnessEvaluator using azure.ai.evaluation.AzureAIProject.
-        
+
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START hate_unfairness_evaluator]
             :end-before: [END hate_unfairness_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call HateUnfairnessEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call HateUnfairnessEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
-    
+
     .. admonition:: Example with Threshold:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py
@@ -80,8 +80,9 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :caption: Initialize with threshold and call a HateUnfairnessEvaluator with a query and response.
     """
 
-    id = "azureml://registries/azureml/models/Hate-and-Unfairness-Evaluator/versions/4"
+    id = "azureai://built-in/evaluators/hate_unfairness"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(
@@ -90,6 +91,7 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
         azure_ai_project,
         *,
         threshold: int = 3,
+        **kwargs,
     ):
         super().__init__(
             eval_metric=EvaluationMetrics.HATE_FAIRNESS,
@@ -98,6 +100,7 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             conversation_aggregation_type=_AggregationType.MAX,
             threshold=threshold,
             _higher_is_better=False,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py

@@ -39,9 +39,9 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param threshold: The threshold for the SelfHarm evaluator. Default is 3.
     :type threshold: int
 
@@ -53,20 +53,21 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call SelfHarmEvaluator using azure.ai.evaluation.AzureAIProject.
-        
+
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START self_harm_evaluator]
             :end-before: [END self_harm_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call SelfHarmEvaluator using Azure AI Project URL in the following format 
-                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.   
+            :caption: Initialize and call SelfHarmEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
     """
 
-    id = "azureml://registries/azureml/models/Self-Harm-Related-Content-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/self_harm"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(
@@ -75,6 +76,7 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
         azure_ai_project,
         *,
         threshold: int = 3,
+        **kwargs,
     ):
         super().__init__(
             eval_metric=EvaluationMetrics.SELF_HARM,
@@ -83,6 +85,7 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             conversation_aggregation_type=_AggregationType.MAX,
             threshold=threshold,
             _higher_is_better=False,
+            **kwargs,
         )
 
     @overload