azure-ai-evaluation 1.8.0__py3-none-any.whl → 1.9.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_code_vulnerability/_code_vulnerability.py

@@ -8,15 +8,16 @@ from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
+
 # cspell:ignore ssrf, vuln
 @experimental
 class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
     """
-    Evaluates code vulnerability for a given query and response for a single-turn evaluation only, 
+    Evaluates code vulnerability for a given query and response for a single-turn evaluation only,
     where query represents the user query or code before the completion, and response represents the code recommended by the assistant.
 
     The code vulnerability evaluation checks for vulnerabilities in the following coding languages:
-    
+
     - Python
     - Java
     - C++
@@ -26,7 +27,7 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
     - SQL
 
     The code vulnerability evaluation identifies the following vulnerabilities:
-    
+
     - path-injection
     - sql-injection
     - code-injection
@@ -49,9 +50,9 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param kwargs: Additional arguments to pass to the evaluator.
     :type kwargs: Any
 
@@ -63,13 +64,13 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call CodeVulnerabilityEvaluator with a query and response using azure.ai.evaluation.AzureAIProject.
-        
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START code_vulnerability_evaluator]
             :end-before: [END code_vulnerability_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call CodeVulnerabilityEvaluator using Azure AI Project URL in following format 
+            :caption: Initialize and call CodeVulnerabilityEvaluator using Azure AI Project URL in following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     .. note::
@@ -99,7 +100,7 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         *,
         query: str,
         response: str,
-    ) -> Dict[str, Union[str, float]]: 
+    ) -> Dict[str, Union[str, float]]:
         """Evaluate a given query/response pair for code vulnerability
 
         :keyword query: The query to be evaluated.
@@ -116,7 +117,7 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         *args,
         **kwargs,
     ):
-        """Evaluate code vulnerability. Accepts query and response for a single-turn evaluation only. 
+        """Evaluate code vulnerability. Accepts query and response for a single-turn evaluation only.
 
         :keyword query: The query to be evaluated.
         :paramtype query: Optional[str]
@@ -124,5 +125,5 @@ class CodeVulnerabilityEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         :paramtype response: Optional[str]
         :rtype: Dict[str, Union[str, bool]]
         """
-        
+
         return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -32,17 +32,17 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call CoherenceEvaluator using azure.ai.evaluation.AzureAIProject
-        
+
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START coherence_evaluator]
             :end-before: [END coherence_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call CoherenceEvaluator using Azure AI Project URL in following format 
+            :caption: Initialize and call CoherenceEvaluator using Azure AI Project URL in following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
-    
+
     .. admonition:: Example with Threshold:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py
@@ -76,7 +76,7 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
             threshold=threshold,
-            _higher_is_better=self._higher_is_better
+            _higher_is_better=self._higher_is_better,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -416,12 +416,12 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
                         threshold_key = f"{base_key}_threshold"
                         result[threshold_key] = self._threshold
                         if self._higher_is_better:
-                            if int(score_value) >= self._threshold:
+                            if float(score_value) >= self._threshold:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[True]
                             else:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[False]
                         else:
-                            if int(score_value) <= self._threshold:
+                            if float(score_value) <= self._threshold:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[True]
                             else:
                                 result[result_key] = EVALUATION_PASS_FAIL_MAPPING[False]
@@ -498,9 +498,19 @@ class AsyncEvaluatorBase:
     # Since we want this to be relatively call-agnostic, we just account for every input that any children
     # are known to throw at this, mash them into kwargs, and then pass them into the real call.
     async def __call__(
-        self, *, query=None, response=None, context=None, conversation=None, ground_truth=None,
-            tool_calls=None, tool_definitions=None, messages=None, retrieval_ground_truth=None,
-            retrieved_documents=None,**kwargs
+        self,
+        *,
+        query=None,
+        response=None,
+        context=None,
+        conversation=None,
+        ground_truth=None,
+        tool_calls=None,
+        tool_definitions=None,
+        messages=None,
+        retrieval_ground_truth=None,
+        retrieved_documents=None,
+        **kwargs,
     ):
         if conversation is not None:
             kwargs["conversation"] = conversation

azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -20,9 +20,14 @@ from ..._common.utils import construct_prompty_model_config, validate_model_conf
 from . import EvaluatorBase
 
 try:
-    from ..._user_agent import USER_AGENT
+    from ..._user_agent import UserAgentSingleton
 except ImportError:
-    USER_AGENT = "None"
+
+    class UserAgentSingleton:
+        @property
+        def value(self) -> str:
+            return "None"
+
 
 T = TypeVar("T")
 
@@ -50,8 +55,17 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
     _LLM_CALL_TIMEOUT = 600
     _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
-    def __init__(self, *, result_key: str, prompty_file: str, model_config: dict, eval_last_turn: bool = False,
-                 threshold: int = 3, _higher_is_better: bool = False, **kwargs) -> None:
+    def __init__(
+        self,
+        *,
+        result_key: str,
+        prompty_file: str,
+        model_config: dict,
+        eval_last_turn: bool = False,
+        threshold: int = 3,
+        _higher_is_better: bool = False,
+        **kwargs,
+    ) -> None:
         self._result_key = result_key
         self._is_reasoning_model = kwargs.get("is_reasoning_model", False)
         self._prompty_file = prompty_file
@@ -60,15 +74,16 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
         super().__init__(eval_last_turn=eval_last_turn, threshold=threshold, _higher_is_better=_higher_is_better)
 
         subclass_name = self.__class__.__name__
-        user_agent = f"{USER_AGENT} (type=evaluator subtype={subclass_name})"
+        user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={subclass_name})"
         prompty_model_config = construct_prompty_model_config(
             validate_model_config(model_config),
             self._DEFAULT_OPEN_API_VERSION,
             user_agent,
         )
 
-        self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config,
-                                       is_reasoning_model=self._is_reasoning_model)
+        self._flow = AsyncPrompty.load(
+            source=self._prompty_file, model=prompty_model_config, is_reasoning_model=self._is_reasoning_model
+        )
 
     # __call__ not overridden here because child classes have such varied signatures that there's no point
     # defining a default here.
@@ -132,7 +147,7 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
                 score = float(match.group())
                 binary_result = self._get_binary_result(score)
             return {
-                self._result_key: float(score), 
+                self._result_key: float(score),
                 f"gpt_{self._result_key}": float(score),
                 f"{self._result_key}_result": binary_result,
                 f"{self._result_key}_threshold": self._threshold,
@@ -140,7 +155,7 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
 
         binary_result = self._get_binary_result(score)
         return {
-            self._result_key: float(score), 
+            self._result_key: float(score),
             f"gpt_{self._result_key}": float(score),
             f"{self._result_key}_result": binary_result,
             f"{self._result_key}_threshold": self._threshold,

azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py

@@ -57,7 +57,12 @@ class RaiServiceEvaluatorBase(EvaluatorBase[T]):
         threshold: int = 3,
         _higher_is_better: Optional[bool] = False,
     ):
-        super().__init__(eval_last_turn=eval_last_turn, conversation_aggregation_type=conversation_aggregation_type, threshold=threshold, _higher_is_better=_higher_is_better)
+        super().__init__(
+            eval_last_turn=eval_last_turn,
+            conversation_aggregation_type=conversation_aggregation_type,
+            threshold=threshold,
+            _higher_is_better=_higher_is_better,
+        )
         self._eval_metric = eval_metric
         self._azure_ai_project = validate_azure_ai_project(azure_ai_project)
         self._credential = credential

azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py

@@ -22,9 +22,9 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param violence_threshold: The threshold for violence evaluation. Default is 3.
     :type violence_threshold: int
     :param sexual_threshold: The threshold for sexual content evaluation. Default is 3.
@@ -45,19 +45,19 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call ContentSafetyEvaluator using azure.ai.evaluation.AzureAIProject.
-    
+
     .. admonition:: Example using Azure AI Project URL:
-    
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START content_safety_evaluator]
             :end-before: [END content_safety_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call ContentSafetyEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call ContentSafetyEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
-    
+
     .. admonition:: Example with Threshold:
-    
+
         .. literalinclude:: ../samples/evaluation_samples_threshold.py
             :start-after: [START threshold_content_safety_evaluator]
             :end-before: [END threshold_content_safety_evaluator]
@@ -70,15 +70,15 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     def __init__(
-        self, 
-        credential, 
+        self,
+        credential,
         azure_ai_project,
-        *, 
+        *,
         violence_threshold: int = 3,
         sexual_threshold: int = 3,
         self_harm_threshold: int = 3,
         hate_unfairness_threshold: int = 3,
-        **kwargs
+        **kwargs,
     ):
         # Type checking
         for name, value in [
@@ -89,7 +89,7 @@ class ContentSafetyEvaluator(MultiEvaluatorBase[Union[str, float]]):
         ]:
             if not isinstance(value, int):
                 raise TypeError(f"{name} must be an int, got {type(value)}")
-        
+
         evaluators = [
             ViolenceEvaluator(credential, azure_ai_project, threshold=violence_threshold),
             SexualEvaluator(credential, azure_ai_project, threshold=sexual_threshold),

azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py

@@ -45,9 +45,9 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param threshold: The threshold for the HateUnfairness evaluator. Default is 3.
     :type threshold: int
 
@@ -59,17 +59,17 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call HateUnfairnessEvaluator using azure.ai.evaluation.AzureAIProject.
-        
+
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START hate_unfairness_evaluator]
             :end-before: [END hate_unfairness_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call HateUnfairnessEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call HateUnfairnessEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
-    
+
     .. admonition:: Example with Threshold:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py

azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py

@@ -39,9 +39,9 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param threshold: The threshold for the SelfHarm evaluator. Default is 3.
     :type threshold: int
 
@@ -53,16 +53,16 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call SelfHarmEvaluator using azure.ai.evaluation.AzureAIProject.
-        
+
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START self_harm_evaluator]
             :end-before: [END self_harm_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call SelfHarmEvaluator using Azure AI Project URL in the following format 
-                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.   
+            :caption: Initialize and call SelfHarmEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}.
     """
 
     id = "azureml://registries/azureml/models/Self-Harm-Related-Content-Evaluator/versions/3"

azure/ai/evaluation/_evaluators/_content_safety/_sexual.py

@@ -41,9 +41,9 @@ class SexualEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param threshold: The threshold for the Sexual evaluator. Default is 3.
     :type threshold: int
 
@@ -55,17 +55,17 @@ class SexualEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call a SexualEvaluator.
-    
+
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START sexual_evaluator]
             :end-before: [END sexual_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call SexualEvaluator using Azure AI Project URL in following format 
+            :caption: Initialize and call SexualEvaluator using Azure AI Project URL in following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
-    
+
     .. admonition:: Example with Threshold:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py

azure/ai/evaluation/_evaluators/_content_safety/_violence.py

@@ -41,9 +41,9 @@ class ViolenceEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
 
     :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
     :param threshold: The threshold for the Violence evaluator. Default is 3.
     :type threshold: int
 
@@ -57,15 +57,15 @@ class ViolenceEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a ViolenceEvaluator.
 
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START violence_evaluator]
             :end-before: [END violence_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call ViolenceEvaluator using Azure AI Project URL in following format 
+            :caption: Initialize and call ViolenceEvaluator using Azure AI Project URL in following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
-    
+
     .. admonition:: Example:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py

azure/ai/evaluation/_evaluators/_document_retrieval/__init__.py

@@ -4,8 +4,4 @@
 
 from ._document_retrieval import DocumentRetrievalEvaluator, RetrievalGroundTruthDocument, RetrievedDocument
 
-__all__ = [
-    "DocumentRetrievalEvaluator",
-    "RetrievalGroundTruthDocument",
-    "RetrievedDocument"
-]
+__all__ = ["DocumentRetrievalEvaluator", "RetrievalGroundTruthDocument", "RetrievedDocument"]