azure-ai-evaluation 1.6.0__tar.gz → 1.7.0__tar.gz

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Release History
 
+## 1.7.0 (2025-05-12)
+
+### Bugs Fixed
+- azure-ai-evaluation failed with module not found [#40992](https://github.com/Azure/azure-sdk-for-python/issues/40992)
+
 ## 1.6.0 (2025-05-07)
 
 ### Features Added

{azure_ai_evaluation-1.6.0/azure_ai_evaluation.egg-info → azure_ai_evaluation-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.6.0
+Version: 1.7.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -30,10 +30,11 @@ Requires-Dist: nltk>=3.9.1
 Requires-Dist: azure-storage-blob>=12.10.0
 Requires-Dist: httpx>=0.25.1
 Requires-Dist: pandas<3.0.0,>=2.1.2
-Requires-Dist: openai>=1.73.0
+Requires-Dist: openai>=1.78.0
 Requires-Dist: ruamel.yaml<1.0.0,>=0.17.10
 Requires-Dist: msrest>=0.6.21
 Requires-Dist: Jinja2>=3.1.6
+Requires-Dist: aiohttp>=3.0
 Provides-Extra: redteam
 Requires-Dist: pyrit==0.8.1; extra == "redteam"
 
@@ -115,13 +116,23 @@ result = relevance_evaluator(
     response="The capital of Japan is Tokyo."
 )
 
-# AI assisted safety evaluator
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project Details 
 azure_ai_project = {
     "subscription_id": "<subscription_id>",
     "resource_group_name": "<resource_group_name>",
     "project_name": "<project_name>",
 }
 
+violence_evaluator = ViolenceEvaluator(azure_ai_project)
+result = violence_evaluator(
+    query="What is the capital of France?",
+    response="Paris."
+)
+
+# Option # 2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 violence_evaluator = ViolenceEvaluator(azure_ai_project)
 result = violence_evaluator(
     query="What is the capital of France?",
@@ -272,11 +283,18 @@ with open("simulator_output.jsonl", "w") as f:
 ```python
 from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario
 from azure.identity import DefaultAzureCredential
+
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project 
 azure_ai_project = {
     "subscription_id": <subscription_id>,
     "resource_group_name": <resource_group_name>,
     "project_name": <project_name>
 }
+
+# Option #2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 scenario = AdversarialScenario.ADVERSARIAL_QA
 simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
 
@@ -382,6 +400,11 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.7.0 (2025-05-12)
+
+### Bugs Fixed
+- azure-ai-evaluation failed with module not found [#40992](https://github.com/Azure/azure-sdk-for-python/issues/40992)
+
 ## 1.6.0 (2025-05-07)
 
 ### Features Added

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/README.md

@@ -76,13 +76,23 @@ result = relevance_evaluator(
     response="The capital of Japan is Tokyo."
 )
 
-# AI assisted safety evaluator
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project Details 
 azure_ai_project = {
     "subscription_id": "<subscription_id>",
     "resource_group_name": "<resource_group_name>",
     "project_name": "<project_name>",
 }
 
+violence_evaluator = ViolenceEvaluator(azure_ai_project)
+result = violence_evaluator(
+    query="What is the capital of France?",
+    response="Paris."
+)
+
+# Option # 2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 violence_evaluator = ViolenceEvaluator(azure_ai_project)
 result = violence_evaluator(
     query="What is the capital of France?",
@@ -233,11 +243,18 @@ with open("simulator_output.jsonl", "w") as f:
 ```python
 from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario
 from azure.identity import DefaultAzureCredential
+
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project 
 azure_ai_project = {
     "subscription_id": <subscription_id>,
     "resource_group_name": <resource_group_name>,
     "project_name": <project_name>
 }
+
+# Option #2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 scenario = AdversarialScenario.ADVERSARIAL_QA
 simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
 

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/__init__.py

@@ -31,6 +31,7 @@ from ._evaluators._xpia import IndirectAttackEvaluator
 from ._evaluators._code_vulnerability import CodeVulnerabilityEvaluator
 from ._evaluators._ungrounded_attributes import UngroundedAttributesEvaluator
 from ._evaluators._tool_call_accuracy import ToolCallAccuracyEvaluator
+from ._evaluators._document_retrieval import DocumentRetrievalEvaluator
 from ._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_aoai/aoai_grader.py

@@ -77,7 +77,7 @@ class AzureOpenAIGrader():
            return AzureOpenAI(
                 azure_endpoint=self._model_config["azure_endpoint"],
                 api_key=self._model_config.get("api_key", None), # Default-style access to appease linters.
-                api_version=self._model_config.get("api_version", DEFAULT_AOAI_API_VERSION),
+                api_version=DEFAULT_AOAI_API_VERSION, # Force a known working version
                 azure_deployment=self._model_config.get("azure_deployment", ""),
             )
         from openai import OpenAI

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_aoai/label_grader.py

@@ -4,7 +4,7 @@
 from typing import Any, Dict, Union, List
 
 from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
-from openai.types.eval_create_params import TestingCriterionLabelModel
+from openai.types.graders import LabelModelGrader
 from azure.ai.evaluation._common._experimental import experimental
 
 from .aoai_grader import AzureOpenAIGrader
@@ -55,7 +55,7 @@ class AzureOpenAILabelGrader(AzureOpenAIGrader):
         passing_labels: List[str],
         **kwargs: Any
     ):
-        grader = TestingCriterionLabelModel(
+        grader = LabelModelGrader(
             input=input,
             labels=labels,
             model=model,

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_aoai/string_check_grader.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, Union
 from typing_extensions import Literal
 
 from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
-from openai.types.eval_string_check_grader import EvalStringCheckGrader
+from openai.types.graders import StringCheckGrader
 from azure.ai.evaluation._common._experimental import experimental
 
 from .aoai_grader import AzureOpenAIGrader
@@ -55,7 +55,7 @@ class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
         reference: str,
         **kwargs: Any
     ):
-        grader = EvalStringCheckGrader(
+        grader = StringCheckGrader(
             input=input,
             name=name,
             operation=operation,

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_aoai/text_similarity_grader.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, Union
 from typing_extensions import Literal
 
 from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
-from openai.types.eval_text_similarity_grader import EvalTextSimilarityGrader
+from openai.types.graders import TextSimilarityGrader
 from azure.ai.evaluation._common._experimental import experimental
 
 from .aoai_grader import AzureOpenAIGrader
@@ -77,7 +77,7 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
         name: str,
         **kwargs: Any
     ):
-        grader = EvalTextSimilarityGrader(
+        grader = TextSimilarityGrader(
             evaluation_metric=evaluation_metric,
             input=input,
             pass_threshold=pass_threshold,

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_common/__init__.py

@@ -9,7 +9,7 @@ from . import constants
 from .rai_service import evaluate_with_rai_service
 from .utils import get_harm_severity_level
 from .evaluation_onedp_client import EvaluationServiceOneDPClient
-from .onedp.models import EvaluationUpload, EvaluationResult
+from .onedp.models import EvaluationUpload, EvaluationResult, RedTeamUpload, ResultType
 
 __all__ = [
     "get_harm_severity_level",
@@ -18,4 +18,6 @@ __all__ = [
     "EvaluationServiceOneDPClient",
     "EvaluationResult",
     "EvaluationUpload",
+    "RedTeamUpload",
+    "ResultType",
 ]

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_common/evaluation_onedp_client.py

@@ -7,7 +7,7 @@ from typing import Union, Any, Dict
 from azure.core.credentials import AzureKeyCredential, TokenCredential
 from azure.ai.evaluation._common.onedp import AIProjectClient as RestEvaluationServiceClient
 from azure.ai.evaluation._common.onedp.models import (PendingUploadRequest, PendingUploadType, EvaluationResult,
-                                                      ResultType, AssetCredentialRequest, EvaluationUpload, InputDataset)
+                                                      ResultType, AssetCredentialRequest, EvaluationUpload, InputDataset, RedTeamUpload)
 from azure.storage.blob import ContainerClient
 from .utils import upload
 
@@ -22,7 +22,8 @@ class EvaluationServiceOneDPClient:
             **kwargs,
         )
 
-    def create_evaluation_result(self, *, name: str, path: str, version=1, metrics: Dict[str, int]=None, **kwargs) -> EvaluationResult:
+    def create_evaluation_result(
+            self, *, name: str, path: str, version=1, metrics: Dict[str, int]=None, result_type: ResultType=ResultType.EVALUATION, **kwargs) -> EvaluationResult:
         """Create and upload evaluation results to Azure evaluation service.
 
         This method uploads evaluation results from a local path to Azure Blob Storage
@@ -39,14 +40,16 @@ class EvaluationServiceOneDPClient:
         :param version: The version number for the evaluation results, defaults to 1
         :type version: int, optional
         :param metrics: Metrics to be added to evaluation result
-        :type version: Dict[str, int], optional
+        :type metrics: Dict[str, int], optional
+        :param result_type: Evaluation Result Type to create
+        :type result_type: ResultType, optional
         :param kwargs: Additional keyword arguments to pass to the underlying API calls
         :return: The response from creating the evaluation result version
         :rtype: EvaluationResult
         :raises: Various exceptions from the underlying API calls or upload process
         """
 
-        LOGGER.debug(f"Creating evaluation result for {name} with version {version} from path {path}")
+        LOGGER.debug(f"Creating evaluation result for {name} with version {version} type {result_type} from path {path}")
         start_pending_upload_response = self.rest_client.evaluation_results.start_pending_upload(
             name=name,
             version=version,
@@ -63,7 +66,7 @@ class EvaluationServiceOneDPClient:
         create_version_response = self.rest_client.evaluation_results.create_or_update_version(
             body=EvaluationResult(
                 blob_uri=start_pending_upload_response.blob_reference_for_consumption.blob_uri,
-                result_type=ResultType.EVALUATION,
+                result_type=result_type,
                 name=name,
                 version=version,
                 metrics=metrics,
@@ -115,4 +118,46 @@ class EvaluationServiceOneDPClient:
             **kwargs
         )
 
+        return update_run_response
+
+    def start_red_team_run(self, *, red_team: RedTeamUpload, **kwargs):
+        """Start a new red team run in the Azure evaluation service.
+
+        This method creates a new red team run with the provided configuration details.
+
+        :param red_team: The red team configuration to upload
+        :type red_team: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The created red team run object
+        :rtype: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        upload_run_response = self.rest_client.red_teams.upload_run(
+            redteam=red_team,
+            **kwargs
+        )
+
+        return upload_run_response
+
+    def update_red_team_run(self, *, name: str, red_team: RedTeamUpload, **kwargs):
+        """Update an existing red team run in the Azure evaluation service.
+
+        This method updates a red team run with new information such as status changes,
+        result references, or other metadata.
+
+        :param name: The identifier of the red team run to update
+        :type name: str
+        :param red_team: The updated red team configuration
+        :type red_team: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The updated red team run object
+        :rtype: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        update_run_response = self.rest_client.red_teams.upload_update_run(
+            name=name,
+            redteam=red_team,
+            **kwargs
+        )
+
         return update_run_response

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_common/onedp/operations/_operations.py

@@ -4267,7 +4267,7 @@ class RedTeamsOperations:
         if isinstance(redteam, (IOBase, bytes)):
             _content = redteam
         else:
-            _content = json.dumps(redteam, cls=SdkJSONEncoder, exclude_readonly=True)  # type: ignore
+            _content = json.dumps(redteam, cls=SdkJSONEncoder, exclude_readonly=False)  # type: ignore
 
         _request = build_red_teams_upload_update_run_request(
             name=name,

{azure_ai_evaluation-1.6.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_common/rai_service.py

@@ -629,8 +629,9 @@ async def evaluate_with_rai_service(
     :type data: dict
     :param metric_name: The evaluation metric to use.
     :type metric_name: str
-    :param project_scope: The Azure AI project scope details.
-    :type project_scope: Dict
+    :param project_scope: The Azure AI project, which can either be a string representing the project endpoint 
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name. 
+    :type project_scope: Union[str, AzureAIProject]
     :param credential: The Azure authentication credential.
     :type credential: ~azure.core.credentials.TokenCredential
     :param annotation_task: The annotation task to use.
@@ -777,11 +778,11 @@ async def evaluate_with_rai_service_multimodal(
        :type messages: str
        :param metric_name: The evaluation metric to use.
        :type metric_name: str
-       :param project_scope: The Azure AI project scope details.
-       :type project_scope: Dict
+       :param project_scope: The Azure AI project, which can either be a string representing the project endpoint 
+            or an instance of AzureAIProject. It contains subscription id, resource group, and project name. 
+       :type project_scope: Union[str, AzureAIProject]
        :param credential: The Azure authentication credential.
-       :type credential:
-    ~azure.core.credentials.TokenCredential
+       :type credential: ~azure.core.credentials.TokenCredential
        :return: The parsed annotation result.
        :rtype: List[List[Dict]]
     """