azure-ai-evaluation 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -2,10 +2,11 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from concurrent.futures import as_completed
-from typing import Callable, Dict, List, Union
+from typing import Union
 
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._evaluators._common import MultiEvaluatorBase
 
 from .._coherence import CoherenceEvaluator
 from .._f1_score import F1ScoreEvaluator
@@ -15,7 +16,7 @@ from .._relevance import RelevanceEvaluator
 from .._similarity import SimilarityEvaluator
 
 
-class QAEvaluator:
+class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
     """
     Initialize a question-answer evaluator configured for a specific Azure OpenAI model.
 
@@ -46,9 +47,7 @@ class QAEvaluator:
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     def __init__(self, model_config, **kwargs):
-        self._parallel = kwargs.pop("_parallel", False)
-
-        self._evaluators: List[Union[Callable[..., Dict[str, Union[str, float]]], Callable[..., Dict[str, float]]]] = [
+        evaluators = [
             GroundednessEvaluator(model_config),
             RelevanceEvaluator(model_config),
             CoherenceEvaluator(model_config),
@@ -56,8 +55,31 @@ class QAEvaluator:
             SimilarityEvaluator(model_config),
             F1ScoreEvaluator(),
         ]
+        super().__init__(evaluators=evaluators, **kwargs)
+
+    @overload  # type: ignore
+    def __call__(self, *, query: str, response: str, context: str, ground_truth: str):
+        """
+        Evaluates question-answering scenario.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword context: The context to be evaluated.
+        :paramtype context: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The scores for QA scenario.
+        :rtype: Dict[str, Union[str, float]]
+        """
 
-    def __call__(self, *, query: str, response: str, context: str, ground_truth: str, **kwargs):
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
         """
         Evaluates question-answering scenario.
 
@@ -72,22 +94,5 @@ class QAEvaluator:
         :return: The scores for QA scenario.
         :rtype: Dict[str, Union[str, float]]
         """
-        results: Dict[str, Union[str, float]] = {}
-        if self._parallel:
-            with ThreadPoolExecutor() as executor:
-                futures = {
-                    executor.submit(
-                        evaluator, query=query, response=response, context=context, ground_truth=ground_truth, **kwargs
-                    ): evaluator
-                    for evaluator in self._evaluators
-                }
-
-                # Collect results as they complete
-                for future in as_completed(futures):
-                    results.update(future.result())
-        else:
-            for evaluator in self._evaluators:
-                result = evaluator(query=query, response=response, context=context, ground_truth=ground_truth, **kwargs)
-                results.update(result)
-
-        return results
+
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_rouge/_rouge.py

@@ -3,9 +3,11 @@
 # ---------------------------------------------------------
 from enum import Enum
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing import Dict
+from typing_extensions import overload, override
 
 from azure.ai.evaluation._vendor.rouge_score import rouge_scorer
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
 
 
 class RougeType(Enum):
@@ -32,21 +34,7 @@ class RougeType(Enum):
     """Overlap of L-grams (L consecutive words) between generated and reference text."""
 
 
-class _AsyncRougeScoreEvaluator:
-    def __init__(self, rouge_type: RougeType):
-        self._rouge_type = rouge_type
-
-    async def __call__(self, *, ground_truth: str, response: str, **kwargs):
-        scorer = rouge_scorer.RougeScorer(rouge_types=[self._rouge_type.value])
-        metrics = scorer.score(ground_truth, response)[self._rouge_type.value]
-        return {
-            "rouge_precision": metrics.precision,
-            "rouge_recall": metrics.recall,
-            "rouge_f1_score": metrics.fmeasure,
-        }
-
-
-class RougeScoreEvaluator:
+class RougeScoreEvaluator(EvaluatorBase):
     """
     Calculates the ROUGE score for a given response and ground truth.
 
@@ -76,10 +64,32 @@ class RougeScoreEvaluator:
     id = "azureml://registries/azureml/models/Rouge-Score-Evaluator/versions/3"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
+    @override
     def __init__(self, rouge_type: RougeType):
-        self._async_evaluator = _AsyncRougeScoreEvaluator(rouge_type)
+        self._rouge_type = rouge_type
+        super().__init__()
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, float]:
+        """Produce a rouge score evaluation result.
 
-    def __call__(self, *, ground_truth: str, response: str, **kwargs):
+        :param eval_input: The input to the evaluation function.
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        ground_truth = eval_input["ground_truth"]
+        response = eval_input["response"]
+        scorer = rouge_scorer.RougeScorer(rouge_types=[self._rouge_type.value])
+        metrics = scorer.score(ground_truth, response)[self._rouge_type.value]
+        return {
+            "rouge_precision": metrics.precision,
+            "rouge_recall": metrics.recall,
+            "rouge_f1_score": metrics.fmeasure,
+        }
+
+    @overload  # type: ignore
+    def __call__(self, *, ground_truth: str, response: str) -> Dict[str, float]:
         """
         Evaluate the ROUGE score between the response and the ground truth.
 
@@ -90,9 +100,20 @@ class RougeScoreEvaluator:
         :return: The ROUGE score.
         :rtype: Dict[str, float]
         """
-        return async_run_allowing_running_loop(
-            self._async_evaluator, ground_truth=ground_truth, response=response, **kwargs
-        )
 
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate route score.
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be compared against.
+        :paramtype ground_truth: str
+        :return: The ROUGE score.
+        :rtype: Dict[str, float]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -2,83 +2,15 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-import math
 import os
-import re
+from typing import Dict
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from promptflow.core import AsyncPrompty
+from typing_extensions import overload, override
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 
-from ..._common.utils import construct_prompty_model_config, validate_model_config
 
-try:
-    from ..._user_agent import USER_AGENT
-except ImportError:
-    USER_AGENT = "None"
-
-
-class _AsyncSimilarityEvaluator:
-    # Constants must be defined within eval's directory to be save/loadable
-    _PROMPTY_FILE = "similarity.prompty"
-    _LLM_CALL_TIMEOUT = 600
-    _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
-
-    def __init__(self, model_config: dict):
-        prompty_model_config = construct_prompty_model_config(
-            validate_model_config(model_config),
-            self._DEFAULT_OPEN_API_VERSION,
-            USER_AGENT,
-        )
-
-        current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
-
-    async def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
-        """
-        Evaluate similarity.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :keyword ground_truth: The ground truth to be evaluated.
-        :paramtype ground_truth: str
-        :return: The similarity score.
-        :rtype: Dict[str, float]
-        """
-        # Validate input parameters
-        query = str(query or "")
-        response = str(response or "")
-        ground_truth = str(ground_truth or "")
-
-        if not (query.strip() and response.strip() and ground_truth.strip()):
-            msg = "'query', 'response' and 'ground_truth' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.SIMILARITY_EVALUATOR,
-            )
-
-        # Run the evaluation flow
-        llm_output = await self._flow(
-            query=query, response=response, ground_truth=ground_truth, timeout=self._LLM_CALL_TIMEOUT, **kwargs
-        )
-
-        score = math.nan
-        if llm_output:
-            match = re.search(r"\d", llm_output)
-            if match:
-                score = float(match.group())
-
-        return {"similarity": float(score), "gpt_similarity": float(score)}
-
-
-class SimilarityEvaluator:
+class SimilarityEvaluator(PromptyEvaluatorBase):
     """
     Evaluates similarity score for a given query, response, and ground truth.
 
@@ -113,13 +45,27 @@ class SimilarityEvaluator:
         however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
-    id = "azureml://registries/azureml/models/Similarity-Evaluator/versions/3"
+    # Constants must be defined within eval's directory to be save/loadable
+
+    _PROMPTY_FILE = "similarity.prompty"
+    _RESULT_KEY = "similarity"
+
+    id = "similarity"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
+    @override
     def __init__(self, model_config):
-        self._async_evaluator = _AsyncSimilarityEvaluator(model_config)
-
-    def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    # Ignoring a mypy error about having only 1 overload function.
+    # We want to use the overload style for all evals, even single-inputs. This is both to make
+    # refactoring to multi-input styles easier, stylistic consistency consistency across evals,
+    # and due to the fact that non-overloaded syntax now causes various parsing issues that
+    # we don't want to deal with.
+    @overload  # type: ignore
+    def __call__(self, *, query: str, response: str, ground_truth: str) -> Dict[str, float]:
         """
         Evaluate similarity.
 
@@ -132,9 +78,23 @@ class SimilarityEvaluator:
         :return: The similarity score.
         :rtype: Dict[str, float]
         """
-        return async_run_allowing_running_loop(
-            self._async_evaluator, query=query, response=response, ground_truth=ground_truth, **kwargs
-        )
 
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate similarity.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The similarity score.
+        :rtype: Dict[str, float]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-VERSION = "1.1.0"
+VERSION = "1.2.0"

azure/ai/evaluation/simulator/_simulator.py

@@ -157,7 +157,7 @@ class Simulator:
                 f"You have specified 'num_queries' < len('tasks') ({num_queries} < {len(tasks)}). "
                 f"Only the first {num_queries} lines of the specified tasks will be simulated."
             )
-        num_queries = min(num_queries, len(tasks))
+
         max_conversation_turns *= 2  # account for both user and assistant turns
 
         prompty_model_config = self.model_config
@@ -586,7 +586,10 @@ class Simulator:
         for i, query_response_pair in enumerate(query_responses):
             query = query_response_pair["q"]
             response = query_response_pair["r"]
-            task = tasks[i]
+            try:
+                task = tasks[i]
+            except IndexError:
+                task = None
 
             conversation = await self._complete_conversation(
                 conversation_starter=query,
@@ -621,7 +624,7 @@ class Simulator:
         *,
         conversation_starter: str,
         max_conversation_turns: int,
-        task: str,
+        task: Optional[str],
         user_simulator_prompty: Optional[str],
         user_simulator_prompty_options: Dict[str, Any],
         target: Callable,
@@ -659,16 +662,21 @@ class Simulator:
                 user_simulator_prompty_options=user_simulator_prompty_options,
             )
             if len(conversation_history) == 0:
-                conversation_starter_from_simulated_user = await user_flow(
-                    task=task,
-                    conversation_history=[
-                        {
-                            "role": "assistant",
-                            "content": conversation_starter,
-                        }
-                    ],
-                    action="rewrite the assistant's message as you have to accomplish the task by asking the right questions. Make sure the original question is not lost in your rewrite.",
-                )
+                if task:
+                    conversation_starter_from_simulated_user = await user_flow(
+                        task=task,
+                        conversation_history=[
+                            {
+                                "role": "assistant",
+                                "content": conversation_starter,
+                            }
+                        ],
+                        action="rewrite the assistant's message as you have to accomplish the task by asking the right questions. Make sure the original question is not lost in your rewrite.",
+                    )
+                else:
+                    conversation_starter_from_simulated_user = {
+                        "content": conversation_starter,
+                    }
             else:
                 conversation_starter_from_simulated_user = await user_flow(
                     task=task,

{azure_ai_evaluation-1.1.0.dist-info → azure_ai_evaluation-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.1.0
+Version: 1.2.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -13,17 +13,16 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: NOTICE.txt
-Requires-Dist: promptflow-devkit >=1.15.0
-Requires-Dist: promptflow-core >=1.15.0
+Requires-Dist: promptflow-devkit >=1.17.1
+Requires-Dist: promptflow-core >=1.17.1
 Requires-Dist: pyjwt >=2.8.0
 Requires-Dist: azure-identity >=1.16.0
 Requires-Dist: azure-core >=1.30.2
@@ -54,7 +53,7 @@ Azure AI SDK provides following to evaluate Generative AI Applications:
 
 ### Prerequisites
 
-- Python 3.8 or later is required to use this package.
+- Python 3.9 or later is required to use this package.
 - [Optional] You must have [Azure AI Project][ai_project] or [Azure Open AI][azure_openai] to use AI-assisted evaluators
 
 ### Install the package
@@ -378,11 +377,76 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
-## 1.1.0 (2024-12-12)
+## 1.2.0 (2025-01-27)
+
+### Features Added
+- CSV files are now supported as data file inputs with `evaluate()` API. The CSV file should have a header row with column names that match the `data` and `target` fields in the `evaluate()` method and the filename should be passed as the `data` parameter. Column name 'Conversation' in CSV file is not fully supported yet.  
+
+### Breaking Changes
+- `ViolenceMultimodalEvaluator`, `SexualMultimodalEvaluator`, `SelfHarmMultimodalEvaluator`, `HateUnfairnessMultimodalEvaluator` and `ProtectedMaterialMultimodalEvaluator` will be removed in next release. 
 
 ### Bugs Fixed
 - Removed `[remote]` extra. This is no longer needed when tracking results in Azure AI Studio.
 - Fixed `AttributeError: 'NoneType' object has no attribute 'get'` while running simulator with 1000+ results
+- Fixed the non adversarial simulator to run in task-free mode
+- Content safety evaluators (violence, self harm, sexual, hate/unfairness) return the maximum result as the
+  main score when aggregating per-turn evaluations from a conversation into an overall
+  evaluation score. Other conversation-capable evaluators still default to a mean for aggregation.
+- Fixed bug in non adversarial simulator sample where `tasks` undefined
+
+### Other Changes
+- Changed minimum required python version to use this package from 3.8 to 3.9
+- Stop dependency on the local promptflow service. No promptflow service will automatically start when running evaluation.
+- Evaluators internally allow for custom aggregation. However, this causes serialization failures if evaluated while the
+  environment variable `AI_EVALS_BATCH_USE_ASYNC` is set to false.
+
+## 1.1.0 (2024-12-12)
+
+### Features Added
+- Added image support in `ContentSafetyEvaluator`, `ViolenceEvaluator`, `SexualEvaluator`, `SelfHarmEvaluator`, `HateUnfairnessEvaluator` and `ProtectedMaterialEvaluator`. Provide image URLs or base64 encoded images in `conversation` input for image evaluation. See below for an example:
+
+```python
+evaluator = ContentSafetyEvaluator(credential=azure_cred, azure_ai_project=project_scope)
+conversation = {
+    "messages": [
+        {
+            "role": "system",
+            "content": [
+                {"type": "text", "text": "You are an AI assistant that understands images."}
+            ],
+        },
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Can you describe this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "https://cdn.britannica.com/68/178268-050-5B4E7FB6/Tom-Cruise-2013.jpg"
+                    },
+                },
+            ],
+        },
+        {
+            "role": "assistant",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "The image shows a man with short brown hair smiling, wearing a dark-colored shirt.",
+                }
+            ],
+        },
+    ]
+}
+print("Calling Content Safety Evaluator for multi-modal")
+score = evaluator(conversation=conversation)
+```
+
+- Please switch to generic evaluators for image evaluations as mentioned above. `ContentSafetyMultimodalEvaluator`, `ContentSafetyMultimodalEvaluatorBase`, `ViolenceMultimodalEvaluator`, `SexualMultimodalEvaluator`, `SelfHarmMultimodalEvaluator`, `HateUnfairnessMultimodalEvaluator` and `ProtectedMaterialMultimodalEvaluator` will be deprecated in the next release.
+
+### Bugs Fixed
+- Removed `[remote]` extra. This is no longer needed when tracking results in Azure AI Foundry portal.
+- Fixed `AttributeError: 'NoneType' object has no attribute 'get'` while running simulator with 1000+ results
 
 ## 1.0.1 (2024-11-15)
 

{azure_ai_evaluation-1.1.0.dist-info → azure_ai_evaluation-1.2.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
 azure/ai/evaluation/__init__.py,sha256=MFxJRoKfSsP_Qlfq0FwynxNf4csNAfTYPQX7jdXc9RU,2757
-azure/ai/evaluation/_constants.py,sha256=d41rQb-w2GmCMHOwiyDD1ieJB1U6JyPPl6APZSJbKzg,2036
+azure/ai/evaluation/_constants.py,sha256=a7eCgdG6Kid79ebAMu0rPNH7foRF5Aii0K5YQI6cNPc,2765
 azure/ai/evaluation/_exceptions.py,sha256=MsTbgsPGYPzIxs7MyLKzSeiVKEoCxYkVjONzNfv2tXA,5162
 azure/ai/evaluation/_http_utils.py,sha256=1bGce6pKAL-vmaUGRPxVX7DVO05XVQ8YPIwIQ3q7mfA,17221
 azure/ai/evaluation/_model_configurations.py,sha256=MNN6cQlz7P9vNfHmfEKsUcly3j1FEOEFsA8WV7GPuKQ,4043
 azure/ai/evaluation/_user_agent.py,sha256=O2y-QPBAcw7w7qQ6M2aRPC3Vy3TKd789u5lcs2yuFaI,290
-azure/ai/evaluation/_version.py,sha256=LzMvSuUB6pmU-LfCPzoYuCoTF0BAqE7ljPjk6r8YaMw,199
+azure/ai/evaluation/_version.py,sha256=aIrrVLGzX0UDxMjpkbe8HTOCqRr6Y9R8tC8XGAOocbE,199
 azure/ai/evaluation/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/_azure/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
-azure/ai/evaluation/_azure/_clients.py,sha256=1mFRSxt-Ld5UBn-m3DJkKc-VPP9CbXQHrqLNdLs9RF0,8201
-azure/ai/evaluation/_azure/_models.py,sha256=tKpxjb5Ou476UasjXPCiuvsxTjLTrnoVnSXy5Bfa51M,12483
+azure/ai/evaluation/_azure/_clients.py,sha256=N1V-LyQkItPuoKl0aieypFPdGSRSld9lQqH1x-n3L7U,9119
+azure/ai/evaluation/_azure/_models.py,sha256=7EHmrCAFOscYY3H90wfmdPPdy0sqnOlgGFvraq_L-2Q,12517
 azure/ai/evaluation/_azure/_token_manager.py,sha256=1NZHwgEc9BMXWPz5Ear_J5-oYjouD77crLHHqNLldEw,5193
 azure/ai/evaluation/_common/__init__.py,sha256=LHTkf6dMLLxikrGNgbUuREBVQcs4ORHR6Eryo4bm9M8,586
 azure/ai/evaluation/_common/_experimental.py,sha256=GVtSn9r1CeR_yEa578dJVNDJ3P24eqe8WYdH7llbiQY,5694
@@ -17,10 +17,10 @@ azure/ai/evaluation/_common/math.py,sha256=d4bwWe35_RWDIZNcbV1BTBbHNx2QHQ4-I3Eof
 azure/ai/evaluation/_common/rai_service.py,sha256=DcakzdOour9qNdMXU-8UFfvLb12oexAoiJXG8XFTRBs,26462
 azure/ai/evaluation/_common/utils.py,sha256=MQIZs95gH5je1L-S3twa_WQi071zRu0Dv54lzCI7ZgU,17642
 azure/ai/evaluation/_evaluate/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
-azure/ai/evaluation/_evaluate/_eval_run.py,sha256=het3cxjK4J-_hT19dT5a0mC2Cdnk93gM3ONQMJb9bxQ,21923
-azure/ai/evaluation/_evaluate/_evaluate.py,sha256=P5aL70eUBKZT9CVRM9RVSfD0DkuljQyc5ECte37Ycmo,36225
-azure/ai/evaluation/_evaluate/_utils.py,sha256=S4LUUDUBo9JNA41ojSezMC-PZzkWcihhhNdyZwZrpr0,13428
-azure/ai/evaluation/_evaluate/_batch_run/__init__.py,sha256=G8McpeLxAS_gFhNShX52_YWvE-arhJn-bVpAfzjWG3Q,427
+azure/ai/evaluation/_evaluate/_eval_run.py,sha256=QBtNBwUxqxsIVmKPU-_H8MDFkF4s_bW7arQYXAniRpo,21965
+azure/ai/evaluation/_evaluate/_evaluate.py,sha256=hj1HG9WCjbvAk8iB0MwnVoV-ceQYKKecfyVTlbc3y4A,38934
+azure/ai/evaluation/_evaluate/_utils.py,sha256=sKj_4iN-QjrRlEkiZwA9UNiWozS4LgJcUZ6AWdHrTY4,14231
+azure/ai/evaluation/_evaluate/_batch_run/__init__.py,sha256=Z-TQdSxKTn0bjsF0YosIJMbQFQHDUv_b9zCBu1TeogQ,474
 azure/ai/evaluation/_evaluate/_batch_run/code_client.py,sha256=XQLaXfswF6ReHLpQthHLuLLa65Pts8uawGp7kRqmMDs,8260
 azure/ai/evaluation/_evaluate/_batch_run/eval_run_context.py,sha256=p3Bsg_shGs5RXvysOlvo0CQb4Te5herSvX1OP6ylFUQ,3543
 azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py,sha256=T_QRHScDMBM4O6ejkkKdBmHPjH2NOF6owW48aVUYF6k,3775
@@ -28,35 +28,37 @@ azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py,sha256=SMos3bVmD7
 azure/ai/evaluation/_evaluate/_telemetry/__init__.py,sha256=fhLqE41qxdjfBOGi23cpk6QgUe-s1Fw2xhAAUjNESF0,7045
 azure/ai/evaluation/_evaluators/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_evaluators/_bleu/__init__.py,sha256=quKKO0kvOSkky5hcoNBvgBuMeeVRFCE9GSv70mAdGP4,260
-azure/ai/evaluation/_evaluators/_bleu/_bleu.py,sha256=iT20SMmEtOnh7RWs55dFfAlKXNkNceXkCUbVyqv6aQ0,2776
+azure/ai/evaluation/_evaluators/_bleu/_bleu.py,sha256=Px3KxTyNIuxy-4U3SE4XJHCd4r144JeVrlIGMdbaqBk,3425
 azure/ai/evaluation/_evaluators/_coherence/__init__.py,sha256=GRqcSCQse02Spyki0UsRNWMIXiea2lLtPPXNGvkJzQ0,258
 azure/ai/evaluation/_evaluators/_coherence/_coherence.py,sha256=uG9hX2XWkMREKfMAWRoosjicoI4Lg3ptR3UcLEgKd0c,4643
 azure/ai/evaluation/_evaluators/_coherence/coherence.prompty,sha256=ANvh9mDFW7KMejrgdWqBLjj4SIqEO5WW9gg5pE0RLJk,6798
-azure/ai/evaluation/_evaluators/_common/__init__.py,sha256=_hPqTkAla_O6s4ebVtTaBrVLEW3KSdDz66WwxjK50cI,423
-azure/ai/evaluation/_evaluators/_common/_base_eval.py,sha256=n6qldJr8d8H0DnS7IwkQPH9Ep9PdZnVeVtSxQiunADc,19424
+azure/ai/evaluation/_evaluators/_common/__init__.py,sha256=xAymP_CZy4aPzWplMdXgQUQVDIUEMI-0nbgdm_umFYY,498
+azure/ai/evaluation/_evaluators/_common/_base_eval.py,sha256=LtlTsA0TUywMXEYj3mVggv43G0TfKnNkDG6ZgA_dWa4,23328
+azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py,sha256=6WFmFMsobJjju3wzVFKx7EjuHqbBV9YXzlhbwu5vzio,2509
 azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py,sha256=hvJD7jR2ESePkRPN17ytoFhFiS0iTotOfeqmTwG2IMs,4531
-azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py,sha256=czyn1MfaxOmrvvFgdeblf6FaauKgKolgPFsP5f7K29w,7331
+azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py,sha256=1ZwWu2dwN2y6bVOU3Ws4VvJoMVQ80tzYNutSEfmpYmg,7830
+azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py,sha256=gjDBjRxJKwaHbshWH0j2idjlzfzNMnT9a9RL0fQiKeM,2129
 azure/ai/evaluation/_evaluators/_content_safety/__init__.py,sha256=PEYMIybfP64f7byhuTaiq4RiqsYbjqejpW1JsJIG1jA,556
-azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py,sha256=CIGfBLNOTVXrlF5HIc2UpuDDG5BfzjD7ubJ23CbvobQ,6341
-azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py,sha256=sjw8FfwxC1f0K1J4TkeA8wkfq88aebiNbaKzS-8DWzk,5919
-azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py,sha256=0zaB-JKm8FU6yoxD1nqoYvxp3gvjuZfcQjb-xhSHoQ0,5156
-azure/ai/evaluation/_evaluators/_content_safety/_sexual.py,sha256=q9bEMu6Dp1wxDlH3h2iTayrWv4ux-izLB0kGkxrgEhM,5396
-azure/ai/evaluation/_evaluators/_content_safety/_violence.py,sha256=W2QwPuWOc3nkLvvWOAhCrpLRDAAo-xG1SvlDhrshzUc,5467
+azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py,sha256=XKnIlxbzkB65cRXXcOGsv0W37QKxo_jsHbR3gijMZ78,4654
+azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py,sha256=LcnJuePAwByoaXAQ5CVKnkO2IVCCRdVnFTUYbOyQTbs,6043
+azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py,sha256=Nl_hTIRXO_UdAjUatPzbCTextsngkgib1ECzsmDHUvE,5280
+azure/ai/evaluation/_evaluators/_content_safety/_sexual.py,sha256=z8bDdkZHW09D6NTY9mlK2abNMOy9hRAJwwTQs5vjvAc,5520
+azure/ai/evaluation/_evaluators/_content_safety/_violence.py,sha256=z9dM3GOBSIw_WoEELPHRE3DSK3ol7MZbDkFJyuYENVk,5591
 azure/ai/evaluation/_evaluators/_eci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/_evaluators/_eci/_eci.py,sha256=a36sLZPHKi3YAdl0JvpL6vboZMqgGjnmz0qZ-o8vcWY,2934
 azure/ai/evaluation/_evaluators/_f1_score/__init__.py,sha256=aEVbO7iMoF20obdpLQKcKm69Yyu3mYnblKELLqu8OGI,260
-azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py,sha256=YtPEG1ZT0jAPvEnOpD2Eaojm-8zS61bxOr3US6vvgqc,5779
+azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py,sha256=nDUAz-vmkIR0Sj7JfMm3mVHfE3XnKrZnTzOUa6QKskk,5399
 azure/ai/evaluation/_evaluators/_fluency/__init__.py,sha256=EEJw39xRa0bOAA1rELTTKXQu2s60n_7CZQRD0Gu2QVw,259
 azure/ai/evaluation/_evaluators/_fluency/_fluency.py,sha256=mHQCismdL4cCeANcqWrDHCiVgr4UAWj0yIYJXt2pFDA,4399
 azure/ai/evaluation/_evaluators/_fluency/fluency.prompty,sha256=n9v0W9eYwgIO-JSsLTSKEM_ApJuxxuKWQpNblrTEkFY,4861
 azure/ai/evaluation/_evaluators/_gleu/__init__.py,sha256=Ae2EvQ7gqiYAoNO3LwGIhdAAjJPJDfT85rQGKrRrmbA,260
-azure/ai/evaluation/_evaluators/_gleu/_gleu.py,sha256=RaY_RZ5A3sMx4yE6uCyjvchB8rRoMvIv0JYYyMBXFM8,2696
+azure/ai/evaluation/_evaluators/_gleu/_gleu.py,sha256=E_HeUuDAW2pPhsbaWLHMMxqgUxPOgBv2Bnr_Z9M6AAs,3359
 azure/ai/evaluation/_evaluators/_groundedness/__init__.py,sha256=UYNJUeRvBwcSVFyZpdsf29un5eyaDzYoo3QvC1gvlLg,274
 azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py,sha256=Zil5S7BXaVvW2wBUlsF3oGzZLOYrvSzGAY4TqKfFUX8,6876
 azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty,sha256=v7TOm75DyW_1gOU6gSiZoPcRnHcJ65DrzR2cL_ucWDY,5814
 azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty,sha256=8kNShdfxQvkII7GnqjmdqQ5TNelA2B6cjnqWZk8FFe4,5296
 azure/ai/evaluation/_evaluators/_meteor/__init__.py,sha256=209na3pPsdmcuYpYHUYtqQybCpc3yZkc93HnRdicSlI,266
-azure/ai/evaluation/_evaluators/_meteor/_meteor.py,sha256=UPNvWpNkMlx8NmOPuSkcXF1DA_daDdrRArhJAbbTQkc,3767
+azure/ai/evaluation/_evaluators/_meteor/_meteor.py,sha256=OpugAjIgcTcNQ6g6Rks_8GVhcRiH524PbmBKH3bTefs,4369
 azure/ai/evaluation/_evaluators/_multimodal/__init__.py,sha256=tPvsY0nv8T3VtiiAwJM6wT5A9FhKP2XXwUlCH994xl4,906
 azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py,sha256=x0l6eLQhxVP85jEyGfFCl27C2okMgD0S3aJ_qrgB3Q8,5219
 azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py,sha256=X2IVw0YvymDD3e4Vx-TfjqgqtYiAKVhUumjBowCpOmA,2441
@@ -68,7 +70,7 @@ azure/ai/evaluation/_evaluators/_multimodal/_violence.py,sha256=t1h3bY6N7SwlSgP_
 azure/ai/evaluation/_evaluators/_protected_material/__init__.py,sha256=eRAQIU9diVXfO5bp6aLWxZoYUvOsrDIfy1gnDOeNTiI,109
 azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py,sha256=IABs1YMBZdIi1u57dPi-aQpSiPWIGxEZ4hyt97jvdNA,4604
 azure/ai/evaluation/_evaluators/_qa/__init__.py,sha256=bcXfT--C0hjym2haqd1B2-u9bDciyM0ThOFtU1Q69sk,244
-azure/ai/evaluation/_evaluators/_qa/_qa.py,sha256=kLkXwkmrXqgfBu7MJwEYAobeqGh4b4zE7cjIkD_1iwA,3854
+azure/ai/evaluation/_evaluators/_qa/_qa.py,sha256=HG4JiSt5S20D70LmzW8t24qWg5_uiIKwIxjJ13ygfDo,3670
 azure/ai/evaluation/_evaluators/_relevance/__init__.py,sha256=JlxytW32Nl8pbE-fI3GRpfgVuY9EG6zxIAn5VZGSwyc,265
 azure/ai/evaluation/_evaluators/_relevance/_relevance.py,sha256=S1J5BR1-ZyCLQOTbdAHLDzzY1ccVnPyy9uVUlivmCx0,5287
 azure/ai/evaluation/_evaluators/_relevance/relevance.prompty,sha256=VHKzVlC2Cv1xuholgIGmerPspspAI0t6IgJ2cxOuYDE,4811
@@ -76,11 +78,11 @@ azure/ai/evaluation/_evaluators/_retrieval/__init__.py,sha256=kMu47ZyTZ7f-4Yh6H3
 azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py,sha256=fmd8zNOVSGQGT5icSAI6PwgnS7kKz_ZMKMnxKIchYl8,5085
 azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty,sha256=_YVoO4Gt_WD42bUcj5n6BDW0dMUqNf0yF3Nj5XMOX2c,16490
 azure/ai/evaluation/_evaluators/_rouge/__init__.py,sha256=kusCDaYcXogDugGefRP8MQSn9xv107oDbrMCqZ6K4GA,291
-azure/ai/evaluation/_evaluators/_rouge/_rouge.py,sha256=SV5rESLVARQqh1n0Pf6EMvJoJH3A0nNKM_U33q1LQoE,4026
+azure/ai/evaluation/_evaluators/_rouge/_rouge.py,sha256=xTsAF9em2QpWcgCHOmDCEcuRwrob5kPUUpkgul9E5jA,4642
 azure/ai/evaluation/_evaluators/_service_groundedness/__init__.py,sha256=0DODUGTOgaYyFbO9_zxuwifixDL3SIm3EkwP1sdwn6M,288
 azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py,sha256=GPvufAgTnoQ2HYs6Xnnpmh23n5E3XxnUV0NGuwjDyU0,6648
 azure/ai/evaluation/_evaluators/_similarity/__init__.py,sha256=V2Mspog99_WBltxTkRHG5NpN5s9XoiTSN4I8POWEkLA,268
-azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=UVBIa1xIlOIJtPctCu-UCOWvXzE4ysaK_XFdokajCuA,5669
+azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=AeqJ_OJUAsdu9Cac4OLVPF2zbrBmpXD7_5oOs_cxKsk,4244
 azure/ai/evaluation/_evaluators/_similarity/similarity.prompty,sha256=eoludASychZoGL625bFCaZai-OY7DIAg90ZLax_o4XE,4594
 azure/ai/evaluation/_evaluators/_xpia/__init__.py,sha256=VMEL8WrpJQeh4sQiOLzP7hRFPnjzsvwfvTzaGCVJPCM,88
 azure/ai/evaluation/_evaluators/_xpia/xpia.py,sha256=Nv14lU7jN0yXKbHgHRXMHEy6pn1rXmesBOYI2Ge9ewk,5849
@@ -96,7 +98,7 @@ azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=FPZ3OdpGuwCHDVoOZ
 azure/ai/evaluation/simulator/_constants.py,sha256=nCL7_1BnYh6k0XvxudxsDVMbiG9MMEvYw5wO9FZHHZ8,857
 azure/ai/evaluation/simulator/_direct_attack_simulator.py,sha256=FTtWf655dHJF5FLJi0xGSBgIlGWNiVWyqaLDJSud9XA,10199
 azure/ai/evaluation/simulator/_indirect_attack_simulator.py,sha256=nweIU_AkUIR50qLQpjmljf_OkpsCPth2Ebf4vusygCA,10226
-azure/ai/evaluation/simulator/_simulator.py,sha256=pWxVfy9ll6gmOyGEk6Ie7Y48X21wJ5DebqY8Re0SIOk,36213
+azure/ai/evaluation/simulator/_simulator.py,sha256=LBzez7qvObpVjTwmlGS_PfhDLo8pRknh5epra2yo9X8,36484
 azure/ai/evaluation/simulator/_tracing.py,sha256=frZ4-usrzINast9F4-ONRzEGGox71y8bYw0UHNufL1Y,3069
 azure/ai/evaluation/simulator/_utils.py,sha256=16NltlywpbMtoFtULwTKqeURguIS1kSKSo3g8uKV8TA,5181
 azure/ai/evaluation/simulator/_conversation/__init__.py,sha256=s8djzJ58_-CiIA8xHB-SbgeZaq1F7ftrc3qJbpUpUdg,17853
@@ -116,8 +118,8 @@ azure/ai/evaluation/simulator/_model_tools/models.py,sha256=bfVm0PV3vfH_8DkdmTMZ
 azure/ai/evaluation/simulator/_prompty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/simulator/_prompty/task_query_response.prompty,sha256=2BzSqDDYilDushvR56vMRDmqFIaIYAewdUlUZg_elMg,2182
 azure/ai/evaluation/simulator/_prompty/task_simulate.prompty,sha256=NE6lH4bfmibgMn4NgJtm9_l3PMoHSFrfjjosDJEKM0g,939
-azure_ai_evaluation-1.1.0.dist-info/METADATA,sha256=zusuZTIcO7487bWQK5V7XZ-Pbqugm7HFzvcCV5yfTmk,28751
-azure_ai_evaluation-1.1.0.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
-azure_ai_evaluation-1.1.0.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
-azure_ai_evaluation-1.1.0.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
-azure_ai_evaluation-1.1.0.dist-info/RECORD,,
+azure_ai_evaluation-1.2.0.dist-info/METADATA,sha256=NM0mPj138_k-6vWuKICoqUBtDq-TaWGXOeaCLpyT2IU,32156
+azure_ai_evaluation-1.2.0.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
+azure_ai_evaluation-1.2.0.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
+azure_ai_evaluation-1.2.0.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
+azure_ai_evaluation-1.2.0.dist-info/RECORD,,