azure-ai-evaluation 1.5.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py

@@ -37,13 +37,26 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
+    
     .. admonition:: Example:
+    
         .. literalinclude:: ../samples/evaluation_samples_evaluate.py
             :start-after: [START completeness_evaluator]
             :end-before: [END completeness_evaluator]
             :language: python
             :dedent: 8
             :caption: Initialize and call a CompletenessEvaluator with a response and groundtruth.
+    
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START completeness_evaluator]
+            :end-before: [END completeness_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call CompletenessEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     """
 
     # Constants must be defined within eval's directory to be save/loadable
@@ -60,11 +73,16 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold: Optional[float] = _DEFAULT_COMPLETENESS_THRESHOLD):
+    def __init__(self, model_config, *,
+                 threshold: Optional[float] = _DEFAULT_COMPLETENESS_THRESHOLD,
+                 **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self.threshold = threshold
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        super().__init__(model_config=model_config,
+                         prompty_file=prompty_path,
+                         result_key=self._RESULT_KEY,
+                         **kwargs)
 
     @overload
     def __call__(

azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty

@@ -22,65 +22,51 @@ inputs:
 ---
 system:
 # Instruction
-## Context
-### You are an expert in evaluating the quality of an answer from an intelligent system based on provided definitions and data. Your goal will involve answering the questions below using the information provided.
-- **Definition**: You are given a definition of the response quality that is being evaluated to help guide your Score.
-- **Data**: Your input data include a response and its ground truth.
-- **Questions**: To complete your evaluation you will be asked to evaluate the Data in different ways.
-
+## Goal
+### You are an expert in evaluating the quality of a Response from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include Response and Ground Truth.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
 
+user:
 # Definition
+**Completeness** refers to how accurately and thoroughly a response represents the information provided in the ground truth. It considers both the inclusion of all relevant statements and the correctness of those statements. Each statement in the ground truth should be evaluated individually to determine if it is accurately reflected in the response without missing any key information. The scale ranges from 1 to 5, with higher numbers indicating greater completeness.
 
-**Level 1: Fully incomplete**
-
-**Definition:**
-A response is considered fully incomplete if it does not contain any the necessary and relevant information with respect to the ground truth. In other words, it completely misses all the information - especially claims and statements - established in the ground truth. 
+# Ratings
+## [Completeness: 1] (Fully Incomplete)
+**Definition:** A response that does not contain any of the necessary and relevant information with respect to the ground truth. It completely misses all the information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot cannot cure cancer. Stay healthy requires sleeping exactly 8 hours a day. A few hours of exercise per week will have little benefits for physical and mental health. Physical and mental health benefits are separate topics. Scientists have not studied any of them."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Staying healthy requires proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
-
-**Level 2: Barely complete**
+  **Response:** "Flu shot cannot cure cancer. Stay healthy requires sleeping exactly 8 hours a day. A few hours of exercise per week will have little benefits for physical and mental health. Physical and mental health benefits are separate topics. Scientists have not studied any of them."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Staying healthy requires proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
-**Definition:**
-A response is considered barely complete if it only contains a small percentage of all the necessary and relevant information with respect to the ground truth. In other words, it misses almost all the information - especially claims and statements - established in the ground truth. 
+## [Completeness: 2] (Barely Complete)
+**Definition:** A response that contains only a small percentage of all the necessary and relevant information with respect to the ground truth. It misses almost all the information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires 2 meals a day. Exercise per week makes not difference to physical and mental health. This is because physical and mental health benefits have low correlation through scientific studies. Scientists are making this observation in studies."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+  **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires 2 meals a day. Exercise per week makes no difference to physical and mental health. This is because physical and mental health benefits have low correlation through scientific studies. Scientists are making this observation in studies."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
-
-**Level 3: Moderately complete**
-
-**Definition:**
-A response is considered moderately complete if it contains half of the necessary and relevant information with respect to the ground truth. In other words, it miss half of the information - especially claims and statements - established in the ground truth. 
+## [Completeness: 3] (Moderately Complete)
+**Definition:** A response that contains half of the necessary and relevant information with respect to the ground truth. It misses half of the information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires a few dollar of investments a day. Even a few dollars of investments per week will not make an impact on physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Fiction writers are starting to discover them through their works."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
-
-**Level 4: Mostly complete**
-
-**Definition:**
-A response is considered mostly complete if it contains most of the necessary and relevant information with respect to the ground truth. In other words, it misses some minor information - especially claims and statements - established in the ground truth. 
+  **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires a few dollars of investments a day. Even a few dollars of investments per week will not make an impact on physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Fiction writers are starting to discover them through their works."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
+## [Completeness: 4] (Mostly Complete)
+**Definition:** A response that contains most of the necessary and relevant information with respect to the ground truth. It misses some minor information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires keto diet and rigorous athletic training. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
+  **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires keto diet and rigorous athletic training. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
-**Level 5: Fully complete**
-
-**Definition:**
-A response is considered complete if it perfectly contains all the necessary and relevant information with respect to the ground truth. In other words, it does not miss any information from statements and claims in the ground truth. 
+## [Completeness: 5] (Fully Complete)
+**Definition:** A response that perfectly contains all the necessary and relevant information with respect to the ground truth. It does not miss any information from statements and claims in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
+  **Response:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
 
 # Data
@@ -89,11 +75,10 @@ Ground Truth: {{ground_truth}}
 
 
 # Tasks
-## Please provide your assessment Score for the previous answer. Your output should include the following information:
-- **ThoughtChain**: To improve the reasoning process, Think Step by Step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and Start your ThoughtChain with "Let's think step by step:".
+## Please provide your assessment Score for the previous RESPONSE in relation to the GROUND TRUTH based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
 - **Explanation**: a very short explanation of why you think the input data should get that Score.
-- **Score**: based on your previous analysis, provide your Score. The answer you give MUST be a integer score ("1", "2", ...) based on the categories of the definitions.
-
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be an integer score (i.e., "1", "2"...) based on the levels of the definitions.
 
 ## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your score</S2>.
 # Output

azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py

@@ -45,6 +45,16 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :dedent: 8
             :caption: Initialize and call a RetrievalEvaluator.
 
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START retrieval_evaluator]
+            :end-before: [END retrieval_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call RetrievalEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. admonition:: Example with Threshold:
     
         .. literalinclude:: ../samples/evaluation_samples_threshold.py

azure/ai/evaluation/_evaluators/_rouge/_rouge.py

@@ -70,6 +70,16 @@ class RougeScoreEvaluator(EvaluatorBase):
             :dedent: 8
             :caption: Initialize and call a RougeScoreEvaluator with a four-gram rouge type.
 
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START rouge_score_evaluator]
+            :end-before: [END rouge_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call RougeScoreEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. admonition:: Example with threshold:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py

azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py

@@ -41,6 +41,16 @@ class GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :dedent: 8
             :caption: Initialize and call a GroundednessProEvaluator with a query, response, and context.
 
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START groundedness_pro_evaluator]
+            :end-before: [END groundedness_pro_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call GroundednessProEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. admonition:: Example with threshold:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py

azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -28,7 +28,7 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :param threshold: The threshold for the similarity evaluator. Default is 5.
+    :param threshold: The threshold for the similarity evaluator. Default is 3.
     :type threshold: int
 
     .. admonition:: Example:
@@ -40,6 +40,16 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
             :dedent: 8
             :caption: Initialize and call a SimilarityEvaluator with a four-gram rouge type.
 
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START similarity_evaluator]
+            :end-before: [END similarity_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call SimilarityEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. admonition:: Example:
 
         .. literalinclude:: ../samples/evaluation_samples_threshold.py

azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py

@@ -42,6 +42,17 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call an TaskAdherenceEvaluator with a query and response.
+    
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START task_adherence_evaluator]
+            :end-before: [END task_adherence_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call TaskAdherenceEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     """
 
     _PROMPTY_FILE = "task_adherence.prompty"
@@ -54,11 +65,14 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold=_DEFAULT_TASK_ADHERENCE_SCORE):
+    def __init__(self, model_config, *, threshold=_DEFAULT_TASK_ADHERENCE_SCORE,
+                 **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self.threshold = threshold
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        super().__init__(model_config=model_config, prompty_file=prompty_path,
+                         result_key=self._RESULT_KEY,
+                         **kwargs)
 
     @overload
     def __call__(

azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py

@@ -45,6 +45,16 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :dedent: 8
             :caption: Initialize and call a ToolCallAccuracyEvaluator.
 
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START tool_call_accuracy_evaluator]
+            :end-before: [END tool_call_accuracy_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call ToolCallAccuracyEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. note::
 
         To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
@@ -64,11 +74,15 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold=_DEFAULT_TOOL_CALL_ACCURACY_SCORE):
+    def __init__(self, model_config, *,
+                 threshold=_DEFAULT_TOOL_CALL_ACCURACY_SCORE,
+                 **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self.threshold = threshold
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        super().__init__(model_config=model_config, prompty_file=prompty_path,
+                         result_key=self._RESULT_KEY,
+                         **kwargs)
 
     @overload
     def __call__(
@@ -210,12 +224,18 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         score = math.nan
         if llm_output:
             score, reason = parse_quality_evaluator_reason_score(llm_output, valid_score_range="[0-1]")
-            return {
-                self._result_key: bool(float(score)),
-                f"{self._result_key}_reason": reason,
-                "tool_call_id" : eval_input.get("tool_call").get("tool_call_id"),
-            }
-        return {self._result_key: float(score)}
+            if score >= 0 and score <= 1:
+                return {
+                    self._result_key: bool(float(score)),
+                    f"{self._result_key}_reason": reason,
+                    "tool_call_id" : eval_input.get("tool_call").get("tool_call_id"),
+                }
+        raise EvaluationException(
+            message="Tool call accuracy evaluator: Invalid score returned from LLM.",
+            blame=ErrorBlame.SYSTEM_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+        )
 
     async def _real_call(self, **kwargs):
         """The asynchronous call where real end-to-end evaluation logic is performed.
@@ -227,13 +247,55 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         """
         # Convert inputs into list of evaluable inputs.
         eval_input_list = self._convert_kwargs_to_eval_input(**kwargs)
+        if len(eval_input_list) == 0:
+            return {self._AGGREGATE_RESULT_KEY: self._NOT_APPLICABLE_RESULT,
+                    f"{self._AGGREGATE_RESULT_KEY}_result": self._NOT_APPLICABLE_RESULT,
+                    f"{self._AGGREGATE_RESULT_KEY}_threshold": self.threshold,
+                    f"{self._AGGREGATE_RESULT_KEY}_reason":
+                        "No tool calls were made.",
+                    "per_tool_call_details": []
+                    }
+
         per_turn_results = []
         # Evaluate all inputs.
         for eval_input in eval_input_list:
-            per_turn_results.append(await self._do_eval(eval_input))
+            if self._is_applicable_tool(eval_input):
+                per_turn_results.append(await self._do_eval(eval_input))
+            else:
+                per_turn_results.append(self._not_applicable_result(eval_input))
 
         return self._aggregate_results(per_turn_results=per_turn_results)
 
+    def _is_applicable_tool(self, eval_input):
+        """Determine if a given tool should be evaluated, since we only evaluate tools that
+        have sufficient context available.
+
+        :type eval_input: Dict
+        :return: True if the tool call should be evaluated
+        :rtype: bool
+        """
+        tool_definition = eval_input.get("tool_definition")
+        if tool_definition is None or len(tool_definition) != 1:
+            return False
+        tool_type = tool_definition[0].get("type")
+        if tool_type is None or tool_type != "function":
+            return False
+        return True
+
+    def _not_applicable_result(self, eval_input):
+        """Return a result indicating that the tool call is not applicable for evaluation.
+
+        :param eval_input: The input to the evaluator.
+        :type eval_input: Dict
+        :return: A dictionary containing the result of the evaluation.
+        :rtype: Dict[str, Union[str, float]]
+        """
+        return {
+            f"{self._result_key}": self._NOT_APPLICABLE_RESULT,
+            f"{self._result_key}_reason": "Tool call not supported for evaluation",
+            "tool_call_id" : eval_input.get("tool_call").get("tool_call_id"),
+        }
+
     def _aggregate_results(self, per_turn_results):
         """Aggregate the evaluation results of each conversation turn into a single result.
 
@@ -256,11 +318,23 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         # Go over each turn, and rotate the results into a
         # metric: List[values] format for the evals_per_turn dictionary.
 
-        score = sum([1 if per_turn_result.get(self._result_key) else 0 for per_turn_result in per_turn_results])/len(per_turn_results)
+        num_evaluated = len([per_turn_result for per_turn_result in per_turn_results
+                             if per_turn_result.get(self._result_key) != self._NOT_APPLICABLE_RESULT])
+        if num_evaluated == 0:
+            # None of the invoked tools were applicable, return not applicable result
+            # (If a tool fails evaluation, we'll throw an exception)
+            return {self._AGGREGATE_RESULT_KEY: self._NOT_APPLICABLE_RESULT,
+                    f"{self._AGGREGATE_RESULT_KEY}_result": self._NOT_APPLICABLE_RESULT,
+                    f"{self._AGGREGATE_RESULT_KEY}_threshold": self.threshold,
+                    f"{self._AGGREGATE_RESULT_KEY}_reason":
+                        "Tool call accuracy evaluation is not yet supported for the invoked tools.",
+                    "per_tool_call_details": []
+                    }
+        # ignore not_applicable results, where the _result_key will be "not applicable"
+        score = sum([per_turn_result.get(self._result_key) == True for per_turn_result in per_turn_results])/num_evaluated
         aggregated[self._AGGREGATE_RESULT_KEY] = score
-        aggregated[f'{self._AGGREGATE_RESULT_KEY}_result'] = 'pass' if score >= self.threshold else 'fail'
+        aggregated[f'{self._AGGREGATE_RESULT_KEY}_result'] = self._PASS_RESULT if score >= self.threshold else self._FAIL_RESULT
         aggregated[f'{self._AGGREGATE_RESULT_KEY}_threshold'] = self.threshold
-
         aggregated["per_tool_call_details"] = per_turn_results
         return aggregated
 

azure/ai/evaluation/_evaluators/_ungrounded_attributes/_ungrounded_attributes.py

@@ -41,6 +41,16 @@ class UngroundedAttributesEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :dedent: 8
             :caption: Initialize and call a UngroundedAttributesEvaluator with a query, response and context.
 
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START ungrounded_attributes_evaluator]
+            :end-before: [END ungrounded_attributes_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call UngroundedAttributesEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. note::
 
         If this evaluator is supplied to the `evaluate` function, the metric

azure/ai/evaluation/_evaluators/_xpia/xpia.py

@@ -54,6 +54,17 @@ class IndirectAttackEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call an IndirectAttackEvaluator.
+    
+    .. admonition:: Example using Azure AI Project URL:
+        
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START indirect_attack_evaluator]
+            :end-before: [END indirect_attack_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call IndirectAttackEvaluator using Azure AI Project URL in the following format 
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     """
 
     id = "azureml://registries/azureml/models/Indirect-Attack-Evaluator/versions/3"

azure/ai/evaluation/_exceptions.py

@@ -38,6 +38,7 @@ class ErrorCategory(Enum):
     FAILED_REMOTE_TRACKING = "FAILED REMOTE TRACKING"
     PROJECT_ACCESS_ERROR = "PROJECT ACCESS ERROR"
     UNKNOWN = "UNKNOWN"
+    UPLOAD_ERROR = "UPLOAD ERROR"
 
 
 class ErrorBlame(Enum):
@@ -85,6 +86,7 @@ class ErrorTarget(Enum):
     CONVERSATION = "Conversation"
     TOOL_CALL_ACCURACY_EVALUATOR = "ToolCallAccuracyEvaluator"
     RED_TEAM = "RedTeam"
+    AOAI_GRADER = "AoaiGrader"
 
 
 class EvaluationException(AzureError):

azure/ai/evaluation/_legacy/_adapters/__init__.py

@@ -5,17 +5,3 @@
 # NOTE: This contains adapters that make the Promptflow dependency optional. In the first phase,
 #       Promptflow will still be installed as part of the azure-ai-evaluation dependencies. This
 #       will be removed in the future once the code migration is complete.
-
-from typing import Final
-
-
-_has_legacy = False
-try:
-    from promptflow.client import PFClient
-
-    _has_legacy = True
-except ImportError:
-    pass
-
-HAS_LEGACY_SDK: Final[bool] = _has_legacy
-MISSING_LEGACY_SDK: Final[bool] = not _has_legacy

azure/ai/evaluation/_legacy/_adapters/_check.py

@@ -0,0 +1,17 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Final
+
+
+_has_legacy = False
+try:
+    from promptflow._constants import FlowType
+
+    _has_legacy = True
+except ImportError:
+    pass
+
+HAS_LEGACY_SDK: Final[bool] = _has_legacy
+MISSING_LEGACY_SDK: Final[bool] = not _has_legacy

azure/ai/evaluation/_legacy/_adapters/_flows.py

@@ -6,7 +6,7 @@ from typing_extensions import TypeAlias
 
 
 try:
-    from promptflow._sdk.entities._flows import AsyncPrompty as _AsyncPrompty
+    from promptflow.core._flow import AsyncPrompty as _AsyncPrompty
     from promptflow._sdk.entities._flows import FlexFlow as _FlexFlow
     from promptflow._sdk.entities._flows.dag import Flow as _Flow
 except ImportError: