azure-ai-evaluation 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

azure/ai/evaluation/_version.py

@@ -3,4 +3,4 @@
 # ---------------------------------------------------------
 # represents upcoming version
 
-VERSION = "1.3.0"
+VERSION = "1.4.0"

azure/ai/evaluation/simulator/_adversarial_scenario.py

@@ -5,7 +5,7 @@
 from enum import Enum
 from azure.ai.evaluation._common._experimental import experimental
 
-
+# cspell:ignore vuln
 @experimental
 class AdversarialScenario(Enum):
     """Adversarial scenario types
@@ -28,6 +28,8 @@ class AdversarialScenario(Enum):
     ADVERSARIAL_CONTENT_GEN_UNGROUNDED = "adv_content_gen_ungrounded"
     ADVERSARIAL_CONTENT_GEN_GROUNDED = "adv_content_gen_grounded"
     ADVERSARIAL_CONTENT_PROTECTED_MATERIAL = "adv_content_protected_material"
+    ADVERSARIAL_CODE_VULNERABILITY = "adv_code_vuln"
+    ADVERSARIAL_UNGROUNDED_ATTRIBUTES = "adv_isa"
 
 
 @experimental

azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -220,33 +220,54 @@ class AdversarialSimulator:
             if randomization_seed is not None:
                 random.seed(randomization_seed)
             random.shuffle(templates)
-        parameter_lists = [t.template_parameters for t in templates]
-        zipped_parameters = list(zip(*parameter_lists))
-        for param_group in zipped_parameters:
-            for template, parameter in zip(templates, param_group):
-                if _jailbreak_type == "upia":
-                    parameter = self._add_jailbreak_parameter(parameter, random.choice(jailbreak_dataset))
-                tasks.append(
-                    asyncio.create_task(
-                        self._simulate_async(
-                            target=target,
-                            template=template,
-                            parameters=parameter,
-                            max_conversation_turns=max_conversation_turns,
-                            api_call_retry_limit=api_call_retry_limit,
-                            api_call_retry_sleep_sec=api_call_retry_sleep_sec,
-                            api_call_delay_sec=api_call_delay_sec,
-                            language=language,
-                            semaphore=semaphore,
-                            scenario=scenario,
-                            simulation_id=simulation_id,
-                        )
+
+        # Prepare task parameters based on scenario - but use a single append call for all scenarios
+        tasks = []
+        template_parameter_pairs = []
+
+        if scenario == AdversarialScenario.ADVERSARIAL_CONVERSATION:
+            # For ADVERSARIAL_CONVERSATION, flatten the parameters
+            for i, template in enumerate(templates):
+                if not template.template_parameters:
+                    continue
+                for parameter in template.template_parameters:
+                    template_parameter_pairs.append((template, parameter))
+        else:
+            # Use original logic for other scenarios - zip parameters
+            parameter_lists = [t.template_parameters for t in templates]
+            zipped_parameters = list(zip(*parameter_lists))
+
+            for param_group in zipped_parameters:
+                for template, parameter in zip(templates, param_group):
+                    template_parameter_pairs.append((template, parameter))
+
+        # Limit to max_simulation_results if needed
+        if len(template_parameter_pairs) > max_simulation_results:
+            template_parameter_pairs = template_parameter_pairs[:max_simulation_results]
+
+        # Single task append loop for all scenarios
+        for template, parameter in template_parameter_pairs:
+            if _jailbreak_type == "upia":
+                parameter = self._add_jailbreak_parameter(parameter, random.choice(jailbreak_dataset))
+
+            tasks.append(
+                asyncio.create_task(
+                    self._simulate_async(
+                        target=target,
+                        template=template,
+                        parameters=parameter,
+                        max_conversation_turns=max_conversation_turns,
+                        api_call_retry_limit=api_call_retry_limit,
+                        api_call_retry_sleep_sec=api_call_retry_sleep_sec,
+                        api_call_delay_sec=api_call_delay_sec,
+                        language=language,
+                        semaphore=semaphore,
+                        scenario=scenario,
+                        simulation_id=simulation_id,
                     )
                 )
-                if len(tasks) >= max_simulation_results:
-                    break
-            if len(tasks) >= max_simulation_results:
-                break
+            )
+
         for task in asyncio.as_completed(tasks):
             sim_results.append(await task)
             progress_bar.update(1)
@@ -305,7 +326,11 @@ class AdversarialSimulator:
         simulation_id: str = "",
     ) -> List[Dict]:
         user_bot = self._setup_bot(
-            role=ConversationRole.USER, template=template, parameters=parameters, scenario=scenario, simulation_id=simulation_id
+            role=ConversationRole.USER,
+            template=template,
+            parameters=parameters,
+            scenario=scenario,
+            simulation_id=simulation_id,
         )
         system_bot = self._setup_bot(
             target=target, role=ConversationRole.ASSISTANT, template=template, parameters=parameters, scenario=scenario
@@ -360,7 +385,9 @@ class AdversarialSimulator:
     ) -> ConversationBot:
         if role is ConversationRole.USER:
             model = self._get_user_proxy_completion_model(
-                template_key=template.template_name, template_parameters=parameters, simulation_id=simulation_id,
+                template_key=template.template_name,
+                template_parameters=parameters,
+                simulation_id=simulation_id,
             )
             return ConversationBot(
                 role=role,

azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py

@@ -0,0 +1,145 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from typing import Dict, List, Optional
+
+from azure.core.credentials import TokenCredential
+from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation.simulator._model_tools import ManagedIdentityAPITokenManager
+from azure.ai.evaluation._common.raiclient import MachineLearningServicesClient
+import jwt
+import time
+import ast
+
+class GeneratedRAIClient:
+    """Client for the Responsible AI Service using the auto-generated MachineLearningServicesClient.
+    
+    :param azure_ai_project: The scope of the Azure AI project. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param token_manager: The token manager
+    :type token_manager: ~azure.ai.evaluation.simulator._model_tools._identity_manager.APITokenManager
+    """
+    
+    def __init__(self, azure_ai_project: AzureAIProject, token_manager: ManagedIdentityAPITokenManager):
+        self.azure_ai_project = azure_ai_project
+        self.token_manager = token_manager
+        
+        # Service URL construction
+        if "RAI_SVC_URL" in os.environ:
+            endpoint = os.environ["RAI_SVC_URL"].rstrip("/")
+        else:
+            endpoint = self._get_service_discovery_url()
+            
+        # Create the autogenerated client
+        self._client = MachineLearningServicesClient(
+            endpoint=endpoint,
+            subscription_id=self.azure_ai_project["subscription_id"],
+            resource_group_name=self.azure_ai_project["resource_group_name"],
+            workspace_name=self.azure_ai_project["project_name"],
+            credential=self.token_manager,
+        )
+        
+    def _get_service_discovery_url(self):
+        """Get the service discovery URL.
+        
+        :return: The service discovery URL
+        :rtype: str
+        """
+        import requests
+        bearer_token = self._fetch_or_reuse_token(self.token_manager)
+        headers = {"Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json"}
+        
+        response = requests.get(
+            f"https://management.azure.com/subscriptions/{self.azure_ai_project['subscription_id']}/"
+            f"resourceGroups/{self.azure_ai_project['resource_group_name']}/"
+            f"providers/Microsoft.MachineLearningServices/workspaces/{self.azure_ai_project['project_name']}?"
+            f"api-version=2023-08-01-preview",
+            headers=headers,
+            timeout=5,
+        )
+        
+        if response.status_code != 200:
+            msg = (
+                f"Failed to connect to your Azure AI project. Please check if the project scope is configured "
+                f"correctly, and make sure you have the necessary access permissions. "
+                f"Status code: {response.status_code}."
+            )
+            raise Exception(msg)
+
+        # Parse the discovery URL
+        from urllib.parse import urlparse
+        base_url = urlparse(response.json()["properties"]["discoveryUrl"])
+        return f"{base_url.scheme}://{base_url.netloc}"
+    
+    async def get_attack_objectives(self, risk_category: Optional[str] = None, application_scenario: str = None, strategy: Optional[str] = None) -> Dict:
+        """Get attack objectives using the auto-generated operations.
+        
+        :param risk_category: Optional risk category to filter the attack objectives
+        :type risk_category: Optional[str]
+        :param application_scenario: Optional description of the application scenario for context
+        :type application_scenario: str
+        :param strategy: Optional strategy to filter the attack objectives
+        :type strategy: Optional[str]
+        :return: The attack objectives
+        :rtype: Dict
+        """ 
+        try:
+            # Send the request using the autogenerated client
+            response = self._client.rai_svc.get_attack_objectives(
+                risk_types=[risk_category],
+                lang="en",
+                strategy=strategy,
+            )
+            return response
+            
+        except Exception as e:
+            # Log the exception for debugging purposes
+            import logging
+            logging.error(f"Error in get_attack_objectives: {str(e)}")
+            raise
+        
+    async def get_jailbreak_prefixes(self) -> List[str]:
+        """Get jailbreak prefixes using the auto-generated operations.
+        
+        :return: The jailbreak prefixes
+        :rtype: List[str]
+        """
+        try:
+            # Send the request using the autogenerated client
+            response = self._client.rai_svc.get_jail_break_dataset_with_type(type="upia")
+            if isinstance(response, list):
+                return response
+            else:
+                self.logger.error("Unexpected response format from get_jail_break_dataset_with_type")
+                raise ValueError("Unexpected response format from get_jail_break_dataset_with_type")
+            
+        except Exception as e:
+            return [""]
+
+    def _fetch_or_reuse_token(self, credential: TokenCredential, token: Optional[str] = None) -> str:
+        """Get token. Fetch a new token if the current token is near expiry
+
+        :param credential: The Azure authentication credential.
+        :type credential:
+        ~azure.core.credentials.TokenCredential
+        :param token: The Azure authentication token. Defaults to None. If none, a new token will be fetched.
+        :type token: str
+        :return: The Azure authentication token.
+        """
+        if token:
+            # Decode the token to get its expiration time
+            try:
+                decoded_token = jwt.decode(token, options={"verify_signature": False})
+            except jwt.PyJWTError:
+                pass
+            else:
+                exp_time = decoded_token["exp"]
+                current_time = time.time()
+
+                # Return current token if not near expiry
+                if (exp_time - current_time) >= 300:
+                    return token
+
+        return credential.get_token("https://management.azure.com/.default").token

azure/ai/evaluation/simulator/_model_tools/_rai_client.py

@@ -2,9 +2,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import os
-from typing import Any
+from typing import Any, Dict, List
 from urllib.parse import urljoin, urlparse
 import base64
+import json
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client, get_http_client
@@ -62,6 +63,7 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
         self.jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak")
         self.simulation_submit_endpoint = urljoin(self.api_url, "simulation/chat/completions/submit")
         self.xpia_jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak/xpia")
+        self.attack_objectives_endpoint = urljoin(self.api_url, "simulation/attackobjectives")
 
     def _get_service_discovery_url(self):
         bearer_token = self.token_manager.get_token()
@@ -206,3 +208,71 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
             category=ErrorCategory.UNKNOWN,
             blame=ErrorBlame.USER_ERROR,
         )
+
+    async def get_attack_objectives(self, risk_categories: List[str], application_scenario: str = None, strategy: str = None) -> Any:
+        """Get the attack objectives based on risk categories and application scenario
+        
+        :param risk_categories: List of risk categories to generate attack objectives for
+        :type risk_categories: List[str]
+        :param application_scenario: Optional description of the application scenario for context
+        :type application_scenario: str
+        :param strategy: Optional attack strategy to get specific objectives for
+        :type strategy: str
+        :return: The attack objectives
+        :rtype: Any
+        """
+        # Create query parameters for the request
+        if application_scenario:
+            raise NotImplementedError("Application scenario is not supported yet")
+        
+        params = {
+            "api-version": "2022-11-01-preview",
+            "riskTypes": ",".join(risk_categories),
+            "lang": "en"  # Default to English
+        }
+        
+        # Add strategy parameter if provided
+        if strategy:
+            params["strategy"] = strategy
+            
+        try:
+            # Make the request using the existing get method
+            result = await self.get(self.attack_objectives_endpoint)
+            # from collections import defaultdict
+            # counts_by_risk = defaultdict(int)
+            # for item in result:
+            #     target_harms = item.get("Metadata", {}).get("TargetHarms", [])
+            #     if not target_harms:
+            #         # No risk type specified
+            #         counts_by_risk["empty"] += 1
+            #     else:
+            #         for harm in target_harms:
+            #             # Use "empty" if the risk type field is missing
+            #             risk_type = harm.get("RiskType", "") or "empty"
+            #             counts_by_risk[risk_type] += 1
+            return result
+        except Exception:
+            # If the API fails or isn't implemented yet, return a mock response
+            # This is temporary until the API endpoint is fully implemented
+            return [{
+                "metadata": {
+                    "lang": "en",
+                    "target_harms": [
+                        {
+                            "risk-type": "violence",
+                            "risk-subtype": ""
+                        }
+                    ]
+                },
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": "Risky content"
+                    }
+                ],
+                "modality": "text",
+                "source": [
+                    "source"
+                ]
+            }]
+ 

{azure_ai_evaluation-1.3.0.dist-info → azure_ai_evaluation-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.3.0
+Version: 1.4.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -21,13 +21,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: NOTICE.txt
-Requires-Dist: promptflow-devkit >=1.17.1
-Requires-Dist: promptflow-core >=1.17.1
-Requires-Dist: pyjwt >=2.8.0
-Requires-Dist: azure-identity >=1.16.0
-Requires-Dist: azure-core >=1.30.2
-Requires-Dist: nltk >=3.9.1
-Requires-Dist: azure-storage-blob >=12.10.0
+Requires-Dist: promptflow-devkit>=1.17.1
+Requires-Dist: promptflow-core>=1.17.1
+Requires-Dist: pyjwt>=2.8.0
+Requires-Dist: azure-identity>=1.16.0
+Requires-Dist: azure-core>=1.30.2
+Requires-Dist: nltk>=3.9.1
+Requires-Dist: azure-storage-blob>=12.10.0
+Provides-Extra: redteam
+Requires-Dist: pyrit>=0.8.0; extra == "redteam"
 
 # Azure AI Evaluation client library for Python
 
@@ -54,7 +56,7 @@ Azure AI SDK provides following to evaluate Generative AI Applications:
 ### Prerequisites
 
 - Python 3.9 or later is required to use this package.
-- [Optional] You must have [Azure AI Project][ai_project] or [Azure Open AI][azure_openai] to use AI-assisted evaluators
+- [Optional] You must have [Azure AI Foundry Project][ai_project] or [Azure Open AI][azure_openai] to use AI-assisted evaluators
 
 ### Install the package
 
@@ -63,10 +65,6 @@ Install the Azure AI Evaluation SDK for Python with [pip][pip_link]:
 ```bash
 pip install azure-ai-evaluation
 ```
-If you want to track results in [AI Studio][ai_studio], install `remote` extra:
-```python
-pip install azure-ai-evaluation[remote]
-```
 
 ## Key concepts
 
@@ -175,9 +173,9 @@ result = evaluate(
             } 
         }
     }
-    # Optionally provide your AI Studio project information to track your evaluation results in your Azure AI Studio project
+    # Optionally provide your AI Foundry project information to track your evaluation results in your Azure AI Foundry project
     azure_ai_project = azure_ai_project,
-    # Optionally provide an output path to dump a json of metric summary, row level data and metric and studio URL
+    # Optionally provide an output path to dump a json of metric summary, row level data and metric and AI Foundry URL
     output_path="./evaluation_results.json"
 )
 ```
@@ -375,8 +373,64 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 [simulate_with_conversation_starter]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter
 [adversarial_jailbreak]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data#simulating-jailbreak-attacks
 
+
 # Release History
 
+## 1.4.0 (2025-03-27)
+
+### Features Added
+- Enhanced binary evaluation results with customizable thresholds
+  - Added threshold support for QA and ContentSafety evaluators
+  - Evaluation results now include both the score and threshold values
+  - Configurable threshold parameter allows custom binary classification boundaries
+  - Default thresholds provided for backward compatibility
+  - Quality evaluators use "higher is better" scoring (score ≥ threshold is positive)
+  - Content safety evaluators use "lower is better" scoring (score ≤ threshold is positive)
+- New Built-in evaluator called CodeVulnerabilityEvaluator is added. 
+  - It provides capabilities to identify the following code vulnerabilities.
+    - path-injection
+    - sql-injection
+    - code-injection
+    - stack-trace-exposure
+    - incomplete-url-substring-sanitization
+    - flask-debug
+    - clear-text-logging-sensitive-data
+    - incomplete-hostname-regexp
+    - server-side-unvalidated-url-redirection
+    - weak-cryptographic-algorithm
+    - full-ssrf
+    - bind-socket-all-network-interfaces
+    - client-side-unvalidated-url-redirection
+    - likely-bugs
+    - reflected-xss
+    - clear-text-storage-sensitive-data
+    - tarslip
+    - hardcoded-credentials
+    - insecure-randomness
+  - It also supports multiple coding languages such as (Python, Java, C++, C#, Go, Javascript, SQL)
+  
+- New Built-in evaluator called UngroundedAttributesEvaluator is added.
+  - It evaluates ungrounded inference of human attributes for a given query, response, and context for a single-turn evaluation only, 
+  - where query represents the user query and response represents the AI system response given the provided context. 
+ 
+  - Ungrounded Attributes checks for whether a response is first, ungrounded, and checks if it contains information about protected class 
+  - or emotional state of a person.
+  
+  - It identifies the following attributes:
+    
+    - emotional_state
+    - protected_class
+    - groundedness
+- New Built-in evaluators for Agent Evaluation (Preview)
+  - IntentResolutionEvaluator - Evaluates the intent resolution of an agent's response to a user query.
+  - ResponseCompletenessEvaluator - Evaluates the response completeness of an agent's response to a user query.
+  - TaskAdherenceEvaluator - Evaluates the task adherence of an agent's response to a user query.
+  - ToolCallAccuracyEvaluator - Evaluates the accuracy of tool calls made by an agent in response to a user query.
+
+### Bugs Fixed
+- Fixed error in `GroundednessProEvaluator` when handling non-numeric values like "n/a" returned from the service.
+- Uploading local evaluation results from `evaluate` with the same run name will no longer result in each online run sharing (and bashing) result files.
+
 ## 1.3.0 (2025-02-28)
 
 ### Breaking Changes