azure-ai-evaluation 1.10.0__py3-none-any.whl → 1.11.1__py3-none-any.whl

azure/ai/evaluation/red_team/_utils/retry_utils.py

@@ -0,0 +1,218 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""
+Retry utilities for Red Team Agent.
+
+This module provides centralized retry logic and decorators for handling
+network errors and other transient failures consistently across the codebase.
+"""
+
+import asyncio
+import logging
+from typing import Any, Callable, Dict, List, Optional, TypeVar
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception,
+    RetryError,
+)
+
+# Retry imports for exception handling
+import httpx
+import httpcore
+
+# Import Azure exceptions if available
+try:
+    from azure.core.exceptions import ServiceRequestError, ServiceResponseError
+
+    AZURE_EXCEPTIONS = (ServiceRequestError, ServiceResponseError)
+except ImportError:
+    AZURE_EXCEPTIONS = ()
+
+
+# Type variable for generic retry decorators
+T = TypeVar("T")
+
+
+class RetryManager:
+    """Centralized retry management for Red Team operations."""
+
+    # Default retry configuration
+    DEFAULT_MAX_ATTEMPTS = 5
+    DEFAULT_MIN_WAIT = 2
+    DEFAULT_MAX_WAIT = 30
+    DEFAULT_MULTIPLIER = 1.5
+
+    # Network-related exceptions that should trigger retries
+    NETWORK_EXCEPTIONS = (
+        httpx.ConnectTimeout,
+        httpx.ReadTimeout,
+        httpx.ConnectError,
+        httpx.HTTPError,
+        httpx.TimeoutException,
+        httpx.HTTPStatusError,
+        httpcore.ReadTimeout,
+        ConnectionError,
+        ConnectionRefusedError,
+        ConnectionResetError,
+        TimeoutError,
+        OSError,
+        IOError,
+        asyncio.TimeoutError,
+    ) + AZURE_EXCEPTIONS
+
+    def __init__(
+        self,
+        logger: Optional[logging.Logger] = None,
+        max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+        min_wait: int = DEFAULT_MIN_WAIT,
+        max_wait: int = DEFAULT_MAX_WAIT,
+        multiplier: float = DEFAULT_MULTIPLIER,
+    ):
+        """Initialize retry manager.
+
+        :param logger: Logger instance for retry messages
+        :param max_attempts: Maximum number of retry attempts
+        :param min_wait: Minimum wait time between retries (seconds)
+        :param max_wait: Maximum wait time between retries (seconds)
+        :param multiplier: Exponential backoff multiplier
+        """
+        self.logger = logger or logging.getLogger(__name__)
+        self.max_attempts = max_attempts
+        self.min_wait = min_wait
+        self.max_wait = max_wait
+        self.multiplier = multiplier
+
+    def should_retry_exception(self, exception: Exception) -> bool:
+        """Determine if an exception should trigger a retry.
+
+        :param exception: The exception to check
+        :return: True if the exception should trigger a retry
+        """
+        if isinstance(exception, self.NETWORK_EXCEPTIONS):
+            return True
+
+        # Special case for HTTP status errors
+        if isinstance(exception, httpx.HTTPStatusError):
+            return exception.response.status_code == 500 or "model_error" in str(exception)
+
+        return False
+
+    def log_retry_attempt(self, retry_state) -> None:
+        """Log retry attempts for visibility.
+
+        :param retry_state: The retry state object from tenacity
+        """
+        exception = retry_state.outcome.exception()
+        if exception:
+            self.logger.warning(
+                f"Retry attempt {retry_state.attempt_number}/{self.max_attempts}: "
+                f"{exception.__class__.__name__} - {str(exception)}. "
+                f"Retrying in {retry_state.next_action.sleep} seconds..."
+            )
+
+    def log_retry_error(self, retry_state) -> Exception:
+        """Log the final error after all retries failed.
+
+        :param retry_state: The retry state object from tenacity
+        :return: The final exception
+        """
+        exception = retry_state.outcome.exception()
+        self.logger.error(
+            f"All retries failed after {retry_state.attempt_number} attempts. "
+            f"Final error: {exception.__class__.__name__}: {str(exception)}"
+        )
+        return exception
+
+    def create_retry_decorator(self, context: str = "") -> Callable:
+        """Create a retry decorator with the configured settings.
+
+        :param context: Optional context string for logging
+        :return: Configured retry decorator
+        """
+        context_prefix = f"[{context}] " if context else ""
+
+        def log_attempt(retry_state):
+            exception = retry_state.outcome.exception()
+            if exception:
+                self.logger.warning(
+                    f"{context_prefix}Retry attempt {retry_state.attempt_number}/{self.max_attempts}: "
+                    f"{exception.__class__.__name__} - {str(exception)}. "
+                    f"Retrying in {retry_state.next_action.sleep} seconds..."
+                )
+
+        def log_final_error(retry_state):
+            exception = retry_state.outcome.exception()
+            self.logger.error(
+                f"{context_prefix}All retries failed after {retry_state.attempt_number} attempts. "
+                f"Final error: {exception.__class__.__name__}: {str(exception)}"
+            )
+            return exception
+
+        return retry(
+            retry=retry_if_exception(self.should_retry_exception),
+            stop=stop_after_attempt(self.max_attempts),
+            wait=wait_exponential(
+                multiplier=self.multiplier,
+                min=self.min_wait,
+                max=self.max_wait,
+            ),
+            before_sleep=log_attempt,
+            retry_error_callback=log_final_error,
+        )
+
+    def get_retry_config(self) -> Dict[str, Any]:
+        """Get retry configuration dictionary for backward compatibility.
+
+        :return: Dictionary containing retry configuration
+        """
+        return {
+            "network_retry": {
+                "retry": retry_if_exception(self.should_retry_exception),
+                "stop": stop_after_attempt(self.max_attempts),
+                "wait": wait_exponential(
+                    multiplier=self.multiplier,
+                    min=self.min_wait,
+                    max=self.max_wait,
+                ),
+                "retry_error_callback": self.log_retry_error,
+                "before_sleep": self.log_retry_attempt,
+            }
+        }
+
+
+def create_standard_retry_manager(logger: Optional[logging.Logger] = None) -> RetryManager:
+    """Create a standard retry manager with default settings.
+
+    :param logger: Optional logger instance
+    :return: Configured RetryManager instance
+    """
+    return RetryManager(logger=logger)
+
+
+# Convenience function for creating retry decorators
+def create_retry_decorator(
+    logger: Optional[logging.Logger] = None,
+    context: str = "",
+    max_attempts: int = RetryManager.DEFAULT_MAX_ATTEMPTS,
+    min_wait: int = RetryManager.DEFAULT_MIN_WAIT,
+    max_wait: int = RetryManager.DEFAULT_MAX_WAIT,
+) -> Callable:
+    """Create a retry decorator with specified parameters.
+
+    :param logger: Optional logger instance
+    :param context: Optional context for logging
+    :param max_attempts: Maximum retry attempts
+    :param min_wait: Minimum wait time between retries
+    :param max_wait: Maximum wait time between retries
+    :return: Configured retry decorator
+    """
+    retry_manager = RetryManager(
+        logger=logger,
+        max_attempts=max_attempts,
+        min_wait=min_wait,
+        max_wait=max_wait,
+    )
+    return retry_manager.create_retry_decorator(context)

azure/ai/evaluation/red_team/_utils/strategy_utils.py

@@ -88,12 +88,15 @@ def get_converter_for_strategy(
 
 
 def get_chat_target(
-    target: Union[PromptChatTarget, Callable, AzureOpenAIModelConfiguration, OpenAIModelConfiguration]
+    target: Union[PromptChatTarget, Callable, AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+    prompt_to_context: Optional[Dict[str, str]] = None,
 ) -> PromptChatTarget:
     """Convert various target types to a PromptChatTarget.
 
     :param target: The target to convert
     :type target: Union[PromptChatTarget, Callable, AzureOpenAIModelConfiguration, OpenAIModelConfiguration]
+    :param prompt_to_context: Optional mapping from prompt content to context
+    :type prompt_to_context: Optional[Dict[str, str]]
     :return: A PromptChatTarget instance
     :rtype: PromptChatTarget
     """
@@ -151,7 +154,7 @@ def get_chat_target(
             has_callback_signature = False
 
         if has_callback_signature:
-            chat_target = _CallbackChatTarget(callback=target)
+            chat_target = _CallbackChatTarget(callback=target, prompt_to_context=prompt_to_context)
         else:
 
             async def callback_target(
@@ -163,8 +166,18 @@ def get_chat_target(
                 messages_list = [_message_to_dict(chat_message) for chat_message in messages]  # type: ignore
                 latest_message = messages_list[-1]
                 application_input = latest_message["content"]
+
+                # Check if target accepts context as a parameter
+                sig = inspect.signature(target)
+                param_names = list(sig.parameters.keys())
+
                 try:
-                    response = target(query=application_input)
+                    if "context" in param_names:
+                        # Pass context if the target function accepts it
+                        response = target(query=application_input, context=context)
+                    else:
+                        # Fallback to original behavior for compatibility
+                        response = target(query=application_input)
                 except Exception as e:
                     response = f"Something went wrong {e!s}"
 
@@ -177,7 +190,7 @@ def get_chat_target(
                 messages_list.append(formatted_response)  # type: ignore
                 return {"messages": messages_list, "stream": stream, "session_state": session_state, "context": {}}
 
-            chat_target = _CallbackChatTarget(callback=callback_target)  # type: ignore
+            chat_target = _CallbackChatTarget(callback=callback_target, prompt_to_context=prompt_to_context)  # type: ignore
 
     return chat_target
 

azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -8,6 +8,7 @@ import logging
 import random
 from typing import Any, Callable, Dict, List, Optional, Union, cast
 import uuid
+import warnings
 
 from tqdm import tqdm
 
@@ -68,6 +69,14 @@ class AdversarialSimulator:
 
     def __init__(self, *, azure_ai_project: Union[str, AzureAIProject], credential: TokenCredential):
         """Constructor."""
+        warnings.warn(
+            "DEPRECATION NOTE: Azure AI Evaluation SDK has discontinued active development on the AdversarialSimulator class."
+            + " While existing functionality remains available in preview, it is no longer recommended for production workloads or future integration. "
+            + "We recommend users migrate to the AI Red Teaming Agent for future use as it supports full parity of functionality."
+            + " See https://aka.ms/airedteamingagent-sample for details on AI Red Teaming Agent.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
 
         if is_onedp_project(azure_ai_project):
             self.azure_ai_project = azure_ai_project

azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py

@@ -30,7 +30,11 @@ class GeneratedRAIClient:
     :type token_manager: ~azure.ai.evaluation.simulator._model_tools._identity_manager.APITokenManager
     """
 
-    def __init__(self, azure_ai_project: Union[AzureAIProject, str], token_manager: ManagedIdentityAPITokenManager):
+    def __init__(
+        self,
+        azure_ai_project: Union[AzureAIProject, str],
+        token_manager: ManagedIdentityAPITokenManager,
+    ):
         self.azure_ai_project = azure_ai_project
         self.token_manager = token_manager
 
@@ -53,10 +57,14 @@ class GeneratedRAIClient:
             ).rai_svc
         else:
             self._client = AIProjectClient(
-                endpoint=azure_ai_project, credential=token_manager, user_agent_policy=user_agent_policy
+                endpoint=azure_ai_project,
+                credential=token_manager,
+                user_agent_policy=user_agent_policy,
             ).red_teams
             self._evaluation_onedp_client = EvaluationServiceOneDPClient(
-                endpoint=azure_ai_project, credential=token_manager, user_agent_policy=user_agent_policy
+                endpoint=azure_ai_project,
+                credential=token_manager,
+                user_agent_policy=user_agent_policy,
             )
 
     def _get_service_discovery_url(self):
@@ -68,7 +76,10 @@ class GeneratedRAIClient:
         import requests
 
         bearer_token = self._fetch_or_reuse_token(self.token_manager)
-        headers = {"Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json"}
+        headers = {
+            "Authorization": f"Bearer {bearer_token}",
+            "Content-Type": "application/json",
+        }
 
         response = requests.get(
             f"https://management.azure.com/subscriptions/{self.azure_ai_project['subscription_id']}/"
@@ -100,6 +111,7 @@ class GeneratedRAIClient:
         risk_category: Optional[str] = None,
         application_scenario: str = None,
         strategy: Optional[str] = None,
+        language: str = "en",
         scan_session_id: Optional[str] = None,
     ) -> Dict:
         """Get attack objectives using the auto-generated operations.
@@ -112,6 +124,8 @@ class GeneratedRAIClient:
         :type application_scenario: str
         :param strategy: Optional strategy to filter the attack objectives
         :type strategy: Optional[str]
+        :param language: Language code for the attack objectives (e.g., "en", "es", "fr")
+        :type language: str
         :param scan_session_id: Optional unique session ID for the scan
         :type scan_session_id: Optional[str]
         :return: The attack objectives
@@ -122,7 +136,7 @@ class GeneratedRAIClient:
             response = self._client.get_attack_objectives(
                 risk_types=[risk_type],
                 risk_category=risk_category,
-                lang="en",
+                lang=language,
                 strategy=strategy,
                 headers={"x-ms-client-request-id": scan_session_id},
             )

azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py

@@ -208,7 +208,7 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
             flag = True
             while flag:
                 try:
-                    response = session.evaluations.operation_results(operation_id, headers=headers)
+                    response = session.red_teams.operation_results(operation_id, headers=headers)
                 except Exception as e:
                     from types import SimpleNamespace  # pylint: disable=forgotten-debug-statement
 
@@ -217,9 +217,10 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
                     response_data = response
                     flag = False
                     break
-                if response.status_code == 200:
-                    response_data = cast(List[Dict], response.json())
+                if not isinstance(response, SimpleNamespace) and response.get("object") == "chat.completion":
+                    response_data = response
                     flag = False
+                    break
                 else:
                     request_count += 1
                     sleep_time = RAIService.SLEEP_TIME**request_count

{azure_ai_evaluation-1.10.0.dist-info → azure_ai_evaluation-1.11.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: azure-ai-evaluation
-Version: 1.10.0
+Version: 1.11.1
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -28,13 +28,28 @@ Requires-Dist: nltk>=3.9.1
 Requires-Dist: azure-storage-blob>=12.10.0
 Requires-Dist: httpx>=0.25.1
 Requires-Dist: pandas<3.0.0,>=2.1.2
-Requires-Dist: openai>=1.78.0
+Requires-Dist: openai>=1.108.0
 Requires-Dist: ruamel.yaml<1.0.0,>=0.17.10
 Requires-Dist: msrest>=0.6.21
 Requires-Dist: Jinja2>=3.1.6
 Requires-Dist: aiohttp>=3.0
 Provides-Extra: redteam
 Requires-Dist: pyrit==0.8.1; extra == "redteam"
+Requires-Dist: duckdb==1.3.2; extra == "redteam"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Azure AI Evaluation client library for Python
 
@@ -398,6 +413,27 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.11.1 (2025-09-17)
+
+### Bugs Fixed
+- Pinning duckdb version to 1.3.2 for redteam extra to fix error `TypeError: unhashable type: '_duckdb.typing.DuckDBPyType'`
+
+## 1.11.0 (2025-09-02)
+
+### Features Added
+- Added support for user-supplied tags in the `evaluate` function. Tags are key-value pairs that can be used for experiment tracking, A/B testing, filtering, and organizing evaluation runs. The function accepts a `tags` parameter.
+- Added support for user-supplied TokenCredentials with LLM based evaluators.
+- Enhanced `GroundednessEvaluator` to support AI agent evaluation with tool calls. The evaluator now accepts agent response data containing tool calls and can extract context from `file_search` tool results for groundedness assessment. This enables evaluation of AI agents that use tools to retrieve information and generate responses. Note: Agent groundedness evaluation is currently supported only when the `file_search` tool is used.
+- Added `language` parameter to `RedTeam` class for multilingual red team scanning support. The parameter accepts values from `SupportedLanguages` enum including English, Spanish, French, German, Italian, Portuguese, Japanese, Korean, and Simplified Chinese, enabling red team attacks to be generated and conducted in multiple languages.
+- Added support for IndirectAttack and UngroundedAttributes risk categories in `RedTeam` scanning. These new risk categories expand red team capabilities to detect cross-platform indirect attacks and evaluate ungrounded inferences about human attributes including emotional state and protected class information.
+
+### Bugs Fixed
+- Fixed issue where evaluation results were not properly aligned with input data, leading to incorrect metrics being reported.
+
+### Other Changes
+- Deprecating `AdversarialSimulator` in favor of the [AI Red Teaming Agent](https://aka.ms/airedteamingagent-sample). `AdversarialSimulator` will be removed in the next minor release. 
+- Moved retry configuration constants (`MAX_RETRY_ATTEMPTS`, `MAX_RETRY_WAIT_SECONDS`, `MIN_RETRY_WAIT_SECONDS`) from `RedTeam` class to new `RetryManager` class for better code organization and configurability.
+
 ## 1.10.0 (2025-07-31)
 
 ### Breaking Changes