azure-ai-evaluation 1.11.1__py3-none-any.whl → 1.12.0__py3-none-any.whl

azure/ai/evaluation/red_team/_red_team_result.py

@@ -1,7 +1,7 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing import Dict, List, Optional, TypedDict
+from typing import Any, Dict, List, Optional, TypedDict
 import json
 from azure.ai.evaluation._common._experimental import experimental
 
@@ -244,6 +244,91 @@ class AttackDetails(TypedDict):
     attack_success_threshold: Optional[int]
 
 
+@experimental
+class RedTeamOutputResultProperties(TypedDict, total=False):
+    """Additional metadata captured for each evaluation result."""
+
+    attack_success: Optional[bool]
+    attack_success_threshold: Optional[int]
+    attack_technique: str
+    attack_complexity: str
+    risk_category: str
+    risk_assessment: Optional[Dict[str, Any]]
+    reason: Optional[str]
+    severity_label: Optional[str]
+    metadata: Optional[Dict[str, Any]]
+
+
+@experimental
+class EvaluationRunOutputItemMessage(TypedDict, total=False):
+    """Representation of a single message within an evaluation sample."""
+
+    role: str
+    content: Any
+    name: Optional[str]
+    tool_calls: Optional[List[Dict[str, Any]]]
+
+
+@experimental
+class RedTeamRunOutputItemResult(TypedDict, total=False):
+    """Flattened evaluation result for a single risk category."""
+
+    # Should extend EvaluationRunOutputItemResult
+
+    object: str
+    type: str
+    name: str
+    passed: Optional[bool]
+    score: Optional[float]
+    metric: Optional[str]
+    threshold: Optional[float]
+    reason: Optional[str]
+    sample: "RedTeamRunOutputItemSample"
+    properties: RedTeamOutputResultProperties
+
+
+@experimental
+class RedTeamDatasourceItem(TypedDict, total=False):
+    """Metadata about the datasource item that produced this conversation."""
+
+    id: Optional[str]
+    input_data: Dict[str, Any]
+
+
+@experimental
+class RedTeamRunOutputItemSample(TypedDict, total=False):
+    """Sample payload containing the red team conversation."""
+
+    # Should extend EvaluationRunOutputItemSample
+
+    object: str
+    input: List[EvaluationRunOutputItemMessage]
+    output: List[EvaluationRunOutputItemMessage]
+    finish_reason: Optional[str]
+    model: Optional[str]
+    error: Optional[Dict[str, Any]]
+    usage: Optional[Dict[str, Any]]
+    seed: Optional[int]
+    temperature: Optional[float]
+    top_p: Optional[float]
+    max_completion_tokens: Optional[float]
+    metadata: Optional[Dict[str, Any]]
+
+
+@experimental
+class RedTeamOutputItem(TypedDict, total=False):
+    """Structured representation of a conversation and its evaluation artifacts."""
+
+    object: str
+    id: str
+    created_time: int
+    status: str
+    datasource_item_id: Optional[str]
+    datasource_item: Optional[RedTeamDatasourceItem]
+    sample: RedTeamRunOutputItemSample
+    results: List[RedTeamRunOutputItemResult]
+
+
 @experimental
 class ScanResult(TypedDict):
     """TypedDict representation of a Red Team Agent evaluation result with the updated structure.
@@ -254,6 +339,12 @@ class ScanResult(TypedDict):
     :type parameters: RedTeamingParameters
     :param attack_details: List of AttackDetails objects representing the conversations in the evaluation
     :type attack_details: List[AttackDetails]
+    :param output_items: List of structured output items from the evaluation
+    :type output_items: List[RedTeamOutputItem]
+    :param AOAI_Compatible_Row_Results: List of evaluation results for each risk category
+    :type AOAI_Compatible_Row_Results: List[RedTeamRunOutputItemResult]
+    :param AOAI_Compatible_Summary: The evaluation run metadata in eval.run format
+    :type AOAI_Compatible_Summary: RedTeamRun
     :param studio_url: Optional URL for the studio
     :type studio_url: Optional[str]
     """
@@ -261,9 +352,147 @@ class ScanResult(TypedDict):
     scorecard: RedTeamingScorecard
     parameters: RedTeamingParameters
     attack_details: List[AttackDetails]
+    output_items: List[RedTeamOutputItem]
+    AOAI_Compatible_Row_Results: List[RedTeamRunOutputItemResult]
+    AOAI_Compatible_Summary: "RedTeamRun"
     studio_url: Optional[str]
 
 
+@experimental
+class ResultCount(TypedDict):
+    """Count of evaluation results by status.
+
+    :param total: Total number of evaluation results
+    :type total: int
+    :param passed: Number of passed evaluation results
+    :type passed: int
+    :param failed: Number of failed evaluation results
+    :type failed: int
+    :param errored: Number of errored evaluation results
+    :type errored: int
+    """
+
+    total: int
+    passed: int
+    failed: int
+    errored: int
+
+
+@experimental
+class PerTestingCriteriaResult(TypedDict):
+    """Result count for a specific testing criteria.
+
+    :param testing_criteria: The name of the testing criteria (e.g., risk category)
+    :type testing_criteria: str
+    :param passed: Number of passed results for this criteria
+    :type passed: int
+    :param failed: Number of failed results for this criteria
+    :type failed: int
+    """
+
+    testing_criteria: str
+    passed: int
+    failed: int
+
+
+@experimental
+class DataSourceItemGenerationParams(TypedDict, total=False):
+    """Parameters for data source item generation.
+
+    :param type: Type of data source generation (e.g., "red_team")
+    :type type: str
+    :param attack_strategies: List of attack strategies used
+    :type attack_strategies: List[str]
+    :param num_turns: Number of turns in the conversation
+    :type num_turns: int
+    """
+
+    type: str
+    attack_strategies: List[str]
+    num_turns: int
+
+
+@experimental
+class DataSource(TypedDict, total=False):
+    """Data source information for the red team evaluation.
+
+    :param type: Type of data source (e.g., "azure_ai_red_team")
+    :type type: str
+    :param target: Target configuration for the data source
+    :type target: Dict[str, Any]
+    :param item_generation_params: Parameters used for generating data items
+    :type item_generation_params: DataSourceItemGenerationParams
+    """
+
+    type: str
+    target: Dict[str, Any]
+    item_generation_params: DataSourceItemGenerationParams
+
+
+@experimental
+class OutputItemsList(TypedDict):
+    """Wrapper for list of output items.
+
+    :param object: Object type identifier (always "list")
+    :type object: str
+    :param data: List of red team output items
+    :type data: List[RedTeamOutputItem]
+    """
+
+    object: str
+    data: List[RedTeamOutputItem]
+
+
+@experimental
+class RedTeamRun(TypedDict, total=False):
+    """TypedDict representation of a Red Team evaluation run in eval.run format.
+
+    :param object: Object type identifier (always "eval.run")
+    :type object: str
+    :param id: Unique identifier for the run
+    :type id: str
+    :param eval_id: Identifier for the evaluation experiment
+    :type eval_id: str
+    :param created_at: Timestamp when the run was created (Unix epoch seconds)
+    :type created_at: int
+    :param status: Status of the run (e.g., "completed", "failed", "in_progress")
+    :type status: str
+    :param name: Display name for the run
+    :type name: str
+    :param report_url: URL to view the run report in Azure AI Studio
+    :type report_url: Optional[str]
+    :param data_source: Information about the data source used for the evaluation
+    :type data_source: DataSource
+    :param metadata: Additional metadata for the run
+    :type metadata: Dict[str, Any]
+    :param result_count: Aggregated counts of evaluation results
+    :type result_count: ResultCount
+    :param per_model_usage: Usage statistics per model (if applicable)
+    :type per_model_usage: List[Any]
+    :param per_testing_criteria_results: Results aggregated by testing criteria
+    :type per_testing_criteria_results: List[PerTestingCriteriaResult]
+    :param output_items: Wrapped list of output items from the evaluation
+    :type output_items: OutputItemsList
+    :param conversations: Optional list of attack details/conversations
+    :type conversations: List[AttackDetails]
+    """
+
+    object: str
+    id: str
+    eval_id: str
+    created_at: int
+    status: str
+    name: str
+    report_url: Optional[str]
+    data_source: DataSource
+    metadata: Dict[str, Any]
+    result_count: ResultCount
+    per_model_usage: List[Any]
+    per_testing_criteria_results: List[PerTestingCriteriaResult]
+    output_items: OutputItemsList
+    conversations: List[AttackDetails]
+
+
 @experimental
 class RedTeamResult:
     def __init__(self, scan_result: Optional[ScanResult] = None, attack_details: Optional[List[AttackDetails]] = None):