azure-ai-evaluation 1.11.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_common/onedp/models/__init__.py

@@ -14,63 +14,139 @@ if TYPE_CHECKING:
 
 
 from ._models import (  # type: ignore
-    AOAIModelConfig,
+    AIContent,
+    AgentClusterInsightResult,
+    AgentClusterInsightsRequest,
     AgentEvaluation,
     AgentEvaluationRedactionConfiguration,
     AgentEvaluationRequest,
     AgentEvaluationResult,
     AgentEvaluationSamplingConfiguration,
+    AgentTaxonomyInput,
     AnnotationDTO,
     ApiKeyCredentials,
     AssetCredentialRequest,
     AssetCredentialResponse,
+    AssistantMessage,
+    AttackMessage,
     AttackObjective,
+    AzureAIAgentTarget,
+    AzureAIEvaluator,
     AzureAISearchIndex,
+    AzureOpenAIModelConfiguration,
     BaseCredentials,
+    BlobReference,
     BlobReferenceForConsumption,
+    ChartCoordinate,
     ChatChoice,
     ChatCompletions,
     ChatResponseMessage,
+    ClusterInsightResult,
+    ClusterTokenUsage,
+    CodeBasedEvaluatorDefinition,
     CompletionsUsage,
     CompletionsUsageDetails,
     Connection,
     Content,
+    ContinuousEvaluationRuleAction,
     CosmosDBIndex,
+    CreateEvalJsonlRunDataSource,
+    CronTrigger,
     CustomCredential,
+    CustomInference,
     CustomizationParameters,
+    DailyRecurrenceSchedule,
     DatasetVersion,
     Deployment,
+    DeveloperMessage,
     EmbeddingConfiguration,
     EntraIDCredentials,
+    EvalCompareReport,
+    EvalJsonlFileContent,
+    EvalJsonlFileContentItem,
+    EvalJsonlFileContentSource,
+    EvalResult,
+    EvalRunOutputItem,
+    EvalRunResultCompareItem,
+    EvalRunResultComparison,
+    EvalRunResultSummary,
     Evaluation,
+    EvaluationComparisonRequest,
     EvaluationResult,
+    EvaluationResultSample,
+    EvaluationRule,
+    EvaluationRuleAction,
+    EvaluationRuleFilter,
+    EvaluationRunClusterInsightResult,
+    EvaluationRunClusterInsightsRequest,
+    EvaluationScheduleTask,
     EvaluationTarget,
+    EvaluationTaxonomy,
+    EvaluationTaxonomyInput,
     EvaluationUpload,
     EvaluatorConfiguration,
+    EvaluatorDefinition,
+    EvaluatorMessage,
+    EvaluatorMetric,
+    EvaluatorVersion,
+    FieldMapping,
     FileDatasetVersion,
     FolderDatasetVersion,
+    HourlyRecurrenceSchedule,
+    HumanEvaluationRuleAction,
+    ImageSource,
+    ImageUrlContent,
     Index,
     InputData,
     InputDataset,
+    Insight,
+    InsightCluster,
+    InsightModelConfiguration,
+    InsightRequest,
+    InsightResult,
+    InsightSample,
+    InsightScheduleTask,
+    InsightSummary,
+    InsightsMetadata,
     LongRunningResponse,
-    MAASModelConfig,
     ManagedAzureAISearchIndex,
     Message,
     Metadata,
     ModelDeployment,
+    ModelResponseGenerationTarget,
+    MonthlyRecurrenceSchedule,
     NoAuthenticationCredentials,
+    OneTimeTrigger,
     PendingUploadRequest,
     PendingUploadResponse,
+    PromptBasedEvaluatorDefinition,
     PromptUsageDetails,
+    QueryResponseInlineMessage,
+    RecurrenceSchedule,
+    RecurrenceTrigger,
     RedTeam,
     RedTeamUpload,
     SASCredentials,
     SasCredential,
+    Schedule,
+    ScheduleRun,
+    ScheduleTask,
     SimulationDTO,
     Sku,
+    SyncEvalInput,
     SystemData,
+    SystemMessage,
+    TargetConfig,
     TargetHarm,
-    TargetModelConfig,
+    TaxonomyCategory,
+    TaxonomySubCategory,
+    TextContent,
+    ToolCallContent,
+    ToolDescription,
+    ToolResultContent,
+    Trigger,
+    UserMessage,
+    WeeklyRecurrenceSchedule,
 )
 
 from ._enums import (  # type: ignore
@@ -80,89 +156,203 @@ from ._enums import (  # type: ignore
     ConnectionType,
     CredentialType,
     DatasetType,
+    DayOfWeek,
     DeploymentType,
+    EvaluationRuleActionType,
+    EvaluationRuleEventType,
+    EvaluationTargetType,
+    EvaluationTaxonomyInputType,
+    EvaluatorCategory,
+    EvaluatorDefinitionType,
+    EvaluatorMetricDirection,
+    EvaluatorMetricType,
+    EvaluatorType,
     IndexType,
+    InsightType,
     ListViewType,
+    OperationState,
     PendingUploadType,
+    RecurrenceType,
     ResultType,
     RiskCategory,
+    SampleType,
+    ScheduleProvisioningStatus,
+    ScheduleTaskType,
     SimulationType,
+    TargetType,
+    TreatmentEffectType,
+    TriggerType,
 )
 from ._patch import __all__ as _patch_all
 from ._patch import *
 from ._patch import patch_sdk as _patch_sdk
 
 __all__ = [
-    "AOAIModelConfig",
+    "AIContent",
+    "AgentClusterInsightResult",
+    "AgentClusterInsightsRequest",
     "AgentEvaluation",
     "AgentEvaluationRedactionConfiguration",
     "AgentEvaluationRequest",
     "AgentEvaluationResult",
     "AgentEvaluationSamplingConfiguration",
+    "AgentTaxonomyInput",
     "AnnotationDTO",
     "ApiKeyCredentials",
     "AssetCredentialRequest",
     "AssetCredentialResponse",
+    "AssistantMessage",
+    "AttackMessage",
     "AttackObjective",
+    "AzureAIAgentTarget",
+    "AzureAIEvaluator",
     "AzureAISearchIndex",
+    "AzureOpenAIModelConfiguration",
     "BaseCredentials",
+    "BlobReference",
     "BlobReferenceForConsumption",
+    "ChartCoordinate",
     "ChatChoice",
     "ChatCompletions",
     "ChatResponseMessage",
+    "ClusterInsightResult",
+    "ClusterTokenUsage",
+    "CodeBasedEvaluatorDefinition",
     "CompletionsUsage",
     "CompletionsUsageDetails",
     "Connection",
     "Content",
+    "ContinuousEvaluationRuleAction",
     "CosmosDBIndex",
+    "CreateEvalJsonlRunDataSource",
+    "CronTrigger",
     "CustomCredential",
+    "CustomInference",
     "CustomizationParameters",
+    "DailyRecurrenceSchedule",
     "DatasetVersion",
     "Deployment",
+    "DeveloperMessage",
     "EmbeddingConfiguration",
     "EntraIDCredentials",
+    "EvalCompareReport",
+    "EvalJsonlFileContent",
+    "EvalJsonlFileContentItem",
+    "EvalJsonlFileContentSource",
+    "EvalResult",
+    "EvalRunOutputItem",
+    "EvalRunResultCompareItem",
+    "EvalRunResultComparison",
+    "EvalRunResultSummary",
     "Evaluation",
+    "EvaluationComparisonRequest",
     "EvaluationResult",
+    "EvaluationResultSample",
+    "EvaluationRule",
+    "EvaluationRuleAction",
+    "EvaluationRuleFilter",
+    "EvaluationRunClusterInsightResult",
+    "EvaluationRunClusterInsightsRequest",
+    "EvaluationScheduleTask",
     "EvaluationTarget",
+    "EvaluationTaxonomy",
+    "EvaluationTaxonomyInput",
     "EvaluationUpload",
     "EvaluatorConfiguration",
+    "EvaluatorDefinition",
+    "EvaluatorMessage",
+    "EvaluatorMetric",
+    "EvaluatorVersion",
+    "FieldMapping",
     "FileDatasetVersion",
     "FolderDatasetVersion",
+    "HourlyRecurrenceSchedule",
+    "HumanEvaluationRuleAction",
+    "ImageSource",
+    "ImageUrlContent",
     "Index",
     "InputData",
     "InputDataset",
+    "Insight",
+    "InsightCluster",
+    "InsightModelConfiguration",
+    "InsightRequest",
+    "InsightResult",
+    "InsightSample",
+    "InsightScheduleTask",
+    "InsightSummary",
+    "InsightsMetadata",
     "LongRunningResponse",
-    "MAASModelConfig",
     "ManagedAzureAISearchIndex",
     "Message",
     "Metadata",
     "ModelDeployment",
+    "ModelResponseGenerationTarget",
+    "MonthlyRecurrenceSchedule",
     "NoAuthenticationCredentials",
+    "OneTimeTrigger",
     "PendingUploadRequest",
     "PendingUploadResponse",
+    "PromptBasedEvaluatorDefinition",
     "PromptUsageDetails",
+    "QueryResponseInlineMessage",
+    "RecurrenceSchedule",
+    "RecurrenceTrigger",
     "RedTeam",
     "RedTeamUpload",
     "SASCredentials",
     "SasCredential",
+    "Schedule",
+    "ScheduleRun",
+    "ScheduleTask",
     "SimulationDTO",
     "Sku",
+    "SyncEvalInput",
     "SystemData",
+    "SystemMessage",
+    "TargetConfig",
     "TargetHarm",
-    "TargetModelConfig",
+    "TaxonomyCategory",
+    "TaxonomySubCategory",
+    "TextContent",
+    "ToolCallContent",
+    "ToolDescription",
+    "ToolResultContent",
+    "Trigger",
+    "UserMessage",
+    "WeeklyRecurrenceSchedule",
     "AttackStrategy",
     "ChatRole",
     "CompletionsFinishReason",
     "ConnectionType",
     "CredentialType",
     "DatasetType",
+    "DayOfWeek",
     "DeploymentType",
+    "EvaluationRuleActionType",
+    "EvaluationRuleEventType",
+    "EvaluationTargetType",
+    "EvaluationTaxonomyInputType",
+    "EvaluatorCategory",
+    "EvaluatorDefinitionType",
+    "EvaluatorMetricDirection",
+    "EvaluatorMetricType",
+    "EvaluatorType",
     "IndexType",
+    "InsightType",
     "ListViewType",
+    "OperationState",
     "PendingUploadType",
+    "RecurrenceType",
     "ResultType",
     "RiskCategory",
+    "SampleType",
+    "ScheduleProvisioningStatus",
+    "ScheduleTaskType",
     "SimulationType",
+    "TargetType",
+    "TreatmentEffectType",
+    "TriggerType",
 ]
 __all__.extend([p for p in _patch_all if p not in __all__])  # pyright: ignore
 _patch_sdk()

azure/ai/evaluation/_common/onedp/models/_enums.py

@@ -149,6 +149,25 @@ class DatasetType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """URI folder."""
 
 
+class DayOfWeek(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Days of the week for recurrence schedule."""
+
+    SUNDAY = "Sunday"
+    """Sunday."""
+    MONDAY = "Monday"
+    """Monday."""
+    TUESDAY = "Tuesday"
+    """Tuesday."""
+    WEDNESDAY = "Wednesday"
+    """Wednesday."""
+    THURSDAY = "Thursday"
+    """Thursday."""
+    FRIDAY = "Friday"
+    """Friday."""
+    SATURDAY = "Saturday"
+    """Saturday."""
+
+
 class DeploymentType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Type of DeploymentType."""
 
@@ -156,6 +175,99 @@ class DeploymentType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Model deployment"""
 
 
+class EvaluationRuleActionType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Type of the evaluation action."""
+
+    CONTINUOUS_EVALUATION = "continuousEvaluation"
+    """Continuous evaluation."""
+    HUMAN_EVALUATION = "humanEvaluation"
+    """Human evaluation."""
+
+
+class EvaluationRuleEventType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Type of the evaluation rule event."""
+
+    RESPONSE_COMPLETED = "response.completed"
+    """Response completed."""
+    MANUAL = "manual"
+    """Manual trigger."""
+
+
+class EvaluationTargetType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Allowed types of evaluation targets."""
+
+    MODEL_RESPONSE_GENERATION = "modelResponseGeneration"
+    """Evaluation target that uses a model for response generation."""
+
+
+class EvaluationTaxonomyInputType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Type of the evaluation taxonomy input."""
+
+    AGENT = "agent"
+    """Agent"""
+    POLICY = "policy"
+    """Policy."""
+
+
+class EvaluatorCategory(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The category of the evaluator."""
+
+    QUALITY = "quality"
+    """Quality"""
+    SAFETY = "safety"
+    """Risk & Safety"""
+    AGENTS = "agents"
+    """Agents"""
+
+
+class EvaluatorDefinitionType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The type of evaluator definition."""
+
+    PROMPT = "prompt"
+    """Prompt-based definition"""
+    CODE = "code"
+    """Code-based definition"""
+    PROMPT_AND_CODE = "prompt_and_code"
+    """Prompt & Code Based definition"""
+    SERVICE = "service"
+    """Service-based evaluator"""
+    OPENAI_GRADERS = "openai_graders"
+    """OpenAI graders"""
+
+
+class EvaluatorMetricDirection(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The direction of the metric indicating whether a higher value is better, a lower value is
+    better, or neutral.
+    """
+
+    INCREASE = "increase"
+    """It indicates a higher value is better for this metric"""
+    DECREASE = "decrease"
+    """It indicates a lower value is better for this metric"""
+    NEUTRAL = "neutral"
+    """It indicates no preference for this metric direction"""
+
+
+class EvaluatorMetricType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The type of the evaluator."""
+
+    ORDINAL = "ordinal"
+    """Ordinal metric representing categories that can be ordered or ranked."""
+    CONTINUOUS = "continuous"
+    """Continuous metric representing values in a continuous range."""
+    BOOLEAN = "boolean"
+    """Boolean metric representing true/false values"""
+
+
+class EvaluatorType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The type of the evaluator."""
+
+    BUILT_IN = "builtin"
+    """Built-in evaluator (Microsoft provided)"""
+    CUSTOM = "custom"
+    """Custom evaluator"""
+
+
 class IndexType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Type of IndexType."""
 
@@ -167,6 +279,17 @@ class IndexType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Managed Azure Search"""
 
 
+class InsightType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The request of the insights."""
+
+    EVALUATION_RUN_CLUSTER_INSIGHT = "EvaluationRunClusterInsight"
+    """Insights on an Evaluation run result."""
+    AGENT_CLUSTER_INSIGHT = "AgentClusterInsight"
+    """Cluster Insight on an Agent."""
+    EVALUATION_COMPARISON = "EvaluationComparison"
+    """Evaluation Comparison."""
+
+
 class ListViewType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """List View Type Definition."""
 
@@ -178,15 +301,45 @@ class ListViewType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """List all items."""
 
 
+class OperationState(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Enum describing allowed operation states."""
+
+    NOT_STARTED = "NotStarted"
+    """The operation has not started."""
+    RUNNING = "Running"
+    """The operation is in progress."""
+    SUCCEEDED = "Succeeded"
+    """The operation has completed successfully."""
+    FAILED = "Failed"
+    """The operation has failed."""
+    CANCELED = "Canceled"
+    """The operation has been canceled by the user."""
+
+
 class PendingUploadType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """The type of pending upload."""
 
     NONE = "None"
     """No pending upload."""
+    BLOB_REFERENCE = "BlobReference"
+    """Blob Reference is the only supported type."""
     TEMPORARY_BLOB_REFERENCE = "TemporaryBlobReference"
     """Temporary Blob Reference is the only supported type."""
 
 
+class RecurrenceType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Recurrence type."""
+
+    HOURLY = "Hourly"
+    """Hourly recurrence pattern."""
+    DAILY = "Daily"
+    """Daily recurrence pattern."""
+    WEEKLY = "Weekly"
+    """Weekly recurrence pattern."""
+    MONTHLY = "Monthly"
+    """Monthly recurrence pattern."""
+
+
 class ResultType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Type of Evaluation result."""
 
@@ -219,6 +372,37 @@ class RiskCategory(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Represents content with ungrounded attributes."""
 
 
+class SampleType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The type of sample used in the analysis."""
+
+    EVALUATION_RESULT_SAMPLE = "EvaluationResultSample"
+    """A sample from the evaluation result."""
+
+
+class ScheduleProvisioningStatus(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Schedule provisioning status."""
+
+    CREATING = "Creating"
+    """Represents the creation status of the schedule."""
+    UPDATING = "Updating"
+    """Represents the updating status of the schedule."""
+    DELETING = "Deleting"
+    """Represents the deleting status of the schedule."""
+    SUCCEEDED = "Succeeded"
+    """Represents the succeeded status of the schedule."""
+    FAILED = "Failed"
+    """Represents the failed status of the schedule."""
+
+
+class ScheduleTaskType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Type of the task."""
+
+    EVALUATION = "Evaluation"
+    """Evaluation task."""
+    INSIGHT = "Insight"
+    """Insight task."""
+
+
 class SimulationType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Simulation type."""
 
@@ -228,3 +412,36 @@ class SimulationType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Custom persona simulation type."""
     HARM_TURN_GENERATOR = "HarmTurnGenerator"
     """Harm turn generator simulation type."""
+
+
+class TargetType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Target type."""
+
+    AZURE_AI_AGENT = "AzureAIAgent"
+    """Azure AI Agent Target"""
+
+
+class TreatmentEffectType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Treatment Effect Type."""
+
+    TOO_FEW_SAMPLES = "TooFewSamples"
+    """Not enough samples to determine treatment effect."""
+    INCONCLUSIVE = "Inconclusive"
+    """No significant difference between treatment and baseline."""
+    CHANGED = "Changed"
+    """Indicates the metric changed with statistical significance, but the direction is neutral."""
+    IMPROVED = "Improved"
+    """Indicates the treatment significantly improved the metric compared to baseline."""
+    DEGRADED = "Degraded"
+    """Indicates the treatment significantly degraded the metric compared to baseline."""
+
+
+class TriggerType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Type of the trigger."""
+
+    CRON = "Cron"
+    """Cron based trigger."""
+    RECURRENCE = "Recurrence"
+    """Recurrence based trigger."""
+    ONE_TIME = "OneTime"
+    """One-time trigger."""