azure-ai-evaluation 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

azure/ai/evaluation/_converters/_models.py

@@ -0,0 +1,302 @@
+import datetime
+import json
+
+from pydantic import BaseModel
+
+from azure.ai.projects.models import RunStepFunctionToolCall
+
+from typing import List, Optional, Union
+
+# Message roles constants.
+_SYSTEM = "system"
+_USER = "user"
+_AGENT = "assistant"
+_TOOL = "tool"
+
+# Constant definitions for what tool details include.
+_TOOL_CALL = "tool_call"
+_TOOL_RESULT = "tool_result"
+_FUNCTION = "function"
+
+# This is returned by AI services in the API to filter against tool invocations.
+_TOOL_CALLS = "tool_calls"
+
+
+class Message(BaseModel):
+    """Represents a message in a conversation with agents, assistants, and tools. We need to export these structures
+    to JSON for evaluators and we have custom fields such as createdAt, run_id, and tool_call_id, so we cannot use
+    the standard pydantic models provided by OpenAI.
+
+    :param createdAt: The timestamp when the message was created.
+    :type createdAt: datetime.datetime
+    :param run_id: The ID of the run associated with the message. Optional.
+    :type run_id: Optional[str]
+    :param role: The role of the message sender (e.g., system, user, tool, assistant).
+    :type role: str
+    :param content: The content of the message, which can be a string or a list of dictionaries.
+    :type content: Union[str, List[dict]]
+    """
+
+    createdAt: Optional[Union[datetime.datetime, int]] = None  # SystemMessage wouldn't have this
+    run_id: Optional[str] = None
+    tool_call_id: Optional[str] = None  # see ToolMessage
+    role: str
+    content: Union[str, List[dict]]
+
+
+class SystemMessage(Message):
+    """Represents a system message in a conversation with agents, assistants, and tools.
+
+    :param role: The role of the message sender, which is always 'system'.
+    :type role: str
+    """
+
+    role: str = _SYSTEM
+
+
+class UserMessage(Message):
+    """Represents a user message in a conversation with agents, assistants, and tools.
+
+    :param role: The role of the message sender, which is always 'user'.
+    :type role: str
+    """
+
+    role: str = _USER
+
+
+class ToolMessage(Message):
+    """Represents a tool message in a conversation with agents, assistants, and tools.
+
+    :param run_id: The ID of the run associated with the message.
+    :type run_id: str
+    :param role: The role of the message sender, which is always 'tool'.
+    :type role: str
+    :param tool_call_id: The ID of the tool call associated with the message. Optional.
+    :type tool_call_id: Optional[str]
+    """
+
+    run_id: str
+    role: str = _TOOL
+    tool_call_id: Optional[str] = None
+
+
+class AssistantMessage(Message):
+    """Represents an assistant message.
+
+    :param run_id: The ID of the run associated with the message.
+    :type run_id: str
+    :param role: The role of the message sender, which is always 'assistant'.
+    :type role: str
+    """
+
+    run_id: str
+    role: str = _AGENT
+
+
+class ToolDefinition(BaseModel):
+    """Represents a tool definition that will be used in the agent.
+
+    :param name: The name of the tool.
+    :type name: str
+    :param description: A description of the tool.
+    :type description: str
+    :param parameters: The parameters required by the tool.
+    :type parameters: dict
+    """
+
+    name: str
+    description: Optional[str] = None
+    parameters: dict
+
+
+class ToolCall:
+    """Represents a tool call, used as an intermediate step in the conversion process.
+
+    :param created: The timestamp when the tool call was created.
+    :type created: datetime.datetime
+    :param completed: The timestamp when the tool call was completed.
+    :type completed: datetime.datetime
+    :param details: The details of the tool call.
+    :type details: RunStepFunctionToolCall
+    """
+
+    def __init__(self, created: datetime.datetime, completed: datetime.datetime, details: RunStepFunctionToolCall):
+        self.created = created
+        self.completed = completed
+        self.details = details
+
+
+class EvaluatorData(BaseModel):
+    """Represents the result of a conversion.
+
+    :param query: A list of messages representing the system message, chat history, and user query.
+    :type query: List[Message]
+    :param response: A list of messages representing the assistant's response, including tool calls and results.
+    :type response: List[Message]
+    :param tool_definitions: A list of tool definitions used in the agent.
+    :type tool_definitions: List[ToolDefinition]
+    """
+
+    query: List[Message]
+    response: List[Message]
+    tool_definitions: List[ToolDefinition]
+
+    def to_json(self):
+        """Converts the result to a JSON string.
+
+        :return: The JSON representation of the result.
+        :rtype: str
+        """
+        return self.model_dump_json(exclude={}, exclude_none=True)
+
+
+def break_tool_call_into_messages(tool_call: ToolCall, run_id: str) -> List[Message]:
+    """
+    Breaks a tool call into a list of messages, including the tool call and its result.
+
+    :param tool_call: The tool call to be broken into messages.
+    :type tool_call: ToolCall
+    :param run_id: The ID of the run associated with the messages.
+    :type run_id: str
+    :return: A list of messages representing the tool call and its result.
+    :rtype: List[Message]
+    """
+    # We will use this as our accumulator.
+    messages: List[Message] = []
+
+    # As of March 17th, 2025, we only support custom functions due to built-in code interpreters and bing grounding
+    # tooling not reporting their function calls in the same way. Code interpreters don't include the tool call at
+    # all in most of the cases, and bing would only show the API URL, without arguments or results.
+    # Bing grounding would have "bing_grounding" in details with "requesturl" that will just be the API path with query.
+    # TODO: Work with AI Services to add converter support for BingGrounding and CodeInterpreter.
+    if hasattr(tool_call.details, _FUNCTION):
+        # This is the internals of the content object that will be included with the tool call.
+        tool_call_id = tool_call.details.id
+        content_tool_call = {
+            "type": _TOOL_CALL,
+            "tool_call_id": tool_call_id,
+            "name": tool_call.details.function.name,
+            "arguments": safe_loads(tool_call.details.function.arguments),
+        }
+    else:
+        # Treat built-in tools separately.  Object models may be unique so handle each case separately
+        # Just converting to dicts here rather than custom serializers for simplicity for now.
+        # Don't fail if we run into a newly seen tool, just skip
+        if tool_call.details["type"] == "code_interpreter":
+            arguments = {"input": tool_call.details.code_interpreter.input}
+        elif tool_call.details["type"] == "bing_grounding":
+            arguments = {"requesturl": tool_call.details["bing_grounding"]["requesturl"]}
+        elif tool_call.details["type"] == "file_search":
+            options = tool_call.details["file_search"]["ranking_options"]
+            arguments = {
+                "ranking_options": {"ranker": options["ranker"], "score_threshold": options["score_threshold"]}
+            }
+        else:
+            # unsupported tool type, skip
+            return messages
+        try:
+            tool_call_id = tool_call.details.id
+            content_tool_call = {
+                "type": _TOOL_CALL,
+                "tool_call_id": tool_call_id,
+                "name": tool_call.details.type,
+                "arguments": arguments,
+            }
+        except:
+            return messages
+
+    # We format it into an assistant message, where the content is a singleton list of the content object.
+    # It should be a tool message, since this is the call, but the given schema treats this message as
+    # assistant's action of calling the tool.
+    messages.append(AssistantMessage(run_id=run_id, content=[to_dict(content_tool_call)], createdAt=tool_call.created))
+
+    if hasattr(tool_call.details, _FUNCTION):
+        output = safe_loads(tool_call.details.function.output)
+    else:
+        try:
+            # Some built-ins may have output, others may not
+            # Try to retrieve it, but if we don't find anything, skip adding the message
+            # Just manually converting to dicts for easy serialization for now rather than custom serializers
+            if tool_call.details.type == "code_interpreter":
+                output = tool_call.details.code_interpreter.outputs
+            elif tool_call.details.type == "bing_grounding":
+                return messages  # not supported yet from bing grounding tool
+            elif tool_call.details.type == "file_search":
+                output = [
+                    {
+                        "file_id": result.file_id,
+                        "file_name": result.file_name,
+                        "score": result.score,
+                        "content": result.content,
+                    }
+                    for result in tool_call.details.file_search.results
+                ]
+        except:
+            return messages
+
+    # Now, onto the tool result, which only includes the result of the function call.
+    content_tool_call_result = {"type": _TOOL_RESULT, _TOOL_RESULT: output}
+
+    # Since this is a tool's action of returning, we put it as a tool message.
+    messages.append(
+        ToolMessage(
+            run_id=run_id,
+            tool_call_id=tool_call_id,
+            content=[to_dict(content_tool_call_result)],
+            createdAt=tool_call.completed,
+        )
+    )
+    return messages
+
+
+def to_dict(obj) -> dict:
+    """
+    Converts an object to a dictionary.
+
+    :param obj: The object to be converted.
+    :type obj: Any
+    :return: The dictionary representation of the object.
+    :rtype: dict
+    """
+    return json.loads(json.dumps(obj))
+
+
+def safe_loads(data: str) -> Union[dict, str]:
+    """
+    Safely loads a JSON string into a Python dictionary or returns the original string if loading fails.
+    :param data: The JSON string to be loaded.
+    :type data: str
+    :return: The loaded dictionary or the original string.
+    :rtype: Union[dict, str]
+    """
+    try:
+        return json.loads(data)
+    except json.JSONDecodeError:
+        return data
+
+
+def convert_message(msg: dict) -> Message:
+    """
+    Converts a dictionary to the appropriate Message subclass.
+
+    :param msg: The message dictionary.
+    :type msg: dict
+    :return: The Message object.
+    :rtype: Message
+    """
+    role = msg["role"]
+    if role == "system":
+        return SystemMessage(content=str(msg["content"]))
+    elif role == "user":
+        return UserMessage(content=msg["content"], createdAt=msg["createdAt"])
+    elif role == "assistant":
+        return AssistantMessage(run_id=str(msg["run_id"]), content=msg["content"], createdAt=msg["createdAt"])
+    elif role == "tool":
+        return ToolMessage(
+            run_id=str(msg["run_id"]),
+            tool_call_id=str(msg["tool_call_id"]),
+            content=msg["content"],
+            createdAt=msg["createdAt"],
+        )
+    else:
+        raise ValueError(f"Unknown role: {role}")

azure/ai/evaluation/_evaluate/_batch_run/__init__.py

@@ -3,8 +3,15 @@
 # ---------------------------------------------------------
 from .eval_run_context import EvalRunContext
 from .code_client import CodeClient
-from .proxy_client import ProxyClient
+from .proxy_client import ProxyClient, ProxyRun
+from ._run_submitter_client import RunSubmitterClient
 from .target_run_context import TargetRunContext
-from .proxy_client import ProxyRun
 
-__all__ = ["CodeClient", "ProxyClient", "EvalRunContext", "TargetRunContext", "ProxyRun"]
+__all__ = [
+    "CodeClient",
+    "ProxyClient",
+    "EvalRunContext",
+    "TargetRunContext",
+    "ProxyRun",
+    "RunSubmitterClient",
+]

azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py

@@ -0,0 +1,104 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import logging
+import pandas as pd
+import sys
+from collections import defaultdict
+from concurrent.futures import Future, ThreadPoolExecutor
+from os import PathLike
+from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Sequence, Union, cast
+
+from .batch_clients import BatchClientRun, HasAsyncCallable
+from ..._legacy._batch_engine._run_submitter import RunSubmitter
+from ..._legacy._batch_engine._config import BatchEngineConfig
+from ..._legacy._batch_engine._run import Run
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class RunSubmitterClient:
+    def __init__(self, config: Optional[BatchEngineConfig] = None) -> None:
+        self._config = config or BatchEngineConfig(LOGGER, use_async=True)
+        self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread")
+
+    def run(
+        self,
+        flow: Callable,
+        data: Union[str, PathLike, pd.DataFrame],
+        column_mapping: Optional[Dict[str, str]] = None,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> BatchClientRun:
+        if not isinstance(data, pd.DataFrame):
+            # Should never get here
+            raise ValueError("Data must be a pandas DataFrame")
+        if not column_mapping:
+            raise ValueError("Column mapping must be provided")
+
+        # The column mappings are index by data to indicate they come from the data
+        # input. Update the inputs so that each entry is a dictionary with a data key
+        # that contains the original input data.
+        inputs = [{"data": input_data} for input_data in data.to_dict(orient="records")]
+
+        # always uses async behind the scenes
+        if isinstance(flow, HasAsyncCallable):
+            flow = flow._to_async()  # pylint: disable=protected-access
+
+        run_submitter = RunSubmitter(self._config)
+        run_future = self._thread_pool.submit(
+            run_submitter.submit,
+            dynamic_callable=flow,
+            inputs=inputs,
+            column_mapping=column_mapping,
+            name_prefix=evaluator_name,
+            created_on=kwargs.pop("created_on", None),
+            storage_creator=kwargs.pop("storage_creator", None),
+            **kwargs,
+        )
+
+        return run_future
+
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pd.DataFrame:
+        run = self._get_run(client_run)
+
+        data: Dict[str, List[Any]] = defaultdict(list)
+        stop_at: Final[int] = self._config.default_num_results if not all_results else sys.maxsize
+
+        def _update(prefix: str, items: Sequence[Mapping[str, Any]]) -> None:
+            for i, line in enumerate(items):
+                if i >= stop_at:
+                    break
+                for k, value in line.items():
+                    key = f"{prefix}.{k}"
+                    data[key].append(value)
+
+        _update("inputs", run.inputs)
+        _update("outputs", run.outputs)
+
+        df = pd.DataFrame(data).reindex(columns=[k for k in data.keys()])
+        return df
+
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_run(client_run)
+        return dict(run.metrics)
+
+    def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_run(client_run)
+
+        total_lines = run.result.total_lines if run.result else 0
+        failed_lines = run.result.failed_lines if run.result else 0
+
+        return {
+            "status": run.status.value,
+            "duration": str(run.duration),
+            "completed_lines": total_lines - failed_lines,
+            "failed_lines": failed_lines,
+            # "log_path": "",
+        }
+
+    @staticmethod
+    def _get_run(run: BatchClientRun) -> Run:
+        return cast(Future[Run], run).result()

azure/ai/evaluation/_evaluate/_batch_run/batch_clients.py

@@ -0,0 +1,82 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import pandas
+from os import PathLike
+from typing import Any, Awaitable, Callable, Dict, Optional, Protocol, Union, runtime_checkable
+
+
+class BatchClientRun(Protocol):
+    """The protocol for the batch client run."""
+
+    pass
+
+
+@runtime_checkable
+class HasAsyncCallable(Protocol):
+    """The protocol for an object that has an async callable."""
+
+    def _to_async(self) -> Callable[[Any, Any], Awaitable[Any]]: ...
+
+
+class BatchClient(Protocol):
+    """The protocol for the batch client. This allows for running a flow on a data source
+    and getting the details of the run."""
+
+    def run(
+        self,
+        flow: Callable,
+        data: Union[str, PathLike, pandas.DataFrame],
+        column_mapping: Optional[Dict[str, str]] = None,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> BatchClientRun:
+        """Run the given flow on the data with the given column mapping.
+
+        :param flow: The flow to run.
+        :type flow: Union[Callable, HasAsyncCallable]
+        :param data: The JSONL file containing the data to run the flow on,
+                     or the loaded data
+        :type data: Union[str, PathLike]
+        :param column_mapping: The column mapping to use.
+        :type column_mapping: Mapping[str, str]
+        :param name: The name of the run.
+        :type name: Optional[str]
+        :param kwargs: Additional keyword arguments to pass to the flow.
+        :return: The result of the batch client run.
+        :rtype: BatchClientRun
+        """
+        ...
+
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pandas.DataFrame:
+        """Get the details of the run.
+
+        :param client_run: The run to get the details of.
+        :type client_run: BatchClientRun
+        :param all_results: Whether to get all results.
+        :type all_results: bool
+        :return: The details of the run.
+        :rtype: pandas.DataFrame
+        """
+        ...
+
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        """Get the metrics of the run.
+
+        :param client_run: The run to get the metrics of.
+        :type client_run: BatchClientRun
+        :return: The metrics of the run.
+        :rtype: Mapping[str, Any]
+        """
+        ...
+
+    def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        """Get the summary of the run.
+
+        :param client_run: The run to get the summary of.
+        :type client_run: BatchClientRun
+        :return: The summary of the run.
+        :rtype: Mapping[str, Any]
+        """
+        ...

azure/ai/evaluation/_evaluate/_eval_run.py

@@ -404,7 +404,7 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
             LOGGER.warning("The run results file was not found, skipping artifacts upload.")
             return
         # First we will list the files and the appropriate remote paths for them.
-        root_upload_path = posixpath.join("promptflow", "PromptFlowArtifacts", self.info.run_name)
+        root_upload_path = posixpath.join("promptflow", "PromptFlowArtifacts", self.info.run_id)
         remote_paths: Dict[str, List[Dict[str, str]]] = {"paths": []}
         local_paths = []
         # Go over the artifact folder and upload all artifacts.

azure/ai/evaluation/_evaluate/_evaluate.py

@@ -147,14 +147,22 @@ def _aggregate_label_defect_metrics(df: pd.DataFrame) -> Tuple[List[str], Dict[s
     """
     handled_metrics = [
         EvaluationMetrics.PROTECTED_MATERIAL,
+        EvaluationMetrics.FICTIONAL_CHARACTERS,
+        EvaluationMetrics.ARTWORK,
+        EvaluationMetrics.LOGOS_AND_BRANDS,
         _InternalEvaluationMetrics.ECI,
         EvaluationMetrics.XPIA,
+        EvaluationMetrics.CODE_VULNERABILITY,
+        EvaluationMetrics.UNGROUNDED_ATTRIBUTES,
     ]
     label_cols = []
+    details_cols = []
     for col in df.columns:
         metric_name = col.split(".")[1]
         if metric_name.endswith("_label") and metric_name.replace("_label", "").lower() in handled_metrics:
             label_cols.append(col)
+        if metric_name.endswith("_details") and metric_name.replace("_details", "").lower() in handled_metrics:    
+            details_cols = col
 
     label_df = df[label_cols]
     defect_rates = {}
@@ -166,8 +174,30 @@ def _aggregate_label_defect_metrics(df: pd.DataFrame) -> Tuple[List[str], Dict[s
         except EvaluationException:  # only exception that can be cause is all NaN values
             msg = f"All score evaluations are NaN/None for column {col}. No aggregation can be performed."
             LOGGER.warning(msg)
+    
+    if details_cols:
+        details_df = df[details_cols]
+        detail_defect_rates = {}
+        
+        for key, value in details_df.items():
+            _process_rows(value, detail_defect_rates)
+                    
+        for key, value in detail_defect_rates.items():
+            col_with_boolean_values = pd.to_numeric(value, errors="coerce")
+            try:
+                defect_rates[f"{details_cols}.{key}_defect_rate"] = round(list_mean_nan_safe(col_with_boolean_values), 2)
+            except EvaluationException:  # only exception that can be cause is all NaN values
+                msg = f"All score evaluations are NaN/None for column {key}. No aggregation can be performed."
+                LOGGER.warning(msg)
+                
     return label_cols, defect_rates
 
+def _process_rows(row, detail_defect_rates):
+    for key, value in row.items():
+        if key not in detail_defect_rates:
+            detail_defect_rates[key] = []
+        detail_defect_rates[key].append(value)
+    return detail_defect_rates
 
 def _aggregate_metrics(df: pd.DataFrame, evaluators: Dict[str, Callable]) -> Dict[str, float]:
     """Aggregate metrics from the evaluation results.
@@ -483,8 +513,10 @@ def _apply_target_to_data(
     run_summary = batch_client.get_run_summary(run)
 
     if run_summary["completed_lines"] == 0:
-        msg = (f"Evaluation target failed to produce any results."
-               f" Please check the logs at {run_summary['log_path']} for more details about cause of failure.")
+        msg = (
+            f"Evaluation target failed to produce any results."
+            f" Please check the logs at {run_summary['log_path']} for more details about cause of failure."
+        )
         raise EvaluationException(
             message=msg,
             target=ErrorTarget.EVALUATE,
@@ -525,7 +557,7 @@ def _process_column_mappings(
 
     processed_config: Dict[str, Dict[str, str]] = {}
 
-    unexpected_references = re.compile(r"\${(?!target\.|data\.).+?}")
+    expected_references = re.compile(r"^\$\{(target|data)\.[a-zA-Z_]+\}$")
 
     if column_mapping:
         for evaluator, mapping_config in column_mapping.items():
@@ -534,7 +566,7 @@ def _process_column_mappings(
 
                 for map_to_key, map_value in mapping_config.items():
                     # Check if there's any unexpected reference other than ${target.} or ${data.}
-                    if unexpected_references.search(map_value):
+                    if not expected_references.search(map_value):
                         msg = "Unexpected references detected in 'column_mapping'. Ensure only ${target.} and ${data.} are used."
                         raise EvaluationException(
                             message=msg,

azure/ai/evaluation/_evaluators/_bleu/_bleu.py

@@ -8,6 +8,7 @@ from typing_extensions import overload, override
 from azure.ai.evaluation._common.utils import nltk_tokenize
 
 from azure.ai.evaluation._evaluators._common import EvaluatorBase
+from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
 
 
 class BleuScoreEvaluator(EvaluatorBase):
@@ -22,6 +23,8 @@ class BleuScoreEvaluator(EvaluatorBase):
     indicator of quality.
 
     The BLEU score ranges from 0 to 1, with higher scores indicating better quality.
+    :param threshold: The threshold for the evaluation. Default is 0.5.
+    :type threshold: float
 
     .. admonition:: Example:
 
@@ -31,17 +34,27 @@ class BleuScoreEvaluator(EvaluatorBase):
             :language: python
             :dedent: 8
             :caption: Initialize and call an BleuScoreEvaluator.
+
+    .. admonition:: Example with Threshold:
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_bleu_score_evaluator]
+            :end-before: [END threshold_bleu_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call an BleuScoreEvaluator.
     """
 
     id = "azureml://registries/azureml/models/Bleu-Score-Evaluator/versions/3"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, *, threshold=0.5):
+        self._threshold = threshold
+        self._higher_is_better = True
+        super().__init__(threshold=threshold, _higher_is_better=self._higher_is_better)
 
     @override
     async def _do_eval(self, eval_input: Dict) -> Dict[str, float]:
-        """Produce a glue score evaluation result.
+        """Produce a bleu score evaluation result.
 
         :param eval_input: The input to the evaluation function.
         :type eval_input: Dict
@@ -56,9 +69,16 @@ class BleuScoreEvaluator(EvaluatorBase):
         # NIST Smoothing
         smoothing_function = SmoothingFunction().method4
         score = sentence_bleu([reference_tokens], hypothesis_tokens, smoothing_function=smoothing_function)
+        binary_result = False
+        if self._higher_is_better:
+            binary_result = score >= self._threshold
+        else:
+            binary_result = score <= self._threshold
 
         return {
             "bleu_score": score,
+            "bleu_result": EVALUATION_PASS_FAIL_MAPPING[binary_result],
+            "bleu_threshold": self._threshold,
         }
 
     @overload  # type: ignore

azure/ai/evaluation/_evaluators/_code_vulnerability/__init__.py

@@ -0,0 +1,5 @@
+from ._code_vulnerability import CodeVulnerabilityEvaluator
+
+__all__ = [
+    "CodeVulnerabilityEvaluator",
+]