azure-ai-evaluation 1.9.0__py3-none-any.whl → 1.11.0__py3-none-any.whl

azure/ai/evaluation/_converters/_models.py

@@ -3,17 +3,31 @@ import json
 
 from pydantic import BaseModel
 
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 # Models moved in a later version of agents SDK, so try a few different locations
-try:
-    from azure.ai.projects.models import RunStepFunctionToolCall
-except ImportError:
-    pass
-try:
-    from azure.ai.agents.models import RunStepFunctionToolCall
-except ImportError:
-    pass
+# Only import for type checking to avoid runtime import errors
+if TYPE_CHECKING:
+    try:
+        from azure.ai.projects.models import RunStepFunctionToolCall
+    except ImportError:
+        try:
+            from azure.ai.agents.models import RunStepFunctionToolCall
+        except ImportError:
+            # Create a protocol for type checking when the real class isn't available
+            from typing import Protocol
+
+            class RunStepFunctionToolCall(Protocol):
+                """Protocol defining the expected interface for RunStepFunctionToolCall."""
+
+                id: str
+                type: str
+
+                def get(self, key: str, default: Any = None) -> Any: ...
+
+else:
+    # At runtime, we don't need the actual class since it's only used in type annotations
+    RunStepFunctionToolCall = Any
 
 # Message roles constants.
 _SYSTEM = "system"
@@ -33,9 +47,12 @@ _TOOL_CALLS = "tool_calls"
 # Constants to only be used internally in this file for the built-in tools.
 _CODE_INTERPRETER = "code_interpreter"
 _BING_GROUNDING = "bing_grounding"
+_BING_CUSTOM_SEARCH = "bing_custom_search"
 _FILE_SEARCH = "file_search"
 _AZURE_AI_SEARCH = "azure_ai_search"
+_SHAREPOINT_GROUNDING = "sharepoint_grounding"
 _FABRIC_DATAAGENT = "fabric_dataagent"
+_OPENAPI = "openapi"
 
 # Built-in tool descriptions and parameters are hidden, but we include basic descriptions
 # for evaluation purposes.
@@ -44,8 +61,10 @@ _BUILT_IN_DESCRIPTIONS = {
     + "generate code, and create graphs and charts using your data. Supports "
     + "up to 20 files.",
     _BING_GROUNDING: "Enhance model output with web data.",
-    _FILE_SEARCH: "Search for data across uploaded files.",
+    _BING_CUSTOM_SEARCH: "Enables agents to retrieve content from a curated subset of websites, enhancing relevance and reducing noise from public web searches.",
+    _FILE_SEARCH: "Search for data across uploaded files. A single call can return multiple results/files in the 'results' field.",
     _AZURE_AI_SEARCH: "Search an Azure AI Search index for relevant data.",
+    _SHAREPOINT_GROUNDING: "Allows agents to access and retrieve relevant content from Microsoft SharePoint document libraries, grounding responses in organizational knowledge.",
     _FABRIC_DATAAGENT: "Connect to Microsoft Fabric data agents to retrieve data across different data sources.",
 }
 
@@ -59,6 +78,15 @@ _BUILT_IN_PARAMS = {
         "type": "object",
         "properties": {"requesturl": {"type": "string", "description": "URL used in Bing Search API."}},
     },
+    _BING_CUSTOM_SEARCH: {
+        "type": "object",
+        "properties": {
+            "requesturl": {
+                "type": "string",
+                "description": "Search queries, along with pre-configured site restrictions or domain filters.",
+            }
+        },
+    },
     _FILE_SEARCH: {
         "type": "object",
         "properties": {
@@ -76,6 +104,12 @@ _BUILT_IN_PARAMS = {
         "type": "object",
         "properties": {"input": {"type": "string", "description": "Search terms to use."}},
     },
+    _SHAREPOINT_GROUNDING: {
+        "type": "object",
+        "properties": {
+            "input": {"type": "string", "description": "A natural language query to search SharePoint content."}
+        },
+    },
     _FABRIC_DATAAGENT: {
         "type": "object",
         "properties": {"input": {"type": "string", "description": "Search terms to use."}},
@@ -217,6 +251,27 @@ class ToolDefinition(BaseModel):
     parameters: dict
 
 
+class OpenAPIToolDefinition(BaseModel):
+    """Represents OpenAPI tool definition that will be used in the agent.
+    :param name: The name of the tool.
+    :type name: str
+    :param type: The type of the tool.
+    :type type: str
+    :param description: A description of the tool.
+    :type description: str
+    :param parameters: The parameters required by the tool.
+    :type parameters: dict
+    """
+
+    name: str
+    type: str
+    description: Optional[str] = None
+    spec: object
+    auth: object
+    default_params: Optional[list[str]] = None
+    functions: list[ToolDefinition]
+
+
 class ToolCall:
     """Represents a tool call, used as an intermediate step in the conversion process.
 
@@ -247,7 +302,7 @@ class EvaluatorData(BaseModel):
 
     query: List[Message]
     response: List[Message]
-    tool_definitions: List[ToolDefinition]
+    tool_definitions: List[Union[ToolDefinition, OpenAPIToolDefinition]]
 
     def to_json(self):
         """Converts the result to a JSON string.
@@ -277,14 +332,16 @@ def break_tool_call_into_messages(tool_call: ToolCall, run_id: str) -> List[Mess
     # all in most of the cases, and bing would only show the API URL, without arguments or results.
     # Bing grounding would have "bing_grounding" in details with "requesturl" that will just be the API path with query.
     # TODO: Work with AI Services to add converter support for BingGrounding and CodeInterpreter.
-    if hasattr(tool_call.details, _FUNCTION):
+    if hasattr(tool_call.details, _FUNCTION) or tool_call.details.get("function"):
         # This is the internals of the content object that will be included with the tool call.
         tool_call_id = tool_call.details.id
         content_tool_call = {
             "type": _TOOL_CALL,
             "tool_call_id": tool_call_id,
-            "name": tool_call.details.function.name,
-            "arguments": safe_loads(tool_call.details.function.arguments),
+            "name": tool_call.details.get(_FUNCTION).get("name") if tool_call.details.get(_FUNCTION) else None,
+            "arguments": safe_loads(
+                tool_call.details.get(_FUNCTION).get("arguments") if tool_call.details.get(_FUNCTION) else None
+            ),
         }
     else:
         # Treat built-in tools separately.  Object models may be unique so handle each case separately
@@ -322,27 +379,19 @@ def break_tool_call_into_messages(tool_call: ToolCall, run_id: str) -> List[Mess
     # assistant's action of calling the tool.
     messages.append(AssistantMessage(run_id=run_id, content=[to_dict(content_tool_call)], createdAt=tool_call.created))
 
-    if hasattr(tool_call.details, _FUNCTION):
-        output = safe_loads(tool_call.details.function["output"])
+    if hasattr(tool_call.details, _FUNCTION) or tool_call.details.get("function"):
+        output = safe_loads(tool_call.details.get("function")["output"])
     else:
         try:
             # Some built-ins may have output, others may not
             # Try to retrieve it, but if we don't find anything, skip adding the message
             # Just manually converting to dicts for easy serialization for now rather than custom serializers
             if tool_call.details.type == _CODE_INTERPRETER:
-                output = tool_call.details.code_interpreter.outputs
+                output = [result.as_dict() for result in tool_call.details.code_interpreter.outputs]
             elif tool_call.details.type == _BING_GROUNDING:
                 return messages  # not supported yet from bing grounding tool
             elif tool_call.details.type == _FILE_SEARCH:
-                output = [
-                    {
-                        "file_id": result.file_id,
-                        "file_name": result.file_name,
-                        "score": result.score,
-                        "content": result.content,
-                    }
-                    for result in tool_call.details.file_search.results
-                ]
+                output = [result.as_dict() for result in tool_call.details.file_search.results]
             elif tool_call.details.type == _AZURE_AI_SEARCH:
                 output = tool_call.details.azure_ai_search["output"]
             elif tool_call.details.type == _FABRIC_DATAAGENT:

azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py

@@ -6,6 +6,7 @@ import asyncio
 import logging
 import pandas as pd
 import sys
+import itertools
 from collections import defaultdict
 from concurrent.futures import Future
 from os import PathLike
@@ -16,15 +17,34 @@ from ..._legacy._batch_engine._run_submitter import RunSubmitter
 from ..._legacy._batch_engine._config import BatchEngineConfig
 from ..._legacy._batch_engine._run import Run
 from ..._legacy._adapters._constants import LINE_NUMBER
+from ..._legacy._adapters.types import AttrDict
 from ..._legacy._common._thread_pool_executor_with_context import ThreadPoolExecutorWithContext
+from ..._evaluate._utils import _has_aggregator
+from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC
 
+from .._utils import get_int_env_var as get_int
 
-LOGGER = logging.getLogger(__name__)
+
+LOGGER = logging.getLogger("run")
+MISSING_VALUE: Final[int] = sys.maxsize
 
 
 class RunSubmitterClient:
-    def __init__(self, config: Optional[BatchEngineConfig] = None) -> None:
-        self._config = config or BatchEngineConfig(LOGGER, use_async=True)
+    def __init__(self, *, raise_on_errors: bool = False, config: Optional[BatchEngineConfig] = None) -> None:
+        if config:
+            self._config = config
+        else:
+            # Generate default config and apply any overrides to the configuration from environment variables
+            self._config = BatchEngineConfig(LOGGER, use_async=True)
+            if (val := get_int(PF_BATCH_TIMEOUT_SEC, MISSING_VALUE)) != MISSING_VALUE:
+                self._config.batch_timeout_seconds = val
+            if (val := get_int("PF_LINE_TIMEOUT_SEC", MISSING_VALUE)) != MISSING_VALUE:
+                self._config.line_timeout_seconds = val
+            if (val := get_int("PF_WORKER_COUNT", MISSING_VALUE)) != MISSING_VALUE:
+                self._config.max_concurrency = val
+
+        self._config.raise_on_error = raise_on_errors
+
         self._thread_pool = ThreadPoolExecutorWithContext(
             thread_name_prefix="evaluators_thread", max_workers=self._config.max_concurrency
         )
@@ -44,7 +64,6 @@ class RunSubmitterClient:
         # input. Update the inputs so that each entry is a dictionary with a data key
         # that contains the original input data.
         inputs = [{"data": input_data} for input_data in data.to_dict(orient="records")]
-
         # Pass the correct previous run to the evaluator
         run: Optional[BatchClientRun] = kwargs.pop("run", None)
         if run:
@@ -75,29 +94,58 @@ class RunSubmitterClient:
     def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pd.DataFrame:
         run = self._get_run(client_run)
 
-        data: Dict[str, List[Any]] = defaultdict(list)
-        stop_at: Final[int] = self._config.default_num_results if not all_results else sys.maxsize
+        def concat(*dataframes: pd.DataFrame) -> pd.DataFrame:
+            return pd.concat(dataframes, axis=1, verify_integrity=True)
 
-        def _update(prefix: str, items: Sequence[Mapping[str, Any]]) -> None:
-            for i, line in enumerate(items):
-                if i >= stop_at:
-                    break
-                for k, value in line.items():
-                    key = f"{prefix}.{k}"
-                    data[key].append(value)
+        def to_dataframe(items: Sequence[Mapping[str, Any]], *, max_length: Optional[int] = None) -> pd.DataFrame:
+            """Convert a sequence of dictionaries to a DataFrame.
 
-        # Go from a list of dictionaries (i.e. a row view of the data) to a dictionary of lists
-        # (i.e. a column view of the data)
-        _update("inputs", run.inputs)
-        _update("inputs", [{LINE_NUMBER: i} for i in range(len(run.inputs))])
-        _update("outputs", run.outputs)
+            :param items: Sequence of dictionaries to convert.
+            :type items: Sequence[Mapping[str, Any]]
+            :param max_length: Maximum number of items to include in the DataFrame. If None, include all items.
+            :type max_length: Optional[int]
+            :return: DataFrame containing the items.
+            :rtype: pd.DataFrame
+            """
+            max_length = None if all_results else self._config.default_num_results
+            return pd.DataFrame(data=items if all_results else itertools.islice(items, max_length))
 
-        df = pd.DataFrame(data).reindex(columns=[k for k in data.keys()])
-        return df
+        inputs = concat(
+            to_dataframe(run.inputs), to_dataframe([{LINE_NUMBER: i} for i in range(len(run.inputs))])
+        ).add_prefix(Prefixes.INPUTS)
+
+        outputs = to_dataframe(run.outputs).add_prefix(Prefixes.OUTPUTS)
+
+        return concat(inputs, outputs)
 
     def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
         run = self._get_run(client_run)
-        return dict(run.metrics)
+        return {**run.metrics, **self._get_aggregated_metrics(client_run)}
+
+    def _get_aggregated_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        aggregated_metrics = None
+        run = self._get_run(client_run)
+        try:
+            if _has_aggregator(run.dynamic_callable):
+                result_df = pd.DataFrame(run.outputs)
+                if len(result_df.columns) == 1 and result_df.columns[0] == "output":
+                    aggregate_input = result_df["output"].tolist()
+                else:
+                    aggregate_input = [AttrDict(item) for item in result_df.to_dict("records")]
+
+                aggr_func = getattr(run.dynamic_callable, "__aggregate__")
+                aggregated_metrics = aggr_func(aggregate_input)
+
+        except Exception as ex:  # pylint: disable=broad-exception-caught
+            LOGGER.warning("Error calculating aggregations for evaluator, failed with error %s", ex)
+
+        if not isinstance(aggregated_metrics, dict):
+            LOGGER.warning(
+                "Aggregated metrics for evaluator is not a dictionary will not be logged as metrics",
+            )
+            return {}
+
+        return aggregated_metrics
 
     def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
         run = self._get_run(client_run)
@@ -110,7 +158,7 @@ class RunSubmitterClient:
             "duration": str(run.duration),
             "completed_lines": total_lines - failed_lines,
             "failed_lines": failed_lines,
-            # "log_path": "",
+            "log_path": None,
         }
 
     @staticmethod

azure/ai/evaluation/_evaluate/_eval_run.py

@@ -81,6 +81,8 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
         ~azure.ai.evaluation._promptflow.azure._lite_azure_management_client.LiteMLClient
     :param promptflow_run: The promptflow run used by the
     :type promptflow_run: Optional[promptflow._sdk.entities.Run]
+    :param tags: A dictionary of tags to be added to the evaluation run for tracking and organization purposes.
+    :type tags: Optional[Dict[str, str]]
     """
 
     _MAX_RETRIES = 5
@@ -98,6 +100,7 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
         workspace_name: str,
         management_client: LiteMLClient,
         promptflow_run: Optional[Run] = None,
+        tags: Optional[Dict[str, str]] = None,
     ) -> None:
         self._tracking_uri: str = tracking_uri
         self._subscription_id: str = subscription_id
@@ -107,6 +110,7 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
         self._is_promptflow_run: bool = promptflow_run is not None
         self._run_name = run_name
         self._promptflow_run = promptflow_run
+        self._tags = tags or {}
         self._status = RunStatus.NOT_STARTED
         self._url_base: Optional[str] = None
         self._info: Optional[RunInfo] = None
@@ -173,11 +177,20 @@ class EvalRun(contextlib.AbstractContextManager):  # pylint: disable=too-many-in
                 )
             else:
                 url = f"https://{self._url_base}/mlflow/v2.0" f"{self._get_scope()}/api/2.0/mlflow/runs/create"
+
+                # Prepare tags: start with user tags, ensure mlflow.user is set
+                run_tags = self._tags.copy()
+                if "mlflow.user" not in run_tags:
+                    run_tags["mlflow.user"] = "azure-ai-evaluation"
+
+                # Convert tags to MLflow format
+                tags_list = [{"key": key, "value": value} for key, value in run_tags.items()]
+
                 body = {
                     "experiment_id": "0",
                     "user_id": "azure-ai-evaluation",
                     "start_time": int(time.time() * 1000),
-                    "tags": [{"key": "mlflow.user", "value": "azure-ai-evaluation"}],
+                    "tags": tags_list,
                 }
                 if self._run_name:
                     body["run_name"] = self._run_name