azure-ai-evaluation 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

azure/ai/evaluation/_converters/_models.py

@@ -0,0 +1,302 @@
+import datetime
+import json
+
+from pydantic import BaseModel
+
+from azure.ai.projects.models import RunStepFunctionToolCall
+
+from typing import List, Optional, Union
+
+# Message roles constants.
+_SYSTEM = "system"
+_USER = "user"
+_AGENT = "assistant"
+_TOOL = "tool"
+
+# Constant definitions for what tool details include.
+_TOOL_CALL = "tool_call"
+_TOOL_RESULT = "tool_result"
+_FUNCTION = "function"
+
+# This is returned by AI services in the API to filter against tool invocations.
+_TOOL_CALLS = "tool_calls"
+
+
+class Message(BaseModel):
+    """Represents a message in a conversation with agents, assistants, and tools. We need to export these structures
+    to JSON for evaluators and we have custom fields such as createdAt, run_id, and tool_call_id, so we cannot use
+    the standard pydantic models provided by OpenAI.
+
+    :param createdAt: The timestamp when the message was created.
+    :type createdAt: datetime.datetime
+    :param run_id: The ID of the run associated with the message. Optional.
+    :type run_id: Optional[str]
+    :param role: The role of the message sender (e.g., system, user, tool, assistant).
+    :type role: str
+    :param content: The content of the message, which can be a string or a list of dictionaries.
+    :type content: Union[str, List[dict]]
+    """
+
+    createdAt: Optional[Union[datetime.datetime, int]] = None  # SystemMessage wouldn't have this
+    run_id: Optional[str] = None
+    tool_call_id: Optional[str] = None  # see ToolMessage
+    role: str
+    content: Union[str, List[dict]]
+
+
+class SystemMessage(Message):
+    """Represents a system message in a conversation with agents, assistants, and tools.
+
+    :param role: The role of the message sender, which is always 'system'.
+    :type role: str
+    """
+
+    role: str = _SYSTEM
+
+
+class UserMessage(Message):
+    """Represents a user message in a conversation with agents, assistants, and tools.
+
+    :param role: The role of the message sender, which is always 'user'.
+    :type role: str
+    """
+
+    role: str = _USER
+
+
+class ToolMessage(Message):
+    """Represents a tool message in a conversation with agents, assistants, and tools.
+
+    :param run_id: The ID of the run associated with the message.
+    :type run_id: str
+    :param role: The role of the message sender, which is always 'tool'.
+    :type role: str
+    :param tool_call_id: The ID of the tool call associated with the message. Optional.
+    :type tool_call_id: Optional[str]
+    """
+
+    run_id: str
+    role: str = _TOOL
+    tool_call_id: Optional[str] = None
+
+
+class AssistantMessage(Message):
+    """Represents an assistant message.
+
+    :param run_id: The ID of the run associated with the message.
+    :type run_id: str
+    :param role: The role of the message sender, which is always 'assistant'.
+    :type role: str
+    """
+
+    run_id: str
+    role: str = _AGENT
+
+
+class ToolDefinition(BaseModel):
+    """Represents a tool definition that will be used in the agent.
+
+    :param name: The name of the tool.
+    :type name: str
+    :param description: A description of the tool.
+    :type description: str
+    :param parameters: The parameters required by the tool.
+    :type parameters: dict
+    """
+
+    name: str
+    description: Optional[str] = None
+    parameters: dict
+
+
+class ToolCall:
+    """Represents a tool call, used as an intermediate step in the conversion process.
+
+    :param created: The timestamp when the tool call was created.
+    :type created: datetime.datetime
+    :param completed: The timestamp when the tool call was completed.
+    :type completed: datetime.datetime
+    :param details: The details of the tool call.
+    :type details: RunStepFunctionToolCall
+    """
+
+    def __init__(self, created: datetime.datetime, completed: datetime.datetime, details: RunStepFunctionToolCall):
+        self.created = created
+        self.completed = completed
+        self.details = details
+
+
+class EvaluatorData(BaseModel):
+    """Represents the result of a conversion.
+
+    :param query: A list of messages representing the system message, chat history, and user query.
+    :type query: List[Message]
+    :param response: A list of messages representing the assistant's response, including tool calls and results.
+    :type response: List[Message]
+    :param tool_definitions: A list of tool definitions used in the agent.
+    :type tool_definitions: List[ToolDefinition]
+    """
+
+    query: List[Message]
+    response: List[Message]
+    tool_definitions: List[ToolDefinition]
+
+    def to_json(self):
+        """Converts the result to a JSON string.
+
+        :return: The JSON representation of the result.
+        :rtype: str
+        """
+        return self.model_dump_json(exclude={}, exclude_none=True)
+
+
+def break_tool_call_into_messages(tool_call: ToolCall, run_id: str) -> List[Message]:
+    """
+    Breaks a tool call into a list of messages, including the tool call and its result.
+
+    :param tool_call: The tool call to be broken into messages.
+    :type tool_call: ToolCall
+    :param run_id: The ID of the run associated with the messages.
+    :type run_id: str
+    :return: A list of messages representing the tool call and its result.
+    :rtype: List[Message]
+    """
+    # We will use this as our accumulator.
+    messages: List[Message] = []
+
+    # As of March 17th, 2025, we only support custom functions due to built-in code interpreters and bing grounding
+    # tooling not reporting their function calls in the same way. Code interpreters don't include the tool call at
+    # all in most of the cases, and bing would only show the API URL, without arguments or results.
+    # Bing grounding would have "bing_grounding" in details with "requesturl" that will just be the API path with query.
+    # TODO: Work with AI Services to add converter support for BingGrounding and CodeInterpreter.
+    if hasattr(tool_call.details, _FUNCTION):
+        # This is the internals of the content object that will be included with the tool call.
+        tool_call_id = tool_call.details.id
+        content_tool_call = {
+            "type": _TOOL_CALL,
+            "tool_call_id": tool_call_id,
+            "name": tool_call.details.function.name,
+            "arguments": safe_loads(tool_call.details.function.arguments),
+        }
+    else:
+        # Treat built-in tools separately.  Object models may be unique so handle each case separately
+        # Just converting to dicts here rather than custom serializers for simplicity for now.
+        # Don't fail if we run into a newly seen tool, just skip
+        if tool_call.details["type"] == "code_interpreter":
+            arguments = {"input": tool_call.details.code_interpreter.input}
+        elif tool_call.details["type"] == "bing_grounding":
+            arguments = {"requesturl": tool_call.details["bing_grounding"]["requesturl"]}
+        elif tool_call.details["type"] == "file_search":
+            options = tool_call.details["file_search"]["ranking_options"]
+            arguments = {
+                "ranking_options": {"ranker": options["ranker"], "score_threshold": options["score_threshold"]}
+            }
+        else:
+            # unsupported tool type, skip
+            return messages
+        try:
+            tool_call_id = tool_call.details.id
+            content_tool_call = {
+                "type": _TOOL_CALL,
+                "tool_call_id": tool_call_id,
+                "name": tool_call.details.type,
+                "arguments": arguments,
+            }
+        except:
+            return messages
+
+    # We format it into an assistant message, where the content is a singleton list of the content object.
+    # It should be a tool message, since this is the call, but the given schema treats this message as
+    # assistant's action of calling the tool.
+    messages.append(AssistantMessage(run_id=run_id, content=[to_dict(content_tool_call)], createdAt=tool_call.created))
+
+    if hasattr(tool_call.details, _FUNCTION):
+        output = safe_loads(tool_call.details.function.output)
+    else:
+        try:
+            # Some built-ins may have output, others may not
+            # Try to retrieve it, but if we don't find anything, skip adding the message
+            # Just manually converting to dicts for easy serialization for now rather than custom serializers
+            if tool_call.details.type == "code_interpreter":
+                output = tool_call.details.code_interpreter.outputs
+            elif tool_call.details.type == "bing_grounding":
+                return messages  # not supported yet from bing grounding tool
+            elif tool_call.details.type == "file_search":
+                output = [
+                    {
+                        "file_id": result.file_id,
+                        "file_name": result.file_name,
+                        "score": result.score,
+                        "content": result.content,
+                    }
+                    for result in tool_call.details.file_search.results
+                ]
+        except:
+            return messages
+
+    # Now, onto the tool result, which only includes the result of the function call.
+    content_tool_call_result = {"type": _TOOL_RESULT, _TOOL_RESULT: output}
+
+    # Since this is a tool's action of returning, we put it as a tool message.
+    messages.append(
+        ToolMessage(
+            run_id=run_id,
+            tool_call_id=tool_call_id,
+            content=[to_dict(content_tool_call_result)],
+            createdAt=tool_call.completed,
+        )
+    )
+    return messages
+
+
+def to_dict(obj) -> dict:
+    """
+    Converts an object to a dictionary.
+
+    :param obj: The object to be converted.
+    :type obj: Any
+    :return: The dictionary representation of the object.
+    :rtype: dict
+    """
+    return json.loads(json.dumps(obj))
+
+
+def safe_loads(data: str) -> Union[dict, str]:
+    """
+    Safely loads a JSON string into a Python dictionary or returns the original string if loading fails.
+    :param data: The JSON string to be loaded.
+    :type data: str
+    :return: The loaded dictionary or the original string.
+    :rtype: Union[dict, str]
+    """
+    try:
+        return json.loads(data)
+    except json.JSONDecodeError:
+        return data
+
+
+def convert_message(msg: dict) -> Message:
+    """
+    Converts a dictionary to the appropriate Message subclass.
+
+    :param msg: The message dictionary.
+    :type msg: dict
+    :return: The Message object.
+    :rtype: Message
+    """
+    role = msg["role"]
+    if role == "system":
+        return SystemMessage(content=str(msg["content"]))
+    elif role == "user":
+        return UserMessage(content=msg["content"], createdAt=msg["createdAt"])
+    elif role == "assistant":
+        return AssistantMessage(run_id=str(msg["run_id"]), content=msg["content"], createdAt=msg["createdAt"])
+    elif role == "tool":
+        return ToolMessage(
+            run_id=str(msg["run_id"]),
+            tool_call_id=str(msg["tool_call_id"]),
+            content=msg["content"],
+            createdAt=msg["createdAt"],
+        )
+    else:
+        raise ValueError(f"Unknown role: {role}")

azure/ai/evaluation/_evaluate/_batch_run/__init__.py

@@ -3,8 +3,15 @@
 # ---------------------------------------------------------
 from .eval_run_context import EvalRunContext
 from .code_client import CodeClient
-from .proxy_client import ProxyClient
+from .proxy_client import ProxyClient, ProxyRun
+from ._run_submitter_client import RunSubmitterClient
 from .target_run_context import TargetRunContext
-from .proxy_client import ProxyRun
 
-__all__ = ["CodeClient", "ProxyClient", "EvalRunContext", "TargetRunContext", "ProxyRun"]
+__all__ = [
+    "CodeClient",
+    "ProxyClient",
+    "EvalRunContext",
+    "TargetRunContext",
+    "ProxyRun",
+    "RunSubmitterClient",
+]

azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py

@@ -0,0 +1,104 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import logging
+import pandas as pd
+import sys
+from collections import defaultdict
+from concurrent.futures import Future, ThreadPoolExecutor
+from os import PathLike
+from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Sequence, Union, cast
+
+from .batch_clients import BatchClientRun, HasAsyncCallable
+from ..._legacy._batch_engine._run_submitter import RunSubmitter
+from ..._legacy._batch_engine._config import BatchEngineConfig
+from ..._legacy._batch_engine._run import Run
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class RunSubmitterClient:
+    def __init__(self, config: Optional[BatchEngineConfig] = None) -> None:
+        self._config = config or BatchEngineConfig(LOGGER, use_async=True)
+        self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread")
+
+    def run(
+        self,
+        flow: Callable,
+        data: Union[str, PathLike, pd.DataFrame],
+        column_mapping: Optional[Dict[str, str]] = None,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> BatchClientRun:
+        if not isinstance(data, pd.DataFrame):
+            # Should never get here
+            raise ValueError("Data must be a pandas DataFrame")
+        if not column_mapping:
+            raise ValueError("Column mapping must be provided")
+
+        # The column mappings are index by data to indicate they come from the data
+        # input. Update the inputs so that each entry is a dictionary with a data key
+        # that contains the original input data.
+        inputs = [{"data": input_data} for input_data in data.to_dict(orient="records")]
+
+        # always uses async behind the scenes
+        if isinstance(flow, HasAsyncCallable):
+            flow = flow._to_async()  # pylint: disable=protected-access
+
+        run_submitter = RunSubmitter(self._config)
+        run_future = self._thread_pool.submit(
+            run_submitter.submit,
+            dynamic_callable=flow,
+            inputs=inputs,
+            column_mapping=column_mapping,
+            name_prefix=evaluator_name,
+            created_on=kwargs.pop("created_on", None),
+            storage_creator=kwargs.pop("storage_creator", None),
+            **kwargs,
+        )
+
+        return run_future
+
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pd.DataFrame:
+        run = self._get_run(client_run)
+
+        data: Dict[str, List[Any]] = defaultdict(list)
+        stop_at: Final[int] = self._config.default_num_results if not all_results else sys.maxsize
+
+        def _update(prefix: str, items: Sequence[Mapping[str, Any]]) -> None:
+            for i, line in enumerate(items):
+                if i >= stop_at:
+                    break
+                for k, value in line.items():
+                    key = f"{prefix}.{k}"
+                    data[key].append(value)
+
+        _update("inputs", run.inputs)
+        _update("outputs", run.outputs)
+
+        df = pd.DataFrame(data).reindex(columns=[k for k in data.keys()])
+        return df
+
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_run(client_run)
+        return dict(run.metrics)
+
+    def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_run(client_run)
+
+        total_lines = run.result.total_lines if run.result else 0
+        failed_lines = run.result.failed_lines if run.result else 0
+
+        return {
+            "status": run.status.value,
+            "duration": str(run.duration),
+            "completed_lines": total_lines - failed_lines,
+            "failed_lines": failed_lines,
+            # "log_path": "",
+        }
+
+    @staticmethod
+    def _get_run(run: BatchClientRun) -> Run:
+        return cast(Future[Run], run).result()

azure/ai/evaluation/_evaluate/_batch_run/batch_clients.py

@@ -0,0 +1,82 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import pandas
+from os import PathLike
+from typing import Any, Awaitable, Callable, Dict, Optional, Protocol, Union, runtime_checkable
+
+
+class BatchClientRun(Protocol):
+    """The protocol for the batch client run."""
+
+    pass
+
+
+@runtime_checkable
+class HasAsyncCallable(Protocol):
+    """The protocol for an object that has an async callable."""
+
+    def _to_async(self) -> Callable[[Any, Any], Awaitable[Any]]: ...
+
+
+class BatchClient(Protocol):
+    """The protocol for the batch client. This allows for running a flow on a data source
+    and getting the details of the run."""
+
+    def run(
+        self,
+        flow: Callable,
+        data: Union[str, PathLike, pandas.DataFrame],
+        column_mapping: Optional[Dict[str, str]] = None,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> BatchClientRun:
+        """Run the given flow on the data with the given column mapping.
+
+        :param flow: The flow to run.
+        :type flow: Union[Callable, HasAsyncCallable]
+        :param data: The JSONL file containing the data to run the flow on,
+                     or the loaded data
+        :type data: Union[str, PathLike]
+        :param column_mapping: The column mapping to use.
+        :type column_mapping: Mapping[str, str]
+        :param name: The name of the run.
+        :type name: Optional[str]
+        :param kwargs: Additional keyword arguments to pass to the flow.
+        :return: The result of the batch client run.
+        :rtype: BatchClientRun
+        """
+        ...
+
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pandas.DataFrame:
+        """Get the details of the run.
+
+        :param client_run: The run to get the details of.
+        :type client_run: BatchClientRun
+        :param all_results: Whether to get all results.
+        :type all_results: bool
+        :return: The details of the run.
+        :rtype: pandas.DataFrame
+        """
+        ...
+
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        """Get the metrics of the run.
+
+        :param client_run: The run to get the metrics of.
+        :type client_run: BatchClientRun
+        :return: The metrics of the run.
+        :rtype: Mapping[str, Any]
+        """
+        ...
+
+    def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        """Get the summary of the run.
+
+        :param client_run: The run to get the summary of.
+        :type client_run: BatchClientRun
+        :return: The summary of the run.
+        :rtype: Mapping[str, Any]
+        """
+        ...

azure/ai/evaluation/_evaluate/_batch_run/code_client.py

@@ -6,17 +6,17 @@ import json
 import logging
 import os
 from concurrent.futures import Future
-from pathlib import Path
-from typing import Any, Callable, Dict, Optional, Union, cast
+from typing import Any, Callable, Dict, Optional, Sequence, Union, cast
 
 import pandas as pd
-from promptflow.contracts.types import AttrDict
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from azure.ai.evaluation._legacy._adapters.types import AttrDict
+from azure.ai.evaluation._legacy._adapters.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
 from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 from ..._constants import PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT
+from .batch_clients import BatchClientRun
 
 LOGGER = logging.getLogger(__name__)
 
@@ -84,7 +84,7 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             for param in inspect.signature(evaluator).parameters.values()
             if param.name not in ["args", "kwargs"]
         }
-        for value in input_df.to_dict("records"):
+        for value in cast(Sequence[Dict[str, Any]], input_df.to_dict("records")):
             # Filter out only the parameters that are present in the input data
             # if no parameters then pass data as is
             filtered_values = {k: v for k, v in value.items() if k in parameters} if len(parameters) > 0 else value
@@ -133,10 +133,10 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
     def run(
         self,  # pylint: disable=unused-argument
         flow: Callable,
-        data: Union[os.PathLike, Path, pd.DataFrame],
-        evaluator_name: Optional[str] = None,
+        data: Union[str, os.PathLike, pd.DataFrame],
         column_mapping: Optional[Dict[str, str]] = None,
-        **kwargs,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
     ) -> CodeRun:
         input_df = data
         if not isinstance(input_df, pd.DataFrame):
@@ -157,7 +157,7 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             evaluator=flow,
             input_df=input_df,
             column_mapping=column_mapping,
-            evaluator_name=evaluator_name,
+            evaluator_name=evaluator_name or "",
         )
 
         return CodeRun(
@@ -169,11 +169,13 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             ),
         )
 
-    def get_details(self, run: CodeRun, all_results: bool = False) -> pd.DataFrame:
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pd.DataFrame:
+        run = self._get_result(client_run)
         result_df = run.get_result_df(exclude_inputs=not all_results)
         return result_df
 
-    def get_metrics(self, run: CodeRun) -> Dict[str, Any]:
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_result(client_run)
         try:
             aggregated_metrics = run.get_aggregated_metrics()
             print("Aggregated metrics")
@@ -183,6 +185,10 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             return {}
         return aggregated_metrics
 
-    def get_run_summary(self, run: CodeRun) -> Any:  # pylint: disable=unused-argument
+    def get_run_summary(self, client_run: BatchClientRun) -> Any:  # pylint: disable=unused-argument
         # Not implemented
         return None
+
+    @staticmethod
+    def _get_result(run: BatchClientRun) -> CodeRun:
+        return cast(CodeRun, run)

azure/ai/evaluation/_evaluate/_batch_run/eval_run_context.py

@@ -5,9 +5,9 @@ import os
 import types
 from typing import Optional, Type, Union
 
-from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP, PF_FLOW_META_LOAD_IN_SUBPROCESS
-from promptflow._utils.user_agent_utils import ClientUserAgentUtil
-from promptflow.tracing._integrations._openai_injector import inject_openai_api, recover_openai_api
+from azure.ai.evaluation._legacy._adapters._constants import PF_FLOW_ENTRY_IN_TMP, PF_FLOW_META_LOAD_IN_SUBPROCESS
+from azure.ai.evaluation._legacy._adapters.utils import ClientUserAgentUtil
+from azure.ai.evaluation._legacy._adapters.tracing import inject_openai_api, recover_openai_api
 
 from azure.ai.evaluation._constants import (
     OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
@@ -19,6 +19,8 @@ from azure.ai.evaluation._constants import (
 
 from ..._user_agent import USER_AGENT
 from .._utils import set_event_loop_policy
+from .batch_clients import BatchClient
+from ._run_submitter_client import RunSubmitterClient
 from .code_client import CodeClient
 from .proxy_client import ProxyClient
 
@@ -33,7 +35,7 @@ class EvalRunContext:
     ]
     """
 
-    def __init__(self, client: Union[CodeClient, ProxyClient]) -> None:
+    def __init__(self, client: BatchClient) -> None:
         self.client = client
         self._is_batch_timeout_set_by_system = False
         self._is_otel_timeout_set_by_system = False
@@ -64,6 +66,9 @@ class EvalRunContext:
             # For addressing the issue of asyncio event loop closed on Windows
             set_event_loop_policy()
 
+        if isinstance(self.client, RunSubmitterClient):
+            set_event_loop_policy()
+
     def __exit__(
         self,
         exc_type: Optional[Type[BaseException]],