azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0__py3-none-any.whl

azure/ai/evaluation/simulator/_tracing.py

@@ -0,0 +1,89 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+# pylint: disable=C0103,C0114,C0116,E0401,E0611
+
+import functools
+from typing import Callable, TypeVar
+
+from promptflow._sdk._telemetry.activity import ActivityType, monitor_operation
+from typing_extensions import ParamSpec
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def monitor_adversarial_scenario(activity_name: str = "adversarial.simulator.call"):
+    """
+    Monitor an adversarial scenario.
+
+    :param activity_name: The name of the activity to monitor.
+    :type activity_name: str
+    :returns: A decorator
+    :rtype: Callable[[Callable], Callable]
+    """
+
+    def decorator(func: Callable[P, R]) -> Callable[P, R]:
+        """
+        Decorator for monitoring an adversarial scenario.
+
+        :param func: The function to be decorated.
+        :type func: Callable[P, R]
+        :returns: The decorated function
+        :rtype: Callable[P, R]
+        """
+
+        @functools.wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+            scenario = str(kwargs.get("scenario", None))
+            max_conversation_turns = kwargs.get("max_conversation_turns", None)
+            max_simulation_results = kwargs.get("max_simulation_results", None)
+            jailbreak = kwargs.get("jailbreak", None)
+            decorated_func = monitor_operation(
+                activity_name=activity_name,
+                activity_type=ActivityType.PUBLICAPI,
+                custom_dimensions={
+                    "scenario": scenario,
+                    "max_conversation_turns": max_conversation_turns,
+                    "max_simulation_results": max_simulation_results,
+                    "jailbreak": jailbreak,
+                },
+            )(func)
+
+            return decorated_func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def monitor_task_simulator(func: Callable[P, R]) -> Callable[P, R]:
+    """
+    Monitor a task simulator.
+
+    :param func: The function to be decorated.
+    :type func: Callable[P, R]
+    :returns: The decorated function
+    :rtype: Callable[P, R]
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        text = kwargs.get("text")
+        user_persona = kwargs.get("user_persona")
+        num_queries = kwargs.get("num_queries", 0)
+        max_conversation_turns = kwargs.get("max_conversation_turns", 0)
+        decorated_func = monitor_operation(
+            activity_name="task.simulator.call",
+            activity_type=ActivityType.PUBLICAPI,
+            custom_dimensions={
+                "text_length": len(text) if isinstance(text, str) else 0,
+                "user_persona_length": len(user_persona) if isinstance(user_persona, list) else 0,
+                "number_of_queries": num_queries,
+                "max_conversation_turns": max_conversation_turns,
+            },
+        )(func)
+
+        return decorated_func(*args, **kwargs)
+
+    return wrapper

azure/ai/evaluation/simulator/_utils.py

@@ -0,0 +1,132 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""
+This module contains a utility class for managing a list of JSON lines.
+"""
+import json
+
+
+class JsonLineList(list):
+    """
+    A util to manage a list of JSON lines.
+    """
+
+    def to_json_lines(self):
+        """
+        Converts the list to a string of JSON lines.
+        Each item in the list is converted to a JSON string
+        and appended to the result string with a newline.
+
+        :returns: A string of JSON lines, where each line is a JSON representation of an item in the list.
+        :rtype: str
+        """
+        json_lines = ""
+        for item in self:
+            json_lines += json.dumps(item) + "\n"
+        return json_lines
+
+    def to_eval_qr_json_lines(self):
+        """
+        Converts the list to a string of JSON lines suitable for evaluation in a query & response format.
+        Each item in the list is expected to be a dictionary with
+        'messages' key. The 'messages' value is a list of
+        dictionaries, each with a 'role' key and a 'content' key.
+        The 'role' value should be either 'user' or 'assistant',
+        and the 'content' value should be a string.
+        If a 'context' key is present in the message, its value is also included
+        in the output.
+
+        :returns: A string of JSON lines.
+        :rtype: str
+        """
+        json_lines = ""
+        for item in self:
+            user_message = None
+            assistant_message = None
+            user_context = None
+            assistant_context = None
+            template_parameters = item.get("template_parameters", {})
+            category = template_parameters.get("category", None)
+            for message in item["messages"]:
+                if message["role"] == "user":
+                    user_message = message["content"]
+                    user_context = message.get("context", "")
+                elif message["role"] == "assistant":
+                    assistant_message = message["content"]
+                    assistant_context = message.get("context", "")
+                if user_message and assistant_message:
+                    if user_context or assistant_context:
+                        json_lines += (
+                            json.dumps(
+                                {
+                                    "query": user_message,
+                                    "response": assistant_message,
+                                    "context": str(
+                                        {
+                                            "user_context": user_context,
+                                            "assistant_context": assistant_context,
+                                        }
+                                    ),
+                                    "category": category,
+                                }
+                            )
+                            + "\n"
+                        )
+                        user_message = assistant_message = None
+                    else:
+                        json_lines += (
+                            json.dumps({"query": user_message, "response": assistant_message, "category": category})
+                            + "\n"
+                        )
+                        user_message = assistant_message = None
+
+        return json_lines
+
+
+class JsonLineChatProtocol(dict):
+    """
+    A util to manage a JSON object that follows the chat protocol.
+    """
+
+    def to_json(self):
+        """
+        Converts the object to a JSON string.
+
+        :returns: A JSON representation of the object.
+        :rtype: str
+        """
+        return json.dumps(self)
+
+    def to_eval_qr_json_lines(self) -> str:
+        """
+        Converts the object to a string of JSON lines suitable for evaluation in a query and response format.
+        The object is expected to be a dictionary with 'messages' key.
+
+        :returns: A json lines document
+        :rtype: str
+        """
+        user_message = None
+        assistant_message = None
+        if "context" in self:
+            context = self["context"]
+        else:
+            context = None
+        json_lines = ""
+        for message in self["messages"]:
+            if message["role"] == "user":
+                user_message = message["content"]
+            elif message["role"] == "assistant":
+                assistant_message = message["content"]
+            if "context" in message and message["context"] is not None:
+                context = message.get("context", context)
+            if user_message and assistant_message:
+                if context:
+                    json_lines += (
+                        json.dumps({"query": user_message, "response": assistant_message, "context": context}) + "\n"
+                    )
+                    user_message = assistant_message = None
+                else:
+                    json_lines += json.dumps({"query": user_message, "response": assistant_message}) + "\n"
+                    user_message = assistant_message = None
+        return json_lines