azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure/ai/evaluation/simulator/_prompty/task_simulate.prompty

@@ -0,0 +1,36 @@
+---
+name: TaskSimulatorWithPersona
+description: Simulates a user to complete a conversation
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+  parameters:
+    temperature: 0.0
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: json_object
+
+inputs:
+  task:
+    type: string
+  conversation_history:
+    type: dict
+
+---
+system:
+You should behave as a user who is planning to accomplish this task: {{ task }} and you continue to interact with a system that responds to your queries.
+Make sure your conversation is engaging and interactive.
+Output must be in JSON format
+Here's a sample output:
+{
+  "content": "Here is my follow-up question.",
+  "user": "user"
+}
+
+Output with a json object that continues the conversation, given the conversation history:
+{{ conversation_history }}

azure/ai/evaluation/simulator/_tracing.py

@@ -0,0 +1,92 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+# pylint: disable=C0103,C0114,C0116,E0401,E0611
+
+import functools
+
+from promptflow._sdk._telemetry.activity import ActivityType, monitor_operation
+
+
+def monitor_adversarial_scenario(activity_name: str = "adversarial.simulator.call"):
+    """
+    Monitor an adversarial scenario.
+
+    Parameters:
+    activity_name (str): The name of the activity to monitor.
+    """
+
+    def decorator(func):
+        """
+        Decorator for monitoring an adversarial scenario.
+
+        Parameters:
+        func (function): The function to be decorated.
+        """
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            """
+            Wrapper for monitoring an adversarial scenario.
+
+            Parameters:
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+            """
+            scenario = str(kwargs.get("scenario", None))
+            max_conversation_turns = kwargs.get("max_conversation_turns", None)
+            max_simulation_results = kwargs.get("max_simulation_results", None)
+            jailbreak = kwargs.get("jailbreak", None)
+            decorated_func = monitor_operation(
+                activity_name=activity_name,
+                activity_type=ActivityType.PUBLICAPI,
+                custom_dimensions={
+                    "scenario": scenario,
+                    "max_conversation_turns": max_conversation_turns,
+                    "max_simulation_results": max_simulation_results,
+                    "jailbreak": jailbreak,
+                },
+            )(func)
+
+            return decorated_func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def monitor_task_simulator(func):
+    """
+    Monitor a task simulator.
+
+    Parameters:
+    func (function): The function to be decorated.
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        """
+        Wrapper for monitoring a task simulator.
+
+        Parameters:
+        *args: Variable length argument list.
+        **kwargs: Arbitrary keyword arguments.
+        """
+        text_length = len(kwargs.get("text", ""))
+        user_persona_length = len(kwargs.get("user_persona", []))
+        num_queries = kwargs.get("num_queries", 0)
+        max_conversation_turns = kwargs.get("max_conversation_turns", 0)
+        decorated_func = monitor_operation(
+            activity_name="task.simulator.call",
+            activity_type=ActivityType.PUBLICAPI,
+            custom_dimensions={
+                "text_length": text_length,
+                "user_persona_length": user_persona_length,
+                "number_of_queries": num_queries,
+                "max_conversation_turns": max_conversation_turns,
+            },
+        )(func)
+
+        return decorated_func(*args, **kwargs)
+
+    return wrapper

azure/ai/evaluation/simulator/_utils.py

@@ -0,0 +1,111 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""
+This module contains a utility class for managing a list of JSON lines.
+"""
+import json
+
+
+class JsonLineList(list):
+    """
+    A util to manage a list of JSON lines.
+    """
+
+    def to_json_lines(self):
+        """
+        Converts the list to a string of JSON lines.
+        Each item in the list is converted to a JSON string
+        and appended to the result string with a newline.
+
+        :returns: A string of JSON lines, where each line is a JSON representation of an item in the list.
+        :rtype: str
+        """
+        json_lines = ""
+        for item in self:
+            json_lines += json.dumps(item) + "\n"
+        return json_lines
+
+    def to_eval_qa_json_lines(self):
+        """
+        Converts the list to a string of JSON lines suitable for evaluation in a Q&A format.
+        Each item in the list is expected to be a dictionary with
+        'messages' key. The 'messages' value is a list of
+        dictionaries, each with a 'role' key and a 'content' key.
+        The 'role' value should be either 'user' or 'assistant',
+        and the 'content' value should be a string.
+        If a 'context' key is present in the message, its value is also included
+        in the output.
+
+        :returns: A string of JSON lines.
+        :rtype: str
+        """
+        json_lines = ""
+        for item in self:
+            user_message = None
+            assistant_message = None
+            context = None
+            for message in item["messages"]:
+                if message["role"] == "user":
+                    user_message = message["content"]
+                elif message["role"] == "assistant":
+                    assistant_message = message["content"]
+                if "context" in message:
+                    context = message.get("context", None)
+                if user_message and assistant_message:
+                    if context:
+                        json_lines += (
+                            json.dumps({"query": user_message, "response": assistant_message, "context": context})
+                            + "\n"
+                        )
+                        user_message = assistant_message = context = None
+                    else:
+                        json_lines += json.dumps({"query": user_message, "response": assistant_message}) + "\n"
+                        user_message = assistant_message = None
+
+        return json_lines
+
+
+class JsonLineChatProtocol(dict):
+    """
+    A util to manage a JSON object that follows the chat protocol.
+    """
+
+    def to_json(self):
+        """
+        Converts the object to a JSON string.
+
+        :returns: A JSON representation of the object.
+        :rtype: str
+        """
+        return json.dumps(self)
+
+    def to_eval_qa_json_lines(self):
+        """
+        Converts the object to a string of JSON lines suitable for evaluation in a Q&A format.
+        The object is expected to be a dictionary with 'messages' key.
+        """
+        user_message = None
+        assistant_message = None
+        if "context" in self:
+            context = self["context"]
+        else:
+            context = None
+        json_lines = ""
+        for message in self["messages"]:
+            if message["role"] == "user":
+                user_message = message["content"]
+            elif message["role"] == "assistant":
+                assistant_message = message["content"]
+            if "context" in message and message["context"] is not None:
+                context = message.get("context", context)
+            if user_message and assistant_message:
+                if context:
+                    json_lines += (
+                        json.dumps({"question": user_message, "answer": assistant_message, "context": context}) + "\n"
+                    )
+                    user_message = assistant_message = None
+                else:
+                    json_lines += json.dumps({"question": user_message, "answer": assistant_message}) + "\n"
+                    user_message = assistant_message = None
+        return json_lines