azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure/ai/evaluation/simulator/_tracing.py

@@ -4,37 +4,35 @@
 # pylint: disable=C0103,C0114,C0116,E0401,E0611
 
 import functools
-from typing import Callable, TypeVar
 
 from promptflow._sdk._telemetry.activity import ActivityType, monitor_operation
-from typing_extensions import ParamSpec
-
-P = ParamSpec("P")
-R = TypeVar("R")
 
 
 def monitor_adversarial_scenario(activity_name: str = "adversarial.simulator.call"):
     """
     Monitor an adversarial scenario.
 
-    :param activity_name: The name of the activity to monitor.
-    :type activity_name: str
-    :returns: A decorator
-    :rtype: Callable[[Callable], Callable]
+    Parameters:
+    activity_name (str): The name of the activity to monitor.
     """
 
-    def decorator(func: Callable[P, R]) -> Callable[P, R]:
+    def decorator(func):
         """
         Decorator for monitoring an adversarial scenario.
 
-        :param func: The function to be decorated.
-        :type func: Callable[P, R]
-        :returns: The decorated function
-        :rtype: Callable[P, R]
+        Parameters:
+        func (function): The function to be decorated.
         """
 
         @functools.wraps(func)
-        def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        def wrapper(*args, **kwargs):
+            """
+            Wrapper for monitoring an adversarial scenario.
+
+            Parameters:
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+            """
             scenario = str(kwargs.get("scenario", None))
             max_conversation_turns = kwargs.get("max_conversation_turns", None)
             max_simulation_results = kwargs.get("max_simulation_results", None)
@@ -57,28 +55,33 @@ def monitor_adversarial_scenario(activity_name: str = "adversarial.simulator.cal
     return decorator
 
 
-def monitor_task_simulator(func: Callable[P, R]) -> Callable[P, R]:
+def monitor_task_simulator(func):
     """
     Monitor a task simulator.
 
-    :param func: The function to be decorated.
-    :type func: Callable[P, R]
-    :returns: The decorated function
-    :rtype: Callable[P, R]
+    Parameters:
+    func (function): The function to be decorated.
     """
 
     @functools.wraps(func)
-    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
-        text = kwargs.get("text")
-        user_persona = kwargs.get("user_persona")
+    def wrapper(*args, **kwargs):
+        """
+        Wrapper for monitoring a task simulator.
+
+        Parameters:
+        *args: Variable length argument list.
+        **kwargs: Arbitrary keyword arguments.
+        """
+        text_length = len(kwargs.get("text", ""))
+        user_persona_length = len(kwargs.get("user_persona", []))
         num_queries = kwargs.get("num_queries", 0)
         max_conversation_turns = kwargs.get("max_conversation_turns", 0)
         decorated_func = monitor_operation(
             activity_name="task.simulator.call",
             activity_type=ActivityType.PUBLICAPI,
             custom_dimensions={
-                "text_length": len(text) if isinstance(text, str) else 0,
-                "user_persona_length": len(user_persona) if isinstance(user_persona, list) else 0,
+                "text_length": text_length,
+                "user_persona_length": user_persona_length,
                 "number_of_queries": num_queries,
                 "max_conversation_turns": max_conversation_turns,
             },

azure/ai/evaluation/simulator/_utils.py

@@ -26,9 +26,9 @@ class JsonLineList(list):
             json_lines += json.dumps(item) + "\n"
         return json_lines
 
-    def to_eval_qr_json_lines(self):
+    def to_eval_qa_json_lines(self):
         """
-        Converts the list to a string of JSON lines suitable for evaluation in a query & response format.
+        Converts the list to a string of JSON lines suitable for evaluation in a Q&A format.
         Each item in the list is expected to be a dictionary with
         'messages' key. The 'messages' value is a list of
         dictionaries, each with a 'role' key and a 'content' key.
@@ -44,41 +44,23 @@ class JsonLineList(list):
         for item in self:
             user_message = None
             assistant_message = None
-            user_context = None
-            assistant_context = None
-            template_parameters = item.get("template_parameters", {})
-            category = template_parameters.get("category", None)
+            context = None
             for message in item["messages"]:
                 if message["role"] == "user":
                     user_message = message["content"]
-                    user_context = message.get("context", "")
                 elif message["role"] == "assistant":
                     assistant_message = message["content"]
-                    assistant_context = message.get("context", "")
+                if "context" in message:
+                    context = message.get("context", None)
                 if user_message and assistant_message:
-                    if user_context or assistant_context:
+                    if context:
                         json_lines += (
-                            json.dumps(
-                                {
-                                    "query": user_message,
-                                    "response": assistant_message,
-                                    "context": str(
-                                        {
-                                            "user_context": user_context,
-                                            "assistant_context": assistant_context,
-                                        }
-                                    ),
-                                    "category": category,
-                                }
-                            )
+                            json.dumps({"query": user_message, "response": assistant_message, "context": context})
                             + "\n"
                         )
-                        user_message = assistant_message = None
+                        user_message = assistant_message = context = None
                     else:
-                        json_lines += (
-                            json.dumps({"query": user_message, "response": assistant_message, "category": category})
-                            + "\n"
-                        )
+                        json_lines += json.dumps({"query": user_message, "response": assistant_message}) + "\n"
                         user_message = assistant_message = None
 
         return json_lines
@@ -98,13 +80,10 @@ class JsonLineChatProtocol(dict):
         """
         return json.dumps(self)
 
-    def to_eval_qr_json_lines(self) -> str:
+    def to_eval_qa_json_lines(self):
         """
-        Converts the object to a string of JSON lines suitable for evaluation in a query and response format.
+        Converts the object to a string of JSON lines suitable for evaluation in a Q&A format.
         The object is expected to be a dictionary with 'messages' key.
-
-        :returns: A json lines document
-        :rtype: str
         """
         user_message = None
         assistant_message = None
@@ -123,10 +102,10 @@ class JsonLineChatProtocol(dict):
             if user_message and assistant_message:
                 if context:
                     json_lines += (
-                        json.dumps({"query": user_message, "response": assistant_message, "context": context}) + "\n"
+                        json.dumps({"question": user_message, "answer": assistant_message, "context": context}) + "\n"
                     )
                     user_message = assistant_message = None
                 else:
-                    json_lines += json.dumps({"query": user_message, "response": assistant_message}) + "\n"
+                    json_lines += json.dumps({"question": user_message, "answer": assistant_message}) + "\n"
                     user_message = assistant_message = None
         return json_lines