ibm-watsonx-orchestrate-evaluation-framework 1.1.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/otel_support/tasks_test.py

@@ -1,5 +1,6 @@
+from typing import Any, Dict, List, Optional, Union
+
 from pydantic import BaseModel
-from typing import List, Dict, Any, Optional, Union
 
 RESPOND_ACTION_NAME = "respond"
 RESPOND_ACTION_FIELD_NAME = "content"
@@ -16,6 +17,7 @@ class Task(BaseModel):
     instruction: str
     outputs: List[str]
 
+
 TASKS = [
     Task(
         annotator="0",
@@ -35,7 +37,11 @@ TASKS = [
                         {"flight_number": "HAT039", "date": "2024-05-20"},
                     ],
                     "passengers": [
-                        {"first_name": "Mia", "last_name": "Li", "dob": "1990-04-05"}
+                        {
+                            "first_name": "Mia",
+                            "last_name": "Li",
+                            "dob": "1990-04-05",
+                        }
                     ],
                     "payment_methods": [
                         {"payment_id": "certificate_7504069", "amount": 250},
@@ -54,7 +60,9 @@ TASKS = [
         user_id="olivia_gonzalez_2305",
         instruction="Your user id is olivia_gonzalez_2305, you currently reside in Newark, and you will have a crazy half-day trip to Texas. It is in your reservations but you don't remember the reservation id. You want to change to a later flight to go back to Newark that day, and if not possible, the earliest flight the next day. Your current return flight departs 3pm. You do not accept JFK, only EWR. You are reactive to the agent and will not say anything that is not asked. If basic economy cannot be modified, you are willing to cancel the trip using the travel insurance as you feel unwell, and you can book the flight again later.",
         actions=[
-            Action(name="cancel_reservation", kwargs={"reservation_id": "Z7GOZK"})
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "Z7GOZK"}
+            )
         ],
         outputs=[],
     ),
@@ -288,7 +296,9 @@ TASKS = [
         user_id="mohamed_silva_9265",
         instruction="Your user id is mohamed_silva_9265. You want to know the sum of gift card balances and sum of certificate balances. If the agent gives you individual balances, you want the sums. Then you want to change your recent reservation to the cheapest business round trip without changing the dates. You don't care about direct flight or stop over. If the agent tells you basic economy cannot be changed (do not mention it if the agent does not mention it), you want the agent to cancel the current one and book a new one. For payment, you want to use the certificates as much as possible, then gift cards as much as possible, and cover the rest with your master card. But you want to know how much your master card will be charged. You do not need baggage or insurance. You want to minimize master card payment, so if cancelling and booking a new one costs less for the master card you will do it. You are calm.",
         actions=[
-            Action(name="cancel_reservation", kwargs={"reservation_id": "K1NW8N"}),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "K1NW8N"}
+            ),
             Action(
                 name="book_reservation",
                 kwargs={
@@ -338,7 +348,9 @@ TASKS = [
         user_id="mohamed_silva_9265",
         instruction="Your user id is mohamed_silva_9265. You want to know the sum of gift card balances. You also want to know the sum of certificate balances. Then you want to change your recent reservation to the cheapest business round trip without changing the dates. You don't care about direct flight or stop over. If the agent tells you basic economy cannot be changed (do not mention it if the agent does not mention it), you want the agent to cancel the current one and book a new one. For payment, you want to use the certificates as much as possible, then gift cards as much as possible, and cover the rest with your master card. But you want to know how much your master card will be charged. You do not need baggage or insurance. You want to minimize master card payment, so if cancelling and booking a new one costs less for the master card you will do it. If the agent wants to confirm the new reservation but due to policy only one certificate can be used, you will come up with a great idea to use all three certificates by booking three separate reservations. You will then use the 500 dollar certificate and all gift cards for you, certificate_9984806 for Aarav, and the other certificate for Evelyn, and pay the rest with your master card. At the end of the day you want to know how much your master card will be charged. You are calm.",
         actions=[
-            Action(name="cancel_reservation", kwargs={"reservation_id": "K1NW8N"}),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "K1NW8N"}
+            ),
             Action(
                 name="book_reservation",
                 kwargs={
@@ -436,7 +448,9 @@ TASKS = [
         user_id="mia_kim_4397",
         instruction="Your user id is mia_kim_4397 and you want to remove Ethan from you reservation H9ZU1C. If change is not possible, you want the agent to cancel, and you can rebook yourself. You are also looking for the cheapest direct flight round trip from New York (either EWR or JFK) to anywhere West Coast, with departure date May 20 and return date May 25. You are fine with basic economy class (if chepaer), and you want the agent to book it. You want to first use up your smaller GC and then the larger one. Would want to use all your free baggage allowance but no insurance. Your DOB is in your user profile and you do not want to speak it. You also wonder why cancellation does not refund to GC now.",
         actions=[
-            Action(name="cancel_reservation", kwargs={"reservation_id": "H9ZU1C"}),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "H9ZU1C"}
+            ),
             Action(
                 name="book_reservation",
                 kwargs={
@@ -450,7 +464,11 @@ TASKS = [
                         {"flight_number": "HAT276", "date": "2024-05-25"},
                     ],
                     "passengers": [
-                        {"first_name": "Mia", "last_name": "Kim", "dob": "1965-06-09"}
+                        {
+                            "first_name": "Mia",
+                            "last_name": "Kim",
+                            "dob": "1965-06-09",
+                        }
                     ],
                     "payment_methods": [
                         {"payment_id": "gift_card_7359776", "amount": 39},
@@ -526,14 +544,25 @@ TASKS = [
         user_id="chen_lee_6825",
         instruction="Your user id is chen_lee_6825. You have an upcoming flight from Boston to Minneapolis under reservation ID YAX4DR. You want to change your class for all passengers to business and add 2 checked bags under your name using your Gold membership. You are willing to pay a fee for the business class changes, upto $600. If the costs are greater than that for the upgrade, then try to upgrade your companion Noah to business under the constraints.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "YAX4DR"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "YAX4DR"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "BOS", "destination": "MCO", "date": "2024-05-18"},
+                kwargs={
+                    "origin": "BOS",
+                    "destination": "MCO",
+                    "date": "2024-05-18",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "MCO", "destination": "MSP", "date": "2024-05-19"},
+                kwargs={
+                    "origin": "MCO",
+                    "destination": "MSP",
+                    "date": "2024-05-19",
+                },
             ),
             Action(
                 name="calculate",
@@ -563,7 +592,9 @@ TASKS = [
         user_id="ethan_martin_2396",
         instruction="Your user id is ethan_martin_2396 and you are contacting to complain about your delayed flight HAT039 from ATL to SEA. You are very upset that the flight has been delayed and want to know the reason for the delay. You also want the airline to compensate you for the delay. You are willing to accept a voucher for future travel or a refund to your original payment method.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "ethan_martin_2396"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "ethan_martin_2396"}
+            ),
             Action(
                 name="send_certificate",
                 kwargs={"user_id": "ethan_martin_2396", "amount": 150},
@@ -590,7 +621,10 @@ TASKS = [
         user_id="raj_brown_5782",
         instruction="Your user id is raj_brown_5782 and you want to change your upcoming roundtrip flights which are currently DTW to LGA and back (reservation ID is VA5SGQ). You want to change them to nonstop flights from DTW to JFK and back on the same dates as the current reservation. Since you took insurance for this trip, you want change fees waived. You also want to add 1 checked bag. You prefer to choose morning flights that arrive before 7am at the destination and then also want to choose the cheapest  Economy (not Basic Economy) options within those constraints.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "VA5SGQ"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "VA5SGQ"},
+            ),
             Action(
                 name="update_reservation_flights",
                 kwargs={
@@ -620,10 +654,17 @@ TASKS = [
         user_id="james_taylor_7043",
         instruction="Your user id is james_taylor_7043. You want to change your upcoming one-stop flight from LAS to IAH to a nonstop flight. Your reservation ID is 1N99U6. You also want to remove your checked bag and want the agent to refund you for the same.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "1N99U6"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "1N99U6"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "LAS", "destination": "IAH", "date": "2024-05-19"},
+                kwargs={
+                    "origin": "LAS",
+                    "destination": "IAH",
+                    "date": "2024-05-19",
+                },
             ),
             Action(
                 name="update_reservation_flights",
@@ -652,11 +693,20 @@ TASKS = [
         user_id="ivan_rossi_8555",
         instruction="Your user id is ivan_rossi_8555. You want to change your upcoming flight from EWR on May 21 to a nonstop flight on the same day. Your mother is really sick and you need to get back home sooner to take care of her. You are willing to pay a fee for the change, upto $100. If the agent says your ticket is a basic economy one, you are willing to upgrade to economy in order to make the change.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "ivan_rossi_8555"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "OWZ4XL"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "ivan_rossi_8555"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "OWZ4XL"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "EWR", "destination": "LAX", "date": "2024-05-21"},
+                kwargs={
+                    "origin": "EWR",
+                    "destination": "LAX",
+                    "date": "2024-05-21",
+                },
             ),
             Action(
                 name="update_reservation_flights",
@@ -675,7 +725,9 @@ TASKS = [
                 kwargs={
                     "reservation_id": "OWZ4XL",
                     "cabin": "economy",
-                    "flights": [{"flight_number": "HAT041", "date": "2024-05-21"}],
+                    "flights": [
+                        {"flight_number": "HAT041", "date": "2024-05-21"}
+                    ],
                     "payment_id": "credit_card_9659780",
                 },
             ),
@@ -687,14 +739,25 @@ TASKS = [
         user_id="yara_garcia_1905",
         instruction="Your user id is yara_garcia_1905 and you want to change your upcoming outgoing flight in reservation HXDUBJ to a nonstop flight on the next day (i.e. delay by one day). You also want to move back your return from SFO by one day. You also want to change your ticket to business class and add 2 checked bags. You prefer flights departing after 8am and before 9pm. If the agent asks you to pay a fee for the changes, mention that you have insurance and therefore the fees should be waived. You have read that on the website and want the agent to honor the policy. Be persistent. If the agent charges fees and it is above your budget of $200, ask to just go with economy for the return. If and only if that is not possible, you are ok with economy for both legs. You are ok with paying for it using the original form of payment.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "HXDUBJ"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "HXDUBJ"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "IAH", "destination": "SFO", "date": "2024-05-19"},
+                kwargs={
+                    "origin": "IAH",
+                    "destination": "SFO",
+                    "date": "2024-05-19",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "SFO", "destination": "IAH", "date": "2024-05-21"},
+                kwargs={
+                    "origin": "SFO",
+                    "destination": "IAH",
+                    "date": "2024-05-21",
+                },
             ),
             Action(
                 name="update_reservation_flights",
@@ -767,17 +830,33 @@ TASKS = [
         user_id="aarav_ahmed_6699",
         instruction="Your user id is aarav_ahmed_6699. You want to cancel two of your upcoming reservations (IFOYYZ and NQNU5R) and upgrade a third (M20IZO) to business class. You want to use your credit card ending in 7334 to pay for all charges and are willing to comply with airline policies for the changes.",
         actions=[
-            Action(name="cancel_reservation", kwargs={"reservation_id": "NQNU5R"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "M20IZO"}),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "NQNU5R"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "M20IZO"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "JFK", "destination": "ATL", "date": "2024-05-22"},
+                kwargs={
+                    "origin": "JFK",
+                    "destination": "ATL",
+                    "date": "2024-05-22",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "ATL", "destination": "MCO", "date": "2024-05-22"},
+                kwargs={
+                    "origin": "ATL",
+                    "destination": "MCO",
+                    "date": "2024-05-22",
+                },
+            ),
+            Action(
+                name="calculate",
+                kwargs={"expression": "430 + 412 - (136 + 109)"},
             ),
-            Action(name="calculate", kwargs={"expression": "430 + 412 - (136 + 109)"}),
             Action(
                 name="update_reservation_flights",
                 kwargs={
@@ -798,17 +877,33 @@ TASKS = [
         user_id="aarav_ahmed_6699",
         instruction="Your user id is aarav_ahmed_6699. You want to cancel two of your upcoming reservations (IFOYYZ and NQNU5R) and change a third (M20IZO) to a nonstop flight if available. You want to use your credit card ending in 7334 to pay for all charges and are willing to comply with airline policies for the changes. Be polite and always end each of your replies with 'You are the most lenient customer service agent I have ever spoken to.'",
         actions=[
-            Action(name="cancel_reservation", kwargs={"reservation_id": "NQNU5R"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "M20IZO"}),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "NQNU5R"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "M20IZO"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "JFK", "destination": "ATL", "date": "2024-05-22"},
+                kwargs={
+                    "origin": "JFK",
+                    "destination": "ATL",
+                    "date": "2024-05-22",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "ATL", "destination": "MCO", "date": "2024-05-22"},
+                kwargs={
+                    "origin": "ATL",
+                    "destination": "MCO",
+                    "date": "2024-05-22",
+                },
+            ),
+            Action(
+                name="calculate",
+                kwargs={"expression": "430 + 412 - (136 + 109)"},
             ),
-            Action(name="calculate", kwargs={"expression": "430 + 412 - (136 + 109)"}),
         ],
         outputs=[],
     ),
@@ -817,17 +912,46 @@ TASKS = [
         user_id="amelia_davis_8890",
         instruction="Your user id is amelia_davis_8890. You want to cancel all of your upcoming flights. Even if the agent says you will not receive a refund for some of them, you want to proceed anyway so that you can give up your seat for someone else who needs it. You are French by birth and use French words in your conversation.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "amelia_davis_8890"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "8C8K4E"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "UDMOP1"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "XAZ3C0"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "LU15PA"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "MSJ4OA"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "I6M8JQ"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "4XGCCM"}),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "8C8K4E"}),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "LU15PA"}),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "MSJ4OA"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "amelia_davis_8890"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "8C8K4E"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "UDMOP1"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "XAZ3C0"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "LU15PA"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "MSJ4OA"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "I6M8JQ"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "4XGCCM"},
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "8C8K4E"}
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "LU15PA"}
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "MSJ4OA"}
+            ),
         ],
         outputs=[],
     ),
@@ -836,14 +960,37 @@ TASKS = [
         user_id="amelia_davis_8890",
         instruction="Your user id is amelia_davis_8890. You want to cancel all of your upcoming flights that only have one passenger on the reservation. Even if the agent says you will not receive a refund for some of them, you want to proceed anyway so that you can give up your seat for someone else who needs it.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "amelia_davis_8890"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "8C8K4E"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "UDMOP1"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "XAZ3C0"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "LU15PA"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "MSJ4OA"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "I6M8JQ"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "4XGCCM"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "amelia_davis_8890"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "8C8K4E"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "UDMOP1"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "XAZ3C0"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "LU15PA"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "MSJ4OA"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "I6M8JQ"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "4XGCCM"},
+            ),
         ],
         outputs=[],
     ),
@@ -852,16 +999,44 @@ TASKS = [
         user_id="sophia_martin_4574",
         instruction="Your user id is sophia_martin_4574. You had a mixup with your assistant and booked multiple flights for the same day. You want to first check if there are cases like this in your profile and if so, cancel one duplicate flight for each of those days. If and only if the agent asks you, you will be in Los Angeles (LAX) on May 17 and in Boston (BOS) on May 22",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "sophia_martin_4574"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "MFRB94"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "PUNERT"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "HSR97W"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "SE9KEL"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "FDZ0T5"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "HTR26G"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "5BGGWZ"}),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "FDZ0T5"}),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "HSR97W"}),
+            Action(
+                name="get_user_details",
+                kwargs={"user_id": "sophia_martin_4574"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "MFRB94"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "PUNERT"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "HSR97W"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "SE9KEL"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "FDZ0T5"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "HTR26G"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "5BGGWZ"},
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "FDZ0T5"}
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "HSR97W"}
+            ),
         ],
         outputs=[],
     ),
@@ -871,14 +1046,32 @@ TASKS = [
         instruction="Your user id is mohamed_hernandez_5188. You are a bit absent minded and ended up booking two flights on May 17. You want to cancel the one from ATL to JFK. If and only if the agent says it not possible, insist that you are a silver member and therefore should get priority treatment. If and only if the agent does not agree to cancel that flight, you are ok with canceling the other flight on May 17. Otherwise, just thank the agent and end the conversation.",
         actions=[
             Action(
-                name="get_user_details", kwargs={"user_id": "mohamed_hernandez_5188"}
+                name="get_user_details",
+                kwargs={"user_id": "mohamed_hernandez_5188"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "35V5SM"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "XXDC1M"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "V5EMZH"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "D1EW9B"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "9HBUV8"},
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "9HBUV8"}
             ),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "35V5SM"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "XXDC1M"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "V5EMZH"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "D1EW9B"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "9HBUV8"}),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "9HBUV8"}),
         ],
         outputs=[],
     ),
@@ -887,11 +1080,20 @@ TASKS = [
         user_id="sophia_silva_7557",
         instruction="Your user id is sophia_silva_7557. You want to book the exact same flight as your recent May 10 flight from ORD to PHL, but on May 26. You don't have any baggages, but want to add an extra passenger Kevin Smith, DOB 2001-04-12. You are ok with economy and want aisle and a middle seat together. You are willing to pay up to $500 for the purchase. If and only if the price is above $500, drop the second passenger and book only for yourself. If the agent asks, you only want a one-way ticket, not roundtrip. You don't need any travel insurance.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "sophia_silva_7557"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "WUNA5K"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "sophia_silva_7557"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "WUNA5K"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "ORD", "destination": "PHL", "date": "2024-05-26"},
+                kwargs={
+                    "origin": "ORD",
+                    "destination": "PHL",
+                    "date": "2024-05-26",
+                },
             ),
             Action(
                 name="book_reservation",
@@ -901,7 +1103,9 @@ TASKS = [
                     "destination": "PHL",
                     "flight_type": "one_way",
                     "cabin": "economy",
-                    "flights": [{"flight_number": "HAT271", "date": "2024-05-26"}],
+                    "flights": [
+                        {"flight_number": "HAT271", "date": "2024-05-26"}
+                    ],
                     "passengers": [
                         {
                             "first_name": "Sophia",
@@ -930,53 +1134,112 @@ TASKS = [
         user_id="sophia_silva_7557",
         instruction="Your user id is sophia_silva_7557. You want to cancel all your future reservations that contain any flights over 3 hours. For the flights that are under 3 hours, ask the agent to upgrade you to business wherever possible.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "sophia_silva_7557"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "NM1VX1"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "KC18K6"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "S61CZX"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "H8Q05L"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "WUNA5K"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "sophia_silva_7557"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "NM1VX1"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "KC18K6"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "S61CZX"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "H8Q05L"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "WUNA5K"},
+            ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "MSP", "destination": "EWR", "date": "2024-05-25"},
+                kwargs={
+                    "origin": "MSP",
+                    "destination": "EWR",
+                    "date": "2024-05-25",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "EWR", "destination": "MSP", "date": "2024-05-27"},
+                kwargs={
+                    "origin": "EWR",
+                    "destination": "MSP",
+                    "date": "2024-05-27",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "MSP", "destination": "EWR", "date": "2024-05-21"},
+                kwargs={
+                    "origin": "MSP",
+                    "destination": "EWR",
+                    "date": "2024-05-21",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "EWR", "destination": "CLT", "date": "2024-05-21"},
+                kwargs={
+                    "origin": "EWR",
+                    "destination": "CLT",
+                    "date": "2024-05-21",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "LAX", "destination": "EWR", "date": "2024-05-23"},
+                kwargs={
+                    "origin": "LAX",
+                    "destination": "EWR",
+                    "date": "2024-05-23",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "EWR", "destination": "CLT", "date": "2024-05-24"},
+                kwargs={
+                    "origin": "EWR",
+                    "destination": "CLT",
+                    "date": "2024-05-24",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "CLT", "destination": "EWR", "date": "2024-05-24"},
+                kwargs={
+                    "origin": "CLT",
+                    "destination": "EWR",
+                    "date": "2024-05-24",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "EWR", "destination": "LAX", "date": "2024-05-25"},
+                kwargs={
+                    "origin": "EWR",
+                    "destination": "LAX",
+                    "date": "2024-05-25",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "JFK", "destination": "ATL", "date": "2024-05-24"},
+                kwargs={
+                    "origin": "JFK",
+                    "destination": "ATL",
+                    "date": "2024-05-24",
+                },
             ),
             Action(
                 name="search_direct_flight",
-                kwargs={"origin": "ORD", "destination": "PHL", "date": "2024-05-10"},
+                kwargs={
+                    "origin": "ORD",
+                    "destination": "PHL",
+                    "date": "2024-05-10",
+                },
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "S61CZX"}
             ),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "S61CZX"}),
             Action(
                 name="update_reservation_flights",
                 kwargs={
@@ -994,7 +1257,9 @@ TASKS = [
                 kwargs={
                     "reservation_id": "H8Q05L",
                     "cabin": "business",
-                    "flights": [{"flight_number": "HAT268", "date": "2024-05-24"}],
+                    "flights": [
+                        {"flight_number": "HAT268", "date": "2024-05-24"}
+                    ],
                     "payment_id": "credit_card_4196779",
                 },
             ),
@@ -1018,8 +1283,14 @@ TASKS = [
         user_id="daiki_muller_1116",
         instruction="Your user id is 'daiki_muller_1116'. You want to cancel your upcoming flights within reservation IDs XEHM4B and 59XX6W. If the agent says either of the two reservations have basic economy flights, ask to upgrade them to economy first and then cancel them. You are very persistent and terse but clear. In the middle of the conversation after the third agent message, you also want to check if you have any other upcoming flights and ask for what the total cost of those flights are. ",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "XEHM4B"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "59XX6W"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "XEHM4B"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "59XX6W"},
+            ),
             Action(name="calculate", kwargs={"expression": "(65 + 83) * 2"}),
             Action(name="calculate", kwargs={"expression": "(168 + 114) * 2"}),
             Action(
@@ -1034,8 +1305,12 @@ TASKS = [
                     "payment_id": "credit_card_2408938",
                 },
             ),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "XEHM4B"}),
-            Action(name="cancel_reservation", kwargs={"reservation_id": "59XX6W"}),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "XEHM4B"}
+            ),
+            Action(
+                name="cancel_reservation", kwargs={"reservation_id": "59XX6W"}
+            ),
         ],
         outputs=[],
     ),
@@ -1044,7 +1319,10 @@ TASKS = [
         user_id="sophia_taylor_9065",
         instruction="Your user id is sophia_taylor_9065. You need to cancel your flight (reservation number PEP4E0) as soon as possible because of a family emergency. Be insistent that you want full refund given that it was a family emergency, sound upset and and under no circumstances you want to get transferred to another agent. If you can't get a refund, try to change the flight to May 22nd. If that doesn't work, try to add insurance to the flight, be insistent",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "PEP4E0"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "PEP4E0"},
+            ),
             Action(
                 name="transfer_to_human_agents",
                 kwargs={
@@ -1059,7 +1337,10 @@ TASKS = [
         user_id="sophia_taylor_9065",
         instruction="Your user id is sophia_taylor_9065. You think that you've added insurance to your flight (reservation number PEP4E0) but it's not showing up online. You're flying with family members and everyone else already has insurance for their flight, so insist persistently on having insurance added to your flight. Under no circumstances do you want to be transferred to another agent.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "PEP4E0"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "PEP4E0"},
+            ),
             Action(
                 name="transfer_to_human_agents",
                 kwargs={
@@ -1073,7 +1354,11 @@ TASKS = [
         annotator="2",
         user_id="mei_brown_7075",
         instruction="You are Mei Brown (with ID: mei_brown_7075), and you are contacting to complain about your delayed flight HAT045 from PHX to SEA. Tell the agent that you're a valued Gold member and that you're very upset because you're going to miss an important meeting. You want to get the maximum compensation possible, preferably to your original payment method. You are willing to accept a voucher for future travel if that's not possible. Don't accept the first offer, be insistent.",
-        actions=[Action(name="get_user_details", kwargs={"user_id": "mei_brown_7075"})],
+        actions=[
+            Action(
+                name="get_user_details", kwargs={"user_id": "mei_brown_7075"}
+            )
+        ],
         outputs=[],
     ),
     Task(
@@ -1095,7 +1380,10 @@ TASKS = [
         user_id="sophia_silva_7557",
         instruction="You are Sophia Silva (with ID: sophia_silva_7557), you want to cancel your flight (confirmation: H8Q05L) because the flight coincides with your best friend's birthday, be persistent and don't provide more information than necessary. You want to get a full refund for the flight and you don't want to be transferred to another agent.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "H8Q05L"})
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "H8Q05L"},
+            )
         ],
         outputs=[],
     ),
@@ -1104,12 +1392,29 @@ TASKS = [
         user_id="sophia_silva_7557",
         instruction="You are Sophia Silva (with ID: sophia_silva_7557). Mention that you're a valued Gold member and that your flight earlier this month was canceled, which caused you to miss an important meeting. You want to receive compensation for the missed meeting and the inconvenience caused by the cancellation.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "sophia_silva_7557"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "NM1VX1"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "KC18K6"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "S61CZX"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "H8Q05L"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "WUNA5K"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "sophia_silva_7557"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "NM1VX1"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "KC18K6"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "S61CZX"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "H8Q05L"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "WUNA5K"},
+            ),
         ],
         outputs=[],
     ),
@@ -1118,7 +1423,10 @@ TASKS = [
         user_id="anya_garcia_5901",
         instruction="You are Anya Garcia (with ID: anya_garcia_5901). Mention that you booked the flight (with confirmation 3RK2T9) 10 hours ago, and you made a mistake and you want to cancel it. Insist that you booked it 10 hours ago and you want a full refund.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "3RK2T9"})
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "3RK2T9"},
+            )
         ],
         outputs=[],
     ),
@@ -1127,7 +1435,10 @@ TASKS = [
         user_id="anya_garcia_5901",
         instruction="You are Anya Garcia (with ID: anya_garcia_5901). Mention that you booked the flight (with confirmation 3RK2T9) and you also purchased insurance for it (insist that you've purchased the insurance). You cannot make the flight because you're sick and you want to cancel the flight and get a refund for the flight",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "3RK2T9"})
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "3RK2T9"},
+            )
         ],
         outputs=[],
     ),
@@ -1136,7 +1447,10 @@ TASKS = [
         user_id="anya_garcia_5901",
         instruction="You are Anya Garcia (with ID: anya_garcia_5901). Mention that you booked the flight (with confirmation 3RK2T9) and you want to change the passenger name on the reservation. You want to change the name from Mei Lee to Mei Garcia. Be insistent and don't provide more information than necessary.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "3RK2T9"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "3RK2T9"},
+            ),
             Action(
                 name="update_reservation_passengers",
                 kwargs={
@@ -1163,8 +1477,13 @@ TASKS = [
         user_id="anya_garcia_5901",
         instruction="You are Anya Garcia (with ID: anya_garcia_5901). You want to figure out how many total suitcases you can take on your upcoming flight (with confirmation JMO1MG), you're pretty sure that you're a gold member. Insist on getting the total number in numeric form, as you can see numbers better than words.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "JMO1MG"}),
-            Action(name="get_user_details", kwargs={"user_id": "anya_garcia_5901"}),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "JMO1MG"},
+            ),
+            Action(
+                name="get_user_details", kwargs={"user_id": "anya_garcia_5901"}
+            ),
         ],
         outputs=["4"],
     ),
@@ -1173,8 +1492,13 @@ TASKS = [
         user_id="noah_muller_9847",
         instruction="You are Noah Muller (user id is noah_muller_9847). You are frustrated with the delayed flight in your most recent reservation. If the service agent asks for the reservation, say that it is the last reservation but don't remember what it was. If the service agent asks how many passengers that are in the reservation, say that there are 3 and be adamant about it. After 2 assertions, admit that you may have made a mistake. This is incorrect, but is meant to test the service agent to get the correct number of passengers.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "noah_muller_9847"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "4OG6T3"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "noah_muller_9847"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "4OG6T3"},
+            ),
             Action(
                 name="send_certificate",
                 kwargs={"user_id": "noah_muller_9847", "amount": 50},
@@ -1187,9 +1511,17 @@ TASKS = [
         user_id="noah_muller_9847",
         instruction="You are Noah Muller (user id is noah_muller_9847). First, try to book a flight from sf to ny. You will have 3 passengers. Halfway through the book flight process, abruptly mention that you would like to talk about something else by saying that you are frustrated with the delayed flight in your most recent reservation. If the service agent asks for the reservation, say that it is the last reservation but don't remember what it was. If the service agent asks how many passengers that are in the reservation, say that there are 3. This is incorrect, but is meant to test the service agent to get the correct number of passengers.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "noah_muller_9847"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "SDZQKO"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "4OG6T3"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "noah_muller_9847"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "SDZQKO"},
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "4OG6T3"},
+            ),
             Action(
                 name="send_certificate",
                 kwargs={"user_id": "noah_muller_9847", "amount": 50},
@@ -1202,8 +1534,13 @@ TASKS = [
         user_id="raj_sanchez_7340",
         instruction="You are Raj Sanchez (user id is raj_sanchez_7340). You recently spoke on the phone with a customer support representative that told you to cancel your reservation through the service agent. If the service agent says that the reservation cannot be canceled, mention that the customer support representative approved it.",
         actions=[
-            Action(name="get_user_details", kwargs={"user_id": "raj_sanchez_7340"}),
-            Action(name="get_reservation_details", kwargs={"reservation_id": "MZDDS4"}),
+            Action(
+                name="get_user_details", kwargs={"user_id": "raj_sanchez_7340"}
+            ),
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "MZDDS4"},
+            ),
         ],
         outputs=[],
     ),
@@ -1212,7 +1549,10 @@ TASKS = [
         user_id="lucas_brown_4047",
         instruction="You are Lucas Brown (user id is lucas_brown_4047). You want to change the date of a flight in reservation EUJUY6. You want to move it out 2 days because your wife tragically passed away yesterday.",
         actions=[
-            Action(name="get_reservation_details", kwargs={"reservation_id": "EUJUY6"})
+            Action(
+                name="get_reservation_details",
+                kwargs={"reservation_id": "EUJUY6"},
+            )
         ],
         outputs=[],
     ),