browsergym-workarena 0.1.0rc7__py3-none-any.whl → 0.2.1__py3-none-any.whl

browsergym/workarena/tasks/scripts/generate_dashboard_configs.py

@@ -0,0 +1,272 @@
+"""
+Generate configurations for the report and dashboard tasks
+
+Notes: sometimes it crashes (e.g., timeout, etc.). Just relaunch and it will be fine.
+
+"""
+
+import json
+import multiprocessing
+import random
+import tenacity
+
+from functools import partial
+from playwright.sync_api import sync_playwright
+
+from browsergym.workarena.api.utils import table_api_call, table_column_info
+from browsergym.workarena.config import (
+    REPORT_DATE_FILTER,
+    REPORT_PATCH_FLAG,
+    REPORT_RETRIEVAL_MINMAX_CONFIG_PATH,
+    REPORT_RETRIEVAL_VALUE_CONFIG_PATH,
+    DASHBOARD_RETRIEVAL_MINMAX_CONFIG_PATH,
+    DASHBOARD_RETRIEVAL_VALUE_CONFIG_PATH,
+)
+from browsergym.workarena.instance import SNowInstance
+from browsergym.workarena.tasks.dashboard import DashboardRetrievalTask
+
+
+N_CPU = 20
+MAX_CONFIGS = 1000
+REPORT = True  # Set to True for reports, False for dashboards
+
+
+class DummyDashboard(DashboardRetrievalTask):
+    def all_configs(self):
+        return [
+            {
+                "url": "",
+                "chart_title": "",
+                "chart_series": "",
+                "question": "max",
+            }
+        ]
+
+
+def get_report_urls(instance):
+    # Generate a bunch of reports on the fly based on valid table fields
+    ON_THE_FLY_REPORTS = []
+    for table in [
+        "alm_asset",
+        "alm_hardware",
+        "asmt_assessment_instance_question",
+        "asmt_m2m_stakeholder",
+        "ast_contract",
+        "change_request",
+        "cmdb_ci_computer",
+        "incident",
+        "sc_cat_item",
+        "sys_user",
+    ]:
+        cols = [
+            x
+            for x, y in table_column_info(instance=instance, table=table).items()
+            if y.get("cangroup", False)
+            and y.get("type", None) == "choice"
+            and "upon" not in x.lower()
+        ]
+        for col in cols:
+            ON_THE_FLY_REPORTS.append({"table": table, "field": col, "type": "pie"})
+            ON_THE_FLY_REPORTS.append({"table": table, "field": col, "type": "bar"})
+
+    # Reports that are already in the instance
+    system_report_tables = "alm_asset,alm_hardware,asmt_assessment_instance_question,asmt_m2m_stakeholder,ast_contract,change_request,cmdb_ci_computer"
+    SYSTEM_REPORTS = table_api_call(
+        instance=instance,
+        table="sys_report",
+        params={
+            "sysparm_query": f"sys_class_name=sys_report^active=true^typeINtrend,donut,vertical_bar,line,horizontal_bar,pie,bar,spline,area^descriptionLIKE{REPORT_PATCH_FLAG}^tableIN{system_report_tables}",
+            "sysparm_fields": "sys_id",
+        },
+    )["result"]
+
+    REPORTS = ON_THE_FLY_REPORTS + SYSTEM_REPORTS
+
+    return [
+        (
+            f"/now/nav/ui/classic/params/target/sys_report_template.do%3Fsysparm_field%3D{report['field']}%26sysparm_type%3D{report['type']}%26sysparm_table%3D{report['table']}%26sysparm_from_list%3Dtrue%26sysparm_chart_size%3Dlarge%26sysparm_manual_labor%3Dtrue%26sysparm_query=sys_created_on<javascript:gs.dateGenerate('{REPORT_DATE_FILTER}','00:00:00')^EQ"
+            if report.get("sys_id", None) is None
+            else f"/now/nav/ui/classic/params/target/sys_report_template.do%3Fjvar_report_id={report['sys_id']}"
+        )
+        for report in REPORTS
+    ]
+
+
+def get_dashboard_urls(instance):
+    # XXX: It's not ideal to use sys_ids but I couldn't find a better way
+    DASHBOARDS = [
+        "812fa4400f1130101527008c07767e1a",  # Assessment overview
+        "fa5fe3e1773130107384c087cc5a99d5",  # Asset overview
+        "68ee1f30770230107384c087cc5a992e",  # Asset contract overview
+        "05b0a8b7c3123010a282a539e540dd69",  # Change overview
+        "18b1f472533130104c90ddeeff7b12a6",  # Incident overview
+        "287d07d1ff3130106c1ef9a7cddcbd5d",  # Request overview
+        "7ab78953eb32011008f2951ff15228e6",  # Service catalog overview
+        "2d297c880f1130101527008c07767e27",  # Survey overview
+        "6b706f448f231110953ddffc9071a4f3",  # Telemetry - Table growth
+        "15c5d2d377213010a435478c4f5a993c",  # Usage overview
+        "85a57f9677100110ba155631dc5a9905",  # Web api usage overview
+        "c38ca3a273031010ae8dd21efaf6a747",  # Data classification
+        "3d48f669538223008329ddeeff7b1253",  # Problem overview
+    ]
+    return [
+        f"/now/nav/ui/classic/params/target/%24pa_dashboard.do%3Fsysparm_dashboard%3D{dashboard}"
+        for dashboard in DASHBOARDS
+    ]
+
+
+@tenacity.retry(
+    wait=tenacity.wait_fixed(1),
+    stop=tenacity.stop_after_attempt(10),
+)
+def get_all_configs_by_url(url, is_report):
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+        task = DummyDashboard(
+            instance=SNowInstance(),
+            fixed_config={
+                "url": url,
+                "chart_title": "",
+                "chart_series": "",
+                "question": "max",
+            },
+        )
+        task.setup(page=page)
+
+        # Handle the case where a dashboard is not found
+        task._wait_for_ready(page)
+        iframe = page.frame(name=task.iframe_id)
+        assert iframe.get_by_text("not found").count() == 0, "Report or dashboard not found"
+
+        # Find all the charts
+        charts = task._get_charts(page)
+
+        # Check enough charts
+        if len(charts) == 0:
+            return []
+
+        questions = []
+        for chart in charts:
+            try:
+                chart_title = chart[0] if not is_report else ""  # No title for reports
+                _, chart_data, _ = task._get_chart_by_title(page, chart_title)
+
+                # Select a series randomly
+                for series_idx in range(len(chart_data)):
+                    series_name = chart_data[series_idx]["name"] if len(chart_data) > 1 else ""
+                    data = chart_data[series_idx]["data"]
+
+                    # Check if the data is interesting
+                    labels = [point["label"] for point in data]
+                    if len(labels) <= 1:
+                        continue
+
+                    if any(l.isdigit() for l in labels):
+                        continue
+
+                    # Generate all value questions
+                    for format in ["count", "percent"]:
+                        for label in labels:
+                            questions.append(
+                                {
+                                    "url": url,
+                                    "chart_title": chart_title,
+                                    "chart_series": series_name,
+                                    "question": f"value; {format}; {label}",
+                                }
+                            )
+
+                    # Generate all other questions
+                    questions.append(
+                        {
+                            "url": url,
+                            "chart_title": chart_title,
+                            "chart_series": series_name,
+                            "question": "min",
+                        }
+                    )
+                    questions.append(
+                        {
+                            "url": url,
+                            "chart_title": chart_title,
+                            "chart_series": series_name,
+                            "question": "max",
+                        }
+                    )
+            except Exception as e:
+                print("Exception in worker", url, chart_title, e)
+                return []
+
+        if len(questions) == 0:
+            return []
+
+        # Test out all questions and keep only those that work
+        valid_questions = []
+        for question in questions:
+            chat_messages = []
+            task.config = question
+
+            try:
+                task.cheat(page=page, chat_messages=chat_messages)
+                valid = task.validate(page=page, chat_messages=chat_messages)[0]
+            except Exception as e:
+                # raise e
+                print("Exception in worker config validations", url, question, e)
+                valid = 0
+
+            if valid == 1:
+                valid_questions.append(question)
+            else:
+                print(f"Failed to validate question {question}")
+
+    print("Worker found", len(valid_questions), "valid questions")
+    return valid_questions
+
+
+if __name__ == "__main__":
+    instance = SNowInstance()
+    reports = get_report_urls(instance)
+    gen_func = partial(get_all_configs_by_url, is_report=REPORT)
+
+    if REPORT:
+        urls = get_report_urls(instance)
+        output_by_question = {
+            "value": REPORT_RETRIEVAL_VALUE_CONFIG_PATH,
+            "min,max": REPORT_RETRIEVAL_MINMAX_CONFIG_PATH,
+        }
+    else:
+        urls = get_dashboard_urls(instance)
+        output_by_question = {
+            "value": DASHBOARD_RETRIEVAL_VALUE_CONFIG_PATH,
+            "min,max": DASHBOARD_RETRIEVAL_MINMAX_CONFIG_PATH,
+        }
+
+    print(f"Generating configs for {len(urls)} URLs")
+    configs = []
+    with multiprocessing.Pool(processes=N_CPU) as pool:
+        results = pool.map(gen_func, urls)
+        pool.close()
+        pool.join()
+
+        # Flatten results list into config
+        for result in results:
+            configs += result
+
+    # Post-process the configs and save them
+    for question_type in output_by_question:
+        types = [x.strip() for x in question_type.split(",")]
+
+        type_configs = [
+            config for config in configs if config["question"].split(";")[0].strip() in types
+        ]
+        type_configs = set(
+            json.dumps(c) for c in type_configs
+        )  # Serialize to string to make unique
+        type_configs = [json.loads(c) for c in type_configs]
+        random.shuffle(type_configs)
+        type_configs = type_configs[:MAX_CONFIGS]
+
+        print("Saving", len(type_configs), "configs for", question_type)
+        with open(output_by_question[question_type], "w") as f:
+            json.dump(type_configs, f)

browsergym/workarena/tasks/scripts/generate_forms.py

@@ -28,12 +28,12 @@ def generate_form_task_configs(task_name, task_class, num_configs=1):
         """Try to setup and cheat a task, and return its configuration if it's new"""
         try:
             with sync_playwright() as p:
-                task = task_class()
+                task = task_class(seed=seed)
                 browser = p.chromium.launch()
                 context = browser.new_context()  # Set the timeout here
                 context.set_default_timeout(5000)
                 page = context.new_page()
-                task._generate_random_config(seed=seed, page=page)
+                task._generate_random_config(page=page)
                 config = {
                     "template_record": task.template_record,
                     "fields": {

browsergym/workarena/tasks/scripts/list.py

@@ -47,12 +47,12 @@ def generate_task_configs(task_class, num_configs=1000, task_type="sort"):
         """Try to setup and cheat a task, and return its configuration if it's new"""
         try:
             with sync_playwright() as p:
-                task = task_class()
+                task = task_class(seed=seed)
                 browser = p.chromium.launch()
                 context = browser.new_context()  # Set the timeout here
                 context.set_default_timeout(5000)
                 page = context.new_page()
-                goal, _ = task._generate_random_config(seed=seed, page=page)
+                goal, _ = task._generate_random_config(page=page)
                 chat_messages = []
                 try:
                     task.cheat(page=page, chat_messages=chat_messages)

browsergym/workarena/tasks/scripts/validate.py

@@ -143,8 +143,8 @@ def validate_on_page(task_class, task_config, page):
     cheat_passed = False
     task_done = False
     reward = 0.0
-    task = task_class(fixed_config=task_config)
-    task.setup(page=page, seed=1)
+    task = task_class(seed=1, fixed_config=task_config)
+    task.setup(page=page)
     chat_messages = []
     task.cheat(page=page, chat_messages=chat_messages)
     cheat_passed = True

browsergym/workarena/tasks/service_catalog.py

@@ -5,13 +5,21 @@ Tasks that require interacting with the service catalog
 
 import json
 import logging
-from time import sleep
-from urllib import parse
-
+from typing import List
 import numpy as np
 import playwright.sync_api
+
 from playwright.sync_api import Page
-from ..instance import SNowInstance
+from time import sleep
+from urllib import parse
+
+from .base import AbstractServiceNowTask
+from .utils.form import fill_text
+
+from ..api.requests import (
+    get_request_by_id,
+    db_delete_from_table,
+)
 from ..config import (
     ORDER_DEVELOPER_LAPTOP_TASK_CONFIG_PATH,
     ORDER_IPAD_MINI_TASK_CONFIG_PATH,
@@ -23,12 +31,8 @@ from ..config import (
     ORDER_DEVELOPMENT_LAPTOP_PC_TASK_CONFIG_PATH,
     ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH,
 )
-from .utils.form import fill_text
-from ..api.requests import (
-    get_request_by_id,
-    db_delete_from_table,
-)
-from .base import AbstractServiceNowTask
+from ..instance import SNowInstance
+from .utils.utils import check_url_suffix_match
 
 ADDITIONAL_SOFTWARE = [
     "Slack",
@@ -142,6 +146,8 @@ class OrderHardwareTask(AbstractServiceNowTask):
 
     Parameters:
     -----------
+    seed: int
+        Random seed
     instance: SNowInstance
         The instance to use.
     fixed_config: dict
@@ -154,12 +160,14 @@ class OrderHardwareTask(AbstractServiceNowTask):
 
     def __init__(
         self,
+        seed: int,
         instance: SNowInstance = None,
         fixed_request_item: str = None,
         fixed_config: dict = None,
         config_path: str = None,
     ):
         super().__init__(
+            seed=seed,
             instance=instance,
             start_rel_url="/now/nav/ui/classic/params/target/catalog_home.do%3Fsysparm_view%3Dcatalog_default",
             final_rel_url="/now/nav/ui/classic/params/target/com.glideapp.servicecatalog_checkout_view_v2.do",
@@ -177,9 +185,9 @@ class OrderHardwareTask(AbstractServiceNowTask):
         with open(config_path, "r") as f:
             self.all_configs = json.load(f)
 
-    def _wait_for_ready(self, page: Page, wait_for_form_api=False) -> None:
+    def _wait_for_ready(self, page: Page, wait_for_form_api: bool = False) -> None:
         """
-        Waits for the main iframe to be fully loaded
+        Waits for the the main iframe to be loaded
 
         """
         logging.debug(f"Waiting for {self.js_prefix} to be fully loaded")
@@ -197,60 +205,86 @@ class OrderHardwareTask(AbstractServiceNowTask):
     def form_js_selector(self):
         return self.js_prefix + "." + self.js_api_forms
 
-    def setup(self, page: playwright.sync_api.Page, seed: int = None) -> None:
-        self.pre_setup(page=page, seed=seed)
-        # the cart is shared for all agents running in parallel. The "Order Now" button
-        # is not affected so we'll make sure the agent can only use that one
-        disable_add_to_cart = """
-        window.addEventListener('DOMContentLoaded', (event) => {
-            const button = document.querySelector('button[aria-label="Add to Cart"]');
-            if (button) {
-                button.disabled = true;
+    def get_init_scripts(self) -> List[str]:
+        return super().get_init_scripts() + [
+            "registerGsftMainLoaded()",
+            self._get_disable_add_to_cart_script(),
+            self._get_remove_top_items_panel_script(),
+        ]
+
+    def _get_disable_add_to_cart_script(self):
+        """
+        Disables the 'Add to Cart' button on the service catalog page
+        This is necessary so that agents running in parallel do not interfere with each other (cart is shared between sessions)
+
+        """
+        script = """
+            function disableAddToCartButton() {
+                waLog('Searching for top items panel...', 'disableAddToCartButton');
+                let button = document.querySelector('button[aria-label="Add to Cart"]');
+                if (button) {
+                    button.disabled = true;
+                    waLog('WorkArena: Disabled the "Add to Cart" button', 'disableAddToCartButton');
+                } else {
+                    waLog('WorkArena: Could not find the "Add to Cart" button', 'disableAddToCartButton');
+                }
             }
-        });
+
+            runInGsftMainOnlyAndProtectByURL(disableAddToCartButton, 'glideapp.servicecatalog_cat_item_view.do');
         """
-        self._add_init_scripts_to_context_and_reload(
-            page, ["registerGsftMainLoaded()", disable_add_to_cart]
-        )
-        self._wait_for_ready(page)
-        self._remove_top_items_panel(page)
+        return script
+
+    def _get_remove_top_items_panel_script(self):
+        """Get script that removes the 'top items' panel that sometimes on the landing page of service catalog
+        Disables the 'Top Requests' panel that sometimes appears on the landing page of the service catalog
+        Runs in a loop to keep checking for the host element and shadow root
+        URL is secured by running only on the catalog_home page; this is a heuristic to avoid running on other pages
+        and does not check that the URL is an exact match, as moving back and forth between pages can cause the URL
+        to change, but catalog_home will always be present.
+        """
+        script = """
+            function removeTopItemsPanel() {
+                waLog('Searching for top items panel...', 'removeTopItemsPanel');
+                let headings = Array.from(document.querySelectorAll('[role="heading"]'));
+                headings.forEach((heading) => {
+                    if (heading.textContent.includes("Top Requests")) {
+                        let parentDiv = heading.closest('div.drag_section');
+                        if (parentDiv) {
+                            parentDiv.remove();
+                            waLog('Removed parent div for heading: ' + heading.textContent, 'removeTopItemsPanel');
+                        }
+                    }
+                });
+            }
+
+            runInGsftMainOnlyAndProtectByURL(removeTopItemsPanel, `catalog_home`);
+            """
+        return script
+
+    def setup_goal(self, page: Page) -> tuple[str, dict]:
+        super().setup_goal(page=page)
+
+        # Get the task configuration
         assert self.all_configs is not None, "No configuration available for the task."
         config = self.fixed_config if self.fixed_config else self.random.choice(self.all_configs)
-        # use fixed config if any
         self.requested_item = config["item"]
         self.short_description = config["description"]
         self.quantity = config["quantity"]
         self.requested_configuration = config["configuration"]
 
-        self.request_sysid = None
-
-        # generate goal
+        # Generate goal
         goal = f'Go to the hardware store and order {self.quantity} "{self.requested_item}"'
         if len(self.requested_configuration) > 0:
             goal += f" with configuration {dict((k, v[1]) for k, v in self.requested_configuration.items())}"
         info = {}
 
-        return goal, info
-
-    def _remove_top_items_panel(self, page: Page):
-        """Removes the 'top items' panel that sometimes on the landing page"""
-        frame = page.wait_for_selector("iframe#gsft_main").content_frame()
+        # Used to keep track of the sysid of the request for validation
+        self.request_sysid = None
 
-        # Use evaluate to find and remove divs containing an element with role="heading" and the text "Top Requests"
-        frame.evaluate(
-            """() => {
-                const headings = Array.from(document.querySelectorAll('[role="heading"]'));
-                headings.forEach((heading) => {
-                    if (heading.textContent.includes("Top Requests")) {
-                        let parentDiv = heading.closest('div.drag_section');
-                        if (parentDiv) parentDiv.remove();
-                    }
-                });
-            }"""
-        )
+        return goal, info
 
     def cheat(self, page: Page, chat_messages: list[str]) -> None:
-        super().cheat(page, chat_messages)
+        super().cheat(page=page, chat_messages=chat_messages)
         self._wait_for_ready(page=page)
 
         iframe = page.frame(self.js_prefix)
@@ -323,22 +357,9 @@ class OrderHardwareTask(AbstractServiceNowTask):
         with page.expect_navigation():
             order_now_button.click()
 
-    def _generate_random_config(self, seed: int, page: Page):
-        self.pre_setup(page=page, seed=seed)
-        # the cart is shared for all agents running in parallel. The "Order Now" button
-        # is not affected so we'll make sure the agent can only use that one
-        disable_add_to_cart = """
-        window.addEventListener('DOMContentLoaded', (event) => {
-            const button = document.querySelector('button[aria-label="Add to Cart"]');
-            if (button) {
-                button.disabled = true;
-            }
-        });
-        """
-        self._add_init_scripts_to_context_and_reload(
-            page, ["registerGsftMainLoaded()", disable_add_to_cart]
-        )
-        self._wait_for_ready(page)
+    def _generate_random_config(self, page: Page):
+        """Generate a random configuration for the task"""
+        self.setup(page=page, do_start=False)
         if self.fixed_request_item:
             self.requested_item = self.fixed_request_item
         else:
@@ -346,15 +367,16 @@ class OrderHardwareTask(AbstractServiceNowTask):
             self.requested_item = self.random.choice(list(META_CONFIGS.keys()))
 
         meta_config = META_CONFIGS[self.requested_item]
-        self.short_description = meta_config["desc"]
-        # ... choose a random quantity and configuration
-        self.quantity = self.random.randint(1, 11)
-        self.requested_configuration = {
-            ctrl_name: (ctrl_type, self.random.choice(values))
-            for ctrl_name, (ctrl_type, values) in meta_config["options"].items()
+        self.fixed_config = {
+            "item": self.requested_item,
+            "description": meta_config["desc"],
+            "quantity": self.random.randint(1, 11),
+            "configuration": {
+                ctrl_name: (ctrl_type, self.random.choice(values))
+                for ctrl_name, (ctrl_type, values) in meta_config["options"].items()
+            },
         }
-
-        self.request_sysid = None
+        self.setup(page=page, do_start=True)
 
     def _get_control_description(self, page, field):
         """
@@ -394,7 +416,17 @@ class OrderHardwareTask(AbstractServiceNowTask):
             )
 
     def validate(self, page: Page, chat_messages: list[str]) -> tuple[int, bool, str, dict]:
-        self._wait_for_ready(page)
+        right_url = check_url_suffix_match(page, expected_url=self.final_url, task=self)
+        if not right_url:
+            return (
+                0,
+                False,
+                "",
+                {
+                    "message": f"The page is not in the right URL to validate task {self.__class__.__name__}."
+                },
+            )
+
         # Retrieve the request sysid from the URL
         current_url = parse.urlparse(
             parse.unquote(self.page.evaluate("() => window.location.href"))

browsergym/workarena/tasks/utils/form.py

@@ -32,19 +32,21 @@ def fill_text(page, input_field, value, iframe=None):
         # Fill in the value using a procedure that triggers the autocomplete
         input_field.fill(value[:-1])
         page.keyboard.press(value[-1])
+        time.sleep(0.5)
 
-        # Wait until the attribute of the locator changes to the desired value
+        # Wait for the autocomplete menu to open and be ready
         max_wait_time = SNOW_BROWSER_TIMEOUT  # maximum time to wait in seconds
         start_time = time.time()
         while True:
-            if input_field.get_attribute("aria-expanded") == "true":
+            if input_field.get_attribute("aria-expanded") == "true" and not input_field.evaluate(
+                "e => e.ac.isResolving()"
+            ):
                 break
             if time.time() - start_time > (max_wait_time / 1000):
                 raise TimeoutError("Timeout waiting for autocompletion menu to open")
             time.sleep(0.5)  # wait for a short period before checking again
 
         # Select the desired value
-        time.sleep(0.5)  # wait for the list to be populated
         options = iframe.locator("[id^='ac_option_']")
         for i in range(options.count()):
             opt = options.nth(i)