browsergym-workarena 0.1.0rc3__py3-none-any.whl → 0.1.0rc5__py3-none-any.whl

browsergym/workarena/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.0rc3"
+__version__ = "0.1.0rc5"
 
 from browsergym.core.registration import register_task
 

browsergym/workarena/config.py

@@ -51,7 +51,7 @@ ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH = str(
 
 # Knowledge base that is included with the benchmark
 KB_NAME = "General Knowledge"
-KB_FILEPATH = str(resources.files(data_files).joinpath("knowledge_base.json"))
+KB_FILEPATH = str(resources.files(data_files).joinpath("setup_files/knowledge_base.json"))
 
 # Form tasks
 CREATE_CHANGE_REQUEST_CONFIG_PATH = str(
@@ -112,6 +112,13 @@ SORT_USER_LIST_CONFIG_PATH = str(
 WORKFLOWS = {
     "kb_publish": {
         "name": "WorkArena Auto-Publish",
-        "update_set": str(resources.files(data_files).joinpath("kb_autopublish_workflow.xml")),
+        "update_set": str(
+            resources.files(data_files).joinpath("setup_files/kb_autopublish_workflow.xml")
+        ),
     }
 }
+
+# Number of columns in the user list; used for setup
+EXPECTED_USER_COLUMNS_PATH = str(
+    resources.files(data_files).joinpath("setup_files/expected_user_columns.json")
+)

browsergym/workarena/data_files/setup_files/expected_user_columns.json

@@ -0,0 +1,59 @@
+[
+    "user_name",
+    "name",
+    "email",
+    "active",
+    "sys_created_on",
+    "sys_updated_on",
+    "avatar",
+    "building",
+    "phone",
+    "calendar_integration",
+    "city",
+    "sys_class_name",
+    "company",
+    "cost_center",
+    "country",
+    "sys_created_by",
+    "date_format",
+    "default_perspective",
+    "department",
+    "sys_domain",
+    "sys_domain_path",
+    "employee_number",
+    "enable_multifactor_authn",
+    "failed_attempts",
+    "first_name",
+    "gender",
+    "home_phone",
+    "internal_integration_user",
+    "ldap_server",
+    "preferred_language",
+    "last_login",
+    "last_login_time",
+    "last_name",
+    "location",
+    "locked_out",
+    "manager",
+    "middle_name",
+    "mobile_phone",
+    "notification",
+    "user_password",
+    "password_needs_reset",
+    "photo",
+    "introduction",
+    "roles",
+    "schedule",
+    "source",
+    "state",
+    "street",
+    "sys_tags",
+    "time_format",
+    "time_zone",
+    "title",
+    "sys_updated_by",
+    "sys_mod_count",
+    "vip",
+    "web_service_access_only",
+    "zip"
+]

browsergym/workarena/install.py

@@ -6,7 +6,7 @@ import re
 from playwright.sync_api import sync_playwright
 
 from .api.utils import table_api_call
-from .config import KB_FILEPATH, KB_NAME, WORKFLOWS
+from .config import KB_FILEPATH, KB_NAME, EXPECTED_USER_COLUMNS_PATH, WORKFLOWS
 from .instance import SNowInstance
 from .utils import ui_login
 
@@ -277,6 +277,60 @@ def install_workflows():
         browser.close()
 
 
+def display_all_columns(url: str):
+    """Display all columns in a given list view."""
+    with sync_playwright() as playwright:
+        instance = SNowInstance()
+        browser = playwright.chromium.launch(headless=True, slow_mo=1000)
+        page = browser.new_page()
+        ui_login(instance, page)
+        page.goto(instance.snow_url + url)
+        frame = page.wait_for_selector("iframe#gsft_main").content_frame()
+        frame.get_by_text("Personalize List").click()
+        available_columns = frame.get_by_label("Available")
+        available_columns.get_by_role("option").first.click()
+        available_columns.get_by_role("option").last.click(modifiers=["Shift"])
+        frame.get_by_text("Add").click()
+        frame.click("#ok_button")
+
+
+def check_all_columns_displayed(url: str, expected_columns_path: str):
+    """Get the visible columns and checks that all expected columns are displayed."""
+    with open(expected_columns_path, "r") as f:
+        expected_columns = set(json.load(f))
+    with sync_playwright() as playwright:
+        instance = SNowInstance()
+        browser = playwright.chromium.launch(headless=True, slow_mo=1000)
+        page = browser.new_page()
+        ui_login(instance, page)
+        page.goto(instance.snow_url + url)
+        iframe = page.frame("gsft_main")
+        lst = iframe.locator("table.data_list_table")
+        lst.wait_for()
+
+        # Validate the number of lists on the page
+        lst = lst.nth(0)
+        js_selector = f"gsft_main.GlideList2.get('{lst.get_attribute('data-list_id')}')"
+        visible_columns = set(page.evaluate(f"{js_selector}.fields").split(","))
+
+        # check if expected columns is contained in the visible columns
+        if not expected_columns.issubset(visible_columns):
+            logging.info(
+                f"Error setting up list at {url} \n Expected {expected_columns} columns, but got {visible_columns}."
+            )
+            return False
+        logging.info(f"All columns properly displayed for {url}.")
+        return True
+
+
+def setup_list_columns(url: str, expected_columns_path: str):
+    """Setup the list view to display the expected number of columns."""
+    display_all_columns(url)
+    assert check_all_columns_displayed(
+        url, expected_columns_path
+    ), f"Error setting up list columns at {url}"
+
+
 def setup():
     """
     Check that WorkArena is installed correctly in the instance.
@@ -285,6 +339,11 @@ def setup():
     # XXX: Install workflows first because they may automate some downstream installations
     setup_workflows()
     setup_knowledge_base()
+    # Setup the user list columns by displaying all columns and checking that the expected number are displayed
+    setup_list_columns(
+        "/now/nav/ui/classic/params/target/sys_user_list.do%3Fsysparm_view%3D%26sysparm_userpref.sys_user_list.view%3D%26sysparm_userpref.sys_user.view%3D%26sysparm_query%3Dactive%253Dtrue%255Ecompany%253D81fd65ecac1d55eb42a426568fc87a63",
+        EXPECTED_USER_COLUMNS_PATH,
+    )
 
 
 def main():

browsergym/workarena/tasks/form.py

@@ -81,7 +81,7 @@ class ServiceNowFormTask(AbstractServiceNowTask):
         # ... augment with rendered metadata
         # XXX: Additional useful info is present in the rendered HTML. We extract it from there.
         for f in self.table_metadata:
-            loc = page.frame(name=self.js_prefix).locator(f"#sys_display\.{self.table_name}\.{f}")
+            loc = page.frame(name=self.js_prefix).locator(f"#sys_display.{self.table_name}.{f}")
             if loc.count() > 0:
                 # Check if the field is dependent on another field
                 self.table_metadata[f]["dependent_on_field"] = loc.first.get_attribute(

browsergym/workarena/tasks/knowledge.py

@@ -33,7 +33,7 @@ class KnowledgeBaseSearchTask(AbstractServiceNowTask):
     def __init__(self, instance=None, fixed_config: dict = None) -> None:
         super().__init__(
             instance=instance,
-            start_rel_url="/now/nav/ui/classic/params/target/knowledge_home_launcher.do",
+            start_rel_url="/now/nav/ui/classic/params/target/%24knowledge.do",
         )
 
         # Load the knowledge base and check its integrity

browsergym/workarena/tasks/scripts/validate.py

@@ -128,7 +128,8 @@ def validate_task(task_config, task_class, page=None):
         chat_messages = []
         task.cheat(page=page, chat_messages=chat_messages)
         page.wait_for_timeout(2000)
-        task_successful = task.validate(page, chat_messages)[1]
+        reward, done, _, _ = task.validate(page, chat_messages)
+        task_successful = done is True and reward == 1.0
         task.teardown()
         tries += 1
         if task_successful:

browsergym/workarena/tasks/service_catalog.py

@@ -23,11 +23,12 @@ from ..config import (
     ORDER_DEVELOPMENT_LAPTOP_PC_TASK_CONFIG_PATH,
     ORDER_LOANER_LAPTOP_TASK_CONFIG_PATH,
 )
-from browsergym.workarena.api.requests import (
+from .utils.form import fill_text
+from ..api.requests import (
     get_request_by_id,
     db_delete_from_table,
 )
-from browsergym.workarena.tasks.base import AbstractServiceNowTask
+from .base import AbstractServiceNowTask
 
 ADDITIONAL_SOFTWARE = [
     "Slack",
@@ -310,18 +311,10 @@ class OrderHardwareTask(AbstractServiceNowTask):
                 element_id = element_control.get_attribute("id")  # this look superfluous
                 text_element = iframe.query_selector(f'[id="{element_id}"]')
                 text_element.click()
-                from .utils.form import fill_text
-
                 fill_text(page=page, input_field=text_element, value=value, iframe=iframe)
+
             elif control_type == "select-one":
-                select_options = iframe.query_selector(f'select[id="{element_id}"]', strict=True)
-                select_options.click()
-                options = select_options.query_selector_all("option")
-                for option in options:
-                    if option.inner_text().startswith(value):
-                        page.keyboard.press("Enter")
-                        break
-                    page.keyboard.press("ArrowDown")
+                iframe.locator(f"id={element_id}").select_option(value)
             else:
                 raise ValueError(f"Unknown control type {control_type}")
 
@@ -475,10 +468,13 @@ class OrderHardwareTask(AbstractServiceNowTask):
 
 
 def option_match_heuristic(value, option):
-    value = str(value).lower()
-    option = str(option).lower()
-    option = option.replace("_", " ")
-    return value == option
+    def _process(x):
+        x = str(x).lower()
+        x = x.replace("_", "")
+        x = x.replace(" ", "")
+        return x
+
+    return _process(value) == _process(option)
 
 
 class OrderDeveloperLaptopTask(OrderHardwareTask):

{browsergym_workarena-0.1.0rc3.dist-info → browsergym_workarena-0.1.0rc5.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: browsergym-workarena
-Version: 0.1.0rc3
+Version: 0.1.0rc5
 Summary: WorkArena benchmark for BrowserGym
 Project-URL: homepage, https://github.com/ServiceNow/WorkArena
 Author: Léo Boisvert, Alex Drouin, Maxime Gasse, Alex Lacoste, Manuel Del Verme
@@ -13,14 +13,14 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >3.7
-Requires-Dist: browsergym-core==0.1.0rc3
+Requires-Dist: browsergym-core==0.1.0rc5
 Requires-Dist: english-words>=2.0.1
 Requires-Dist: numpy>=1.14
 Requires-Dist: requests>=2.31
 Requires-Dist: tenacity>=8.2.3
 Description-Content-Type: text/markdown
 
-# WorkArena - How Capable are Web Agents at Solving Common Knowledge Work Tasks?
+# WorkArena: How Capable are Web Agents at Solving Common Knowledge Work Tasks?
 
 [[Paper]](https://arxiv.org/abs/2403.07718) ♦ [[Benchmark Contents]](#benchmark-contents) ♦ [[Getting Started]](#getting-started) ♦ [[Live Demo]](#live-demo) ♦ [[BrowserGym]](https://github.com/ServiceNow/BrowserGym) ♦ [[Citing This Work]](#citing-this-work)
 
@@ -78,12 +78,13 @@ To setup WorkArena, you will need to get your own ServiceNow instance, install o
 ### a) Create a ServiceNow Developer Instance
 
 1. Go to https://developer.servicenow.com/ and create an account.
-2. Click on `Request an instance` and select the `Vancouver` release (initializing the instance will take a few minutes)
-3. Once the instance is ready, you will see a popup showing its URL and credentials. You will also receive a copy by email. Based on this information, set the following environment variables:
-    * `SNOW_INSTANCE_URL`: URL of your ServiceNow developer instance
-    * `SNOW_INSTANCE_UNAME`: Just use "admin"
-    * `SNOW_INSTANCE_PWD`: The password for your instance. Make sure you place the value in quotes "" since it might contain special characters.
-4. Log into your instance via a browser using the admin credentials. Close any popup that appears on the main screen (e.g., agreeing to analytics).
+2. Click on `Request an instance` and select the `Utah` release (initializing the instance will take a few minutes)
+3. Once the instance is ready, click _Return to the Developer Portal_, then navigate to _Manage instance password_ and click _Reset instance password_.
+4. You should now see your URL and credentials. Based on this information, set the following environment variables:
+    * `SNOW_INSTANCE_URL`: The URL of your ServiceNow developer instance
+    * `SNOW_INSTANCE_UNAME`: The username, should be "admin"
+    * `SNOW_INSTANCE_PWD`: The password, make sure you place the value in quotes "" since it might contain special characters.
+6. Log into your instance via a browser using the admin credentials. Close any popup that appears on the main screen (e.g., agreeing to analytics).
 
 **Warning:** Feel free to look around the platform, but please make sure you revert any changes (e.g., changes to list views, pinning some menus, etc.) as these changes will be persistent and affect the benchmarking process.
 

{browsergym_workarena-0.1.0rc3.dist-info → browsergym_workarena-0.1.0rc5.dist-info}/RECORD

@@ -1,13 +1,14 @@
-browsergym/workarena/__init__.py,sha256=UOnLE54QQ90EnokWr8-z5f5peLUNtBSarE2HX60YvU8,599
-browsergym/workarena/config.py,sha256=cCdWzQ4h9qXC106PQKptWzZdxdI2RwccDlcwI0LlmoQ,4610
-browsergym/workarena/install.py,sha256=-pfRzPRmuN7RZElS2LFmm2Zo8uy6ZD0nJeGZA4TOXKs,9789
+browsergym/workarena/__init__.py,sha256=nQzWa5xwZr2SGZLk747NUqSDN8h_VbKr-s5ytxTH5Dg,599
+browsergym/workarena/config.py,sha256=dFOwG8EiAyWohmk0iq-1pcV8keHJlACaDeDAuWP2ac8,4829
+browsergym/workarena/install.py,sha256=nxYO4CarQ2AnLcVLmPg3NlcR-uAUzlXJULSkDgMumR8,12558
 browsergym/workarena/instance.py,sha256=W_2C4QM_D7l36bXgXP_BtJwItGCW4uUJuFxmjdari1I,3071
 browsergym/workarena/utils.py,sha256=UXV-loPmwFCGF8E874QcyACmVS4BBHpPrW6BofKMADY,2289
 browsergym/workarena/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 browsergym/workarena/api/requests.py,sha256=z2vQTTkm1HR04_d_pce6jopuN--Oh0gUaT-YhrVa9-8,4148
 browsergym/workarena/api/utils.py,sha256=0S6-yWg8mIk-2zT2fvP_Xakvwz9sKGuNFTSAUoNVDTI,3307
-browsergym/workarena/data_files/kb_autopublish_workflow.xml,sha256=zMSoA3lfg4-Fvs7egnR286CPzzDHoTk6ffKnODWrunM,11500
-browsergym/workarena/data_files/knowledge_base.json,sha256=HHUH6ctyjea8R5oyhVCDedC__lGa_W9jaCGNfUVst50,474939
+browsergym/workarena/data_files/setup_files/expected_user_columns.json,sha256=L4yWNhOczhgC4gKOikrRgX2p2jFxaJ5XSsMM20zwjV4,1068
+browsergym/workarena/data_files/setup_files/kb_autopublish_workflow.xml,sha256=zMSoA3lfg4-Fvs7egnR286CPzzDHoTk6ffKnODWrunM,11500
+browsergym/workarena/data_files/setup_files/knowledge_base.json,sha256=HHUH6ctyjea8R5oyhVCDedC__lGa_W9jaCGNfUVst50,474939
 browsergym/workarena/data_files/task_configs/all_menu.json,sha256=sm5nMArqRdV16YCzd_UbSs2PbEE2Aofi42c5vG-0QoY,269871
 browsergym/workarena/data_files/task_configs/create_change_request_task.json,sha256=ik2qFv8Kj-CH7WDyOqL4ZCUEqQx9C2euipb20LNr9ns,4486789
 browsergym/workarena/data_files/task_configs/create_hardware_asset_task.json,sha256=LMqLfHvZkJOok--Xm2q4yqXWjupsphvcAc9zVlaZ-1A,3439223
@@ -39,11 +40,11 @@ browsergym/workarena/data_files/task_configs/sort_service_catalog_item_list_task
 browsergym/workarena/data_files/task_configs/sort_user_list_task.json,sha256=-M0hhooaS5NXnP1cDMBg8T3BAT4asU-ATdFPxRa21Uk,210213
 browsergym/workarena/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 browsergym/workarena/tasks/base.py,sha256=O0IDwvSIRrHUvFRzIsr0Yl9ph59IacpC52O_xlFmjTU,3419
-browsergym/workarena/tasks/form.py,sha256=_sBj0d1a0s_zmlIc6KVfP-evNduYaQjYc-pLUw6GSic,29340
-browsergym/workarena/tasks/knowledge.py,sha256=kPV8roBb2JqMnAPoxagH1qKtAMROzziY-8AVup9eQ7E,5782
+browsergym/workarena/tasks/form.py,sha256=z7OrJ1hgJt2bCzfCXuZPcO6UJkee0UEaBU5hAVnUyvE,29338
+browsergym/workarena/tasks/knowledge.py,sha256=lVs058sceJS_DkoBw3lRymmy5rTjLH38Fr7BqaGZikI,5771
 browsergym/workarena/tasks/list.py,sha256=8An0v13vgWvgiP0xA1wo31hVZuvWO4D6O8Iih6yAfs0,32776
 browsergym/workarena/tasks/navigation.py,sha256=5n6wIXxF-vZK5OXl-runw6tFdBeH6JdvnBO5dtEJrbw,7384
-browsergym/workarena/tasks/service_catalog.py,sha256=F-D92vHzfcXqtzeKPSe4pz5GMMMtZ8XiB3F14o-uW5s,22048
+browsergym/workarena/tasks/service_catalog.py,sha256=36XsIsKX5OowEJQySoI8-pIuSC34rge8bnkvKS_vU5I,21661
 browsergym/workarena/tasks/scripts/README.md,sha256=-jOtGf9k2zoBhrBkesmfDnr9-s0PhOEi0KWI9xfOw1s,296
 browsergym/workarena/tasks/scripts/extract_all_menu_items.py,sha256=oqSH2Sx5P_tgvC51SbvbgaY8oCa9w6IDR2BEsK-HsmI,9729
 browsergym/workarena/tasks/scripts/generate_forms.py,sha256=gABzW4ya1H2CVuGMqU38cJ5nWcGMVtv6gXQVqn6Ycxg,3590
@@ -51,13 +52,13 @@ browsergym/workarena/tasks/scripts/knowledge.py,sha256=A--1BRXbRgwlHGU-JIBRv3ozb
 browsergym/workarena/tasks/scripts/list.py,sha256=VQpoc9cjqkShtG-g-Yx6ZVjVrZOfB-__BrRgb7QFVMU,3475
 browsergym/workarena/tasks/scripts/navigation.py,sha256=bvNv2pLTHdstKDowowbP7qeI8IlkDexxCJLZxpRbgbc,823
 browsergym/workarena/tasks/scripts/service_catalog.py,sha256=HwLNQ3qxYxJJ_uXmA5_Rn1Xntv0V2tQzfgBBflWlfMc,4678
-browsergym/workarena/tasks/scripts/validate.py,sha256=o6IAxQmJWzvMX1zmUbs_q4TwWdzXi0Djo1OXkBC2l08,6686
+browsergym/workarena/tasks/scripts/validate.py,sha256=tJ6a1VC0u7F1bbszwnRLj2ef7dWumuYgpOt3Vt-ceTQ,6743
 browsergym/workarena/tasks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 browsergym/workarena/tasks/utils/debug.py,sha256=EEy7Rt2ETwkorPm0s6y_u0wutl5rDp_PVuM7BvnsbIA,670
 browsergym/workarena/tasks/utils/form.py,sha256=EMi_L6WBvIX9loczyVMGqBbYcb34r3zwVphr2KPh69M,2487
 browsergym/workarena/tasks/utils/js_utils.js,sha256=DTU9oiO1IC-8sAb_CY5KwVWx9jrjdmP8aAEMZZEww_k,1642
-browsergym_workarena-0.1.0rc3.dist-info/METADATA,sha256=FIRsZiZUSLXS8XKr_d6OTEApEvdNabRB2gpL8Gh9hj8,6976
-browsergym_workarena-0.1.0rc3.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-browsergym_workarena-0.1.0rc3.dist-info/entry_points.txt,sha256=rjnc1GaWB89r1PO0P_Uwriv-iVzWMhRM7EqUtXluDPs,72
-browsergym_workarena-0.1.0rc3.dist-info/licenses/LICENSE,sha256=sZLFiZHo_1hcxXRhXUDnQYVATUuWwRCdQjBxqxNnNEs,579
-browsergym_workarena-0.1.0rc3.dist-info/RECORD,,
+browsergym_workarena-0.1.0rc5.dist-info/METADATA,sha256=fA7i8dBUvF64XU661W629Be4hW6egnsld7R6extawoI,7047
+browsergym_workarena-0.1.0rc5.dist-info/WHEEL,sha256=LL0B1KxSLwaTWceo7tT-0aDLd-qq9dmbnsnk1DnXlg8,87
+browsergym_workarena-0.1.0rc5.dist-info/entry_points.txt,sha256=rjnc1GaWB89r1PO0P_Uwriv-iVzWMhRM7EqUtXluDPs,72
+browsergym_workarena-0.1.0rc5.dist-info/licenses/LICENSE,sha256=sZLFiZHo_1hcxXRhXUDnQYVATUuWwRCdQjBxqxNnNEs,579
+browsergym_workarena-0.1.0rc5.dist-info/RECORD,,

{browsergym_workarena-0.1.0rc3.dist-info → browsergym_workarena-0.1.0rc5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.21.1
+Generator: hatchling 1.22.2
 Root-Is-Purelib: true
 Tag: py3-none-any