browsergym-workarena 0.1.0rc7__tar.gz → 0.2.1__tar.gz

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/.github/workflows/pypi.yml

@@ -1,4 +1,4 @@
-name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
+name: Build and Publish
 
 on: [push, workflow_dispatch]
 
@@ -48,10 +48,11 @@ jobs:
           uses: pypa/gh-action-pypi-publish@release/v1
 
     github-release:
-      name: Sign with Sigstore and upload them to GitHub Release
+      name: Sign packages with Sigstore and upload them to GitHub Release
       needs:
       - publish-to-pypi
       runs-on: ubuntu-latest
+
       permissions:
         contents: write  # IMPORTANT: mandatory for making GitHub Releases
         id-token: write  # IMPORTANT: mandatory for sigstore
@@ -64,7 +65,7 @@ jobs:
           path: dist/
 
       - name: Sign the dists with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v1.2.3
+        uses: sigstore/gh-action-sigstore-python@v2.1.1
         with:
           inputs: >-
             ./dist/*.tar.gz

browsergym_workarena-0.2.1/.github/workflows/unit_tests.yml

@@ -0,0 +1,104 @@
+name: Unit tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+
+  code-format:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip' # caching pip dependencies
+
+      - name: Pip install
+        run: pip install black[jupyter]==24.2.0 blacken-docs
+
+      - name: Pip list
+        run: pip list
+
+      - name: Code Formatting
+        run: black . --check
+
+  browsergym-workarena-fast:
+    runs-on: ubuntu-latest
+  
+    defaults:
+      run:
+        shell: bash -l {0}
+  
+    steps:
+  
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+  
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip' # caching pip dependencies
+  
+      - name: Pip install
+        working-directory: ./dev
+        run: pip install -r requirements.txt
+  
+      - name: Pip list
+        run: pip list
+  
+      - name: Install Playwright
+        run: playwright install --with-deps
+  
+      - name: Run non-slow browsergym-workarena Unit Tests
+        env:
+          SNOW_INSTANCE_URL: ${{ secrets.SNOW_INSTANCE_URL }}
+          SNOW_INSTANCE_UNAME: ${{ secrets.SNOW_INSTANCE_UNAME }}
+          SNOW_INSTANCE_PWD: ${{ secrets.SNOW_INSTANCE_PWD }}
+        run: pytest -n 5 --durations=10 -m 'not slow and not pricy' --slowmo 1000 -v tests
+
+  browsergym-workarena-slow:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip' # caching pip dependencies
+
+      - name: Pip install
+        working-directory: ./dev
+        run: pip install -r requirements.txt
+
+      - name: Pip list
+        run: pip list
+
+      - name: Install Playwright
+        run: playwright install --with-deps
+
+      - name: Run slow browsergym-workarena Unit Tests
+        env:
+          SNOW_INSTANCE_URL: ${{ secrets.SNOW_INSTANCE_URL }}
+          SNOW_INSTANCE_UNAME: ${{ secrets.SNOW_INSTANCE_UNAME }}
+          SNOW_INSTANCE_PWD: ${{ secrets.SNOW_INSTANCE_PWD }}
+        run: pytest -n 5 --durations=10 -m 'slow and not pricy' --slowmo 1000 -v tests

browsergym_workarena-0.2.1/.gitignore

@@ -0,0 +1,3 @@
+.DS_store
+__pycache__/
+*.py[cod]

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: browsergym-workarena
-Version: 0.1.0rc7
+Version: 0.2.1
 Summary: WorkArena benchmark for BrowserGym
 Project-URL: homepage, https://github.com/ServiceNow/WorkArena
-Author: Léo Boisvert, Alex Drouin, Maxime Gasse, Alex Lacoste, Manuel Del Verme
+Author: Léo Boisvert, Alex Drouin, Maxime Gasse, Alex Lacoste, Manuel Del Verme, Megh Thakkar
 License: Apache-2.0
 License-File: LICENSE
 Classifier: Development Status :: 2 - Pre-Alpha
@@ -13,11 +13,13 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >3.7
-Requires-Dist: browsergym-core==0.1.0rc7
+Requires-Dist: browsergym-core>=0.2
 Requires-Dist: english-words>=2.0.1
+Requires-Dist: faker>=24.11.0
 Requires-Dist: numpy>=1.14
 Requires-Dist: requests>=2.31
 Requires-Dist: tenacity>=8.2.3
+Requires-Dist: tqdm>=4.66.2
 Description-Content-Type: text/markdown
 
 # WorkArena: How Capable are Web Agents at Solving Common Knowledge Work Tasks?
@@ -32,10 +34,12 @@ WorkArena is included in [BrowserGym](https://github.com/ServiceNow/BrowserGym),
 
 https://github.com/ServiceNow/WorkArena/assets/2374980/68640f09-7d6f-4eb1-b556-c294a6afef70
 
+## ⚠️ Pre-Release warning ⚠️
+Please note that the WorkArena benchmark is still undergoing minor bug fixes and updates, which may cause discrepancies with results reported in our latest arXiv preprint. We plan to release soon a stable version of WorkArena with enhanced stability, and a final version v1.0.0 with a new suite of tasks.
 
 ## Benchmark Contents
 
-At the moment, WorkArena includes `23,150` task instances drawn from `29` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point.
+At the moment, WorkArena includes `18,050` task instances drawn from `33` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point.
 
 ### Knowledge Bases
 
@@ -71,6 +75,12 @@ https://github.com/ServiceNow/WorkArena/assets/1726818/7538b3ef-d39b-4978-b9ea-8
 
 https://github.com/ServiceNow/WorkArena/assets/1726818/ca26dfaf-2358-4418-855f-80e482435e6e
 
+### Dashboards
+
+**Goal:** The agent must extract information from a dashboard.
+
+
+
 ## Getting Started
 
 To setup WorkArena, you will need to get your own ServiceNow instance, install our Python package, and upload some data to your instance. Follow the steps below to achieve this.
@@ -78,7 +88,7 @@ To setup WorkArena, you will need to get your own ServiceNow instance, install o
 ### a) Create a ServiceNow Developer Instance
 
 1. Go to https://developer.servicenow.com/ and create an account.
-2. Click on `Request an instance` and select the `Utah` release (initializing the instance will take a few minutes)
+2. Click on `Request an instance` and select the `Washington` release (initializing the instance will take a few minutes)
 3. Once the instance is ready, you should see your instance URL and credentials. If not, click _Return to the Developer Portal_, then navigate to _Manage instance password_ and click _Reset instance password_.
 4. You should now see your URL and credentials. Based on this information, set the following environment variables:
     * `SNOW_INSTANCE_URL`: The URL of your ServiceNow developer instance

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/README.md

@@ -10,10 +10,12 @@ WorkArena is included in [BrowserGym](https://github.com/ServiceNow/BrowserGym),
 
 https://github.com/ServiceNow/WorkArena/assets/2374980/68640f09-7d6f-4eb1-b556-c294a6afef70
 
+## ⚠️ Pre-Release warning ⚠️
+Please note that the WorkArena benchmark is still undergoing minor bug fixes and updates, which may cause discrepancies with results reported in our latest arXiv preprint. We plan to release soon a stable version of WorkArena with enhanced stability, and a final version v1.0.0 with a new suite of tasks.
 
 ## Benchmark Contents
 
-At the moment, WorkArena includes `23,150` task instances drawn from `29` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point.
+At the moment, WorkArena includes `18,050` task instances drawn from `33` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point.
 
 ### Knowledge Bases
 
@@ -49,6 +51,12 @@ https://github.com/ServiceNow/WorkArena/assets/1726818/7538b3ef-d39b-4978-b9ea-8
 
 https://github.com/ServiceNow/WorkArena/assets/1726818/ca26dfaf-2358-4418-855f-80e482435e6e
 
+### Dashboards
+
+**Goal:** The agent must extract information from a dashboard.
+
+
+
 ## Getting Started
 
 To setup WorkArena, you will need to get your own ServiceNow instance, install our Python package, and upload some data to your instance. Follow the steps below to achieve this.
@@ -56,7 +64,7 @@ To setup WorkArena, you will need to get your own ServiceNow instance, install o
 ### a) Create a ServiceNow Developer Instance
 
 1. Go to https://developer.servicenow.com/ and create an account.
-2. Click on `Request an instance` and select the `Utah` release (initializing the instance will take a few minutes)
+2. Click on `Request an instance` and select the `Washington` release (initializing the instance will take a few minutes)
 3. Once the instance is ready, you should see your instance URL and credentials. If not, click _Return to the Developer Portal_, then navigate to _Manage instance password_ and click _Reset instance password_.
 4. You should now see your URL and credentials. Based on this information, set the following environment variables:
     * `SNOW_INSTANCE_URL`: The URL of your ServiceNow developer instance

browsergym_workarena-0.2.1/dev/environment.yaml

@@ -0,0 +1,13 @@
+name: workarena-dev
+
+channels:
+  - huggingface
+  - conda-forge
+  - defaults
+
+dependencies:
+  - python>=3.10
+  - pip
+
+  - pip:
+      - -r requirements.txt

browsergym_workarena-0.2.1/dev/requirements.txt

@@ -0,0 +1,9 @@
+black[jupyter]==24.2.0
+blacken-docs
+pre-commit
+pytest==7.3.2
+pytest-xdist
+pytest-playwright
+tenacity
+browsergym-core
+-e .. # local package

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/pyproject.toml

@@ -11,6 +11,7 @@ authors = [
     {name = "Maxime Gasse"},
     {name = "Alex Lacoste"},
     {name = "Manuel Del Verme"},
+    {name = "Megh Thakkar"},
 ]
 readme = "README.md"
 requires-python = ">3.7"
@@ -39,3 +40,30 @@ files = ["requirements.txt"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/browsergym"]
+
+[tool.black]
+line-length = 100
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.nox
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[tool.pytest.ini_options]
+filterwarnings = [
+    'ignore::UserWarning:gymnasium.*:',  # too many "The obs is not within the observation space." warnings.
+]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+]

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/requirements.txt

@@ -1,5 +1,7 @@
-browsergym-core==0.1.0rc7
+browsergym-core>=0.2
 english-words>=2.0.1
+faker>=24.11.0
 numpy>=1.14
 requests>=2.31
 tenacity>=8.2.3  # only used in cheat() -> move to tests?
+tqdm>=4.66.2

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/src/browsergym/workarena/__init__.py

@@ -1,7 +1,8 @@
-__version__ = "0.1.0rc7"
+__version__ = "0.2.1"
 
 from browsergym.core.registration import register_task
 
+from .tasks.dashboard import __TASKS__ as DASHBOARD_TASKS
 from .tasks.form import __TASKS__ as FORM_TASKS
 from .tasks.knowledge import __TASKS__ as KB_TASKS
 from .tasks.list import __TASKS__ as LIST_TASKS
@@ -9,6 +10,7 @@ from .tasks.navigation import __TASKS__ as NAVIGATION_TASKS
 from .tasks.service_catalog import __TASKS__ as SERVICE_CATALOG_TASKS
 
 ALL_WORKARENA_TASKS = [
+    *DASHBOARD_TASKS,
     *FORM_TASKS,
     *KB_TASKS,
     *LIST_TASKS,
@@ -21,5 +23,4 @@ for task in ALL_WORKARENA_TASKS:
     register_task(
         task.get_task_id(),
         task,
-        kwargs={"viewport": {"width": 1280, "height": 720}, "timeout": 10000},
     )

browsergym_workarena-0.2.1/src/browsergym/workarena/api/ui_themes.py

@@ -0,0 +1,35 @@
+"""
+Utility functions for UI themes
+
+"""
+
+from .utils import table_api_call
+
+
+def get_workarena_theme_variants(instance):
+    """
+    Get the list of available WorkArena UI themes
+
+    Parameters:
+    -----------
+    instance: SNowInstance
+        The ServiceNow instance to get the UI themes from
+
+    Returns:
+    --------
+    list[dict]
+        The list of available WorkArena UI themes and their information
+
+    """
+    themes = table_api_call(
+        instance=instance,
+        table="m2m_theme_style",
+        params={
+            "sysparm_query": "style.type=variant",
+            "sysparm_fields": "theme.name,theme.sys_id,style.name,style.sys_id",
+            "sysparm_display_value": True,
+        },
+        method="GET",
+    )["result"]
+    themes = [t for t in themes if t["theme.name"] == "WorkArena"]
+    return themes

browsergym_workarena-0.2.1/src/browsergym/workarena/api/user.py

@@ -0,0 +1,153 @@
+import random
+from faker import Faker
+import time
+
+fake = Faker()
+
+from ..instance import SNowInstance
+from .ui_themes import get_workarena_theme_variants
+from .utils import table_api_call
+
+
+def create_user(
+    instance: SNowInstance,
+    first_name: str = None,
+    last_name: str = None,
+    user_name: str = None,
+    admin=True,
+) -> list[str]:
+    """
+    Create a user with a random username and password with an admin role
+
+    Parameters:
+    -----------
+    first_name: str
+        The first name of the user, defaults to a random first name
+    last_name: str
+        The last name of the user, defaults to a random last name
+    user_name: str
+        The user name of the user, defaults to first_name.last_name
+    admin: bool
+        Whether to give the user admin permissions
+
+    Returns:
+    --------
+    username, password, sys_id
+
+    """
+    user_idx = str(random.randint(1000, 9999))
+    user_password = "aStrongPassword!"
+    first_name = fake.first_name() if not first_name else first_name
+    last_name = fake.last_name() if not last_name else last_name
+
+    # Create user
+    user_data = {
+        "user_name": f"{first_name}.{last_name}.{user_idx}" if not user_name else user_name,
+        "first_name": first_name,
+        "last_name": last_name,
+        "email": f"{first_name}.{last_name}.{user_idx}@workarena.com".lower(),
+        "user_password": user_password,
+        "active": True,
+    }
+    user_params = {"sysparm_input_display_value": True}
+    user_response = table_api_call(
+        instance=instance, table="sys_user", json=user_data, params=user_params, method="POST"
+    )["result"]
+    user_name = user_response["user_name"]
+    user_sys_id = user_response["sys_id"]
+
+    # Get admin role sys_id
+    if admin:
+        role_sys_id = table_api_call(
+            instance=instance,
+            table="sys_user_role",
+            params={"sysparm_query": "name=admin", "sysparm_fields": "sys_id"},
+            method="GET",
+        )["result"][0]["sys_id"]
+
+        # Give admin permissions
+        association_data = {"user": user_sys_id, "role": role_sys_id}
+        table_api_call(
+            instance=instance, table="sys_user_has_role", json=association_data, method="POST"
+        )
+
+    # Randomly pick a UI theme and set it for the user
+    themes = get_workarena_theme_variants(instance)
+    theme = random.choice(themes)
+    set_user_preference(
+        instance, "glide.ui.polaris.theme.variant", theme["style.sys_id"], user=user_sys_id
+    )
+
+    return user_name, user_password, user_sys_id
+
+
+def set_user_preference(instance: SNowInstance, key: str, value: str, user=None) -> dict:
+    """
+    Set a user preference in the ServiceNow instance
+
+    Parameters:
+    -----------
+    key: str
+        The name of the preference
+    value: str
+        The value of the preference
+    user: str
+        The sys_id of the user. If None, the preference will be set globally.
+
+    Returns:
+    --------
+    dict
+        The preference that was set
+
+    """
+    if user is None:
+        # make it global
+        user = ""
+        system = True
+    else:
+        system = False
+
+    # Try to get the preference's sys_id
+    preference = table_api_call(
+        instance=instance,
+        table="sys_user_preference",
+        params={"sysparm_query": f"name={key},user={user}", "sysparm_fields": "sys_id"},
+    )["result"]
+
+    if not preference:
+        # ... The preference key doesn't exist, create it
+        pref_sysid = ""
+        method = "POST"
+    else:
+        # ... The preference key exists, update it
+        pref_sysid = "/" + preference[0]["sys_id"]
+        method = "PUT"
+
+    property = table_api_call(
+        instance=instance,
+        table=f"sys_user_preference{pref_sysid}",
+        method=method,
+        json={
+            "name": key,
+            "value": value,
+            "user": user,
+            "system": system,
+            "description": "Updated by WorkArena",
+        },
+    )["result"]
+
+    # Verify that the property was updated
+    property["user"] = (
+        property["user"].get("value") if isinstance(property["user"], dict) else property["user"]
+    )
+    assert (
+        property["value"] == value
+    ), f"Error setting system property {key}, incorrect value {property['value']}, while expecting {value}."
+    assert (
+        property["user"] == user
+    ), f"Error setting system property {key}, incorrect user {property['user']}, while expecting {user}."
+    assert (
+        property["system"] == str(system).lower()
+    ), f"Error setting {key}, incorrect system {property['system']}, while expecting {system}."
+
+    return property

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/src/browsergym/workarena/api/utils.py

@@ -86,7 +86,7 @@ def table_column_info(instance: SNowInstance, table: str) -> dict:
 
     # Clean column value choices
     for info in meta_info.values():
-        if "choices" in info:
+        if info.get("choices", None):
             info["choices"] = {c["value"]: c["label"] for c in info["choices"]}
 
     # Query the sys_dictionnary table to find more info (e.g., is this column dependent on another)

{browsergym_workarena-0.1.0rc7 → browsergym_workarena-0.2.1}/src/browsergym/workarena/config.py

@@ -7,10 +7,25 @@ from ..workarena.tasks import utils
 SNOW_DATA_LOOKBACK_MINUTES = 5
 SNOW_BROWSER_TIMEOUT = 30000  # Milliseconds
 SNOW_JS_UTILS_FILEPATH = str(resources.files(utils).joinpath("js_utils.js"))
-SNOW_SUPPORTED_RELEASES = ["utah"]
+SNOW_SUPPORTED_RELEASES = ["washingtondc"]
 
 # Path to the Menu navigation task configuration
 ALL_MENU_PATH = str(resources.files(data_files).joinpath("task_configs/all_menu.json"))
+
+# Path to the dashboard/report retrieval task configurations
+DASHBOARD_RETRIEVAL_MINMAX_CONFIG_PATH = str(
+    resources.files(data_files).joinpath("task_configs/dashboard_retrieval_minmax_task.json")
+)
+DASHBOARD_RETRIEVAL_VALUE_CONFIG_PATH = str(
+    resources.files(data_files).joinpath("task_configs/dashboard_retrieval_value_task.json")
+)
+REPORT_RETRIEVAL_MINMAX_CONFIG_PATH = str(
+    resources.files(data_files).joinpath("task_configs/report_retrieval_minmax_task.json")
+)
+REPORT_RETRIEVAL_VALUE_CONFIG_PATH = str(
+    resources.files(data_files).joinpath("task_configs/report_retrieval_value_task.json")
+)
+
 # Path to knowledge base task configurations
 KB_CONFIG_PATH = str(
     resources.files(data_files).joinpath("task_configs/knowledge_base_configs.json")
@@ -119,6 +134,28 @@ WORKFLOWS = {
     }
 }
 
+
+# Custom UI Themes
+UI_THEMES_UPDATE_SET = {
+    "name": "WorkArena UI Themes",
+    "update_set": str(
+        resources.files(data_files).joinpath("setup_files/ui_themes/workarena_themes.xml")
+    ),
+    "variants": [
+        "Astranova",
+        "Charlies",
+        "Great pasta",
+        "Mighty capital",
+        "Speedy tires",
+        "Skyward",
+        "Turbobots",
+        "Ultrashoes",
+        "Vitasphere",
+        "Workarena",
+    ],
+}
+
+
 # Expected columns for list tasks; used in setup
 EXPECTED_ASSET_LIST_COLUMNS_PATH = str(
     resources.files(data_files).joinpath("setup_files/lists/expected_asset_list_columns.json")
@@ -163,3 +200,8 @@ EXPECTED_PROBLEM_FORM_FIELDS_PATH = str(
 EXPECTED_USER_FORM_FIELDS_PATH = str(
     resources.files(data_files).joinpath("setup_files/forms/expected_user_form_fields.json")
 )
+
+
+# Report date filter patch flag
+REPORT_PATCH_FLAG = "WORKARENA_DATE_FILTER_PATCH"
+REPORT_DATE_FILTER = "2024-04-01"