browsergym-workarena 0.4.4__tar.gz → 0.5.1__tar.gz

browsergym_workarena-0.5.1/.github/workflows/instance_pool_ci.yml

@@ -0,0 +1,82 @@
+name: Monitor the pool of WorkArena instances
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 3 * * *"       # daily at 03:00 UTC
+
+jobs:
+
+  test-l1-tasks:
+    name: Test L1 tasks
+    runs-on: ubuntu-22.04
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install Python dependencies
+        working-directory: ./dev
+        run: |
+          pip install -r requirements.txt
+          pip install huggingface_hub
+
+      - name: Pip list
+        run: pip list
+
+      - name: Install Playwright
+        run: playwright install chromium --with-deps
+
+      - name: Run L1 tests
+        run: pytest -n 20 --durations=10 --slowmo 1000 -v tests/test_task_general.py
+
+
+  test-snow-instance:
+    name: Test snow instance
+    runs-on: ubuntu-22.04
+    needs: test-l1-tasks   # remove this line if you want both jobs to run in parallel
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install Python dependencies
+        working-directory: ./dev
+        run: |
+          pip install -r requirements.txt
+          pip install huggingface_hub
+
+      - name: Pip list
+        run: pip list
+
+      - name: Install Playwright
+        run: playwright install chromium --with-deps
+
+      - name: Run snow instance tests
+        run: pytest -n 20 --durations=10 --slowmo 1000 -v tests/test_snow_instance.py

{browsergym_workarena-0.4.4 → browsergym_workarena-0.5.1}/.github/workflows/unit_tests.yml

@@ -36,7 +36,7 @@ jobs:
         run: black . --check
 
   browsergym-workarena-fast:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
   
     defaults:
       run:
@@ -50,7 +50,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
           cache: 'pip' # caching pip dependencies
   
       - name: Pip install
@@ -59,9 +59,9 @@ jobs:
   
       - name: Pip list
         run: pip list
-  
+        
       - name: Install Playwright
-        run: playwright install --with-deps
+        run: playwright install chromium --with-deps
   
       - name: Run non-slow browsergym-workarena Unit Tests
         env:

browsergym_workarena-0.5.1/CITATION.cff

@@ -0,0 +1,81 @@
+cff-version: 1.2.0
+message: "If you use WorkArena in your research, please cite both of the following papers."
+title: "WorkArena"
+date-released: 2024-03-12
+url: "https://github.com/ServiceNow/workarena"
+license: "Apache-2.0"
+
+authors:
+  - family-names: Drouin
+    given-names: Alexandre
+  - family-names: Gasse
+    given-names: Maxime
+  - family-names: Caccia
+    given-names: Massimo
+  - family-names: Laradji
+    given-names: Issam H.
+  - family-names: Del Verme
+    given-names: Manuel
+  - family-names: Marty
+    given-names: Tom
+  - family-names: Vazquez
+    given-names: David
+  - family-names: Chapados
+    given-names: Nicolas
+  - family-names: Lacoste
+    given-names: Alexandre
+
+preferred-citation:
+  - type: inproceedings
+    title: "WorkArena: How Capable are Web Agents at Solving Common Knowledge Work Tasks?"
+    authors:
+      - family-names: Drouin
+        given-names: Alexandre
+      - family-names: Gasse
+        given-names: Maxime
+      - family-names: Caccia
+        given-names: Massimo
+      - family-names: Laradji
+        given-names: Issam H.
+      - family-names: Del Verme
+        given-names: Manuel
+      - family-names: Marty
+        given-names: Tom
+      - family-names: Vazquez
+        given-names: David
+      - family-names: Chapados
+        given-names: Nicolas
+      - family-names: Lacoste
+        given-names: Alexandre
+    booktitle: "Proceedings of the 41st International Conference on Machine Learning (ICML)"
+    series: "Proceedings of Machine Learning Research"
+    volume: 235
+    pages: "11642–11662"
+    year: 2024
+    url: "https://proceedings.mlr.press/v235/drouin24a.html"
+
+  - type: inproceedings
+    title: "WorkArena++: Towards Compositional Planning and Reasoning-based Common Knowledge Work Tasks"
+    authors:
+      - family-names: Boisvert
+        given-names: Léo
+      - family-names: Thakkar
+        given-names: Megh
+      - family-names: Gasse
+        given-names: Maxime
+      - family-names: Caccia
+        given-names: Massimo
+      - family-names: Le Sellier De Chezelles
+        given-names: Thibault
+      - family-names: Cappart
+        given-names: Quentin
+      - family-names: Chapados
+        given-names: Nicolas
+      - family-names: Lacoste
+        given-names: Alexandre
+      - family-names: Drouin
+        given-names: Alexandre
+    booktitle: "Advances in Neural Information Processing Systems 37 (NeurIPS 2024), Datasets & Benchmarks Track"
+    year: 2024
+    url: "https://proceedings.neurips.cc/paper_files/paper/2024/hash/0b82662b6c32e887bb252a74d8cb2d5e-Paper-Datasets_and_Benchmarks_Track.pdf"
+    doi: "10.52202/079017-0195"

{browsergym_workarena-0.4.4 → browsergym_workarena-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: browsergym-workarena
-Version: 0.4.4
+Version: 0.5.1
 Summary: WorkArena benchmark for BrowserGym
 Project-URL: homepage, https://github.com/ServiceNow/WorkArena
 Author: Léo Boisvert, Alex Drouin, Maxime Gasse, Alex Lacoste, Manuel Del Verme, Megh Thakkar
@@ -16,6 +16,7 @@ Requires-Python: >3.7
 Requires-Dist: browsergym-core>=0.2
 Requires-Dist: english-words>=2.0.1
 Requires-Dist: faker>=24.8.0
+Requires-Dist: huggingface-hub>=0.23
 Requires-Dist: numpy>=1.14
 Requires-Dist: requests>=2.31
 Requires-Dist: tenacity>=8.2.3
@@ -65,28 +66,19 @@ https://github.com/ServiceNow/WorkArena/assets/2374980/68640f09-7d6f-4eb1-b556-c
 
 ## Getting Started
 
-To setup WorkArena, you will need to get your own ServiceNow instance, install our Python package, and upload some data to your instance. Follow the steps below to achieve this.
+To setup WorkArena, you will need to gain access to ServiceNow instances and install our Python package locally. Follow the steps below to achieve this.
 
-### a) Create a ServiceNow Developer Instance
+### a) Gain Access to ServiceNow Instances
 
-1. Go to https://developer.servicenow.com/ and create an account.
-2. Click on `Request an instance` and select the `Washington` release (initializing the instance will take a few minutes)
-3. Once the instance is ready, you should see your instance URL and credentials. If not, click _Return to the Developer Portal_, then navigate to _Manage instance password_ and click _Reset instance password_.
-4. Change the role of the user to admin in yoyr instance parameters ![image](https://github.com/user-attachments/assets/6f0fbf8e-f40f-411a-84cb-fead93d85f60)
+1. Navigate to https://huggingface.co/datasets/ServiceNow/WorkArena-Instances.
+2. Fill the form, accept the terms to gain access to the gated repository and wait for approval.
+3. Ensure that the machine where you will run WorkArena is [authenticated with Hugging Face](https://huggingface.co/docs/hub/en/datasets-polars-auth) (e.g., via huggingface-cli login or the HUGGING_FACE_HUB_TOKEN environment variable).
 
-5. You should now see your URL and credentials. Based on this information, set the following environment variables:
-    * `SNOW_INSTANCE_URL`: The URL of your ServiceNow developer instance
-    * `SNOW_INSTANCE_UNAME`: The username, should be "admin"
-    * `SNOW_INSTANCE_PWD`: The password, make sure you place the value in quotes "" and be mindful of [escaping special shell characters](https://onlinelinuxtools.com/escape-shell-characters). Running `echo $SNOW_INSTANCE_PWD` should print the correct password.
-6. Log into your instance via a browser using the admin credentials. Close any popup that appears on the main screen (e.g., agreeing to analytics).
-
-**Warning:** Feel free to look around the platform, but please make sure you revert any changes (e.g., changes to list views, pinning some menus, etc.) as these changes will be persistent and affect the benchmarking process.
-
-### b) Install WorkArena and Initialize your Instance
+### b) Install WorkArena
 
 Run the following command to install WorkArena in the [BrowswerGym](https://github.com/servicenow/browsergym) environment:
 ```
-pip install browsergym
+pip install browsergym-workarena
 ```
 
 Then, install [Playwright](https://github.com/microsoft/playwright):
@@ -94,10 +86,6 @@ Then, install [Playwright](https://github.com/microsoft/playwright):
 playwright install
 ```
 
-Finally, run this command in a terminal to upload the benchmark data to your ServiceNow instance:
-```
-workarena-install
-```
 Your installation is now complete! 🎉
 
 

{browsergym_workarena-0.4.4 → browsergym_workarena-0.5.1}/README.md

@@ -41,28 +41,19 @@ https://github.com/ServiceNow/WorkArena/assets/2374980/68640f09-7d6f-4eb1-b556-c
 
 ## Getting Started
 
-To setup WorkArena, you will need to get your own ServiceNow instance, install our Python package, and upload some data to your instance. Follow the steps below to achieve this.
+To setup WorkArena, you will need to gain access to ServiceNow instances and install our Python package locally. Follow the steps below to achieve this.
 
-### a) Create a ServiceNow Developer Instance
+### a) Gain Access to ServiceNow Instances
 
-1. Go to https://developer.servicenow.com/ and create an account.
-2. Click on `Request an instance` and select the `Washington` release (initializing the instance will take a few minutes)
-3. Once the instance is ready, you should see your instance URL and credentials. If not, click _Return to the Developer Portal_, then navigate to _Manage instance password_ and click _Reset instance password_.
-4. Change the role of the user to admin in yoyr instance parameters ![image](https://github.com/user-attachments/assets/6f0fbf8e-f40f-411a-84cb-fead93d85f60)
+1. Navigate to https://huggingface.co/datasets/ServiceNow/WorkArena-Instances.
+2. Fill the form, accept the terms to gain access to the gated repository and wait for approval.
+3. Ensure that the machine where you will run WorkArena is [authenticated with Hugging Face](https://huggingface.co/docs/hub/en/datasets-polars-auth) (e.g., via huggingface-cli login or the HUGGING_FACE_HUB_TOKEN environment variable).
 
-5. You should now see your URL and credentials. Based on this information, set the following environment variables:
-    * `SNOW_INSTANCE_URL`: The URL of your ServiceNow developer instance
-    * `SNOW_INSTANCE_UNAME`: The username, should be "admin"
-    * `SNOW_INSTANCE_PWD`: The password, make sure you place the value in quotes "" and be mindful of [escaping special shell characters](https://onlinelinuxtools.com/escape-shell-characters). Running `echo $SNOW_INSTANCE_PWD` should print the correct password.
-6. Log into your instance via a browser using the admin credentials. Close any popup that appears on the main screen (e.g., agreeing to analytics).
-
-**Warning:** Feel free to look around the platform, but please make sure you revert any changes (e.g., changes to list views, pinning some menus, etc.) as these changes will be persistent and affect the benchmarking process.
-
-### b) Install WorkArena and Initialize your Instance
+### b) Install WorkArena
 
 Run the following command to install WorkArena in the [BrowswerGym](https://github.com/servicenow/browsergym) environment:
 ```
-pip install browsergym
+pip install browsergym-workarena
 ```
 
 Then, install [Playwright](https://github.com/microsoft/playwright):
@@ -70,10 +61,6 @@ Then, install [Playwright](https://github.com/microsoft/playwright):
 playwright install
 ```
 
-Finally, run this command in a terminal to upload the benchmark data to your ServiceNow instance:
-```
-workarena-install
-```
 Your installation is now complete! 🎉
 
 

{browsergym_workarena-0.4.4 → browsergym_workarena-0.5.1}/requirements.txt

@@ -5,3 +5,4 @@ numpy>=1.14
 requests>=2.31
 tenacity>=8.2.3  # only used in cheat() -> move to tests?
 tqdm>=4.66.2
+huggingface_hub>=0.23

{browsergym_workarena-0.4.4 → browsergym_workarena-0.5.1}/src/browsergym/workarena/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.4.4"
+__version__ = "0.5.1"
 
 import inspect
 from logging import warning

{browsergym_workarena-0.4.4 → browsergym_workarena-0.5.1}/src/browsergym/workarena/config.py

@@ -11,6 +11,12 @@ SNOW_BROWSER_TIMEOUT = 30000  # Milliseconds
 SNOW_JS_UTILS_FILEPATH = str(resources.files(utils).joinpath("js_utils.js"))
 SNOW_SUPPORTED_RELEASES = ["washingtondc"]
 
+# Hugging Face dataset containing available instances
+INSTANCE_REPO_ID = "ServiceNow/WorkArena-Instances"
+INSTANCE_REPO_FILENAME = "instances.json"
+INSTANCE_REPO_TYPE = "dataset"
+INSTANCE_XOR_SEED = "x3!+-9mi#nhlo%a02$9hna{]"
+
 # Path to the Menu navigation task configuration
 ALL_MENU_PATH = str(resources.files(data_files).joinpath("task_configs/all_menu.json"))