browsergym-workarena 0.5.1__tar.gz → 0.5.2__tar.gz

browsergym_workarena-0.5.2/.github/workflows/pool-telemetry.yml

@@ -0,0 +1,41 @@
+name: Pool Usage Telemetry
+
+on:
+  schedule:
+    # Run daily at 00:00 UTC
+    - cron: '0 0 * * *'
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  telemetry:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install Python dependencies
+        working-directory: ./dev
+        run: |
+          pip install -r requirements.txt
+          pip install huggingface_hub
+          pip install wandb>=0.16
+
+      - name: Run telemetry script
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+        run: |
+          python monitor_pool_usage.py

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: browsergym-workarena
-Version: 0.5.1
+Version: 0.5.2
 Summary: WorkArena benchmark for BrowserGym
 Project-URL: homepage, https://github.com/ServiceNow/WorkArena
 Author: Léo Boisvert, Alex Drouin, Maxime Gasse, Alex Lacoste, Manuel Del Verme, Megh Thakkar
@@ -30,16 +30,6 @@ Description-Content-Type: text/markdown
 # WorkArena: A Benchmark for Evaluating Agents on Knowledge Work Tasks 
 [[Benchmark Contents]](#benchmark-contents) ♦ [[Getting Started]](#getting-started) ♦ [[Live Demo]](#live-demo) ♦ [[BrowserGym]](https://github.com/ServiceNow/BrowserGym) ♦ [[Citing This Work]](#citing-this-work) ♦ [Join us on Discord!](https://discord.gg/rDkP69X7)
 
-## Join Our Discord Community
-
-Want to brainstorm ideas, troubleshoot issues, or just geek out with fellow agent builders? Our official Discord server is the perfect place to connect and collaborate. Come hang out with us to:
-
-- Exchange tips, tricks, and success stories
-- Get real-time support and feedback
-- Stay updated on the latest features and announcements
-
-[Join us on Discord!](https://discord.gg/rDkP69X7)
-
 ---
 
 ### Explore the BrowserGym Ecosystem
@@ -73,6 +63,7 @@ To setup WorkArena, you will need to gain access to ServiceNow instances and ins
 1. Navigate to https://huggingface.co/datasets/ServiceNow/WorkArena-Instances.
 2. Fill the form, accept the terms to gain access to the gated repository and wait for approval.
 3. Ensure that the machine where you will run WorkArena is [authenticated with Hugging Face](https://huggingface.co/docs/hub/en/datasets-polars-auth) (e.g., via huggingface-cli login or the HUGGING_FACE_HUB_TOKEN environment variable).
+4. Unset any previous WorkArena environment variables if you are upgrading from a previous install (`SNOW_INSTANCE_URL`, etc.)
 
 ### b) Install WorkArena
 

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/README.md

@@ -5,16 +5,6 @@
 # WorkArena: A Benchmark for Evaluating Agents on Knowledge Work Tasks 
 [[Benchmark Contents]](#benchmark-contents) ♦ [[Getting Started]](#getting-started) ♦ [[Live Demo]](#live-demo) ♦ [[BrowserGym]](https://github.com/ServiceNow/BrowserGym) ♦ [[Citing This Work]](#citing-this-work) ♦ [Join us on Discord!](https://discord.gg/rDkP69X7)
 
-## Join Our Discord Community
-
-Want to brainstorm ideas, troubleshoot issues, or just geek out with fellow agent builders? Our official Discord server is the perfect place to connect and collaborate. Come hang out with us to:
-
-- Exchange tips, tricks, and success stories
-- Get real-time support and feedback
-- Stay updated on the latest features and announcements
-
-[Join us on Discord!](https://discord.gg/rDkP69X7)
-
 ---
 
 ### Explore the BrowserGym Ecosystem
@@ -48,6 +38,7 @@ To setup WorkArena, you will need to gain access to ServiceNow instances and ins
 1. Navigate to https://huggingface.co/datasets/ServiceNow/WorkArena-Instances.
 2. Fill the form, accept the terms to gain access to the gated repository and wait for approval.
 3. Ensure that the machine where you will run WorkArena is [authenticated with Hugging Face](https://huggingface.co/docs/hub/en/datasets-polars-auth) (e.g., via huggingface-cli login or the HUGGING_FACE_HUB_TOKEN environment variable).
+4. Unset any previous WorkArena environment variables if you are upgrading from a previous install (`SNOW_INSTANCE_URL`, etc.)
 
 ### b) Install WorkArena
 

browsergym_workarena-0.5.2/monitor_pool_usage.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+"""
+Count how many users were created recently on each ServiceNow instance in the pool.
+
+This reuses the instance loader and table API helper from the codebase.
+"""
+
+import logging
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+from typing import Dict, List, Tuple
+
+WAND_ENTITY = "alexdrouin"
+WAND_PROJECT = "workarena-monitoring"
+RUN_VERSION = "v3"  # Increment if you need to recreate runs after deletion
+
+from browsergym.workarena.api.utils import table_api_call
+from browsergym.workarena.instance import SNowInstance, fetch_instances
+
+
+def _time_window(hours: int = 24) -> Tuple[str, str]:
+    end = datetime.now(timezone.utc)
+    start = end - timedelta(hours=hours)
+    ts_format = "%Y-%m-%d %H:%M:%S"
+    return start.strftime(ts_format), end.strftime(ts_format)
+
+
+def _fetch_user_creations(
+    instance: SNowInstance, start_ts: str, end_ts: str
+) -> List[Dict[str, str]]:
+    # Query the audit log directly so deleted users are still counted.
+    page_size = 10000  # avoid the default 100-row limit
+    offset = 0
+    seen: Dict[str, Dict[str, str]] = {}
+    while True:
+        params = {
+            "sysparm_query": f"tablename=sys_user^sys_created_on>={start_ts}^sys_created_on<{end_ts}",
+            "sysparm_fields": "documentkey,sys_created_on,user,fieldname,newvalue",
+            "sysparm_limit": page_size,
+            "sysparm_offset": offset,
+        }
+        response = table_api_call(instance=instance, table="sys_audit", params=params)
+        batch = response.get("result", [])
+        for audit in batch:
+            doc = audit.get("documentkey")
+            if not doc:
+                continue
+            # Keep the earliest audit entry per user record.
+            if doc not in seen or audit.get("sys_created_on", "") < seen[doc].get(
+                "sys_created_on", ""
+            ):
+                seen[doc] = audit
+        if len(batch) < page_size:
+            break
+        offset += page_size
+    return list(seen.values())
+
+
+def _parse_sys_created(ts: str | None) -> datetime | None:
+    if not ts:
+        return None
+    ts = ts.replace("Z", "+00:00")
+    # Try ISO parsing with timezone if provided
+    try:
+        dt = datetime.fromisoformat(ts)
+    except ValueError:
+        dt = None
+    if dt is None:
+        for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"):
+            try:
+                dt = datetime.strptime(ts, fmt)
+                break
+            except ValueError:
+                continue
+    if dt is None:
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt.astimezone(timezone.utc)
+
+
+def _hourly_counts(records: List[Dict[str, str]]) -> Dict[datetime, int]:
+    buckets: Dict[datetime, int] = defaultdict(int)
+    for record in records:
+        ts = _parse_sys_created(record.get("sys_created_on"))
+        if ts is None:
+            continue
+        bucket = ts.replace(minute=0, second=0, microsecond=0)
+        buckets[bucket] += 1
+    return buckets
+
+
+def _daily_counts(records: List[Dict[str, str]]) -> Dict[datetime, int]:
+    buckets: Dict[datetime, int] = defaultdict(int)
+    for record in records:
+        ts = _parse_sys_created(record.get("sys_created_on"))
+        if ts is None:
+            continue
+        bucket = ts.replace(hour=0, minute=0, second=0, microsecond=0)
+        buckets[bucket] += 1
+    return buckets
+
+
+def _init_wandb(instance_name: str | None = None):
+    try:
+        import wandb
+    except ImportError as exc:
+        raise SystemExit(
+            "wandb is required for logging; install it to enable W&B logging."
+        ) from exc
+
+    # Use instance name or "total" as the display name
+    display_name = instance_name or "total"
+    # Add version suffix to run ID to avoid conflicts with deleted runs
+    run_id = f"{display_name}-{RUN_VERSION}"
+
+    run = wandb.init(
+        project=WAND_PROJECT,
+        entity=WAND_ENTITY,
+        name=display_name,  # Clean name for display
+        mode="online",
+        id=run_id,  # Versioned ID for persistence
+        resume="allow",
+        settings=wandb.Settings(init_timeout=180),
+        config={
+            "hours": 24,
+            "instance": display_name,
+        },
+    )
+    return run
+
+
+def _log_time_series_to_wandb(
+    run,
+    hourly_data: Dict[datetime, int],
+    daily_data: Dict[datetime, int],
+):
+    """Log time series data to a W&B run, ensuring chronological order."""
+    if run is None:
+        return
+
+    import wandb
+
+    # Define metrics to allow out-of-order logging based on timestamp
+    run.define_metric("daily_tasks_run", step_metric="timestamp", summary="last")
+    run.define_metric("hourly_tasks_run", step_metric="timestamp", summary="last")
+    run.define_metric("date", step_metric="timestamp")
+
+    # Combine all timestamps and sort them chronologically
+    all_data = []
+
+    # Add daily data points
+    for bucket, count in daily_data.items():
+        all_data.append((bucket, "daily_tasks_run", count))
+
+    # Add hourly data points
+    for bucket, count in hourly_data.items():
+        all_data.append((bucket, "hourly_tasks_run", count))
+
+    # Sort by timestamp
+    all_data.sort(key=lambda x: x[0])
+
+    # Log in chronological order with human-readable date
+    for bucket, metric_name, count in all_data:
+        run.log(
+            {
+                "timestamp": int(bucket.timestamp()),
+                metric_name: count,
+                "date": bucket,  # Pass datetime object directly for W&B to format
+            }
+        )
+
+    run.finish()
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+    start_ts, end_ts = _time_window()
+    logging.info("Checking user creations between %s and %s (UTC)", start_ts, end_ts)
+
+    instances = fetch_instances()
+    if not instances:
+        raise SystemExit("No ServiceNow instances available.")
+
+    summaries: List[Tuple[str, int]] = []
+    hourly_totals: Dict[datetime, int] = defaultdict(int)
+    hourly_per_instance: Dict[str, Dict[datetime, int]] = {}
+    daily_totals: Dict[datetime, int] = defaultdict(int)
+    daily_per_instance: Dict[str, Dict[datetime, int]] = {}
+
+    # Fetch data from all instances
+    for entry in instances:
+        url = entry["url"]
+        logging.info("Querying %s", url)
+        try:
+            instance = SNowInstance(snow_url=url, snow_credentials=("admin", entry["password"]))
+            creations = _fetch_user_creations(instance=instance, start_ts=start_ts, end_ts=end_ts)
+            summaries.append((url, len(creations)))
+            hourly = _hourly_counts(creations)
+            for bucket, count in hourly.items():
+                hourly_totals[bucket] += count
+            hourly_per_instance[url] = hourly
+            daily = _daily_counts(creations)
+            for bucket, count in daily.items():
+                daily_totals[bucket] += count
+            daily_per_instance[url] = daily
+            logging.info("...found %s tasks run", len(creations))
+        except Exception:
+            logging.exception("Failed to fetch data for %s", url)
+
+    # Log total data to a separate W&B run
+    logging.info("Logging total usage to W&B")
+    total_run = _init_wandb(instance_name=None)
+    _log_time_series_to_wandb(total_run, hourly_totals, daily_totals)
+
+    # Log each instance's data to separate W&B runs
+    for url, hourly_data in hourly_per_instance.items():
+        instance_name = url.split("//")[-1].replace(".service-now.com", "")
+        logging.info(f"Logging {instance_name} usage to W&B")
+
+        instance_run = _init_wandb(instance_name=instance_name)
+        daily_data = daily_per_instance[url]
+        _log_time_series_to_wandb(instance_run, hourly_data, daily_data)
+
+    # Print summary
+    total_created = sum(count for _, count in summaries)
+    print(f"\nTotal tasks run across instances: {total_created}")
+
+    for url, count in summaries:
+        print(f"{url}: {count} task(s) run")
+
+    if daily_totals:
+        print("\nDaily task runs (UTC):")
+        for bucket in sorted(daily_totals.keys()):
+            ts_str = bucket.strftime("%Y-%m-%d")
+            print(f"{ts_str}: {daily_totals[bucket]}")
+
+    if hourly_totals:
+        print("\nHourly task runs (UTC):")
+        for bucket in sorted(hourly_totals.keys()):
+            ts_str = bucket.strftime("%Y-%m-%d %H:%M")
+            print(f"{ts_str}: {hourly_totals[bucket]}")
+
+
+if __name__ == "__main__":
+    main()

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/src/browsergym/workarena/__init__.py

@@ -1,4 +1,14 @@
-__version__ = "0.5.1"
+__version__ = "0.5.2"
+
+# Check playwright version early to avoid cryptic errors
+import importlib.metadata
+
+_playwright_version = importlib.metadata.version("playwright")
+if _playwright_version != "1.44.0":
+    raise RuntimeError(
+        f"browsergym-workarena requires playwright==1.44.0, but found {_playwright_version}. "
+        f"Please install the correct version: pip install playwright==1.44.0"
+    )
 
 import inspect
 from logging import warning

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/src/browsergym/workarena/config.py

@@ -13,7 +13,7 @@ SNOW_SUPPORTED_RELEASES = ["washingtondc"]
 
 # Hugging Face dataset containing available instances
 INSTANCE_REPO_ID = "ServiceNow/WorkArena-Instances"
-INSTANCE_REPO_FILENAME = "instances.json"
+INSTANCE_REPO_FILENAME = "instances_v2.json"
 INSTANCE_REPO_TYPE = "dataset"
 INSTANCE_XOR_SEED = "x3!+-9mi#nhlo%a02$9hna{]"
 

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/src/browsergym/workarena/instance.py

@@ -45,10 +45,18 @@ def encrypt_instance_password(password: str) -> str:
     return base64.b64encode(cipher_bytes).decode("utf-8")
 
 
-def fetch_instances():
+def fetch_instances(filename: str = None):
     """
     Load the latest instances from either a custom pool (SNOW_INSTANCE_POOL env var) or the gated HF dataset.
+
+    Parameters:
+    -----------
+    filename: str
+        Optional filename to fetch from the HF dataset. Defaults to INSTANCE_REPO_FILENAME.
     """
+    if filename is None:
+        filename = INSTANCE_REPO_FILENAME
+
     pool_path = os.getenv("SNOW_INSTANCE_POOL")
     if pool_path:
         path = os.path.expanduser(pool_path)
@@ -62,13 +70,13 @@ def fetch_instances():
             disable_progress_bars()
             path = hf_hub_download(
                 repo_id=INSTANCE_REPO_ID,
-                filename=INSTANCE_REPO_FILENAME,
+                filename=filename,
                 repo_type=INSTANCE_REPO_TYPE,
             )
             logging.info("Loaded ServiceNow instances from the default instance pool.")
         except Exception as e:
             raise RuntimeError(
-                f"Could not access {INSTANCE_REPO_ID}/{INSTANCE_REPO_FILENAME}. "
+                f"Could not access {INSTANCE_REPO_ID}/{filename}. "
                 "Make sure you have been granted access to the gated repo and that you are "
                 "authenticated (run `huggingface-cli login` or set HUGGING_FACE_HUB_TOKEN)."
             ) from e
@@ -77,6 +85,8 @@ def fetch_instances():
         entries = json.load(f)
 
     for entry in entries:
+        if entry.get("error"):
+            raise RuntimeError(entry.get("message", "Unknown error from instance pool"))
         entry["url"] = entry["u"]
         entry["password"] = decrypt_instance_password(entry["p"])
         del entry["u"]

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/src/browsergym/workarena/tasks/form.py

@@ -371,6 +371,30 @@ class ServiceNowFormTask(AbstractServiceNowTask):
 
                 runInGsftMainOnlyAndProtectByURL(monitorChangeOnFields, '{url_suffix}');
             """,
+            f"""
+                function removePersonalizeFormButton() {{
+                    waLog('Searching for Personalize Form button...', 'removePersonalizeFormButton');
+                    let button = document.querySelector('#togglePersonalizeForm');
+                    if (button) {{
+                        button.remove();
+                        waLog('Removed Personalize Form button', 'removePersonalizeFormButton');
+                    }}
+                }}
+
+                runInGsftMainOnlyAndProtectByURL(removePersonalizeFormButton, '{url_suffix}');
+            """,
+            f"""
+                function removeAdditionalActionsButton() {{
+                    waLog('Searching for Additional Actions button...', 'removeAdditionalActionsButton');
+                    let button = document.querySelector('button.additional-actions-context-menu-button');
+                    if (button) {{
+                        button.remove();
+                        waLog('Removed Additional Actions button', 'removeAdditionalActionsButton');
+                    }}
+                }}
+
+                runInGsftMainOnlyAndProtectByURL(removeAdditionalActionsButton, '{url_suffix}');
+            """,
         ]
 
     def start(self, page: Page) -> None:

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/src/browsergym/workarena/tasks/list.py

@@ -113,7 +113,28 @@ class ServiceNowListTask(AbstractServiceNowTask):
             return json.load(f)
 
     def get_init_scripts(self) -> List[str]:
-        return super().get_init_scripts() + ["registerGsftMainLoaded();"]
+        return super().get_init_scripts() + [
+            "registerGsftMainLoaded();",
+            self._get_remove_personalize_list_button_script(),
+        ]
+
+    def _get_remove_personalize_list_button_script(self):
+        """
+        Removes the 'Personalize List' button on list pages.
+        """
+        script = """
+            function removePersonalizeListButton() {
+                waLog('Searching for Personalize List buttons...', 'removePersonalizeListButton');
+                let buttons = document.querySelectorAll('i[data-type="list_mechanic2_open"]');
+                buttons.forEach((button) => {
+                    button.remove();
+                });
+                waLog('Removed ' + buttons.length + ' Personalize List buttons', 'removePersonalizeListButton');
+            }
+
+            runInGsftMainOnlyAndProtectByURL(removePersonalizeListButton, '_list.do');
+        """
+        return script
 
     def _get_visible_list(self, page: Page):
         self._wait_for_ready(page)

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.2}/src/browsergym/workarena/tasks/service_catalog.py

@@ -225,6 +225,9 @@ class OrderHardwareTask(AbstractServiceNowTask):
             "registerGsftMainLoaded()",
             self._get_disable_add_to_cart_script(),
             self._get_remove_top_items_panel_script(),
+            self._get_remove_add_content_button_script(),
+            self._get_remove_header_decorations_script(),
+            self._get_remove_more_options_buttons_script(),
         ]
 
     def _get_disable_add_to_cart_script(self):
@@ -276,6 +279,60 @@ class OrderHardwareTask(AbstractServiceNowTask):
             """
         return script
 
+    def _get_remove_add_content_button_script(self):
+        """
+        Removes the 'Add content' button from the service catalog page.
+        """
+        script = """
+            function removeAddContentButton() {
+                waLog('Searching for Add content button...', 'removeAddContentButton');
+                let button = document.querySelector('button[aria-label="Add content"]');
+                if (button) {
+                    button.remove();
+                    waLog('Removed Add content button', 'removeAddContentButton');
+                }
+            }
+
+            runInGsftMainOnlyAndProtectByURL(removeAddContentButton, 'catalog_home');
+        """
+        return script
+
+    def _get_remove_header_decorations_script(self):
+        """
+        Removes all header decoration panels (edit/settings/close buttons) from the service catalog page.
+        """
+        script = """
+            function removeHeaderDecorations() {
+                waLog('Searching for header decoration panels...', 'removeHeaderDecorations');
+                let panels = document.querySelectorAll('div.header_decorations');
+                panels.forEach((panel) => {
+                    panel.remove();
+                });
+                waLog('Removed ' + panels.length + ' header decoration panels', 'removeHeaderDecorations');
+            }
+
+            runInGsftMainOnlyAndProtectByURL(removeHeaderDecorations, 'catalog_home');
+        """
+        return script
+
+    def _get_remove_more_options_buttons_script(self):
+        """
+        Removes all 'More Options' buttons from the service catalog page.
+        """
+        script = """
+            function removeMoreOptionsButtons() {
+                waLog('Searching for More Options buttons...', 'removeMoreOptionsButtons');
+                let buttons = document.querySelectorAll('button.btn.btn-icon.icon-ellipsis');
+                buttons.forEach((button) => {
+                    button.remove();
+                });
+                waLog('Removed ' + buttons.length + ' More Options buttons', 'removeMoreOptionsButtons');
+            }
+
+            runInGsftMainOnlyAndProtectByURL(removeMoreOptionsButtons, 'com.glideapp.servicecatalog');
+        """
+        return script
+
     def setup_goal(self, page: Page) -> tuple[str, dict]:
         super().setup_goal(page=page)