browsergym-workarena 0.5.1__tar.gz → 0.5.3__tar.gz

browsergym_workarena-0.5.3/.github/workflows/pool-telemetry.yml

@@ -0,0 +1,41 @@
+name: Pool Usage Telemetry
+
+on:
+  schedule:
+    # Run daily at 00:00 UTC
+    - cron: '0 0 * * *'
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  telemetry:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install Python dependencies
+        working-directory: ./dev
+        run: |
+          pip install -r requirements.txt
+          pip install huggingface_hub
+          pip install wandb>=0.16
+
+      - name: Run telemetry script
+        env:
+          WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+        run: |
+          python monitor_pool_usage.py

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: browsergym-workarena
-Version: 0.5.1
+Version: 0.5.3
 Summary: WorkArena benchmark for BrowserGym
 Project-URL: homepage, https://github.com/ServiceNow/WorkArena
 Author: Léo Boisvert, Alex Drouin, Maxime Gasse, Alex Lacoste, Manuel Del Verme, Megh Thakkar
@@ -30,16 +30,6 @@ Description-Content-Type: text/markdown
 # WorkArena: A Benchmark for Evaluating Agents on Knowledge Work Tasks 
 [[Benchmark Contents]](#benchmark-contents) ♦ [[Getting Started]](#getting-started) ♦ [[Live Demo]](#live-demo) ♦ [[BrowserGym]](https://github.com/ServiceNow/BrowserGym) ♦ [[Citing This Work]](#citing-this-work) ♦ [Join us on Discord!](https://discord.gg/rDkP69X7)
 
-## Join Our Discord Community
-
-Want to brainstorm ideas, troubleshoot issues, or just geek out with fellow agent builders? Our official Discord server is the perfect place to connect and collaborate. Come hang out with us to:
-
-- Exchange tips, tricks, and success stories
-- Get real-time support and feedback
-- Stay updated on the latest features and announcements
-
-[Join us on Discord!](https://discord.gg/rDkP69X7)
-
 ---
 
 ### Explore the BrowserGym Ecosystem
@@ -73,6 +63,7 @@ To setup WorkArena, you will need to gain access to ServiceNow instances and ins
 1. Navigate to https://huggingface.co/datasets/ServiceNow/WorkArena-Instances.
 2. Fill the form, accept the terms to gain access to the gated repository and wait for approval.
 3. Ensure that the machine where you will run WorkArena is [authenticated with Hugging Face](https://huggingface.co/docs/hub/en/datasets-polars-auth) (e.g., via huggingface-cli login or the HUGGING_FACE_HUB_TOKEN environment variable).
+4. Unset any previous WorkArena environment variables if you are upgrading from a previous install (`SNOW_INSTANCE_URL`, etc.)
 
 ### b) Install WorkArena
 

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.3}/README.md

@@ -5,16 +5,6 @@
 # WorkArena: A Benchmark for Evaluating Agents on Knowledge Work Tasks 
 [[Benchmark Contents]](#benchmark-contents) ♦ [[Getting Started]](#getting-started) ♦ [[Live Demo]](#live-demo) ♦ [[BrowserGym]](https://github.com/ServiceNow/BrowserGym) ♦ [[Citing This Work]](#citing-this-work) ♦ [Join us on Discord!](https://discord.gg/rDkP69X7)
 
-## Join Our Discord Community
-
-Want to brainstorm ideas, troubleshoot issues, or just geek out with fellow agent builders? Our official Discord server is the perfect place to connect and collaborate. Come hang out with us to:
-
-- Exchange tips, tricks, and success stories
-- Get real-time support and feedback
-- Stay updated on the latest features and announcements
-
-[Join us on Discord!](https://discord.gg/rDkP69X7)
-
 ---
 
 ### Explore the BrowserGym Ecosystem
@@ -48,6 +38,7 @@ To setup WorkArena, you will need to gain access to ServiceNow instances and ins
 1. Navigate to https://huggingface.co/datasets/ServiceNow/WorkArena-Instances.
 2. Fill the form, accept the terms to gain access to the gated repository and wait for approval.
 3. Ensure that the machine where you will run WorkArena is [authenticated with Hugging Face](https://huggingface.co/docs/hub/en/datasets-polars-auth) (e.g., via huggingface-cli login or the HUGGING_FACE_HUB_TOKEN environment variable).
+4. Unset any previous WorkArena environment variables if you are upgrading from a previous install (`SNOW_INSTANCE_URL`, etc.)
 
 ### b) Install WorkArena
 

browsergym_workarena-0.5.3/monitor_pool_usage.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+"""
+Count how many users were created recently on each ServiceNow instance in the pool.
+
+This reuses the instance loader and table API helper from the codebase.
+"""
+
+import logging
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+from typing import Dict, List, Tuple
+
+WAND_ENTITY = "alexdrouin"
+WAND_PROJECT = "workarena-monitoring"
+RUN_VERSION = "v3"  # Increment if you need to recreate runs after deletion
+
+from browsergym.workarena.api.utils import table_api_call
+from browsergym.workarena.instance import SNowInstance, fetch_instances
+
+
+def _time_window(hours: int = 24) -> Tuple[str, str]:
+    end = datetime.now(timezone.utc)
+    start = end - timedelta(hours=hours)
+    ts_format = "%Y-%m-%d %H:%M:%S"
+    return start.strftime(ts_format), end.strftime(ts_format)
+
+
+def _fetch_user_creations(
+    instance: SNowInstance, start_ts: str, end_ts: str
+) -> List[Dict[str, str]]:
+    # Query the audit log directly so deleted users are still counted.
+    page_size = 10000  # avoid the default 100-row limit
+    offset = 0
+    seen: Dict[str, Dict[str, str]] = {}
+    while True:
+        params = {
+            "sysparm_query": f"tablename=sys_user^sys_created_on>={start_ts}^sys_created_on<{end_ts}",
+            "sysparm_fields": "documentkey,sys_created_on,user,fieldname,newvalue",
+            "sysparm_limit": page_size,
+            "sysparm_offset": offset,
+        }
+        response = table_api_call(instance=instance, table="sys_audit", params=params)
+        batch = response.get("result", [])
+        for audit in batch:
+            doc = audit.get("documentkey")
+            if not doc:
+                continue
+            # Keep the earliest audit entry per user record.
+            if doc not in seen or audit.get("sys_created_on", "") < seen[doc].get(
+                "sys_created_on", ""
+            ):
+                seen[doc] = audit
+        if len(batch) < page_size:
+            break
+        offset += page_size
+    return list(seen.values())
+
+
+def _parse_sys_created(ts: str | None) -> datetime | None:
+    if not ts:
+        return None
+    ts = ts.replace("Z", "+00:00")
+    # Try ISO parsing with timezone if provided
+    try:
+        dt = datetime.fromisoformat(ts)
+    except ValueError:
+        dt = None
+    if dt is None:
+        for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"):
+            try:
+                dt = datetime.strptime(ts, fmt)
+                break
+            except ValueError:
+                continue
+    if dt is None:
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt.astimezone(timezone.utc)
+
+
+def _hourly_counts(records: List[Dict[str, str]]) -> Dict[datetime, int]:
+    buckets: Dict[datetime, int] = defaultdict(int)
+    for record in records:
+        ts = _parse_sys_created(record.get("sys_created_on"))
+        if ts is None:
+            continue
+        bucket = ts.replace(minute=0, second=0, microsecond=0)
+        buckets[bucket] += 1
+    return buckets
+
+
+def _daily_counts(records: List[Dict[str, str]]) -> Dict[datetime, int]:
+    buckets: Dict[datetime, int] = defaultdict(int)
+    for record in records:
+        ts = _parse_sys_created(record.get("sys_created_on"))
+        if ts is None:
+            continue
+        bucket = ts.replace(hour=0, minute=0, second=0, microsecond=0)
+        buckets[bucket] += 1
+    return buckets
+
+
+def _init_wandb(instance_name: str | None = None):
+    try:
+        import wandb
+    except ImportError as exc:
+        raise SystemExit(
+            "wandb is required for logging; install it to enable W&B logging."
+        ) from exc
+
+    # Use instance name or "total" as the display name
+    display_name = instance_name or "total"
+    # Add version suffix to run ID to avoid conflicts with deleted runs
+    run_id = f"{display_name}-{RUN_VERSION}"
+
+    run = wandb.init(
+        project=WAND_PROJECT,
+        entity=WAND_ENTITY,
+        name=display_name,  # Clean name for display
+        mode="online",
+        id=run_id,  # Versioned ID for persistence
+        resume="allow",
+        settings=wandb.Settings(init_timeout=180),
+        config={
+            "hours": 24,
+            "instance": display_name,
+        },
+    )
+    return run
+
+
+def _log_time_series_to_wandb(
+    run,
+    hourly_data: Dict[datetime, int],
+    daily_data: Dict[datetime, int],
+):
+    """Log time series data to a W&B run, ensuring chronological order."""
+    if run is None:
+        return
+
+    import wandb
+
+    # Define metrics to allow out-of-order logging based on timestamp
+    run.define_metric("daily_tasks_run", step_metric="timestamp", summary="last")
+    run.define_metric("hourly_tasks_run", step_metric="timestamp", summary="last")
+    run.define_metric("date", step_metric="timestamp")
+
+    # Combine all timestamps and sort them chronologically
+    all_data = []
+
+    # Add daily data points
+    for bucket, count in daily_data.items():
+        all_data.append((bucket, "daily_tasks_run", count))
+
+    # Add hourly data points
+    for bucket, count in hourly_data.items():
+        all_data.append((bucket, "hourly_tasks_run", count))
+
+    # Sort by timestamp
+    all_data.sort(key=lambda x: x[0])
+
+    # Log in chronological order with human-readable date
+    for bucket, metric_name, count in all_data:
+        run.log(
+            {
+                "timestamp": int(bucket.timestamp()),
+                metric_name: count,
+                "date": bucket,  # Pass datetime object directly for W&B to format
+            }
+        )
+
+    run.finish()
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+    start_ts, end_ts = _time_window()
+    logging.info("Checking user creations between %s and %s (UTC)", start_ts, end_ts)
+
+    instances = fetch_instances()
+    if not instances:
+        raise SystemExit("No ServiceNow instances available.")
+
+    summaries: List[Tuple[str, int]] = []
+    hourly_totals: Dict[datetime, int] = defaultdict(int)
+    hourly_per_instance: Dict[str, Dict[datetime, int]] = {}
+    daily_totals: Dict[datetime, int] = defaultdict(int)
+    daily_per_instance: Dict[str, Dict[datetime, int]] = {}
+
+    # Fetch data from all instances
+    for entry in instances:
+        url = entry["url"]
+        logging.info("Querying %s", url)
+        try:
+            instance = SNowInstance(snow_url=url, snow_credentials=("admin", entry["password"]))
+            creations = _fetch_user_creations(instance=instance, start_ts=start_ts, end_ts=end_ts)
+            summaries.append((url, len(creations)))
+            hourly = _hourly_counts(creations)
+            for bucket, count in hourly.items():
+                hourly_totals[bucket] += count
+            hourly_per_instance[url] = hourly
+            daily = _daily_counts(creations)
+            for bucket, count in daily.items():
+                daily_totals[bucket] += count
+            daily_per_instance[url] = daily
+            logging.info("...found %s tasks run", len(creations))
+        except Exception:
+            logging.exception("Failed to fetch data for %s", url)
+
+    # Log total data to a separate W&B run
+    logging.info("Logging total usage to W&B")
+    total_run = _init_wandb(instance_name=None)
+    _log_time_series_to_wandb(total_run, hourly_totals, daily_totals)
+
+    # Log each instance's data to separate W&B runs
+    for url, hourly_data in hourly_per_instance.items():
+        instance_name = url.split("//")[-1].replace(".service-now.com", "")
+        logging.info(f"Logging {instance_name} usage to W&B")
+
+        instance_run = _init_wandb(instance_name=instance_name)
+        daily_data = daily_per_instance[url]
+        _log_time_series_to_wandb(instance_run, hourly_data, daily_data)
+
+    # Print summary
+    total_created = sum(count for _, count in summaries)
+    print(f"\nTotal tasks run across instances: {total_created}")
+
+    for url, count in summaries:
+        print(f"{url}: {count} task(s) run")
+
+    if daily_totals:
+        print("\nDaily task runs (UTC):")
+        for bucket in sorted(daily_totals.keys()):
+            ts_str = bucket.strftime("%Y-%m-%d")
+            print(f"{ts_str}: {daily_totals[bucket]}")
+
+    if hourly_totals:
+        print("\nHourly task runs (UTC):")
+        for bucket in sorted(hourly_totals.keys()):
+            ts_str = bucket.strftime("%Y-%m-%d %H:%M")
+            print(f"{ts_str}: {hourly_totals[bucket]}")
+
+
+if __name__ == "__main__":
+    main()

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.3}/src/browsergym/workarena/__init__.py

@@ -1,4 +1,14 @@
-__version__ = "0.5.1"
+__version__ = "0.5.3"
+
+# Check playwright version early to avoid cryptic errors
+import importlib.metadata
+
+_playwright_version = importlib.metadata.version("playwright")
+if _playwright_version != "1.44.0":
+    raise RuntimeError(
+        f"browsergym-workarena requires playwright==1.44.0, but found {_playwright_version}. "
+        f"Please install the correct version: pip install playwright==1.44.0"
+    )
 
 import inspect
 from logging import warning

{browsergym_workarena-0.5.1 → browsergym_workarena-0.5.3}/src/browsergym/workarena/config.py

@@ -13,7 +13,7 @@ SNOW_SUPPORTED_RELEASES = ["washingtondc"]
 
 # Hugging Face dataset containing available instances
 INSTANCE_REPO_ID = "ServiceNow/WorkArena-Instances"
-INSTANCE_REPO_FILENAME = "instances.json"
+INSTANCE_REPO_FILENAME = "instances_v2.json"
 INSTANCE_REPO_TYPE = "dataset"
 INSTANCE_XOR_SEED = "x3!+-9mi#nhlo%a02$9hna{]"