workarena-cube 1.0.0__tar.gz

workarena_cube-1.0.0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.3
+Name: workarena-cube
+Version: 1.0.0
+Summary: WorkArena ServiceNow benchmark for cube
+Requires-Dist: cube-standard>=0.1.0rc5
+Requires-Dist: browsergym-workarena
+Requires-Dist: termcolor
+Requires-Dist: cube-browser-tool>=0.2.0
+Requires-Dist: cube-browser-playwright>=0.2.0
+Requires-Dist: cube-chat-tool>=0.1.0
+Requires-Dist: playwright==1.44
+Requires-Python: >=3.12, <3.13

workarena_cube-1.0.0/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "workarena-cube"
+version = "1.0.0"
+description = "WorkArena ServiceNow benchmark for cube"
+requires-python = ">=3.12,<3.13"
+dependencies = [
+    "cube-standard>=0.1.0rc5",
+    "browsergym-workarena",
+    "termcolor",
+    "cube-browser-tool>=0.2.0",
+    "cube-browser-playwright>=0.2.0",
+    "cube-chat-tool>=0.1.0",
+    "playwright==1.44",
+]
+
+[project.entry-points."cube.benchmarks"]
+workarena-cube = "workarena_cube.benchmark:WorkArenaBenchmark"
+
+[build-system]
+requires = ["uv_build>=0.8,<0.9"]
+build-backend = "uv_build"
+
+[tool.uv-build]
+include = ["src/workarena_cube/task_metadata.json"]
+
+[tool.ruff]
+fix = true
+line-length = 120
+indent-width = 4

workarena_cube-1.0.0/src/workarena_cube/__init__.py

@@ -0,0 +1,27 @@
+from workarena_cube.benchmark import WorkArenaBenchmark
+from workarena_cube.debug import CheatAgent, make_debug_agent, get_debug_benchmark
+from workarena_cube.task import WorkArenaTask, WorkArenaTaskConfig, WorkArenaTaskMetadata
+from workarena_cube.tools import (
+    WorkArenaBrowserTool,
+    WorkArenaCheatTool,
+    WorkArenaInfeasibleTool,
+    WorkarenaBrowserToolConfig,
+    WorkArenaInfeasibleToolConfig,
+    WorkArenaCheatToolConfig,
+)
+
+__all__ = [
+    "WorkArenaBenchmark",
+    "WorkArenaTask",
+    "WorkArenaTaskConfig",
+    "WorkArenaTaskMetadata",
+    "CheatAgent",
+    "make_debug_agent",
+    "get_debug_benchmark",
+    "WorkArenaBrowserTool",
+    "WorkArenaCheatTool",
+    "WorkArenaInfeasibleTool",
+    "WorkarenaBrowserToolConfig",
+    "WorkArenaInfeasibleToolConfig",
+    "WorkArenaCheatToolConfig",
+]

workarena_cube-1.0.0/src/workarena_cube/benchmark.py

@@ -0,0 +1,125 @@
+"""WorkArena benchmark implementation for the CUBE framework."""
+
+import logging
+from typing import ClassVar
+
+from browsergym.workarena import get_all_tasks_agents
+from cube.benchmark import Benchmark, BenchmarkMetadata
+from cube.seed import AbstractSeedGenerator
+from cube.task import TaskConfig, TaskMetadata
+from pydantic import PrivateAttr, model_validator
+
+from workarena_cube.task import WorkArenaTaskConfig, WorkArenaTaskMetadata
+
+logger = logging.getLogger(__name__)
+
+
+class WorkArenaSeedGenerator(AbstractSeedGenerator):
+    """Generates seeds for WorkArena tasks by delegating to get_all_tasks_agents().
+
+    Covers all three levels (l1, l2, l3) so it works naturally with any subset
+    produced by named_subset() or subset_from_glob().
+
+    Seeds are derived from WorkArena's own RNG (seeded by meta_seed) to maintain
+    compatibility with the original benchmark's evaluation protocol.
+
+    Lazily loads on first call and caches {task_id: [seeds]} for the lifetime
+    of this generator.
+    """
+
+    meta_seed: int = 42
+    n_seeds_l1: int = 10
+    is_agent_curriculum: bool = True
+
+    _cache: dict[str, list[int]] | None = PrivateAttr(default=None)
+
+    def _ensure_loaded(self) -> None:
+        if self._cache is not None:
+            return
+        cache: dict[str, list[int]] = {}
+        for level in ("l1", "l2", "l3"):
+            for task_class, seed in get_all_tasks_agents(
+                filter=level,
+                meta_seed=self.meta_seed,
+                n_seed_l1=self.n_seeds_l1,
+                is_agent_curriculum=self.is_agent_curriculum,
+            ):
+                task_id = task_class.get_task_id()
+                cache.setdefault(task_id, []).append(seed)
+        self._cache = cache
+
+    def __call__(self, task_metadata: TaskMetadata) -> list[int]:
+        self._ensure_loaded()
+        assert self._cache
+        return self._cache.get(task_metadata.id, [])
+
+
+class WorkArenaBenchmark(Benchmark):
+    """CUBE Benchmark for WorkArena ServiceNow tasks.
+
+    By default loads all task types from all levels (l1, l2, l3).
+    Use named_subset() or subset_from_glob() in user-land to filter:
+
+        bench.named_subset("l1")                                   # L1 only
+        bench.named_subset("l2").subset_from_glob("in_human_curriculum", "True")  # L2 human curriculum
+
+    Required environment variables:
+        SNOW_INSTANCE_URL, SNOW_INSTANCE_UNAME, SNOW_INSTANCE_PWD
+        or HUGGING_FACE_HUB_TOKEN for the hosted instance pool.
+
+    task_metadata.json is a shipped package resource containing lightweight public fields
+    (level, in_human_curriculum, task_class_path). No heavy execution data exists — all
+    task logic is available from the browsergym-workarena library at runtime.
+
+    To regenerate task_metadata.json (developer use only), run:
+        scripts/generate_task_metadata.py
+    """
+
+    benchmark_metadata: ClassVar[BenchmarkMetadata] = BenchmarkMetadata(
+        name="workarena-cube",
+        version="1.0.0",
+        description=(
+            "WorkArena ServiceNow benchmark tasks across three levels. "
+            "By default all task types from all levels are loaded. "
+            "Use named_subset('l1'/'l2'/'l3') to filter by level. "
+            "For human curriculum: bench.named_subset('l2').subset_from_glob('in_human_curriculum', 'True')."
+        ),
+        tags=["browser", "web", "servicenow"],
+        named_subsets={
+            "l1": ("level", "l1"),
+            "l2": ("level", "l2"),
+            "l3": ("level", "l3"),
+        },
+        num_tasks=333,
+    )
+    task_metadata: ClassVar[dict[str, WorkArenaTaskMetadata]]  # type: ignore - populated automatically at import time in Benchmark.__init_subclass__
+    task_config_class: ClassVar[type[TaskConfig]] = WorkArenaTaskConfig
+
+    meta_seed: int = 42
+    n_seeds_l1: int = 10
+    is_agent_curriculum: bool = True
+
+    @model_validator(mode="after")
+    def _init_seed_generator(self) -> "WorkArenaBenchmark":
+        """Initialize seed_generator at construction time from benchmark fields."""
+        if self.seed_generator is None:
+            object.__setattr__(
+                self,
+                "seed_generator",
+                WorkArenaSeedGenerator(
+                    meta_seed=self.meta_seed,
+                    n_seeds_l1=self.n_seeds_l1,
+                    is_agent_curriculum=self.is_agent_curriculum,
+                ),
+            )
+        return self
+
+    # ── lifecycle ──────────────────────────────────────────────────
+
+    def _setup(self) -> None:
+        """No shared infrastructure needed — WorkArena tasks connect to a remote ServiceNow instance."""
+        logger.info(f"WorkArena benchmark ready with {len(self.task_metadata)} tasks")
+
+    def close(self) -> None:
+        """No-op: WorkArena has no server process to shut down."""
+        logger.info("WorkArena benchmark closed.")

workarena_cube-1.0.0/src/workarena_cube/debug.py

@@ -0,0 +1,69 @@
+"""Smoke-test script for workarena-cube — validates infrastructure without an LLM.
+
+Verifies that WorkArena task configs can be enumerated, tasks can be instantiated,
+and the tool + WorkArena episode lifecycle run without errors.
+
+Requires ServiceNow credentials (SNOW_INSTANCE_URL, SNOW_INSTANCE_UNAME,
+SNOW_INSTANCE_PWD) or HUGGING_FACE_HUB_TOKEN for the hosted instance pool.
+
+Public API (cube.testing protocol)
+-----------------------------------
+get_debug_benchmark()              -> WorkArenaBenchmark
+make_debug_agent(task_id: str)     -> CheatAgent
+
+Usage:
+    uv run cube test workarena-cube
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from cube.core import Action, ActionSchema, Observation
+from cube.testing import run_debug_suite
+
+from workarena_cube.benchmark import WorkArenaBenchmark
+from workarena_cube.tools import WorkArenaCheatToolConfig
+
+logger = logging.getLogger(__name__)
+
+_DEBUG_N_TASKS = 2
+
+
+class CheatAgent:
+    """Agent that calls WorkArena's cheat action to solve the task, then stops."""
+
+    def __init__(self, task_id: str) -> None:
+        self._task_id = task_id
+        self._cheated: bool = False
+
+    def __call__(self, obs: Observation, action_set: list[ActionSchema]) -> Action:
+        if not self._cheated:
+            self._cheated = True
+            return Action(name="workarena_cheat", arguments={})
+        return Action(name="final_step", arguments={})
+
+
+def make_debug_agent(task_id: str) -> CheatAgent:
+    return CheatAgent(task_id)
+
+
+def get_debug_benchmark() -> WorkArenaBenchmark:
+    bench = WorkArenaBenchmark(
+        n_seeds_l1=1,
+        default_tool_config=WorkArenaCheatToolConfig(),
+    )
+    l1_bench = bench.named_subset("l1")
+    task_ids = list(l1_bench.task_metadata.keys())[:_DEBUG_N_TASKS]
+    return l1_bench.subset_from_list(task_ids)  # type: ignore
+
+
+if __name__ == "__main__":
+    import workarena_cube.debug as _this_module
+
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(levelname)-8s  %(name)s  %(message)s")
+
+    results = run_debug_suite("workarena-cube", _this_module)
+    failed = [r for r in results if r["error"]]
+
+    sys.exit(1 if failed else 0)

workarena_cube-1.0.0/src/workarena_cube/task.py

@@ -0,0 +1,235 @@
+"""WorkArena task implementation for the CUBE framework."""
+
+import importlib
+import logging
+import time
+from typing import Any, List, Literal, override
+
+from browsergym.workarena.tasks.base import AbstractServiceNowTask
+from cube.benchmark import RuntimeContext
+from cube.container import ContainerBackend
+from cube.core import Action, ActionSchema, EnvironmentOutput, Observation
+from cube.task import Task, TaskConfig, TaskMetadata
+from cube.tool import Toolbox
+from cube.tools.browser import BrowserTool
+from cube_browser_playwright import Viewport
+from cube_chat_tool import ChatTool
+from workarena_cube.tools import WorkArenaCheatTool, WorkArenaInfeasibleTool, WorkArenaBrowserTool
+from pydantic import PrivateAttr
+
+
+logger = logging.getLogger(__name__)
+
+
+class WorkArenaTaskMetadata(TaskMetadata):
+    """TaskMetadata subclass for WorkArena ServiceNow tasks.
+
+    Public fields shipped in task_metadata.json (available at import time).
+    WorkArena has no heavy execution data — all task logic is available from
+    the browsergym-workarena library at runtime via task_class_path.
+    """
+
+    level: Literal["l1", "l2", "l3"]
+    """Task level: l1 = atomic, l2 = compositional, l3 = extended compositional."""
+
+    in_human_curriculum: bool
+    """Whether this task type is part of the human evaluation curriculum."""
+
+    task_class_path: str
+    """Dotted path to the WorkArena task class, e.g. 'browsergym.workarena.tasks.dashboard.MultiChartValueRetrievalTask'."""
+
+
+class WorkArenaTask(Task):
+    """CUBE Task wrapper for WorkArena ServiceNow tasks."""
+
+    metadata: WorkArenaTaskMetadata  # type: ignore[assignment]
+    seed: int
+    wait_first_page_time: float = 10.0
+    validate_per_step: bool = True
+
+    _workarena_task: AbstractServiceNowTask | None = PrivateAttr(default=None)
+    _validate_cache: tuple[Any, ...] | None = PrivateAttr(default=None)
+
+    @property
+    def _browser_tool(self) -> WorkArenaBrowserTool:
+        """Resolve the browser tool whether it's direct or inside a Toolbox."""
+        if isinstance(self.tool, Toolbox):
+            tool = self.tool.find_tool(BrowserTool)
+            if tool is None:
+                raise RuntimeError("No BrowserTool found in Toolbox")
+        else:
+            tool = self.tool
+        if not isinstance(tool, WorkArenaBrowserTool):
+            raise RuntimeError(
+                f"The browser tool must satisfy the WorkArenaBrowserTool protocol (e.g., BrowsergymTool or SyncPlaywrightTool), got {type(tool).__name__}"
+            )
+        return tool
+
+    @property
+    def _chat_tool(self) -> ChatTool | None:
+        """Return the ChatTool if present in a Toolbox, else None."""
+        if isinstance(self.tool, Toolbox):
+            return self.tool.find_tool(ChatTool)  # type: ignore
+        return None
+
+    @property
+    def _infeasible_tool(self) -> WorkArenaInfeasibleTool | None:
+        """Return the WorkArenaInfeasibleTool if present in a Toolbox, else None."""
+        if isinstance(self.tool, Toolbox):
+            tool = self.tool.find_tool(WorkArenaInfeasibleTool)
+            return tool if isinstance(tool, WorkArenaInfeasibleTool) else None
+        return None
+
+    def reset(self) -> tuple[Observation, dict[str, Any]]:
+        """Instantiate and set up the WorkArena task, returning the initial observation."""
+        task_class = _load_task_class(self.metadata.task_class_path)
+        self._workarena_task = task_class(seed=self.seed)
+        if self._workarena_task is None:
+            raise RuntimeError("Failed to initialize WorkArena task.")
+        _apply_task_runtime_preferences(self._browser_tool, self._workarena_task)
+        self.tool.reset()
+        self._validate_cache = None
+        if isinstance(self._browser_tool, WorkArenaCheatTool):
+            self._browser_tool._workarena_task = self._workarena_task
+        page = self._browser_tool.page
+        goal, task_info = self._workarena_task.setup(page)
+
+        logger.info(f"WorkArena page URL after setup: {page.url}")
+        logger.info(f"WorkArena page title: {page.title()}")
+        logger.info(f"WorkArena task class: {self._workarena_task.__class__.__name__}")
+
+        self._browser_tool.noop()
+        time.sleep(self.wait_first_page_time)
+        logger.info(f"WorkArena task goal: {goal}")
+
+        page_obs = self._browser_tool.page_obs()
+        if self._chat_tool is not None:
+            self._chat_tool.add_message("user", goal)
+            obs = Observation.from_text(self._chat_tool.chat_obs()) + page_obs
+        else:
+            obs = Observation.from_text(goal) + page_obs
+        info = {
+            "task_id": self.id,
+            "task_class": task_class.__name__,
+            "seed": self.seed,
+            "goal": goal,
+            **task_info,
+        }
+        return obs, info
+
+    @property
+    def _chat_messages(self) -> list[dict]:
+        """
+        Return combined chat and infeasible messages.
+
+        Normal path (ChatTool): a copy of session history — safe for parallel episodes,
+        always current because send_message() writes before evaluate() runs.
+
+        Cheat path (WorkArenaCheatTool, no ChatTool): the live _chat_messages_ref list.
+        cheat() appends directly to whatever list it receives, so cheat() and validate()
+        must share the same list instance.
+        """
+        messages: list[dict] = []
+        if self._chat_tool is None and isinstance(self._browser_tool, WorkArenaCheatTool):
+            messages.extend(self._browser_tool._chat_messages_ref)
+        elif (chat := self._chat_tool) is not None:
+            messages.extend(chat.messages)
+        if (infeasible := self._infeasible_tool) is not None:
+            messages.extend(infeasible.messages)
+        return messages
+
+    def _validate(self) -> tuple[float, bool, str, dict]:
+        """Call WorkArena's validate() with per-step caching.
+
+        Both evaluate() and finished() call this on every step. The cache avoids
+        duplicate ServiceNow REST calls within the same step. It is cleared after
+        the first consumer reads it, so the next step gets a fresh call.
+        """
+        if self._workarena_task is None:
+            raise RuntimeError("WorkArena task is not initialized. Call reset() first.")
+        if self._validate_cache is None:
+            page = self._browser_tool.page
+            self._validate_cache = self._workarena_task.validate(page, self._chat_messages)  # type: ignore : Workarena validators expect list[dict] despite the protocol specifying list[str].
+        return self._validate_cache  # type: ignore[return-value]
+
+    @override
+    def step(self, action: Action | List[Action]) -> EnvironmentOutput:
+        self._validate_cache = None
+        return super().step(action)
+
+    def evaluate(self, obs: Observation | None = None) -> tuple[float, dict[str, Any]]:
+        """Score the current task state via WorkArena's validate()."""
+        reward, done, _user_message, task_info = self._validate()
+        return reward, {"done": done, **task_info}
+
+    def finished(self, obs: Observation | None = None) -> bool:
+        """Check if the task is done via WorkArena's validate()."""
+        if self._workarena_task is None:
+            return False
+        _reward, done, _user_message, _task_info = self._validate()
+        return done
+
+    def filter_actions(self, actions: list[ActionSchema]) -> list[ActionSchema]:
+        """Filter actions based on available tools."""
+        if self._chat_tool is None:
+            actions = [a for a in actions if a.name != "send_message"]
+        if self._infeasible_tool is None:
+            actions = [a for a in actions if a.name != "report_infeasible"]
+        return actions
+
+    def close(self) -> None:
+        """Teardown the WorkArena task and close the tool."""
+        if self._workarena_task is not None:
+            try:
+                self._workarena_task.teardown()
+            except Exception as e:
+                logger.warning(f"Error during WorkArena task teardown: {e}")
+            finally:
+                self._workarena_task = None
+        super().close()
+
+
+class WorkArenaTaskConfig(TaskConfig):
+    """Serializable configuration for a single WorkArena task."""
+
+    def make(
+        self,
+        runtime_context: RuntimeContext | None = None,
+        container_backend: ContainerBackend | None = None,
+    ) -> WorkArenaTask:
+        # Import here to avoid circular import (benchmark imports task)
+        from workarena_cube.benchmark import WorkArenaBenchmark
+
+        _ = runtime_context, container_backend
+        meta = WorkArenaBenchmark.task_metadata[self.task_id]
+        assert self.tool_config, f"WorkArenaTaskConfig requires a tool_config, got {self.tool_config}"
+        return WorkArenaTask(
+            metadata=meta,
+            tool_config=self.tool_config,
+            seed=self.seed if self.seed is not None else 42,
+        )
+
+
+def _load_task_class(class_path: str) -> type:
+    """Reconstruct a task class from its dotted module-qualified name."""
+    module_name, class_name = class_path.rsplit(".", 1)
+    module = importlib.import_module(module_name)
+    return getattr(module, class_name)
+
+
+def _apply_task_runtime_preferences(tool: WorkArenaBrowserTool, workarena_task: AbstractServiceNowTask) -> None:
+    """Apply WorkArena task runtime defaults to the tool config when not explicitly set."""
+    browser_config = tool.config.browser
+    explicitly_set = browser_config.model_fields_set
+    updates: dict[str, Any] = {}
+    for field in ("slow_mo", "timeout", "locale", "timezone_id"):
+        if field not in explicitly_set and getattr(workarena_task, field, None) is not None:
+            updates[field] = getattr(workarena_task, field)
+    if "viewport" not in explicitly_set:
+        raw_vp = getattr(workarena_task, "viewport", None)
+        if isinstance(raw_vp, dict):
+            updates["viewport"] = Viewport(**raw_vp)
+        elif isinstance(raw_vp, Viewport):
+            updates["viewport"] = raw_vp
+    if updates:
+        tool.config.browser = browser_config.model_copy(update=updates)