cudag 0.3.10__py3-none-any.whl

cudag/core/screen.py

@@ -0,0 +1,402 @@
+# Copyright (c) 2025 Tylt LLC. All rights reserved.
+# CONFIDENTIAL AND PROPRIETARY. Unauthorized use, copying, or distribution
+# is strictly prohibited. For licensing inquiries: hello@claimhawk.app
+
+"""Rails-like DSL for screen definitions.
+
+Simple, readable screen definitions with DSL functions.
+
+Example:
+    class CalendarScreen(Screen):
+        name = "calendar"
+        base_image = "calendar.png"
+        size = (224, 208)
+
+        day_grid = grid((10, 50, 210, 150), rows=6, cols=7)
+        back_month = button((7, 192, 20, 12), label="Back Month")
+        scroll_area = scrollable((0, 0, 224, 208), step=100)
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+from dataclasses import dataclass
+from dataclasses import field as dataclass_field
+from pathlib import Path
+from typing import Any, ClassVar
+
+# =============================================================================
+# Bounds - represents a rectangular area
+# =============================================================================
+
+
+@dataclass
+class Bounds:
+    """Rectangular bounds for a region on screen."""
+
+    x: int
+    y: int
+    width: int
+    height: int
+
+    @property
+    def center(self) -> tuple[int, int]:
+        """Center point of the bounds."""
+        return (self.x + self.width // 2, self.y + self.height // 2)
+
+    @property
+    def right(self) -> int:
+        """Right edge x coordinate."""
+        return self.x + self.width
+
+    @property
+    def bottom(self) -> int:
+        """Bottom edge y coordinate."""
+        return self.y + self.height
+
+    def contains(self, point: tuple[int, int]) -> bool:
+        """Check if a point is within bounds."""
+        px, py = point
+        return self.x <= px < self.right and self.y <= py < self.bottom
+
+    @classmethod
+    def from_tuple(cls, t: tuple[int, int, int, int]) -> Bounds:
+        """Create from (x, y, width, height) tuple."""
+        return cls(x=t[0], y=t[1], width=t[2], height=t[3])
+
+
+# =============================================================================
+# Region Base Class
+# =============================================================================
+
+
+@dataclass
+class Region(ABC):
+    """Base class for interactive screen regions."""
+
+    bounds: Bounds
+    name: str = ""
+
+    @abstractmethod
+    def get_action_point(self, target: Any = None) -> tuple[int, int]:
+        """Get the pixel coordinate for performing an action."""
+        pass
+
+
+# =============================================================================
+# Region Types
+# =============================================================================
+
+
+@dataclass
+class ClickRegion(Region):
+    """A simple clickable region."""
+
+    def get_action_point(self, target: Any = None) -> tuple[int, int]:
+        return self.bounds.center
+
+
+@dataclass
+class ButtonRegion(ClickRegion):
+    """A clickable button."""
+
+    label: str = ""
+    description: str = ""
+    tolerance: tuple[int, int] = (5, 5)
+
+
+@dataclass
+class GridRegion(Region):
+    """A grid of clickable cells."""
+
+    rows: int = 1
+    cols: int = 1
+    cell_width: int | None = None
+    cell_height: int | None = None
+    row_gap: int = 0
+    col_gap: int = 0
+
+    def __post_init__(self) -> None:
+        if self.cell_width is None:
+            total_gaps = self.col_gap * (self.cols - 1) if self.cols > 1 else 0
+            self.cell_width = (self.bounds.width - total_gaps) // self.cols
+        if self.cell_height is None:
+            total_gaps = self.row_gap * (self.rows - 1) if self.rows > 1 else 0
+            self.cell_height = (self.bounds.height - total_gaps) // self.rows
+
+    def get_action_point(self, target: tuple[int, int] | int | None = None) -> tuple[int, int]:
+        if target is None:
+            return self.bounds.center
+
+        if isinstance(target, int):
+            row, col = divmod(target, self.cols)
+        else:
+            row, col = target
+
+        assert self.cell_width is not None
+        assert self.cell_height is not None
+
+        x = self.bounds.x + col * (self.cell_width + self.col_gap) + self.cell_width // 2
+        y = self.bounds.y + row * (self.cell_height + self.row_gap) + self.cell_height // 2
+        return (x, y)
+
+    def cell_bounds(self, row: int, col: int) -> Bounds:
+        assert self.cell_width is not None
+        assert self.cell_height is not None
+
+        x = self.bounds.x + col * (self.cell_width + self.col_gap)
+        y = self.bounds.y + row * (self.cell_height + self.row_gap)
+        return Bounds(x=x, y=y, width=self.cell_width, height=self.cell_height)
+
+
+@dataclass
+class ScrollRegion(Region):
+    """A scrollable region."""
+
+    scroll_step: int = 100
+    direction: str = "vertical"
+
+    def get_action_point(self, target: Any = None) -> tuple[int, int]:
+        return self.bounds.center
+
+    def get_scroll_pixels(self, direction: str = "down") -> int:
+        amount = self.scroll_step
+        if direction in ("up", "left"):
+            return -amount
+        return amount
+
+
+@dataclass
+class DropdownRegion(Region):
+    """A dropdown/select field."""
+
+    items: Sequence[str] = dataclass_field(default_factory=list)
+    item_height: int = 20
+
+    def get_action_point(self, target: str | int | None = None) -> tuple[int, int]:
+        if target is None:
+            return self.bounds.center
+
+        if isinstance(target, str):
+            try:
+                idx = list(self.items).index(target)
+            except ValueError:
+                return self.bounds.center
+        else:
+            idx = target
+
+        x = self.bounds.center[0]
+        y = self.bounds.bottom + (idx * self.item_height) + self.item_height // 2
+        return (x, y)
+
+
+# =============================================================================
+# DSL Functions - the Rails-like interface
+# =============================================================================
+
+BoundsTuple = tuple[int, int, int, int]
+
+
+def region(bounds: BoundsTuple) -> ClickRegion:
+    """Define a simple clickable region.
+
+    Example:
+        header = region((0, 0, 100, 50))
+    """
+    return ClickRegion(bounds=Bounds.from_tuple(bounds))
+
+
+def button(
+    bounds: BoundsTuple,
+    label: str = "",
+    description: str = "",
+    tolerance: tuple[int, int] = (5, 5),
+) -> ButtonRegion:
+    """Define a button.
+
+    Example:
+        back_month = button((7, 192, 20, 12), label="Back Month")
+    """
+    return ButtonRegion(
+        bounds=Bounds.from_tuple(bounds),
+        label=label,
+        description=description,
+        tolerance=tolerance,
+    )
+
+
+def grid(
+    bounds: BoundsTuple,
+    rows: int = 1,
+    cols: int = 1,
+    cell_width: int | None = None,
+    cell_height: int | None = None,
+    row_gap: int = 0,
+    col_gap: int = 0,
+) -> GridRegion:
+    """Define a grid region.
+
+    Example:
+        day_grid = grid((10, 50, 210, 150), rows=6, cols=7)
+        data_grid = grid((0, 100, 800, 400), rows=10, cols=5, row_gap=2)
+    """
+    return GridRegion(
+        bounds=Bounds.from_tuple(bounds),
+        rows=rows,
+        cols=cols,
+        cell_width=cell_width,
+        cell_height=cell_height,
+        row_gap=row_gap,
+        col_gap=col_gap,
+    )
+
+
+def scrollable(
+    bounds: BoundsTuple,
+    step: int = 100,
+    direction: str = "vertical",
+) -> ScrollRegion:
+    """Define a scrollable region.
+
+    Example:
+        content = scrollable((0, 100, 800, 500), step=100)
+    """
+    return ScrollRegion(
+        bounds=Bounds.from_tuple(bounds),
+        scroll_step=step,
+        direction=direction,
+    )
+
+
+def dropdown(
+    bounds: BoundsTuple,
+    items: Sequence[str] | None = None,
+    item_height: int = 20,
+) -> DropdownRegion:
+    """Define a dropdown region.
+
+    Example:
+        month_select = dropdown((100, 10, 80, 25), items=["Jan", "Feb", "Mar"])
+    """
+    return DropdownRegion(
+        bounds=Bounds.from_tuple(bounds),
+        items=items or [],
+        item_height=item_height,
+    )
+
+
+# =============================================================================
+# Screen Meta - configuration for screens
+# =============================================================================
+
+
+class ScreenMeta:
+    """Metadata for a Screen class."""
+
+    name: str = ""
+    base_image: str | Path = ""
+    size: tuple[int, int] = (1000, 1000)
+    task_types: list[str]
+
+    def __init__(self) -> None:
+        self.task_types = []
+
+
+# =============================================================================
+# Screen Base Class
+# =============================================================================
+
+
+class ScreenBase(ABC):
+    """Base class for Screen definitions.
+
+    A Screen has many Tasks - this is the core 1:N relationship.
+    Each Screen defines the UI layout and can declare which task types it supports.
+
+    Example:
+        class CalendarScreen(Screen):
+            name = "calendar"
+            base_image = "calendar.png"
+            size = (224, 208)
+
+            # Regions
+            day_grid = grid((10, 50, 210, 150), rows=6, cols=7)
+            back_month = button((7, 192, 20, 12), label="Back")
+
+            # Tasks that belong to this screen
+            task_types = ["click-day", "click-month", "scroll-calendar"]
+    """
+
+    # Class-level attributes that can be set directly
+    name: ClassVar[str] = ""
+    base_image: ClassVar[str | Path] = ""
+    size: ClassVar[tuple[int, int]] = (1000, 1000)
+    task_types: ClassVar[list[str]] = []
+
+    # Collected metadata
+    _regions: ClassVar[dict[str, Region]] = {}
+    _meta: ClassVar[ScreenMeta]
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        # Collect regions from class attributes
+        regions: dict[str, Region] = {}
+        for attr_name, value in cls.__dict__.items():
+            if isinstance(value, Region):
+                value.name = attr_name
+                regions[attr_name] = value
+
+        cls._regions = regions
+
+        # Build meta from class attributes or inner Meta class
+        cls._meta = ScreenMeta()
+
+        # Check for inner Meta class (Rails style)
+        inner_meta = getattr(cls, "Meta", None)
+        if inner_meta:
+            for attr in ("name", "base_image", "size", "task_types"):
+                if hasattr(inner_meta, attr):
+                    setattr(cls._meta, attr, getattr(inner_meta, attr))
+
+        # Also check class-level attributes (simpler style)
+        for attr in ("name", "base_image", "size", "task_types"):
+            val = cls.__dict__.get(attr)
+            if val is not None and val != "" and val != []:
+                setattr(cls._meta, attr, val)
+
+        # Default name from class name
+        if not cls._meta.name:
+            cls._meta.name = cls.__name__.lower().replace("screen", "")
+
+    @classmethod
+    def get_region(cls, name: str) -> Region:
+        """Get a region by name."""
+        if name not in cls._regions:
+            raise KeyError(f"Region '{name}' not found in {cls.__name__}")
+        return cls._regions[name]
+
+    @classmethod
+    def regions(cls) -> dict[str, Region]:
+        """Get all regions."""
+        return cls._regions.copy()
+
+    @classmethod
+    def meta(cls) -> ScreenMeta:
+        """Get screen metadata."""
+        return cls._meta
+
+    @classmethod
+    def get_task_types(cls) -> list[str]:
+        """Get task types that belong to this screen."""
+        return cls._meta.task_types.copy()
+
+    @abstractmethod
+    def render(self, state: Any) -> tuple[Any, dict[str, Any]]:
+        """Render the screen with given state."""
+        pass
+
+
+# Alias for cleaner imports
+Screen = ScreenBase

cudag/core/scroll_task.py

@@ -0,0 +1,254 @@
+# Copyright (c) 2025 Tylt LLC. All rights reserved.
+# CONFIDENTIAL AND PROPRIETARY. Unauthorized use, copying, or distribution
+# is strictly prohibited. For licensing inquiries: hello@claimhawk.app
+
+"""Base class for scroll interaction tasks.
+
+This module provides an abstract base class for scroll tasks, reducing
+boilerplate code when implementing scroll-up/scroll-down tasks across
+different generators.
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from random import Random
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from cudag.core.coords import normalize_coord
+from cudag.core.task import BaseTask, TaskContext, TaskSample, TestCase
+from cudag.prompts.tools import ToolCall
+
+if TYPE_CHECKING:
+    from cudag.core.renderer import BaseRenderer
+
+
+@dataclass
+class ScrollTaskConfig:
+    """Configuration for a scroll task.
+
+    This dataclass encapsulates all the static configuration for a scroll
+    task, making it easy to create multiple scroll tasks with different
+    configurations.
+
+    Attributes:
+        task_type: Unique task type identifier (e.g., "scroll-page-down")
+        scroll_pixels: Number of pixels to scroll (positive=down, negative=up)
+        direction: Human-readable direction ("up" or "down")
+        prompt: Prompt text for the training sample
+        tolerance: Default tolerance in RU units (x, y)
+
+    Example:
+        >>> config = ScrollTaskConfig(
+        ...     task_type="scroll-page-down",
+        ...     scroll_pixels=300,
+        ...     direction="down",
+        ...     prompt="Scroll down one page",
+        ... )
+    """
+
+    task_type: str
+    scroll_pixels: int
+    direction: str
+    prompt: str
+    tolerance: tuple[int, int] = (100, 6)
+
+
+class ScrollTaskBase(BaseTask):
+    """Abstract base class for scroll direction tasks.
+
+    This base class encapsulates the common pattern for scroll interaction
+    tasks, reducing boilerplate in individual task implementations. Instead
+    of implementing the full generate_sample() logic, subclasses only need to:
+
+    1. Set the `config` class variable with a ScrollTaskConfig
+    2. Implement `get_scroll_center()` to return the target coordinates
+    3. Implement `generate_state()` to create the appropriate state
+
+    The base class handles:
+    - Rendering the image
+    - Saving the image
+    - Creating the tool call
+    - Building the TaskSample with proper metadata
+    - Creating test cases
+
+    Example:
+        class ScrollPageDownTask(ScrollTaskBase):
+            config = ScrollTaskConfig(
+                task_type="scroll-page-down",
+                scroll_pixels=300,
+                direction="down",
+                prompt="Scroll down one page",
+            )
+
+            def get_scroll_center(self, metadata: dict) -> tuple[int, int]:
+                return metadata["grid_center"]
+
+            def generate_state(self, rng: Random):
+                return MyState.generate_for_scroll(rng, "middle")
+    """
+
+    config: ClassVar[ScrollTaskConfig]
+    """Configuration for this scroll task. Must be set by subclass."""
+
+    @property
+    def task_type(self) -> str:
+        """Return the task type from config."""
+        return self.config.task_type
+
+    @abstractmethod
+    def get_scroll_center(self, metadata: dict[str, Any]) -> tuple[int, int]:
+        """Return the pixel coordinates for the scroll action.
+
+        Args:
+            metadata: Rendering metadata from the renderer, typically contains
+                      information about element positions and grid dimensions.
+
+        Returns:
+            (x, y) pixel coordinates for scroll center
+        """
+        ...
+
+    @abstractmethod
+    def generate_state(self, rng: Random) -> Any:
+        """Generate the state for this scroll task.
+
+        The state should represent a scrollable position appropriate for
+        the scroll direction. For example:
+        - scroll-down tasks should generate states near the top/middle
+        - scroll-up tasks should generate states near the middle/bottom
+
+        Args:
+            rng: Random number generator for reproducibility
+
+        Returns:
+            State object appropriate for the scroll position
+        """
+        ...
+
+    def generate_sample(self, ctx: TaskContext) -> TaskSample:
+        """Generate a training sample for this scroll task.
+
+        This method orchestrates the sample generation:
+        1. Generates state using generate_state()
+        2. Renders the image using the renderer
+        3. Saves the image
+        4. Gets scroll coordinates from get_scroll_center()
+        5. Creates and returns the TaskSample
+
+        Args:
+            ctx: Task context with RNG, index, output directory, etc.
+
+        Returns:
+            TaskSample with image, prompt, and scroll tool call
+        """
+        state = self.generate_state(ctx.rng)
+        image, metadata = self.renderer.render(state)
+
+        image_path = self.save_image(image, ctx)
+        scroll_center = self.get_scroll_center(metadata)
+        normalized = normalize_coord(scroll_center, image.size)
+
+        return TaskSample(
+            id=self.build_id(ctx),
+            image_path=image_path,
+            human_prompt=self.config.prompt,
+            tool_call=ToolCall.scroll(normalized, pixels=self.config.scroll_pixels),
+            pixel_coords=scroll_center,
+            metadata={
+                "task_type": self.config.task_type,
+                "scroll_pixels": self.config.scroll_pixels,
+                "scroll_direction": self.config.direction,
+                "tolerance": list(self.config.tolerance),
+                **metadata,
+            },
+            image_size=image.size,
+        )
+
+    def generate_test(self, ctx: TaskContext) -> TestCase:
+        """Generate a test case for this scroll task.
+
+        Creates a test case by first generating a sample, then wrapping
+        it in a TestCase with the appropriate tolerance.
+
+        Args:
+            ctx: Task context with RNG, index, output directory, etc.
+
+        Returns:
+            TestCase ready for evaluation
+        """
+        sample = self.generate_sample(ctx)
+        return TestCase(
+            test_id=f"test_{sample.id}",
+            screenshot=sample.image_path,
+            prompt=sample.human_prompt,
+            expected_action=sample.tool_call.to_dict(),
+            tolerance=self.config.tolerance,
+            metadata=sample.metadata,
+            pixel_coords=sample.pixel_coords,
+        )
+
+
+def create_scroll_task_pair(
+    base_task_type: str,
+    scroll_pixels: int,
+    up_prompt: str,
+    down_prompt: str,
+    tolerance: tuple[int, int] = (100, 6),
+) -> tuple[type[ScrollTaskBase], type[ScrollTaskBase]]:
+    """Factory function to create a pair of scroll up/down task classes.
+
+    This is a convenience function for creating complementary scroll tasks
+    that share the same configuration except for direction.
+
+    Args:
+        base_task_type: Base name for task types (e.g., "scroll-page")
+        scroll_pixels: Number of pixels to scroll
+        up_prompt: Prompt for scroll-up task
+        down_prompt: Prompt for scroll-down task
+        tolerance: Tolerance in RU units
+
+    Returns:
+        Tuple of (ScrollUpTask, ScrollDownTask) classes
+
+    Example:
+        >>> ScrollUp, ScrollDown = create_scroll_task_pair(
+        ...     "scroll-page",
+        ...     300,
+        ...     "Scroll up one page",
+        ...     "Scroll down one page",
+        ... )
+    """
+
+    class _ScrollUpTask(ScrollTaskBase):
+        config = ScrollTaskConfig(
+            task_type=f"{base_task_type}-up",
+            scroll_pixels=-scroll_pixels,
+            direction="up",
+            prompt=up_prompt,
+            tolerance=tolerance,
+        )
+
+        def get_scroll_center(self, metadata: dict[str, Any]) -> tuple[int, int]:
+            raise NotImplementedError("Subclass must implement get_scroll_center()")
+
+        def generate_state(self, rng: Random) -> Any:
+            raise NotImplementedError("Subclass must implement generate_state()")
+
+    class _ScrollDownTask(ScrollTaskBase):
+        config = ScrollTaskConfig(
+            task_type=f"{base_task_type}-down",
+            scroll_pixels=scroll_pixels,
+            direction="down",
+            prompt=down_prompt,
+            tolerance=tolerance,
+        )
+
+        def get_scroll_center(self, metadata: dict[str, Any]) -> tuple[int, int]:
+            raise NotImplementedError("Subclass must implement get_scroll_center()")
+
+        def generate_state(self, rng: Random) -> Any:
+            raise NotImplementedError("Subclass must implement generate_state()")
+
+    return _ScrollUpTask, _ScrollDownTask