cudag 0.3.10__py3-none-any.whl

cudag/core/icon.py

@@ -0,0 +1,207 @@
+# Copyright (c) 2025 Tylt LLC. All rights reserved.
+# CONFIDENTIAL AND PROPRIETARY. Unauthorized use, copying, or distribution
+# is strictly prohibited. For licensing inquiries: hello@claimhawk.app
+
+"""Icon abstraction for clickable UI icons.
+
+Provides reusable Icon classes for any icon-based UI element:
+- Desktop icons (large, with labels)
+- Taskbar icons (small, no labels)
+- Application icons
+- Toolbar buttons
+
+The Icon class handles:
+- Placement (position, size)
+- Tolerance calculation
+- Center point for clicks
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class IconSpec:
+    """Specification for an icon type.
+
+    Defines the size and characteristics of a class of icons
+    (e.g., all desktop icons are 54x54 with labels).
+    """
+
+    width: int
+    """Icon width in pixels."""
+
+    height: int
+    """Icon height in pixels."""
+
+    has_label: bool = False
+    """Whether this icon type has a text label."""
+
+    label_height: int = 0
+    """Height of the label area below icon."""
+
+    @property
+    def total_height(self) -> int:
+        """Total height including label."""
+        return self.height + (self.label_height if self.has_label else 0)
+
+    @property
+    def tolerance_pixels(self) -> tuple[int, int]:
+        """Natural tolerance for this icon size (70% of dimensions)."""
+        return (int(self.width * 0.7), int(self.height * 0.7))
+
+    def tolerance_ru(self, image_size: tuple[int, int]) -> tuple[int, int]:
+        """Tolerance in RU units for normalized coordinates.
+
+        Args:
+            image_size: (width, height) of the image in pixels
+
+        Returns:
+            (x_tolerance, y_tolerance) in RU units (0-1000 scale)
+        """
+        x_ru = (self.width * 0.7 / image_size[0]) * 1000
+        y_ru = (self.height * 0.7 / image_size[1]) * 1000
+        return (int(x_ru), int(y_ru))
+
+
+# Common icon specs
+DESKTOP_ICON = IconSpec(width=54, height=54, has_label=True, label_height=20)
+TASKBAR_ICON = IconSpec(width=27, height=28, has_label=False)
+TOOLBAR_ICON = IconSpec(width=24, height=24, has_label=False)
+APP_ICON_LARGE = IconSpec(width=48, height=48, has_label=True, label_height=16)
+APP_ICON_SMALL = IconSpec(width=32, height=32, has_label=False)
+
+
+@dataclass
+class IconPlacement:
+    """An icon placed at a specific location.
+
+    Represents a single icon instance with its position and metadata.
+    """
+
+    icon_id: str
+    """Unique identifier for this icon (e.g., 'chrome', 'od')."""
+
+    x: int
+    """X position of icon top-left corner."""
+
+    y: int
+    """Y position of icon top-left corner."""
+
+    spec: IconSpec
+    """Icon specification (size, etc.)."""
+
+    label: str = ""
+    """Display label for the icon."""
+
+    image_file: str | Path = ""
+    """Path to the icon image file."""
+
+    metadata: dict[str, Any] = field(default_factory=dict)
+    """Additional metadata."""
+
+    @property
+    def width(self) -> int:
+        """Icon width."""
+        return self.spec.width
+
+    @property
+    def height(self) -> int:
+        """Icon height (excluding label)."""
+        return self.spec.height
+
+    @property
+    def center(self) -> tuple[int, int]:
+        """Center point of the icon (for clicking)."""
+        return (
+            self.x + self.spec.width // 2,
+            self.y + self.spec.height // 2,
+        )
+
+    @property
+    def bounds(self) -> tuple[int, int, int, int]:
+        """Bounding box as (x, y, width, height)."""
+        return (self.x, self.y, self.spec.width, self.spec.height)
+
+    @property
+    def tolerance_pixels(self) -> tuple[int, int]:
+        """Natural tolerance for this icon."""
+        return self.spec.tolerance_pixels
+
+    def tolerance_ru(self, image_size: tuple[int, int]) -> tuple[int, int]:
+        """Tolerance in RU units."""
+        return self.spec.tolerance_ru(image_size)
+
+
+@dataclass
+class IconLayout:
+    """Layout manager for icons in a region.
+
+    Handles placement of multiple icons in a grid or list layout.
+    """
+
+    spec: IconSpec
+    """Icon specification for all icons in this layout."""
+
+    start_x: int = 0
+    """Starting X position."""
+
+    start_y: int = 0
+    """Starting Y position."""
+
+    padding: int = 10
+    """Padding between icons."""
+
+    direction: str = "vertical"
+    """Layout direction: 'vertical', 'horizontal', or 'grid'."""
+
+    cols: int = 1
+    """Number of columns for grid layout."""
+
+    def place_icons(
+        self,
+        icon_ids: list[str],
+        labels: dict[str, str] | None = None,
+        image_files: dict[str, str | Path] | None = None,
+    ) -> list[IconPlacement]:
+        """Place icons according to layout rules.
+
+        Args:
+            icon_ids: List of icon identifiers to place
+            labels: Optional dict mapping icon_id to label
+            image_files: Optional dict mapping icon_id to image path
+
+        Returns:
+            List of IconPlacement objects
+        """
+        labels = labels or {}
+        image_files = image_files or {}
+        placements: list[IconPlacement] = []
+
+        for i, icon_id in enumerate(icon_ids):
+            if self.direction == "vertical":
+                x = self.start_x
+                y = self.start_y + i * (self.spec.total_height + self.padding)
+            elif self.direction == "horizontal":
+                x = self.start_x + i * (self.spec.width + self.padding)
+                y = self.start_y
+            else:  # grid
+                row, col = divmod(i, self.cols)
+                x = self.start_x + col * (self.spec.width + self.padding)
+                y = self.start_y + row * (self.spec.total_height + self.padding)
+
+            placements.append(
+                IconPlacement(
+                    icon_id=icon_id,
+                    x=x,
+                    y=y,
+                    spec=self.spec,
+                    label=labels.get(icon_id, ""),
+                    image_file=image_files.get(icon_id, ""),
+                )
+            )
+
+        return placements

cudag/core/iconlist_task.py

@@ -0,0 +1,301 @@
+# Copyright (c) 2025 Tylt LLC. All rights reserved.
+# CONFIDENTIAL AND PROPRIETARY. Unauthorized use, copying, or distribution
+# is strictly prohibited. For licensing inquiries: hello@claimhawk.app
+
+"""Base class for iconlist element tasks.
+
+This module provides an abstract base class for tasks that target iconlist
+elements in annotation.json. When a task targets an iconlist, it generates
+one sample per icon in that list.
+
+Element type determines generation behavior:
+- iconlist -> generate one sample per icon
+- button -> generate one sample (not handled here)
+- loading -> handled separately by wait tasks
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from random import Random
+from typing import TYPE_CHECKING, Any, Protocol
+
+from cudag.annotation.config import AnnotatedElement, AnnotatedTask, AnnotationConfig
+from cudag.core.task import BaseTask, TaskContext, TaskSample, TestCase
+from cudag.prompts.tools import ToolCall
+
+if TYPE_CHECKING:
+    from cudag.core.renderer import BaseRenderer
+
+
+class IconInfo(Protocol):
+    """Protocol for icon placement info from state."""
+
+    icon_id: str
+    center: tuple[int, int]
+    bounds: tuple[int, int, int, int]
+
+
+def make_tool_call(
+    action: str, coord: tuple[int, int], wait_time: float = 0
+) -> ToolCall:
+    """Create a ToolCall based on action string from annotation.
+
+    Args:
+        action: Action string from annotation (double_click, left_click, etc.)
+        coord: (x, y) pixel coordinates
+        wait_time: Wait time in seconds for wait actions
+
+    Returns:
+        ToolCall instance for the action
+    """
+    if action == "double_click":
+        return ToolCall.double_click(coord)
+    elif action == "left_click":
+        return ToolCall.left_click(coord)
+    elif action == "right_click":
+        return ToolCall.right_click(coord)
+    elif action == "wait":
+        return ToolCall.wait(wait_time)
+    else:
+        raise ValueError(f"Unknown action '{action}' - must be one of: double_click, left_click, right_click, wait")
+
+
+class IconListTaskBase(BaseTask):
+    """Abstract base class for tasks targeting iconlist elements.
+
+    This base class handles the common pattern for iconlist tasks:
+    1. Iterate over tasks in annotation
+    2. For each task targeting an iconlist element, generate one sample per icon
+    3. Use task.prompt_template, task.action, task.task_type from annotation
+    4. Use element.tolerance_x/y from annotation
+
+    Subclasses must implement:
+    - get_annotation_config(): Return the AnnotationConfig
+    - get_icons_for_element(): Return icon placements and info for an element
+    - generate_state(): Generate the screen state
+
+    Example:
+        class ClickIconTask(IconListTaskBase):
+            def get_annotation_config(self) -> AnnotationConfig:
+                return ANNOTATION_CONFIG
+
+            def get_icons_for_element(
+                self, element: AnnotatedElement, state: Any
+            ) -> tuple[list[IconInfo], dict[str, dict]]:
+                if element.label == "desktop":
+                    return state.desktop_icons, get_desktop_icons()
+                elif element.label == "taskbar":
+                    return state.taskbar_icons, get_taskbar_icons()
+                return [], {}
+
+            def generate_state(self, rng: Random, **kwargs) -> Any:
+                return DesktopState.generate(rng=rng, **kwargs)
+    """
+
+    task_type: str = "iconlist"
+
+    @abstractmethod
+    def get_annotation_config(self) -> AnnotationConfig | None:
+        """Return the annotation config for this generator.
+
+        Returns:
+            AnnotationConfig loaded from annotation.json, or None
+        """
+        ...
+
+    @abstractmethod
+    def get_icons_for_element(
+        self, element: AnnotatedElement, state: Any
+    ) -> tuple[list[Any], dict[str, dict[str, Any]]]:
+        """Get icons and their info for an element.
+
+        Args:
+            element: The annotated element (iconlist type)
+            state: The generated screen state
+
+        Returns:
+            Tuple of (icon_placements, icons_info_dict)
+            - icon_placements: List of IconPlacement-like objects with icon_id, center, bounds
+            - icons_info_dict: Dict mapping icon_id to {label, required, ...}
+        """
+        ...
+
+    @abstractmethod
+    def generate_state(self, rng: Random, **kwargs: Any) -> Any:
+        """Generate the screen state.
+
+        Args:
+            rng: Random number generator
+            **kwargs: Additional arguments for state generation
+
+        Returns:
+            State object for rendering
+        """
+        ...
+
+    def generate_samples(self, ctx: TaskContext) -> list[TaskSample]:
+        """Generate samples for ALL iconlist tasks in annotation.
+
+        Iterates over tasks in annotation.json. For each task targeting an
+        iconlist element, generates one sample per icon in that list.
+
+        Args:
+            ctx: Task context with RNG, index, output directory
+
+        Returns:
+            List of TaskSample, one per icon per iconlist task
+        """
+        config = self.get_annotation_config()
+        if config is None:
+            return []
+
+        # Generate state - subclass provides kwargs
+        state = self.generate_state(ctx.rng)
+
+        # Render once - shared by all samples
+        image, metadata = self.renderer.render(state)
+        image_path = self.save_image(image, ctx)
+
+        samples: list[TaskSample] = []
+
+        # Iterate over ALL tasks in annotation (except wait and grounding tasks)
+        for task in config.tasks:
+            if task.action == "wait":
+                continue  # Wait tasks handled separately
+            if task.action == "grounding":
+                continue  # Grounding tasks handled by GroundingTask
+
+            # Get target element
+            element = config.get_element(task.target_element_id)
+            if element is None:
+                continue
+
+            # Only handle iconlist elements
+            if element.element_type != "iconlist":
+                continue
+
+            # Get icons for this element from state
+            icons_in_state, icons_info = self.get_icons_for_element(element, state)
+            if not icons_in_state:
+                continue
+
+            tolerance = (element.tolerance_x, element.tolerance_y)
+
+            # Generate a sample for each icon
+            for icon in icons_in_state:
+                icon_info = icons_info.get(icon.icon_id, {})
+                label = icon_info.get("label", icon.icon_id)
+
+                prompt = task.prompt_template.replace("[icon_label]", label)
+                click_x, click_y = icon.center
+
+                tool_call = make_tool_call(task.action, (click_x, click_y))
+
+                samples.append(
+                    TaskSample(
+                        id=self.build_id(ctx, f"_{element.label}_{icon.icon_id}"),
+                        image_path=image_path,
+                        human_prompt=prompt,
+                        tool_call=tool_call,
+                        pixel_coords=(click_x, click_y),
+                        metadata={
+                            "task_type": task.task_type,
+                            "element_type": element.element_type,
+                            "element_label": element.label,
+                            "icon_id": icon.icon_id,
+                            "icon_label": label,
+                            "icon_bounds": icon.bounds,
+                            "ground_truth": (
+                                state.to_ground_truth()
+                                if hasattr(state, "to_ground_truth")
+                                else {}
+                            ),
+                            "tolerance": list(tolerance),
+                        },
+                        image_size=config.image_size,
+                    )
+                )
+
+        return samples
+
+    def generate_sample(self, ctx: TaskContext) -> TaskSample:
+        """Generate samples - returns first sample."""
+        samples = self.generate_samples(ctx)
+        if not samples:
+            raise ValueError("No samples generated")
+        return samples[0]
+
+    def generate_tests(self, ctx: TaskContext) -> list[TestCase]:
+        """Generate test cases for required icons in iconlist tasks.
+
+        Only generates tests for icons marked as required=true in annotation.
+
+        Args:
+            ctx: Task context
+
+        Returns:
+            List of TestCase for required icons
+        """
+        config = self.get_annotation_config()
+        if config is None:
+            return []
+
+        state = self.generate_state(ctx.rng)
+        image, metadata = self.renderer.render(state)
+        image_path = self.save_image(image, ctx)
+
+        tests: list[TestCase] = []
+
+        for task in config.tasks:
+            if task.action == "wait":
+                continue
+
+            element = config.get_element(task.target_element_id)
+            if element is None or element.element_type != "iconlist":
+                continue
+
+            icons_in_state, icons_info = self.get_icons_for_element(element, state)
+            tolerance = (element.tolerance_x, element.tolerance_y)
+
+            # Test on required icons only
+            for icon in icons_in_state:
+                icon_info = icons_info.get(icon.icon_id, {})
+                if not icon_info.get("required", False):
+                    continue
+
+                label = icon_info.get("label", icon.icon_id)
+                prompt = task.prompt_template.replace("[icon_label]", label)
+                click_x, click_y = icon.center
+
+                tool_call = make_tool_call(task.action, (click_x, click_y))
+
+                tests.append(
+                    TestCase(
+                        test_id=f"test_{ctx.index:04d}_{element.label}_{icon.icon_id}",
+                        screenshot=image_path,
+                        prompt=prompt,
+                        expected_action=tool_call.to_dict(),
+                        tolerance=tolerance,
+                        metadata={
+                            "task_type": task.task_type,
+                            "element_type": element.element_type,
+                            "element_label": element.label,
+                            "icon_id": icon.icon_id,
+                            "icon_label": label,
+                            "icon_bounds": icon.bounds,
+                            "image_size": image.size,
+                        },
+                        pixel_coords=(click_x, click_y),
+                    )
+                )
+
+        return tests
+
+    def generate_test(self, ctx: TaskContext) -> TestCase:
+        """Generate test - returns first."""
+        tests = self.generate_tests(ctx)
+        if not tests:
+            raise ValueError("No tests generated")
+        return tests[0]