cudag 0.3.10__py3-none-any.whl

cudag/prompts/tools.py

@@ -0,0 +1,382 @@
+# Copyright (c) 2025 Tylt LLC. All rights reserved.
+# CONFIDENTIAL AND PROPRIETARY. Unauthorized use, copying, or distribution
+# is strictly prohibited. For licensing inquiries: hello@claimhawk.app
+
+"""Computer use tool definition and tool_call formatting.
+
+This module codifies the canonical tool_call format used in VLM training datasets.
+All tool calls must use this format for consistency across generators.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+# Valid actions for computer_use tool
+TOOL_ACTIONS = Literal[
+    "key",
+    "type",
+    "mouse_move",
+    "left_click",
+    "left_click_drag",
+    "right_click",
+    "middle_click",
+    "double_click",
+    "triple_click",
+    "scroll",
+    "hscroll",
+    "wait",
+    "terminate",
+    "answer",
+]
+
+# Action descriptions for system prompt
+ACTION_DESCRIPTIONS = {
+    "key": "Press keys in order, release in reverse.",
+    "type": "Type a string of text.",
+    "mouse_move": "Move the cursor to (x, y).",
+    "left_click": "Left click at (x, y).",
+    "left_click_drag": "Click and drag from current to (x, y).",
+    "right_click": "Right click at (x, y).",
+    "middle_click": "Middle click at (x, y).",
+    "double_click": "Double-click at (x, y).",
+    "triple_click": "Triple-click at (x, y) (simulated as double-click).",
+    "scroll": "Scroll the mouse wheel.",
+    "hscroll": "Horizontal scroll.",
+    "wait": "Wait N seconds.",
+    "terminate": "End the task with a status.",
+    "answer": "Answer a question.",
+}
+
+# Actions that require coordinate parameter
+COORDINATE_ACTIONS = {
+    "mouse_move",
+    "left_click",
+    "left_click_drag",
+    "right_click",
+    "middle_click",
+    "double_click",
+    "triple_click",
+    "scroll",
+    "hscroll",
+}
+
+# Actions that require specific parameters
+ACTION_REQUIRED_PARAMS: dict[str, list[str]] = {
+    "key": ["keys"],
+    "type": ["text"],
+    "scroll": ["coordinate", "pixels"],
+    "hscroll": ["coordinate", "pixels"],
+    "wait": ["time"],
+    "terminate": ["status"],
+}
+
+# Canonical computer_use tool definition (JSON schema)
+COMPUTER_USE_TOOL: dict[str, Any] = {
+    "type": "function",
+    "function": {
+        "name_for_human": "computer_use",
+        "name": "computer_use",
+        "description": "Perform computer actions",
+        "parameters": {
+            "properties": {
+                "action": {
+                    "description": "\n".join(
+                        f"* `{action}`: {desc}" for action, desc in ACTION_DESCRIPTIONS.items()
+                    ),
+                    "enum": list(ACTION_DESCRIPTIONS.keys()),
+                    "type": "string",
+                },
+                "keys": {
+                    "description": "Required only by `action=key`.",
+                    "type": "array",
+                },
+                "text": {
+                    "description": "Required only by `action=type`.",
+                    "type": "string",
+                },
+                "coordinate": {
+                    "description": "Mouse coordinates (1000x1000 normalized).",
+                    "type": "array",
+                },
+                "pixels": {
+                    "description": "The amount of scrolling.",
+                    "type": "number",
+                },
+                "time": {
+                    "description": "The seconds to wait.",
+                    "type": "number",
+                },
+                "status": {
+                    "description": "The status of the task.",
+                    "type": "string",
+                    "enum": ["success", "failure"],
+                },
+            },
+            "required": ["action"],
+            "type": "object",
+        },
+        "args_format": "Format the arguments as a JSON object.",
+    },
+}
+
+
+@dataclass
+class BboxCall:
+    """Represents a get_bbox tool call for element grounding.
+
+    This is the canonical format for bounding box detection in VLMGen datasets.
+    Used for "grounding" task types that identify element locations.
+    """
+
+    bbox_2d: tuple[int, int, int, int]
+    """Bounding box coordinates [x1, y1, x2, y2] in RU (0-1000)."""
+
+    label: str | None = None
+    """Optional human-readable label of the element being located."""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        args: dict[str, Any] = {"bbox_2d": list(self.bbox_2d)}
+        if self.label:
+            args["label"] = self.label
+        return {
+            "name": "get_bbox",
+            "arguments": args,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> BboxCall:
+        """Create from dictionary."""
+        if data.get("name") != "get_bbox":
+            raise ValueError(f"Expected get_bbox tool, got: {data.get('name')}")
+
+        args = data.get("arguments", {})
+        bbox = args.get("bbox_2d", [0, 0, 0, 0])
+
+        return cls(
+            bbox_2d=tuple(bbox),  # type: ignore[arg-type]
+            label=args.get("label"),
+        )
+
+    @classmethod
+    def create(
+        cls, bbox_2d: tuple[int, int, int, int], label: str | None = None
+    ) -> BboxCall:
+        """Create a get_bbox tool call.
+
+        Args:
+            bbox_2d: Bounding box [x1, y1, x2, y2] in RU units (0-1000)
+            label: Optional human-readable label of the element (e.g., "Appts")
+
+        Returns:
+            BboxCall instance
+        """
+        return cls(bbox_2d=bbox_2d, label=label)
+
+
+@dataclass
+class ToolCall:
+    """Represents a computer_use tool call.
+
+    This is the canonical format for all tool calls in VLMGen datasets.
+    """
+
+    action: str
+    coordinate: tuple[int, int] | None = None
+    pixels: int | None = None
+    keys: list[str] | None = None
+    text: str | None = None
+    time: float | None = None
+    status: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        args: dict[str, Any] = {"action": self.action}
+
+        if self.coordinate is not None:
+            args["coordinate"] = list(self.coordinate)
+        if self.pixels is not None:
+            args["pixels"] = self.pixels
+        if self.keys is not None:
+            args["keys"] = self.keys
+        if self.text is not None:
+            args["text"] = self.text
+        if self.time is not None:
+            args["time"] = self.time
+        if self.status is not None:
+            args["status"] = self.status
+
+        # Include any extra fields
+        args.update(self.extra)
+
+        return {"name": "computer_use", "arguments": args}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ToolCall:
+        """Create from dictionary."""
+        if data.get("name") != "computer_use":
+            raise ValueError(f"Expected computer_use tool, got: {data.get('name')}")
+
+        args = data.get("arguments", {})
+        coord = args.get("coordinate")
+
+        # Extract known fields
+        known_fields = {"action", "coordinate", "pixels", "keys", "text", "time", "status"}
+        extra = {k: v for k, v in args.items() if k not in known_fields}
+
+        return cls(
+            action=args["action"],
+            coordinate=tuple(coord) if coord else None,
+            pixels=args.get("pixels"),
+            keys=args.get("keys"),
+            text=args.get("text"),
+            time=args.get("time"),
+            status=args.get("status"),
+            extra=extra,
+        )
+
+    @classmethod
+    def left_click(cls, coordinate: tuple[int, int]) -> ToolCall:
+        """Create a left_click tool call."""
+        return cls(action="left_click", coordinate=coordinate)
+
+    @classmethod
+    def double_click(cls, coordinate: tuple[int, int]) -> ToolCall:
+        """Create a double_click tool call."""
+        return cls(action="double_click", coordinate=coordinate)
+
+    @classmethod
+    def right_click(cls, coordinate: tuple[int, int]) -> ToolCall:
+        """Create a right_click tool call."""
+        return cls(action="right_click", coordinate=coordinate)
+
+    @classmethod
+    def scroll(cls, coordinate: tuple[int, int], pixels: int) -> ToolCall:
+        """Create a scroll tool call. Negative pixels = scroll up."""
+        return cls(action="scroll", coordinate=coordinate, pixels=pixels)
+
+    @classmethod
+    def key_press(cls, keys: list[str]) -> ToolCall:
+        """Create a key press tool call."""
+        return cls(action="key", keys=keys)
+
+    @classmethod
+    def type_text(cls, text: str) -> ToolCall:
+        """Create a type tool call."""
+        return cls(action="type", text=text)
+
+    @classmethod
+    def wait(cls, seconds: float) -> ToolCall:
+        """Create a wait tool call."""
+        return cls(action="wait", time=seconds)
+
+    @classmethod
+    def terminate(cls, status: str = "success") -> ToolCall:
+        """Create a terminate tool call."""
+        return cls(action="terminate", status=status)
+
+
+def format_tool_call(tool_call: ToolCall | BboxCall | dict[str, Any]) -> str:
+    """Format a tool call as XML-wrapped JSON string.
+
+    This is the canonical output format for GPT responses in training data.
+
+    Args:
+        tool_call: ToolCall, BboxCall instance, or dict with {name, arguments}
+
+    Returns:
+        Formatted string like:
+        <tool_call>
+        {"name": "computer_use", "arguments": {...}}
+        </tool_call>
+
+        or for bounding box:
+        <tool_call>
+        {"name": "get_bbox", "arguments": {"label": "...", "bbox_2d": [...]}}
+        </tool_call>
+    """
+    if isinstance(tool_call, (ToolCall, BboxCall)):
+        data = tool_call.to_dict()
+    else:
+        data = tool_call
+
+    json_str = json.dumps(data)
+    return f"<tool_call>\n{json_str}\n</tool_call>"
+
+
+# Regex pattern for parsing tool calls
+TOOL_CALL_PATTERN = re.compile(
+    r"<tool_call>\s*(?P<json>\{.*?\})\s*</tool_call>",
+    re.DOTALL | re.IGNORECASE,
+)
+
+
+def parse_tool_call(text: str) -> ToolCall | None:
+    """Parse a tool call from model output text.
+
+    Args:
+        text: Model output containing <tool_call>...</tool_call>
+
+    Returns:
+        Parsed ToolCall or None if not found
+    """
+    match = TOOL_CALL_PATTERN.search(text)
+    if not match:
+        return None
+
+    try:
+        data = json.loads(match.group("json"))
+        return ToolCall.from_dict(data)
+    except (json.JSONDecodeError, ValueError, KeyError):
+        return None
+
+
+def validate_tool_call(tool_call: ToolCall) -> list[str]:
+    """Validate a tool call and return list of errors.
+
+    Args:
+        tool_call: ToolCall to validate
+
+    Returns:
+        List of error messages (empty if valid)
+    """
+    errors: list[str] = []
+
+    # Check action is valid
+    if tool_call.action not in ACTION_DESCRIPTIONS:
+        errors.append(f"Invalid action: {tool_call.action}")
+        return errors  # Can't validate further without valid action
+
+    # Check coordinate is provided for coordinate-requiring actions
+    if tool_call.action in COORDINATE_ACTIONS and tool_call.coordinate is None:
+        errors.append(f"Action '{tool_call.action}' requires coordinate")
+
+    # Check coordinate values are in valid range
+    if tool_call.coordinate is not None:
+        x, y = tool_call.coordinate
+        if not (0 <= x <= 1000 and 0 <= y <= 1000):
+            errors.append(f"Coordinate out of range [0, 1000]: ({x}, {y})")
+
+    # Check required parameters
+    required = ACTION_REQUIRED_PARAMS.get(tool_call.action, [])
+    for param in required:
+        value = getattr(tool_call, param, None)
+        if value is None:
+            errors.append(f"Action '{tool_call.action}' requires '{param}'")
+
+    # Validate scroll pixels
+    if tool_call.action in ("scroll", "hscroll") and tool_call.pixels is not None:
+        if not isinstance(tool_call.pixels, (int, float)):
+            errors.append(f"Invalid pixels value: {tool_call.pixels}")
+
+    # Validate terminate status
+    if tool_call.action == "terminate" and tool_call.status is not None:
+        if tool_call.status not in ("success", "failure"):
+            errors.append(f"Invalid terminate status: {tool_call.status}")
+
+    return errors

cudag/schemas/filesystem.json

@@ -0,0 +1,90 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://claimhawk.com/schemas/cudag/filesystem.json",
+  "title": "CUDAG Dataset Filesystem Structure",
+  "description": "Defines the required filesystem structure for CUDAG datasets",
+  "filesystem": {
+    "root": {
+      "description": "Dataset root directory (e.g., datasets/calendar--mike--20251203_123456)",
+      "required_files": [
+        {
+          "name": "config.json",
+          "description": "Generation configuration and metadata"
+        },
+        {
+          "name": "data.jsonl",
+          "description": "All training samples (JSONL format)"
+        },
+        {
+          "name": "train.jsonl",
+          "description": "Training split samples (JSONL format)"
+        },
+        {
+          "name": "val.jsonl",
+          "description": "Validation split samples (JSONL format)"
+        }
+      ],
+      "optional_files": [
+        {
+          "name": "held_out.jsonl",
+          "description": "Held-out samples for evaluation (JSONL format)"
+        }
+      ],
+      "required_directories": [
+        {
+          "name": "images",
+          "description": "Training images directory",
+          "contents": "*.jpg or *.png files referenced by train.jsonl/val.jsonl"
+        },
+        {
+          "name": "test",
+          "description": "Test cases directory"
+        }
+      ]
+    },
+    "test": {
+      "description": "Test directory structure (dataset_root/test/)",
+      "required_files": [
+        {
+          "name": "test.json",
+          "description": "Test case definitions (JSON array)"
+        }
+      ],
+      "required_directories": [
+        {
+          "name": "images",
+          "description": "Test screenshot images",
+          "contents": "*.png files referenced by test.json"
+        }
+      ],
+      "optional_directories": [
+        {
+          "name": "annotated",
+          "description": "Annotated test images with crosshairs and tool call output",
+          "contents": "*_annotated.png files"
+        }
+      ]
+    }
+  },
+  "naming_conventions": {
+    "dataset_directory": {
+      "pattern": "{prefix}--{user}--{timestamp}",
+      "example": "calendar--mike--20251203_123456"
+    },
+    "training_images": {
+      "pattern": "{dataset_name}_{index:05d}.{ext}",
+      "example": "calendar_00001.jpg"
+    },
+    "test_images": {
+      "pattern": "{prefix}_{index:05d}.png",
+      "example": "test_00001.png"
+    }
+  },
+  "constraints": [
+    "All image paths in JSONL/JSON files must be relative to their parent directory",
+    "Training images are relative to dataset root (e.g., 'images/sample.jpg')",
+    "Test screenshots are relative to test directory (e.g., 'images/test.png')",
+    "Coordinates in training data use RU (Resolution Units) normalized to [0, 1000]",
+    "Test tolerance values are in RU units"
+  ]
+}

cudag/schemas/test_record.schema.json

@@ -0,0 +1,113 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://claimhawk.com/schemas/cudag/test_record.schema.json",
+  "title": "CUDAG Test Record",
+  "description": "Schema for test case records in test/test.json",
+  "type": "object",
+  "required": ["test_id", "screenshot", "prompt", "expected_action", "tolerance", "metadata"],
+  "properties": {
+    "test_id": {
+      "type": "string",
+      "description": "Unique identifier for this test case",
+      "pattern": "^[a-zA-Z0-9_-]+$"
+    },
+    "screenshot": {
+      "type": "string",
+      "description": "Relative path to screenshot from test directory (e.g., 'images/test_00001.png')",
+      "pattern": "^images/[^/]+\\.(jpg|jpeg|png)$"
+    },
+    "prompt": {
+      "type": "string",
+      "description": "Human instruction prompt",
+      "minLength": 1
+    },
+    "expected_action": {
+      "type": "object",
+      "required": ["name", "arguments"],
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Tool name",
+          "const": "computer_use"
+        },
+        "arguments": {
+          "type": "object",
+          "required": ["action"],
+          "properties": {
+            "action": {
+              "type": "string",
+              "description": "Action type",
+              "enum": ["left_click", "right_click", "double_click", "scroll", "type", "key"]
+            },
+            "coordinate": {
+              "type": "array",
+              "description": "Click coordinates [x, y] in RU units (0-1000)",
+              "items": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 1000
+              },
+              "minItems": 2,
+              "maxItems": 2
+            },
+            "text": {
+              "type": "string",
+              "description": "Text to type (for type action)"
+            },
+            "direction": {
+              "type": "string",
+              "description": "Scroll direction",
+              "enum": ["up", "down", "left", "right"]
+            },
+            "amount": {
+              "type": "integer",
+              "description": "Scroll amount in pixels"
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "tolerance": {
+      "type": "array",
+      "description": "Allowed coordinate tolerance [tol_x, tol_y] in RU units",
+      "items": {
+        "type": "integer",
+        "minimum": 0
+      },
+      "minItems": 2,
+      "maxItems": 2
+    },
+    "metadata": {
+      "type": "object",
+      "required": ["task_type"],
+      "properties": {
+        "task_type": {
+          "type": "string",
+          "description": "Task type identifier"
+        },
+        "real_coords": {
+          "type": ["array", "null"],
+          "description": "Original pixel coordinates [x, y]",
+          "items": {
+            "type": "integer"
+          },
+          "minItems": 2,
+          "maxItems": 2
+        },
+        "image_size": {
+          "type": "array",
+          "description": "Image dimensions [width, height]",
+          "items": {
+            "type": "integer"
+          },
+          "minItems": 2,
+          "maxItems": 2
+        }
+      },
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": false
+}

cudag/schemas/train_record.schema.json

@@ -0,0 +1,90 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://claimhawk.com/schemas/cudag/train_record.schema.json",
+  "title": "CUDAG Training Record",
+  "description": "Schema for training/validation records in data.jsonl, train.jsonl, val.jsonl",
+  "type": "object",
+  "required": ["id", "image", "conversations", "metadata"],
+  "properties": {
+    "id": {
+      "type": "string",
+      "description": "Unique identifier for this sample",
+      "pattern": "^[a-zA-Z0-9_-]+$"
+    },
+    "image": {
+      "type": "string",
+      "description": "Relative path to image file from dataset root (e.g., 'images/sample_00001.jpg')",
+      "pattern": "^images/[^/]+\\.(jpg|jpeg|png)$"
+    },
+    "conversations": {
+      "type": "array",
+      "description": "Human-GPT conversation turns",
+      "minItems": 2,
+      "maxItems": 2,
+      "items": [
+        {
+          "type": "object",
+          "required": ["from", "value"],
+          "properties": {
+            "from": {
+              "type": "string",
+              "const": "human"
+            },
+            "value": {
+              "type": "string",
+              "description": "Human prompt, must start with <image>\\n",
+              "pattern": "^<image>\\n.+"
+            }
+          },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "required": ["from", "value"],
+          "properties": {
+            "from": {
+              "type": "string",
+              "const": "gpt"
+            },
+            "value": {
+              "type": "string",
+              "description": "GPT response in <tool_call> format",
+              "pattern": "^<tool_call>"
+            }
+          },
+          "additionalProperties": false
+        }
+      ]
+    },
+    "metadata": {
+      "type": "object",
+      "required": ["task_type", "real_coords"],
+      "properties": {
+        "task_type": {
+          "type": "string",
+          "description": "Task type identifier (e.g., 'click-day', 'click-appointment')"
+        },
+        "real_coords": {
+          "type": "array",
+          "description": "Original pixel coordinates [x, y]",
+          "items": {
+            "type": "integer"
+          },
+          "minItems": 2,
+          "maxItems": 2
+        },
+        "tolerance": {
+          "type": "array",
+          "description": "Coordinate tolerance [tol_x, tol_y] in RU units",
+          "items": {
+            "type": "integer"
+          },
+          "minItems": 2,
+          "maxItems": 2
+        }
+      },
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": false
+}