openadapt-ml 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

openadapt_ml/experiments/demo_prompt/format_demo.py

@@ -0,0 +1,226 @@
+"""Demo formatting utilities for few-shot prompting."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from openadapt_ml.schema import Action, ActionType, Episode, Step
+
+
+def format_action(action: "Action") -> str:
+    """Format an Action as a string for the prompt.
+
+    Args:
+        action: Action to format.
+
+    Returns:
+        String representation like "CLICK(0.5, 0.3)" or "TYPE('hello')".
+    """
+    # Get action type value (handle both enum and string)
+    action_type = action.type.value if hasattr(action.type, 'value') else action.type
+
+    if action_type == "click":
+        if action.normalized_coordinates is not None:
+            x, y = action.normalized_coordinates
+            return f"CLICK({x:.3f}, {y:.3f})"
+        return "CLICK()"
+
+    elif action_type == "double_click":
+        if action.normalized_coordinates is not None:
+            x, y = action.normalized_coordinates
+            return f"DOUBLE_CLICK({x:.3f}, {y:.3f})"
+        return "DOUBLE_CLICK()"
+
+    elif action_type == "type":
+        text = action.text or ""
+        # Escape quotes and truncate if very long
+        text = text.replace('"', '\\"')
+        if len(text) > 50:
+            text = text[:47] + "..."
+        return f'TYPE("{text}")'
+
+    elif action_type == "key":
+        key = action.key or "unknown"
+        if action.modifiers:
+            mods = "+".join(action.modifiers)
+            return f"KEY({mods}+{key})"
+        return f"KEY({key})"
+
+    elif action_type == "scroll":
+        direction = action.scroll_direction or "down"
+        return f"SCROLL({direction})"
+
+    elif action_type == "drag":
+        if action.normalized_coordinates is not None and action.normalized_end is not None:
+            x, y = action.normalized_coordinates
+            end_x, end_y = action.normalized_end
+            return f"DRAG({x:.3f}, {y:.3f}, {end_x:.3f}, {end_y:.3f})"
+        return "DRAG()"
+
+    else:
+        return f"{action_type.upper()}()"
+
+
+def format_step(step: "Step", step_num: int) -> str:
+    """Format a single step for the demo.
+
+    Args:
+        step: Step to format.
+        step_num: Step number (1-indexed).
+
+    Returns:
+        Formatted step string.
+    """
+    lines = [f"Step {step_num}:"]
+
+    # Add window context if available
+    if step.observation and step.observation.window_title:
+        lines.append(f"  Window: {step.observation.window_title}")
+
+    # Add action
+    if step.action:
+        action_str = format_action(step.action)
+        lines.append(f"  Action: {action_str}")
+
+    return "\n".join(lines)
+
+
+def format_episode_as_demo(
+    episode: "Episode",
+    max_steps: int = 10,
+    include_screenshots: bool = False,
+) -> str:
+    """Convert an Episode to a few-shot demo format.
+
+    Args:
+        episode: Episode containing the demonstration.
+        max_steps: Maximum number of steps to include.
+        include_screenshots: Whether to include screenshot paths (for multi-image).
+
+    Returns:
+        Formatted demo string for prompt injection.
+    """
+    lines = [
+        "DEMONSTRATION:",
+        f"Task: {episode.instruction}",
+        "",
+    ]
+
+    for i, step in enumerate(episode.steps[:max_steps], 1):
+        lines.append(format_step(step, i))
+
+        # Optionally include screenshot reference
+        if include_screenshots and step.observation and step.observation.screenshot_path:
+            lines.append(f"  [Screenshot: {step.observation.screenshot_path}]")
+
+        lines.append("")
+
+    lines.append("---")
+    return "\n".join(lines)
+
+
+def format_episode_verbose(
+    episode: "Episode",
+    max_steps: int = 10,
+) -> str:
+    """Format episode with more context per step.
+
+    Includes:
+    - Screen summary
+    - User intent (inferred)
+    - Action taken
+    - Observed result
+
+    Args:
+        episode: Episode to format.
+        max_steps: Maximum steps to include.
+
+    Returns:
+        Verbose demo string.
+    """
+    lines = [
+        "DEMONSTRATION:",
+        f"Goal: {episode.instruction}",
+        "",
+        "The following shows the step-by-step procedure:",
+        "",
+    ]
+
+    for i, step in enumerate(episode.steps[:max_steps], 1):
+        lines.append(f"Step {i}:")
+
+        # Screen summary
+        if step.observation:
+            if step.observation.window_title:
+                lines.append(f"  [Screen: {step.observation.window_title}]")
+
+        # Action taken
+        if step.action:
+            action_str = format_action(step.action)
+            lines.append(f"  [Action: {action_str}]")
+
+        # Observed result (inferred from next step's observation)
+        if i < len(episode.steps):
+            next_step = episode.steps[i]
+            if next_step.observation and next_step.observation.window_title:
+                if (
+                    not step.observation
+                    or next_step.observation.window_title != step.observation.window_title
+                ):
+                    lines.append(f"  [Result: Window changed to {next_step.observation.window_title}]")
+
+        lines.append("")
+
+    lines.append("---")
+    return "\n".join(lines)
+
+
+def get_demo_screenshot_paths(
+    episode: "Episode",
+    max_steps: int = 10,
+) -> list[str]:
+    """Get screenshot paths from episode for multi-image prompting.
+
+    Args:
+        episode: Episode to extract screenshots from.
+        max_steps: Maximum steps to include.
+
+    Returns:
+        List of screenshot paths.
+    """
+    paths = []
+    for step in episode.steps[:max_steps]:
+        if step.observation and step.observation.screenshot_path:
+            path = step.observation.screenshot_path
+            if Path(path).exists():
+                paths.append(path)
+    return paths
+
+
+def generate_length_matched_control(demo: str) -> str:
+    """Generate a control prompt with the same token count but no trajectory info.
+
+    Used to control for prompt length effects.
+
+    Args:
+        demo: The demo string to match length of.
+
+    Returns:
+        Control string of similar length with irrelevant content.
+    """
+    # Use generic placeholder text
+    placeholder = (
+        "This is placeholder text that serves as a control condition. "
+        "It contains no relevant information about the task or demonstration. "
+        "The purpose is to match the token count of the demonstration prompt. "
+    )
+
+    # Repeat to match approximate length
+    target_len = len(demo)
+    control = ""
+    while len(control) < target_len:
+        control += placeholder
+
+    return control[:target_len]

openadapt_ml/experiments/demo_prompt/results/experiment_20251231_002125.json

@@ -0,0 +1,83 @@
+{
+  "timestamp": "20251231_002125",
+  "provider": "anthropic",
+  "screenshot": "/Users/abrichr/oa/src/openadapt-capture/turn-off-nightshift/screenshots/capture_31807990_step_0.png",
+  "demo": "DEMONSTRATION:\nGoal: Turn off Night Shift in macOS System Settings\n\nThe following shows the step-by-step procedure:\n\nStep 1:\n  [Screen: Desktop with Terminal window visible]\n  [Action: CLICK(0.01, 0.01) - Click Apple menu icon in top-left]\n  [Result: Apple menu dropdown opened]\n\nStep 2:\n  [Screen: Apple menu visible with options]\n  [Action: CLICK on \"System Settings...\" menu item]\n  [Result: System Settings application opened]\n\nStep 3:\n  [Screen: System Settings window with sidebar]\n  [Action: CLICK on \"Displays\" in the sidebar]\n  [Result: Displays panel shown in main area]\n\nStep 4:\n  [Screen: Displays panel showing display settings]\n  [Action: CLICK on \"Night Shift...\" button]\n  [Result: Night Shift popup/sheet appeared]\n\nStep 5:\n  [Screen: Night Shift popup with Schedule dropdown]\n  [Action: CLICK on Schedule dropdown, select \"Off\"]\n  [Result: Night Shift schedule set to Off, Night Shift disabled]\n\n---",
+  "test_cases": [
+    {
+      "test_case": {
+        "name": "near_toggle",
+        "task": "Turn ON Night Shift in macOS System Settings",
+        "similarity": "near",
+        "notes": "Same procedure, just toggle to opposite state"
+      },
+      "results": {
+        "zero_shot": {
+          "action": "CLICK(20, 8)",
+          "response": "I need to turn ON Night Shift in macOS. Based on the terminal window showing an attempt to control Night Shift, I should access System Settings to properly enable it. Let me click on the Apple menu.\n\nACTION: CLICK(20, 8)",
+          "error": null
+        },
+        "with_demo": {
+          "action": "CLICK(20, 8)",
+          "response": "I need to turn ON Night Shift in macOS System Settings. Let me start by clicking on the Apple menu icon to access System Settings.\n\nACTION: CLICK(20, 8)",
+          "error": null
+        },
+        "control": {
+          "action": "CLICK(1243, 8)",
+          "response": "I need to analyze the screenshot to understand how to turn on Night Shift in macOS System Settings.\n\nThe screenshot shows a Terminal window with some command output. To access Night Shift settings in macOS, I need to open System Settings. I can see various system icons in the menu bar at the top of the screen.\n\nACTION: CLICK(1243, 8)",
+          "error": null
+        }
+      }
+    },
+    {
+      "test_case": {
+        "name": "medium_same_panel",
+        "task": "Adjust Night Shift color temperature to warmer setting",
+        "similarity": "medium",
+        "notes": "Same navigation path, different final action"
+      },
+      "results": {
+        "zero_shot": {
+          "action": "CLICK(1218, 8)",
+          "response": "The task requires adjusting the Night Shift color temperature to a warmer setting. Looking at the screenshot, I can see a macOS Terminal window. The system menu bar at the top right contains various control icons. I need to find the Night Shift or display settings controls.\n\nACTION: CLICK(1218, 8)",
+          "error": null
+        },
+        "with_demo": {
+          "action": "CLICK(19, 8)",
+          "response": "I need to adjust the Night Shift color temperature to make it warmer. First, I should access the System Settings to find the Night Shift controls.\n\nACTION: CLICK(19, 8)",
+          "error": null
+        },
+        "control": {
+          "action": "CLICK(1114, 8)",
+          "response": "I need to analyze the screenshot and the goal to determine the next action.\n\nThe screenshot shows a Terminal window on macOS. The goal is to \"Adjust Night Shift color temperature to warmer setting.\"\n\nLooking at the top menu bar, I can see various system icons. To adjust Night Shift settings, I would need to access System Preferences or use the Control Center. I can see what appears to be system control icons in the top-right corner of the screen.\n\nThe most direct way to adjust Night Shift would be to click on the Control Center icon or the display settings in the menu bar.\n\nACTION: CLICK(1114, 8)",
+          "error": null
+        }
+      }
+    },
+    {
+      "test_case": {
+        "name": "far_different_setting",
+        "task": "Turn on True Tone display in macOS System Settings",
+        "similarity": "far",
+        "notes": "Same app but different panel (True Tone is in Displays)"
+      },
+      "results": {
+        "zero_shot": {
+          "action": "CLICK(1217, 8)",
+          "response": "I need to access System Settings to turn on True Tone display. Let me analyze the screen to find how to access it.\n\nACTION: CLICK(1217, 8)",
+          "error": null
+        },
+        "with_demo": {
+          "action": "CLICK(20, 8)",
+          "response": "I need to turn on True Tone display in macOS System Settings. Let me start by clicking on the Apple menu in the top-left corner.\n\nACTION: CLICK(20, 8)",
+          "error": null
+        },
+        "control": {
+          "action": "CLICK(20, 8)",
+          "response": "I need to open System Settings to enable True Tone display. I'll click on the Apple menu in the top-left corner to access System Settings.\n\nACTION: CLICK(20, 8)",
+          "error": null
+        }
+      }
+    }
+  ]
+}