openadapt-ml 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

openadapt_ml/experiments/waa_demo/tasks.py

@@ -0,0 +1,151 @@
+"""Task definitions for WAA demo experiment.
+
+10 carefully selected tasks across 4 enterprise-relevant domains.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+
+class Difficulty(Enum):
+    EASY = "easy"
+    MEDIUM = "medium"
+    HARD = "hard"
+
+
+class Domain(Enum):
+    BROWSER = "msedge"
+    OFFICE_CALC = "libreoffice_calc"
+    OFFICE_WRITER = "libreoffice_writer"
+    SETTINGS = "settings"
+    FILE_EXPLORER = "file_explorer"
+
+
+@dataclass
+class WATask:
+    """A Windows Agent Arena task definition."""
+
+    task_id: str
+    instruction: str
+    domain: Domain
+    difficulty: Difficulty
+    first_action_hint: str
+    demo_method: str  # "manual" or "recorded"
+    json_path: str  # Path in WAA repo
+
+
+TASKS: dict[str, WATask] = {
+    "1": WATask(
+        task_id="004587f8-6028-4656-94c1-681481abbc9c-wos",
+        instruction="Enable the 'Do Not Track' feature in Edge",
+        domain=Domain.BROWSER,
+        difficulty=Difficulty.MEDIUM,
+        first_action_hint="Click three-dot menu in Edge",
+        demo_method="manual",
+        json_path="examples/msedge/004587f8-6028-4656-94c1-681481abbc9c-wos.json",
+    ),
+    "2": WATask(
+        task_id="049d3788-c979-4ea6-934d-3a35c4630faf-WOS",
+        instruction="Save this webpage to bookmarks bar",
+        domain=Domain.BROWSER,
+        difficulty=Difficulty.EASY,
+        first_action_hint="Click star/bookmark icon or Ctrl+D",
+        demo_method="manual",
+        json_path="examples/msedge/049d3788-c979-4ea6-934d-3a35c4630faf-WOS.json",
+    ),
+    "3": WATask(
+        task_id="2acd62b4-a2ab-44a7-a7e3-f5227bbd8324-wos",
+        instruction="Set default font size to largest for grandmother",
+        domain=Domain.BROWSER,
+        difficulty=Difficulty.MEDIUM,
+        first_action_hint="Open Settings > Appearance",
+        demo_method="manual",
+        json_path="examples/msedge/2acd62b4-a2ab-44a7-a7e3-f5227bbd8324-wos.json",
+    ),
+    "4": WATask(
+        task_id="01b269ae-2111-4a07-81fd-3fcd711993b0-WOS",
+        instruction="Fill all blank cells with value from cell above",
+        domain=Domain.OFFICE_CALC,
+        difficulty=Difficulty.HARD,
+        first_action_hint="Select cells, use Go To Special > Blanks",
+        demo_method="recorded",
+        json_path="examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-WOS.json",
+    ),
+    "5": WATask(
+        task_id="0a2e43bf-b26c-4631-a966-af9dfa12c9e5-WOS",
+        instruction="Calculate monthly totals and create line chart",
+        domain=Domain.OFFICE_CALC,
+        difficulty=Difficulty.HARD,
+        first_action_hint="Click cell for SUM formula",
+        demo_method="recorded",
+        json_path="examples/libreoffice_calc/0a2e43bf-b26c-4631-a966-af9dfa12c9e5-WOS.json",
+    ),
+    "6": WATask(
+        task_id="3ef2b351-8a84-4ff2-8724-d86eae9b842e-WOS",
+        instruction="Center align the heading in LibreOffice Writer",
+        domain=Domain.OFFICE_WRITER,
+        difficulty=Difficulty.EASY,
+        first_action_hint="Select text, click center align button",
+        demo_method="manual",
+        json_path="examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e-WOS.json",
+    ),
+    "7": WATask(
+        task_id="37e10fc4-b4c5-4b02-a65c-bfae8bc51d3f-wos",
+        instruction="Turn off notifications for system",
+        domain=Domain.SETTINGS,
+        difficulty=Difficulty.MEDIUM,
+        first_action_hint="Open Settings > System > Notifications",
+        demo_method="manual",
+        json_path="examples/settings/37e10fc4-b4c5-4b02-a65c-bfae8bc51d3f-wos.json",
+    ),
+    "8": WATask(
+        task_id="46adf721-2949-4426-b069-010b7c128d8f-wos",
+        instruction="Enable Night Light: on at 7PM, off at 7AM",
+        domain=Domain.SETTINGS,
+        difficulty=Difficulty.MEDIUM,
+        first_action_hint="Open Settings > Display > Night Light",
+        demo_method="manual",
+        json_path="examples/settings/46adf721-2949-4426-b069-010b7c128d8f-wos.json",
+    ),
+    "9": WATask(
+        task_id="0c9dda13-428c-492b-900b-f48562111f93-WOS",
+        instruction="Create Archive folder and move all .docx files",
+        domain=Domain.FILE_EXPLORER,
+        difficulty=Difficulty.MEDIUM,
+        first_action_hint="Right-click > New Folder, then select and move files",
+        demo_method="recorded",
+        json_path="examples/file_explorer/0c9dda13-428c-492b-900b-f48562111f93-WOS.json",
+    ),
+    "10": WATask(
+        task_id="34a4fee9-e52e-4a4a-96d2-68d35091504a-WOS",
+        instruction="Change view to Details view",
+        domain=Domain.FILE_EXPLORER,
+        difficulty=Difficulty.EASY,
+        first_action_hint="Click View menu or dropdown",
+        demo_method="manual",
+        json_path="examples/file_explorer/34a4fee9-e52e-4a4a-96d2-68d35091504a-WOS.json",
+    ),
+}
+
+
+def get_task(task_num: str | int) -> Optional[WATask]:
+    """Get a task by its number (1-10)."""
+    return TASKS.get(str(task_num))
+
+
+def get_tasks_by_method(method: str) -> list[WATask]:
+    """Get all tasks that use a specific demo method."""
+    return [t for t in TASKS.values() if t.demo_method == method]
+
+
+def get_manual_tasks() -> list[WATask]:
+    """Get tasks requiring manual demo writing."""
+    return get_tasks_by_method("manual")
+
+
+def get_recorded_tasks() -> list[WATask]:
+    """Get tasks requiring recorded demos."""
+    return get_tasks_by_method("recorded")

openadapt_ml/export/__init__.py

@@ -0,0 +1,9 @@
+"""Export utilities for Episode data.
+
+This module provides tools to export Episode trajectories to various formats
+for analytics, training, and sharing.
+"""
+
+from openadapt_ml.export.parquet import to_parquet, from_parquet
+
+__all__ = ["to_parquet", "from_parquet"]

openadapt_ml/export/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running export module as python -m openadapt_ml.export."""
+
+from openadapt_ml.export.cli import main
+
+if __name__ == "__main__":
+    main()

openadapt_ml/export/cli.py

@@ -0,0 +1,89 @@
+"""CLI for export utilities."""
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    """Main entry point for export CLI."""
+    parser = argparse.ArgumentParser(
+        description="Export Episode data to various formats",
+        prog="python -m openadapt_ml.export",
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Export format")
+
+    # Parquet subcommand
+    parquet_parser = subparsers.add_parser(
+        "parquet",
+        help="Export to Parquet format for analytics",
+    )
+    parquet_parser.add_argument(
+        "--input",
+        "-i",
+        required=True,
+        help="Directory containing Episode JSON files",
+    )
+    parquet_parser.add_argument(
+        "--output",
+        "-o",
+        required=True,
+        help="Output path for .parquet file",
+    )
+    parquet_parser.add_argument(
+        "--include-summary",
+        action="store_true",
+        help="Also generate episode-level summary table",
+    )
+
+    args = parser.parse_args()
+
+    if args.command == "parquet":
+        return export_parquet(args)
+    else:
+        parser.print_help()
+        return 1
+
+
+def export_parquet(args: argparse.Namespace) -> int:
+    """Export Episodes to Parquet."""
+    try:
+        from openadapt_ml.export import to_parquet
+        from openadapt_ml.ingest import load_episodes
+    except ImportError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        print(f"Error: Input path does not exist: {input_path}", file=sys.stderr)
+        return 1
+
+    print(f"Loading episodes from: {input_path}")
+    episodes = load_episodes(str(input_path))
+    print(f"Loaded {len(episodes)} episodes")
+
+    if not episodes:
+        print("Warning: No episodes found", file=sys.stderr)
+        return 1
+
+    total_steps = sum(len(ep.steps) for ep in episodes)
+    print(f"Total steps: {total_steps}")
+
+    print(f"Exporting to: {args.output}")
+    to_parquet(
+        episodes,
+        args.output,
+        include_summary=args.include_summary,
+    )
+
+    print("Done!")
+    if args.include_summary:
+        summary_path = args.output.replace(".parquet", "_summary.parquet")
+        print(f"Summary written to: {summary_path}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

openadapt_ml/export/parquet.py

@@ -0,0 +1,277 @@
+"""Parquet export utilities for Episode trajectories.
+
+Parquet is a derived format for analytics and governance.
+Episode JSON remains the canonical representation.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from openadapt_ml.schema import Episode
+
+
+def to_parquet(
+    episodes: list[Episode],
+    output_path: str,
+    flatten_steps: bool = True,
+    include_summary: bool = False,
+) -> None:
+    """Export Episodes to Parquet for analytics.
+
+    Creates a step-level Parquet file with one row per step.
+    Episode-level fields are repeated for each step.
+
+    Args:
+        episodes: List of Episode objects to export.
+        output_path: Path to output .parquet file.
+        flatten_steps: If True, one row per step. If False, one row per episode
+            with steps as nested structure (not yet implemented).
+        include_summary: If True, also generate {output_path}_summary.parquet
+            with episode-level aggregations.
+
+    Raises:
+        ImportError: If pyarrow is not installed.
+        ValueError: If flatten_steps is False (not yet implemented).
+
+    Example:
+        >>> from openadapt_ml.ingest import load_episodes
+        >>> from openadapt_ml.export import to_parquet
+        >>> episodes = load_episodes("workflow_exports/")
+        >>> to_parquet(episodes, "episodes.parquet")
+    """
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError:
+        raise ImportError(
+            "Parquet export requires pyarrow. "
+            "Install with: pip install openadapt-ml[parquet]"
+        )
+
+    if not flatten_steps:
+        raise ValueError(
+            "flatten_steps=False is not yet implemented. "
+            "Use flatten_steps=True for step-level rows."
+        )
+
+    rows = []
+    for episode in episodes:
+        episode_metadata = None
+        if hasattr(episode, "metadata") and episode.metadata:
+            episode_metadata = json.dumps(episode.metadata)
+
+        for step in episode.steps:
+            # Extract normalized coordinates if available
+            x, y = None, None
+            if step.action and step.action.normalized_coordinates:
+                x, y = step.action.normalized_coordinates
+
+            # Extract action type value (enum -> string)
+            action_type = None
+            if step.action:
+                action_type = (
+                    step.action.type.value
+                    if hasattr(step.action.type, "value")
+                    else step.action.type
+                )
+
+            row = {
+                "episode_id": episode.episode_id,
+                "instruction": episode.instruction,
+                "task_id": getattr(episode, "task_id", None),
+                "step_index": step.step_index,
+                "timestamp": step.timestamp,
+                "action_type": action_type,
+                "x": x,
+                "y": y,
+                "end_x": step.action.normalized_end[0]
+                if step.action and step.action.normalized_end
+                else None,
+                "end_y": step.action.normalized_end[1]
+                if step.action and step.action.normalized_end
+                else None,
+                "text": getattr(step.action, "text", None) if step.action else None,
+                "key": getattr(step.action, "key", None) if step.action else None,
+                "scroll_direction": (
+                    getattr(step.action, "scroll_direction", None)
+                    if step.action
+                    else None
+                ),
+                "screenshot_path": (
+                    step.observation.screenshot_path if step.observation else None
+                ),
+                "window_title": (
+                    getattr(step.observation, "window_title", None)
+                    if step.observation
+                    else None
+                ),
+                "app_name": (
+                    None  # Not in new schema at Observation level
+                ),
+                "url": (
+                    None  # Not in new schema at Observation level
+                ),
+                "reasoning": getattr(step, "reasoning", None),
+                "episode_metadata": episode_metadata,
+            }
+            rows.append(row)
+
+    table = pa.Table.from_pylist(rows)
+    pq.write_table(table, output_path)
+
+    if include_summary:
+        _write_summary(episodes, output_path)
+
+
+def _write_summary(episodes: list[Episode], output_path: str) -> None:
+    """Write episode-level summary Parquet file."""
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError:
+        return
+
+    summary_rows = []
+    for episode in episodes:
+        first_t = episode.steps[0].timestamp if episode.steps else None
+        last_t = episode.steps[-1].timestamp if episode.steps else None
+        duration = (
+            (last_t - first_t) if first_t is not None and last_t is not None else None
+        )
+
+        # Extract action type values (enum -> string)
+        first_action_type = None
+        last_action_type = None
+        if episode.steps and episode.steps[0].action:
+            t = episode.steps[0].action.type
+            first_action_type = t.value if hasattr(t, "value") else t
+        if episode.steps and episode.steps[-1].action:
+            t = episode.steps[-1].action.type
+            last_action_type = t.value if hasattr(t, "value") else t
+
+        summary_rows.append(
+            {
+                "episode_id": episode.episode_id,
+                "instruction": episode.instruction,
+                "task_id": getattr(episode, "task_id", None),
+                "step_count": len(episode.steps),
+                "duration": duration,
+                "success": getattr(episode, "success", None),
+                "first_action_type": first_action_type,
+                "last_action_type": last_action_type,
+                "metadata": (
+                    json.dumps(episode.metadata)
+                    if hasattr(episode, "metadata") and episode.metadata
+                    else None
+                ),
+            }
+        )
+
+    summary_table = pa.Table.from_pylist(summary_rows)
+    summary_path = str(output_path).replace(".parquet", "_summary.parquet")
+    pq.write_table(summary_table, summary_path)
+
+
+def from_parquet(parquet_path: str) -> list[Episode]:
+    """Load Episodes from Parquet (inverse of to_parquet).
+
+    This is a lossy reconstruction. For full fidelity, always keep
+    Episode JSON as the source of truth.
+
+    Args:
+        parquet_path: Path to the Parquet file created by to_parquet().
+
+    Returns:
+        List of reconstructed Episode objects.
+
+    Raises:
+        ImportError: If pyarrow is not installed.
+
+    Note:
+        - Metadata fields are deserialized from JSON strings
+        - Step ordering is recovered from step_index
+        - Episode boundaries are recovered from episode_id grouping
+    """
+    try:
+        import pyarrow.parquet as pq
+    except ImportError:
+        raise ImportError(
+            "Parquet import requires pyarrow. "
+            "Install with: pip install openadapt-ml[parquet]"
+        )
+
+    from openadapt_ml.schema import Action, ActionType, Episode, Observation, Step
+
+    table = pq.read_table(parquet_path)
+    df = table.to_pandas()
+
+    episodes = []
+    for episode_id, group in df.groupby("episode_id"):
+        group = group.sort_values("step_index")
+
+        steps = []
+        for _, row in group.iterrows():
+            observation = Observation(
+                screenshot_path=row.get("screenshot_path") or row.get("image_path"),
+                window_title=row.get("window_title"),
+            )
+
+            action = None
+            if row.get("action_type"):
+                # Convert string action type to ActionType enum
+                action_type_str = row["action_type"]
+                try:
+                    action_type = ActionType(action_type_str)
+                except ValueError:
+                    action_type = ActionType.CLICK  # Default fallback
+
+                # Build normalized coordinates tuple if x and y are present
+                normalized_coords = None
+                if row.get("x") is not None and row.get("y") is not None:
+                    normalized_coords = (float(row["x"]), float(row["y"]))
+
+                # Build normalized end coordinates for drag
+                normalized_end = None
+                if row.get("end_x") is not None and row.get("end_y") is not None:
+                    normalized_end = (float(row["end_x"]), float(row["end_y"]))
+
+                action = Action(
+                    type=action_type,
+                    normalized_coordinates=normalized_coords,
+                    normalized_end=normalized_end,
+                    text=row.get("text"),
+                    key=row.get("key"),
+                    scroll_direction=row.get("scroll_direction"),
+                )
+
+            step = Step(
+                step_index=int(row.get("step_index", 0)),
+                observation=observation,
+                action=action,
+                reasoning=row.get("reasoning") or row.get("thought"),
+                timestamp=row.get("timestamp"),
+            )
+            steps.append(step)
+
+        # Parse metadata if present
+        metadata = None
+        if group.iloc[0].get("episode_metadata"):
+            try:
+                metadata = json.loads(group.iloc[0]["episode_metadata"])
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+        episode = Episode(
+            episode_id=str(episode_id),
+            instruction=group.iloc[0].get("instruction")
+            or group.iloc[0].get("goal", ""),
+            steps=steps,
+            task_id=group.iloc[0].get("task_id"),
+            metadata=metadata,
+        )
+        episodes.append(episode)
+
+    return episodes

openadapt_ml/grounding/detector.py

@@ -20,7 +20,7 @@ from openadapt_ml.config import settings
 from openadapt_ml.grounding.base import GroundingModule, RegionCandidate
 
 if TYPE_CHECKING:
-    from PIL import Image, ImageDraw, ImageFont
+    from PIL import Image
 
 
 class GeminiGrounder(GroundingModule):
@@ -104,7 +104,7 @@ class GeminiGrounder(GroundingModule):
 
         # Try to parse JSON from the response
         # Look for JSON array or object in the response
-        json_match = re.search(r'\[[\s\S]*\]|\{[\s\S]*\}', response_text)
+        json_match = re.search(r"\[[\s\S]*\]|\{[\s\S]*\}", response_text)
         if not json_match:
             return candidates
 
@@ -340,11 +340,11 @@ Example output format:
         response_text = response.text
 
         # Try to extract JSON array from response
-        json_match = re.search(r'\[[\s\S]*\]', response_text)
+        json_match = re.search(r"\[[\s\S]*\]", response_text)
         if not json_match:
             # Maybe it's just a plain array
-            if response_text.strip().startswith('['):
-                json_match = re.match(r'.*', response_text)
+            if response_text.strip().startswith("["):
+                json_match = re.match(r".*", response_text)
             else:
                 return []
 
@@ -369,13 +369,18 @@ Example output format:
                         max(0, min(1, y2 / screenshot.height)),
                     ]
 
-                normalized_elements.append({
-                    "id": elem.get("id", len(normalized_elements) + 1),
-                    "label": elem.get("label", f"Element {elem.get('id', len(normalized_elements) + 1)}"),
-                    "bbox": norm_bbox,
-                    "type": elem.get("type", "other"),
-                    "text": elem.get("text", ""),
-                })
+                normalized_elements.append(
+                    {
+                        "id": elem.get("id", len(normalized_elements) + 1),
+                        "label": elem.get(
+                            "label",
+                            f"Element {elem.get('id', len(normalized_elements) + 1)}",
+                        ),
+                        "bbox": norm_bbox,
+                        "type": elem.get("type", "other"),
+                        "text": elem.get("text", ""),
+                    }
+                )
 
         return normalized_elements
 
@@ -549,8 +554,7 @@ class DetectorGrounder(GroundingModule):
             self._backend = GeminiGrounder(**kwargs)
         elif backend == "omniparser":
             raise NotImplementedError(
-                "OmniParser backend not yet implemented. "
-                "Use backend='gemini' for now."
+                "OmniParser backend not yet implemented. Use backend='gemini' for now."
             )
         else:
             raise ValueError(f"Unknown backend: {backend}")

openadapt_ml/ingest/__init__.py

@@ -6,7 +6,6 @@ and converting them to the format used for training.
 Data Model:
     - Episode: A single task attempt (e.g., "log into the app"). Contains a sequence
       of Steps, each with an Observation (screenshot) and Action (click/type/etc).
-    - Session: A container grouping one or more Episodes with shared metadata.
 
 Functions:
     - load_episodes(): Load Episodes from JSON files (primary entry point)
@@ -14,30 +13,32 @@ Functions:
     - capture_to_episode(): Converts one openadapt-capture recording → one Episode
     - capture_to_session(): Converts one recording → Session containing one Episode
     - load_captures_as_sessions(): Loads multiple recordings → list of Sessions
-    - generate_synthetic_sessions(): Creates synthetic training data
+    - generate_synthetic_episodes(): Creates synthetic training data
 """
 
 from openadapt_ml.ingest.loader import load_episodes, save_episodes
-from openadapt_ml.ingest.synthetic import generate_synthetic_sessions
+from openadapt_ml.ingest.synthetic import generate_synthetic_episodes
 
 __all__ = [
     "load_episodes",
     "save_episodes",
-    "generate_synthetic_sessions",
+    "generate_synthetic_episodes",
 ]
 
 # Conditionally export capture functions if openadapt-capture is installed
 try:
-    from openadapt_ml.ingest.capture import (
+    from openadapt_ml.ingest.capture import (  # noqa: F401
         capture_to_episode,
         capture_to_session,
         load_captures_as_sessions,
     )
 
-    __all__.extend([
-        "capture_to_episode",
-        "capture_to_session",
-        "load_captures_as_sessions",
-    ])
+    __all__.extend(
+        [
+            "capture_to_episode",
+            "capture_to_session",
+            "load_captures_as_sessions",
+        ]
+    )
 except ImportError:
     pass