openadapt-ml 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

openadapt_ml/baselines/__init__.py

@@ -0,0 +1,121 @@
+"""Unified baseline adapters for VLM comparison.
+
+This module provides tools for comparing different VLM providers
+(Claude, GPT, Gemini) across multiple evaluation tracks:
+
+- Track A: Direct coordinate prediction
+- Track B: ReAct-style reasoning with coordinates
+- Track C: Set-of-Mark element selection
+
+Based on SOTA patterns from:
+- Claude Computer Use (Anthropic)
+- Microsoft UFO/UFO2
+- OSWorld benchmark
+- Agent-S/Agent-S2 (Simular AI)
+
+Usage:
+    from openadapt_ml.baselines import UnifiedBaselineAdapter, BaselineConfig, TrackConfig
+
+    # Quick start with model alias
+    adapter = UnifiedBaselineAdapter.from_alias("claude-opus-4.5")
+    action = adapter.predict(screenshot, "Click the submit button")
+
+    # With explicit configuration
+    config = BaselineConfig(
+        provider="anthropic",
+        model="claude-opus-4-5-20251101",
+        track=TrackConfig.track_c(),
+    )
+    adapter = UnifiedBaselineAdapter(config)
+
+    # OSWorld-compatible configuration
+    config = BaselineConfig(
+        provider="openai",
+        model="gpt-5.2",
+        track=TrackConfig.osworld_compatible(),
+    )
+
+    # Parse responses directly
+    from openadapt_ml.baselines import UnifiedResponseParser, ElementRegistry
+
+    parser = UnifiedResponseParser()
+    action = parser.parse('{"action": "CLICK", "x": 0.5, "y": 0.3}')
+
+    # With element ID to coordinate conversion
+    registry = ElementRegistry.from_a11y_tree(tree)
+    parser = UnifiedResponseParser(element_registry=registry)
+    action = parser.parse_and_resolve('{"action": "CLICK", "element_id": 17}')
+"""
+
+from openadapt_ml.baselines.adapter import UnifiedBaselineAdapter
+from openadapt_ml.baselines.config import (
+    # Enums
+    ActionOutputFormat,
+    CoordinateSystem,
+    TrackType,
+    # Config dataclasses
+    BaselineConfig,
+    ModelSpec,
+    ReActConfig,
+    ScreenConfig,
+    SoMConfig,
+    TrackConfig,
+    # Registry
+    MODELS,
+    # Helper functions
+    get_default_model,
+    get_model_spec,
+)
+from openadapt_ml.baselines.parser import (
+    ElementRegistry,
+    ParsedAction,
+    UIElement,
+    UnifiedResponseParser,
+)
+from openadapt_ml.baselines.prompts import (
+    # System prompts
+    FORMAT_PROMPTS,
+    SYSTEM_PROMPT_OSWORLD,
+    SYSTEM_PROMPT_TRACK_A,
+    SYSTEM_PROMPT_TRACK_B,
+    SYSTEM_PROMPT_TRACK_C,
+    SYSTEM_PROMPT_UFO,
+    SYSTEM_PROMPTS,
+    # Builder class
+    PromptBuilder,
+)
+
+__all__ = [
+    # Main adapter
+    "UnifiedBaselineAdapter",
+    # Configuration - Enums
+    "ActionOutputFormat",
+    "CoordinateSystem",
+    "TrackType",
+    # Configuration - Dataclasses
+    "BaselineConfig",
+    "ModelSpec",
+    "ReActConfig",
+    "ScreenConfig",
+    "SoMConfig",
+    "TrackConfig",
+    # Configuration - Registry
+    "MODELS",
+    # Configuration - Functions
+    "get_default_model",
+    "get_model_spec",
+    # Parsing
+    "ElementRegistry",
+    "ParsedAction",
+    "UIElement",
+    "UnifiedResponseParser",
+    # Prompts
+    "FORMAT_PROMPTS",
+    "PromptBuilder",
+    "SYSTEM_PROMPT_OSWORLD",
+    "SYSTEM_PROMPT_TRACK_A",
+    "SYSTEM_PROMPT_TRACK_B",
+    "SYSTEM_PROMPT_TRACK_C",
+    "SYSTEM_PROMPT_UFO",
+    "SYSTEM_PROMPTS",
+]

openadapt_ml/baselines/adapter.py

@@ -0,0 +1,185 @@
+"""Unified baseline adapter for comparing VLMs across tracks.
+
+Main entry point for baseline evaluations.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Any
+
+from openadapt_ml.baselines.config import BaselineConfig, TrackConfig
+from openadapt_ml.baselines.parser import ParsedAction, UnifiedResponseParser
+from openadapt_ml.baselines.prompts import PromptBuilder
+from openadapt_ml.config import settings
+from openadapt_ml.models.providers import get_provider
+
+if TYPE_CHECKING:
+    from PIL import Image
+
+
+class UnifiedBaselineAdapter:
+    """Adapter for running baseline evaluations across VLM providers.
+
+    Provides a unified interface for Claude, GPT, and Gemini models
+    across multiple evaluation tracks (coordinates, ReAct, SoM).
+
+    Example:
+        adapter = UnifiedBaselineAdapter(BaselineConfig.from_alias("claude-opus-4.5"))
+        result = adapter.predict(screenshot, "Click the submit button")
+        print(result.x, result.y)
+
+        # With Track C (SoM)
+        adapter = UnifiedBaselineAdapter(BaselineConfig.from_alias(
+            "gemini-3-pro",
+            track=TrackConfig.track_c(),
+        ))
+        result = adapter.predict(screenshot, "Click the login button", a11y_tree=tree)
+        print(result.element_id)
+    """
+
+    def __init__(self, config: BaselineConfig):
+        """Initialize the baseline adapter.
+
+        Args:
+            config: Baseline configuration including model, track, etc.
+        """
+        self.config = config
+        self._provider = get_provider(config.provider)
+        self._client = None
+        self._prompt_builder = PromptBuilder(config.track)
+        self._parser = UnifiedResponseParser()
+
+    @property
+    def client(self) -> Any:
+        """Lazy-load API client."""
+        if self._client is None:
+            api_key = self._resolve_api_key()
+            self._client = self._provider.create_client(api_key)
+        return self._client
+
+    def _resolve_api_key(self) -> str:
+        """Resolve API key from config, settings, or environment."""
+        if self.config.api_key:
+            return self.config.api_key
+
+        # Try settings first
+        if self.config.provider == "anthropic":
+            key = settings.anthropic_api_key or os.getenv("ANTHROPIC_API_KEY")
+        elif self.config.provider == "openai":
+            key = settings.openai_api_key or os.getenv("OPENAI_API_KEY")
+        elif self.config.provider == "google":
+            key = settings.google_api_key or os.getenv("GOOGLE_API_KEY")
+        else:
+            key = None
+
+        if not key:
+            raise RuntimeError(
+                f"API key for {self.config.provider} not found. "
+                "Set in .env file or environment variable."
+            )
+        return key
+
+    def predict(
+        self,
+        screenshot: "Image",
+        goal: str,
+        a11y_tree: str | dict[str, Any] | None = None,
+        history: list[dict[str, Any]] | None = None,
+    ) -> ParsedAction:
+        """Predict the next action given current state.
+
+        Args:
+            screenshot: Current screenshot as PIL Image.
+            goal: Task goal/instruction.
+            a11y_tree: Optional accessibility tree (string or dict).
+            history: Optional list of previous actions.
+
+        Returns:
+            ParsedAction with predicted action.
+        """
+        # Build system prompt
+        system_prompt = self._prompt_builder.get_system_prompt(self.config.demo)
+
+        # Build user content
+        content = self._prompt_builder.build_user_content(
+            goal=goal,
+            screenshot=screenshot,
+            a11y_tree=a11y_tree,
+            history=history,
+            encode_image_fn=self._provider.encode_image,
+        )
+
+        # Call API
+        response = self._provider.send_message(
+            client=self.client,
+            model=self.config.model,
+            system=system_prompt,
+            content=content,
+            max_tokens=self.config.max_tokens,
+            temperature=self.config.temperature,
+        )
+
+        if self.config.verbose:
+            print(f"[{self.config.provider}] Response: {response[:200]}...")
+
+        # Parse response
+        action = self._parser.parse(response)
+
+        return action
+
+    def predict_batch(
+        self,
+        samples: list[dict[str, Any]],
+    ) -> list[ParsedAction]:
+        """Predict actions for multiple samples.
+
+        Note: Currently runs sequentially. Future optimization could
+        use async/parallel calls for providers that support it.
+
+        Args:
+            samples: List of dicts with keys: screenshot, goal, a11y_tree, history.
+
+        Returns:
+            List of ParsedActions.
+        """
+        results = []
+        for sample in samples:
+            action = self.predict(
+                screenshot=sample.get("screenshot"),
+                goal=sample.get("goal", ""),
+                a11y_tree=sample.get("a11y_tree"),
+                history=sample.get("history"),
+            )
+            results.append(action)
+        return results
+
+    @classmethod
+    def from_alias(
+        cls,
+        model_alias: str,
+        track: TrackConfig | None = None,
+        **kwargs: Any,
+    ) -> "UnifiedBaselineAdapter":
+        """Create adapter from model alias.
+
+        Convenience constructor that resolves model aliases.
+
+        Args:
+            model_alias: Model alias (e.g., 'claude-opus-4.5', 'gpt-5.2').
+            track: Track config (defaults to Track A).
+            **kwargs: Additional config options.
+
+        Returns:
+            UnifiedBaselineAdapter instance.
+        """
+        config = BaselineConfig.from_alias(model_alias, track=track, **kwargs)
+        return cls(config)
+
+    def __repr__(self) -> str:
+        return (
+            f"UnifiedBaselineAdapter("
+            f"provider={self.config.provider}, "
+            f"model={self.config.model}, "
+            f"track={self.config.track.track_type.value})"
+        )

openadapt_ml/baselines/cli.py

@@ -0,0 +1,314 @@
+"""CLI for baseline adapter operations.
+
+Provides commands for comparing VLMs across tracks.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from openadapt_ml.baselines.config import MODELS
+
+
+@click.group()
+def baselines():
+    """Baseline adapter commands for VLM comparison."""
+    pass
+
+
+@baselines.command()
+@click.option(
+    "--model",
+    "-m",
+    required=True,
+    type=click.Choice(list(MODELS.keys())),
+    help="Model alias to use",
+)
+@click.option(
+    "--track",
+    "-t",
+    type=click.Choice(["A", "B", "C"]),
+    default="A",
+    help="Evaluation track (A=coords, B=ReAct, C=SoM)",
+)
+@click.option(
+    "--image",
+    "-i",
+    type=click.Path(exists=True),
+    required=True,
+    help="Screenshot image path",
+)
+@click.option(
+    "--goal",
+    "-g",
+    required=True,
+    help="Task goal/instruction",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    help="Output JSON file path",
+)
+@click.option(
+    "--verbose",
+    "-v",
+    is_flag=True,
+    help="Enable verbose output",
+)
+def run(
+    model: str,
+    track: str,
+    image: str,
+    goal: str,
+    output: str | None,
+    verbose: bool,
+):
+    """Run a single baseline prediction.
+
+    Example:
+        uv run python -m openadapt_ml.baselines.cli run \\
+            --model claude-opus-4.5 \\
+            --track A \\
+            --image screenshot.png \\
+            --goal "Click the submit button"
+    """
+    from PIL import Image
+
+    from openadapt_ml.baselines import UnifiedBaselineAdapter, TrackConfig
+
+    # Select track config
+    track_configs = {
+        "A": TrackConfig.track_a(),
+        "B": TrackConfig.track_b(),
+        "C": TrackConfig.track_c(),
+    }
+    track_config = track_configs[track]
+
+    click.echo(f"Model: {model}")
+    click.echo(f"Track: {track} ({track_config.track_type.value})")
+    click.echo(f"Image: {image}")
+    click.echo(f"Goal: {goal}")
+    click.echo()
+
+    # Load image
+    screenshot = Image.open(image)
+
+    # Create adapter
+    adapter = UnifiedBaselineAdapter.from_alias(
+        model,
+        track=track_config,
+        verbose=verbose,
+    )
+
+    # Run prediction
+    click.echo("Running prediction...")
+    action = adapter.predict(screenshot, goal)
+
+    # Display result
+    click.echo()
+    click.echo("=" * 50)
+    click.echo("RESULT")
+    click.echo("=" * 50)
+
+    if action.is_valid:
+        click.echo(f"Action: {action.action_type.upper()}")
+        if action.x is not None and action.y is not None:
+            click.echo(f"Coordinates: ({action.x:.4f}, {action.y:.4f})")
+        if action.element_id is not None:
+            click.echo(f"Element ID: {action.element_id}")
+        if action.text is not None:
+            click.echo(f"Text: {action.text}")
+        if action.thought is not None:
+            click.echo(f"Thought: {action.thought}")
+    else:
+        click.echo(f"Parse Error: {action.parse_error}")
+        click.echo(
+            f"Raw Response: {action.raw_response[:200] if action.raw_response else 'None'}..."
+        )
+
+    # Save output if requested
+    if output:
+        result = {
+            "model": model,
+            "track": track,
+            "goal": goal,
+            "action": action.to_dict(),
+            "raw_response": action.raw_response,
+            "parse_error": action.parse_error,
+        }
+        Path(output).write_text(json.dumps(result, indent=2))
+        click.echo(f"\nSaved to: {output}")
+
+
+@baselines.command()
+@click.option(
+    "--models",
+    "-m",
+    required=True,
+    help="Comma-separated model aliases",
+)
+@click.option(
+    "--track",
+    "-t",
+    type=click.Choice(["A", "B", "C"]),
+    default="A",
+    help="Evaluation track",
+)
+@click.option(
+    "--image",
+    "-i",
+    type=click.Path(exists=True),
+    required=True,
+    help="Screenshot image path",
+)
+@click.option(
+    "--goal",
+    "-g",
+    required=True,
+    help="Task goal/instruction",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    help="Output JSON file path",
+)
+def compare(
+    models: str,
+    track: str,
+    image: str,
+    goal: str,
+    output: str | None,
+):
+    """Compare multiple models on the same task.
+
+    Example:
+        uv run python -m openadapt_ml.baselines.cli compare \\
+            --models claude-opus-4.5,gpt-5.2,gemini-3-pro \\
+            --track C \\
+            --image screenshot.png \\
+            --goal "Click the login button"
+    """
+    from PIL import Image
+
+    from openadapt_ml.baselines import UnifiedBaselineAdapter, TrackConfig
+
+    model_list = [m.strip() for m in models.split(",")]
+
+    # Validate models
+    for m in model_list:
+        if m not in MODELS:
+            click.echo(f"Error: Unknown model '{m}'", err=True)
+            click.echo(f"Available: {', '.join(MODELS.keys())}", err=True)
+            sys.exit(1)
+
+    # Select track config
+    track_configs = {
+        "A": TrackConfig.track_a(),
+        "B": TrackConfig.track_b(),
+        "C": TrackConfig.track_c(),
+    }
+    track_config = track_configs[track]
+
+    click.echo(f"Comparing {len(model_list)} models on Track {track}")
+    click.echo(f"Image: {image}")
+    click.echo(f"Goal: {goal}")
+    click.echo()
+
+    # Load image
+    screenshot = Image.open(image)
+
+    results = []
+
+    # Run each model
+    for model in model_list:
+        click.echo(f"Running {model}...")
+
+        try:
+            adapter = UnifiedBaselineAdapter.from_alias(model, track=track_config)
+            action = adapter.predict(screenshot, goal)
+
+            results.append(
+                {
+                    "model": model,
+                    "success": action.is_valid,
+                    "action": action.to_dict(),
+                    "error": action.parse_error,
+                }
+            )
+
+            status = "OK" if action.is_valid else "FAILED"
+            click.echo(f"  {status}: {action.action_type}")
+
+        except Exception as e:
+            results.append(
+                {
+                    "model": model,
+                    "success": False,
+                    "action": None,
+                    "error": str(e),
+                }
+            )
+            click.echo(f"  ERROR: {e}")
+
+    # Summary table
+    click.echo()
+    click.echo("=" * 60)
+    click.echo("COMPARISON SUMMARY")
+    click.echo("=" * 60)
+    click.echo(f"{'Model':<25} {'Status':<10} {'Action':<25}")
+    click.echo("-" * 60)
+
+    for r in results:
+        model = r["model"]
+        status = "OK" if r["success"] else "FAILED"
+        action = r["action"]
+        if action:
+            if action.get("x") is not None:
+                action_str = f"CLICK({action['x']:.3f}, {action['y']:.3f})"
+            elif action.get("element_id") is not None:
+                action_str = f"CLICK([{action['element_id']}])"
+            else:
+                action_str = action.get("type", "unknown").upper()
+        else:
+            action_str = r.get("error", "Unknown error")[:25]
+
+        click.echo(f"{model:<25} {status:<10} {action_str:<25}")
+
+    # Save output if requested
+    if output:
+        full_results = {
+            "models": model_list,
+            "track": track,
+            "goal": goal,
+            "results": results,
+        }
+        Path(output).write_text(json.dumps(full_results, indent=2))
+        click.echo(f"\nSaved to: {output}")
+
+
+@baselines.command()
+def list_models():
+    """List available models and their providers."""
+    click.echo("Available models:")
+    click.echo()
+    click.echo(f"{'Alias':<20} {'Provider':<12} {'Model ID':<35} {'Default'}")
+    click.echo("-" * 75)
+
+    for alias, spec in MODELS.items():
+        default = "*" if spec.is_default else ""
+        click.echo(f"{alias:<20} {spec.provider:<12} {spec.model_id:<35} {default}")
+
+
+# Entry point for direct execution
+def main():
+    baselines()
+
+
+if __name__ == "__main__":
+    main()