openadapt-ml 0.1.0__py3-none-any.whl

openadapt_ml/scripts/demo_policy.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import argparse
+
+from openadapt_ml.datasets.next_action import build_next_action_sft_samples
+from openadapt_ml.ingest.synthetic import generate_synthetic_sessions
+from openadapt_ml.models.dummy_adapter import DummyAdapter
+from openadapt_ml.models.qwen_vl import QwenVLAdapter
+from openadapt_ml.models.api_adapter import ApiVLMAdapter
+from openadapt_ml.runtime.policy import AgentPolicy
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--backend",
+        choices=["dummy", "qwen3", "qwen2_5", "claude", "openai"],
+        default="dummy",
+    )
+    args = parser.parse_args()
+
+    # Use synthetic data to build one SFT-style sample
+    sessions = generate_synthetic_sessions(num_sessions=1, seed=99, output_dir="synthetic/demo")
+    episodes = [ep for sess in sessions for ep in sess.episodes]
+    samples = build_next_action_sft_samples(episodes)
+
+    # Load first sample and overwrite assistant content so the dummy adapter
+    # doesn't depend on any particular target.
+    sample = samples[0]
+
+    if args.backend == "dummy":
+        adapter = DummyAdapter()
+    elif args.backend == "qwen3":
+        adapter = QwenVLAdapter.from_pretrained(
+            "Qwen/Qwen3-VL-8B-Instruct",
+            lora_config=None,
+            load_in_4bit=False,
+        )
+    elif args.backend == "qwen2_5":
+        adapter = QwenVLAdapter.from_pretrained(
+            "Qwen/Qwen2.5-VL-7B-Instruct",
+            lora_config=None,
+            load_in_4bit=False,
+        )
+    elif args.backend == "claude":
+        adapter = ApiVLMAdapter(provider="anthropic")
+    else:  # openai
+        adapter = ApiVLMAdapter(provider="openai")
+    policy = AgentPolicy(adapter)
+
+    action, thought, state, raw_text = policy.predict_action_from_sample(sample)
+    print("Raw sample messages:")
+    for m in sample["messages"]:
+        print(f"[{m['role']}] {m['content']}")
+
+    print("\nPredicted action:", action)
+    print("Thought:", thought)
+    print("State:", state)
+    print("Raw output:", raw_text)
+
+if __name__ == "__main__":
+    main()

openadapt_ml/scripts/eval_policy.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import yaml
+
+from openadapt_ml.datasets.next_action import build_next_action_sft_samples, parse_action_som
+from openadapt_ml.evals.trajectory_matching import evaluate_policy_on_episodes
+from openadapt_ml.ingest.synthetic import generate_synthetic_sessions
+from openadapt_ml.models.dummy_adapter import DummyAdapter
+from openadapt_ml.models.qwen_vl import QwenVLAdapter
+from openadapt_ml.models.api_adapter import ApiVLMAdapter
+from openadapt_ml.runtime.policy import AgentPolicy
+
+
+def _load_config(path: str | Path) -> dict:
+    with open(path, "r", encoding="utf-8") as f:
+        return yaml.safe_load(f)
+
+
+def main(
+    config_path: str,
+    backend: str,
+    output_json: str | None,
+    ignore_lora: bool = False,
+    log_samples: Optional[str] = None,
+    log_limit: Optional[int] = None,
+    dsl_mode: str = "coord",
+    eval_on_training_data: bool = False,
+    no_jitter: bool = False,
+    scenario: Optional[str] = None,
+) -> None:
+    cfg = _load_config(config_path)
+
+    # Determine if using Set-of-Marks (SoM) mode
+    use_som = dsl_mode == "som"
+
+    # Synthetic data config
+    synth_cfg: Dict[str, Any] = cfg.get("synthetic_data", {})
+    num_sessions = synth_cfg.get("num_sessions", 4)
+    seed = synth_cfg.get("seed", 999)
+
+    # Determine output directory and jitter setting
+    if eval_on_training_data:
+        # Use the SAME data directory as training to test memorization
+        output_dir = synth_cfg.get("output_dir", "synthetic_train")
+        # When evaluating on training data, use same jitter setting as training
+        # (default True unless explicitly set)
+        jitter = synth_cfg.get("jitter", True) and not no_jitter
+        print(f"[INFO] Evaluating on TRAINING data from: {output_dir}")
+    else:
+        # Generate fresh data for generalization testing
+        output_dir = synth_cfg.get("output_dir", "synthetic_eval") + "_eval"
+        jitter = not no_jitter
+        print(f"[INFO] Evaluating on FRESH data in: {output_dir}")
+
+    if no_jitter:
+        print("[INFO] Jitter disabled - using deterministic layouts")
+
+    # Determine scenario: CLI arg takes precedence, then config, then default "login"
+    scenario_to_use = scenario if scenario else synth_cfg.get("scenario", "login")
+
+    # Generate sessions with SoM if requested
+    sessions = generate_synthetic_sessions(
+        num_sessions=num_sessions,
+        seed=seed,
+        output_dir=output_dir,
+        use_som=use_som,
+        jitter=jitter,
+        scenario=scenario_to_use,
+    )
+    print(f"[INFO] Scenario: {scenario_to_use}")
+    episodes = [ep for sess in sessions for ep in sess.episodes]
+
+    # Build samples with appropriate DSL mode
+    samples = build_next_action_sft_samples(episodes, use_som=use_som)
+
+    # Backend / adapter selection
+    if backend == "dummy":
+        adapter = DummyAdapter()
+    elif backend == "qwen3":
+        model_cfg = cfg.get("model", {})
+        model_name = model_cfg.get("name", "Qwen/Qwen3-VL-8B-Instruct")
+        load_in_4bit = model_cfg.get("load_in_4bit", False)
+
+        # Optionally ignore LoRA to evaluate the base model only.
+        if ignore_lora:
+            lora_cfg = None
+        else:
+            lora_cfg = cfg.get("lora")
+
+        adapter = QwenVLAdapter.from_pretrained(
+            model_name,
+            lora_config=lora_cfg,
+            load_in_4bit=load_in_4bit,
+        )
+    elif backend == "qwen2_5":
+        adapter = QwenVLAdapter.from_pretrained(
+            "Qwen/Qwen2.5-VL-7B-Instruct",
+            lora_config=None,
+            load_in_4bit=False,
+        )
+    elif backend == "claude":
+        adapter = ApiVLMAdapter(provider="anthropic")
+    elif backend == "openai":
+        adapter = ApiVLMAdapter(provider="openai")
+    else:
+        raise ValueError(f"Unsupported backend: {backend}")
+
+    policy = AgentPolicy(adapter)
+
+    log_fn: Optional[callable] = None
+    log_file_handle = None
+    if log_samples is not None:
+        log_path = Path(log_samples)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        log_file_handle = open(log_path, "w", encoding="utf-8")
+
+        def _log(record: Dict[str, Any]) -> None:
+            assert log_file_handle is not None
+            log_file_handle.write(json.dumps(record) + "\n")
+
+        log_fn = _log
+
+    try:
+        metrics = evaluate_policy_on_episodes(
+            policy,
+            episodes,
+            samples,
+            log_fn=log_fn,
+            log_limit=log_limit,
+            use_som=use_som,
+        )
+    finally:
+        if log_file_handle is not None:
+            log_file_handle.close()
+
+    print(f"Evaluation results (DSL mode: {dsl_mode}):")
+    print(f"  num_episodes: {metrics.num_episodes}")
+    print(f"  num_steps: {metrics.num_steps}")
+    print(f"  action_type_accuracy: {metrics.action_type_accuracy:.4f}")
+    if metrics.mean_coord_error is not None:
+        print(
+            "  mean_coord_error (normalized): "
+            f"{metrics.mean_coord_error:.4f} (n={metrics.coord_error_count})"
+        )
+    else:
+        print("  mean_coord_error (normalized): N/A")
+    if metrics.episode_success_rate is not None:
+        print(f"  episode_success_rate: {metrics.episode_success_rate:.4f}")
+    else:
+        print("  episode_success_rate: N/A")
+    if metrics.click_hit_rate is not None:
+        print(f"  click_hit_rate: {metrics.click_hit_rate:.4f}")
+    else:
+        print("  click_hit_rate: N/A")
+    if metrics.mean_episode_progress is not None:
+        print(f"  mean_episode_progress: {metrics.mean_episode_progress:.4f}")
+    else:
+        print("  mean_episode_progress: N/A")
+    if metrics.mean_episode_step_score is not None:
+        print(f"  mean_episode_step_score: {metrics.mean_episode_step_score:.4f}")
+    else:
+        print("  mean_episode_step_score: N/A")
+    if metrics.weak_episode_success_rate is not None:
+        print(f"  weak_episode_success_rate: {metrics.weak_episode_success_rate:.4f}")
+    else:
+        print("  weak_episode_success_rate: N/A")
+    if metrics.state_success_rate is not None:
+        print(f"  state_success_rate: {metrics.state_success_rate:.4f}")
+    else:
+        print("  state_success_rate: N/A")
+    if metrics.bbox_hit_rate is not None:
+        print(f"  bbox_hit_rate: {metrics.bbox_hit_rate:.4f}")
+    else:
+        print("  bbox_hit_rate: N/A")
+    if metrics.element_accuracy is not None:
+        print(f"  element_accuracy: {metrics.element_accuracy:.4f}")
+    else:
+        print("  element_accuracy: N/A")
+
+    if output_json is not None:
+        payload = {
+            "config_path": str(config_path),
+            "backend": backend,
+            "dsl_mode": dsl_mode,
+            "metrics": {
+                "num_episodes": metrics.num_episodes,
+                "num_steps": metrics.num_steps,
+                "action_type_accuracy": metrics.action_type_accuracy,
+                "mean_coord_error": metrics.mean_coord_error,
+                "coord_error_count": metrics.coord_error_count,
+                "episode_success_rate": metrics.episode_success_rate,
+                "click_hit_rate": metrics.click_hit_rate,
+                "bbox_hit_rate": metrics.bbox_hit_rate,
+                "mean_episode_progress": metrics.mean_episode_progress,
+                "mean_episode_step_score": metrics.mean_episode_step_score,
+                "weak_episode_success_rate": metrics.weak_episode_success_rate,
+                "state_success_rate": metrics.state_success_rate,
+                "element_accuracy": metrics.element_accuracy if hasattr(metrics, 'element_accuracy') else None,
+            },
+        }
+        out_path = Path(output_json)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(out_path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, indent=2)
+        print(f"Metrics written to {output_json}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Evaluate a policy on synthetic episodes.")
+    parser.add_argument("--config", type=str, required=True, help="Path to YAML config file.")
+    parser.add_argument(
+        "--backend",
+        type=str,
+        choices=["dummy", "qwen3", "qwen2_5", "claude", "openai"],
+        default="qwen2_5",
+        help="Backend adapter to use for evaluation.",
+    )
+    parser.add_argument(
+        "--output-json",
+        type=str,
+        default=None,
+        help="Optional path to write metrics as JSON.",
+    )
+    parser.add_argument(
+        "--ignore-lora",
+        action="store_true",
+        help="Ignore any LoRA config in the YAML and evaluate the base model only.",
+    )
+    parser.add_argument(
+        "--log-samples",
+        type=str,
+        default=None,
+        help="Optional path to write per-step eval logs as JSONL.",
+    )
+    parser.add_argument(
+        "--log-limit",
+        type=int,
+        default=None,
+        help="Maximum number of steps to log (default: no limit).",
+    )
+    parser.add_argument(
+        "--dsl-mode",
+        type=str,
+        choices=["coord", "som"],
+        default="coord",
+        help="DSL mode: 'coord' for coordinate-based (CLICK(x=..., y=...)), "
+             "'som' for Set-of-Marks index-based (CLICK([1])). Default: coord.",
+    )
+    parser.add_argument(
+        "--overfit",
+        action="store_true",
+        help="Evaluate on training data to check memorization/overfitting. "
+             "If not set, generates fresh data to test generalization.",
+    )
+    parser.add_argument(
+        "--no-jitter",
+        action="store_true",
+        help="Disable jitter for deterministic UI layouts. "
+             "Useful for testing memorization of fixed layouts.",
+    )
+    parser.add_argument(
+        "--scenario",
+        type=str,
+        choices=["login", "registration"],
+        default=None,
+        help="Scenario type: 'login' (6 steps, 3 elements) or 'registration' (12 steps, 6 elements). "
+             "Overrides config if provided.",
+    )
+    args = parser.parse_args()
+
+    main(
+        config_path=args.config,
+        backend=args.backend,
+        output_json=args.output_json,
+        ignore_lora=args.ignore_lora,
+        log_samples=args.log_samples,
+        log_limit=args.log_limit,
+        dsl_mode=args.dsl_mode,
+        eval_on_training_data=args.overfit,
+        no_jitter=args.no_jitter,
+        scenario=args.scenario,
+    )

openadapt_ml/scripts/make_gif.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from PIL import Image, ImageDraw, ImageFont
+
+
+def _load_font(size: int = 16) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:  # type: ignore[name-defined]
+    try:
+        return ImageFont.truetype("arial.ttf", size)
+    except OSError:
+        return ImageFont.load_default()
+
+
+FONT = _load_font(16)
+
+
+def _load_frames(frames_dir: Path, pattern: str) -> List[Path]:
+    paths = sorted(Path(p) for p in glob.glob(str(frames_dir / pattern)))
+    if not paths:
+        raise ValueError(f"No frames matched pattern '{pattern}' under {frames_dir}")
+    return paths
+
+
+def _default_login_caption(filename: str, index: int) -> str:
+    # Heuristic captions for the synthetic login script based on step index
+    # and the conventional *_step_{k}.png naming.
+    name = os.path.basename(filename)
+    # Try to extract step index from name if present.
+    step_idx = index
+    for part in name.split("_"):
+        if part.startswith("step"):
+            try:
+                step_idx = int(part.replace("step", "").replace(".png", ""))
+            except ValueError:
+                pass
+    if step_idx == 0:
+        return "Step 0: Initial login screen (WAIT)"
+    if step_idx == 1:
+        return "Step 1: CLICK username field"
+    if step_idx == 2:
+        return "Step 2: TYPE username"
+    if step_idx == 3:
+        return "Step 3: CLICK password field"
+    if step_idx == 4:
+        return "Step 4: TYPE password (masked)"
+    if step_idx == 5:
+        return "Step 5: CLICK Login button"
+    if step_idx == 6:
+        return "Step 6: DONE (logged in)"
+    return f"Step {step_idx}: synthetic step"
+
+
+def _draw_caption(image: Image.Image, text: str) -> Image.Image:
+    img = image.convert("RGB").copy()
+    draw = ImageDraw.Draw(img)
+    width, height = img.size
+
+    # Draw a semi-transparent rectangle at the bottom for text background
+    padding = 8
+    text_width, text_height = draw.textbbox((0, 0), text, font=FONT)[2:4]  # type: ignore[assignment]
+    rect_height = text_height + 2 * padding
+    y0 = height - rect_height
+    draw.rectangle([(0, y0), (width, height)], fill=(0, 0, 0, 180))
+    x_text = max(padding, (width - text_width) // 2)
+    y_text = y0 + padding
+    draw.text((x_text, y_text), text, font=FONT, fill=(255, 255, 255))
+    return img
+
+
+def make_gif(
+    frames_dir: Path,
+    pattern: str,
+    output: Path,
+    duration_ms: int = 1000,
+    scenario: Optional[str] = None,
+) -> None:
+    frame_paths = _load_frames(frames_dir, pattern)
+
+    frames: List[Image.Image] = []
+    for idx, frame_path in enumerate(frame_paths):
+        img = Image.open(frame_path)
+        if scenario == "login":
+            caption = _default_login_caption(frame_path.name, idx)
+            img = _draw_caption(img, caption)
+        frames.append(img)
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    frames[0].save(
+        output,
+        save_all=True,
+        append_images=frames[1:],
+        duration=duration_ms,
+        loop=0,
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Generate an animated GIF from a sequence of PNG frames.",
+    )
+    parser.add_argument(
+        "--frames-dir",
+        type=str,
+        required=True,
+        help="Directory containing frame PNGs (e.g. synthetic_demo/session_0000)",
+    )
+    parser.add_argument(
+        "--pattern",
+        type=str,
+        default="*step_*.png",
+        help="Glob pattern for frame filenames inside frames-dir (default: *step_*.png)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        required=True,
+        help="Output GIF path",
+    )
+    parser.add_argument(
+        "--duration-ms",
+        type=int,
+        default=1000,
+        help="Frame duration in milliseconds (default: 1000)",
+    )
+    parser.add_argument(
+        "--scenario",
+        type=str,
+        default=None,
+        choices=["login", None],  # type: ignore[list-item]
+        help="Optional built-in captioning scenario (e.g. 'login')",
+    )
+
+    args = parser.parse_args()
+
+    frames_dir = Path(args.frames_dir)
+    output = Path(args.output)
+
+    make_gif(
+        frames_dir=frames_dir,
+        pattern=args.pattern,
+        output=output,
+        duration_ms=args.duration_ms,
+        scenario=args.scenario,
+    )
+
+
+if __name__ == "__main__":
+    main()

openadapt_ml/scripts/prepare_synthetic.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from openadapt_ml.ingest.synthetic import generate_synthetic_sessions
+
+
+def main() -> None:
+    output_dir = Path("synthetic") / "debug"
+    sessions = generate_synthetic_sessions(num_sessions=2, seed=42, output_dir=output_dir)
+
+    print(f"Generated {len(sessions)} sessions into {output_dir.resolve()}")
+
+    total_episodes = 0
+    total_steps = 0
+    missing_images: list[str] = []
+
+    for session in sessions:
+        total_episodes += len(session.episodes)
+        for episode in session.episodes:
+            total_steps += len(episode.steps)
+            for step in episode.steps:
+                path = step.observation.image_path
+                if not path:
+                    missing_images.append(f"[no path] in episode {episode.id}")
+                    continue
+                if not os.path.exists(path):
+                    missing_images.append(path)
+
+    print(f"Episodes: {total_episodes}, Steps: {total_steps}")
+
+    if missing_images:
+        print("Missing images:")
+        for p in missing_images:
+            print(" -", p)
+        raise SystemExit(1)
+
+    print("All observation image paths exist.")
+
+
+if __name__ == "__main__":
+    main()