vco 0.1.0__py3-none-any.whl

vco/__init__.py

@@ -0,0 +1,35 @@
+"""vco — a clicker for LLMs: operate web pages and desktop screens."""
+
+from .geometry import GridMapper
+from .models import Action, GridPoint, GridSpec, Region, parse_action
+from .ocr import (
+    HTTPOCRBackend,
+    OCRBackend,
+    OCRBox,
+    OCRRequest,
+    OCRResult,
+    RapidOCRBackend,
+    create_ocr_backend,
+    register_ocr_backend,
+)
+from .ocr_assist import OCRAssistProvider
+
+__all__ = [
+    "Action",
+    "GridMapper",
+    "GridPoint",
+    "GridSpec",
+    "OCRBackend",
+    "OCRAssistProvider",
+    "OCRBox",
+    "OCRRequest",
+    "OCRResult",
+    "HTTPOCRBackend",
+    "RapidOCRBackend",
+    "Region",
+    "create_ocr_backend",
+    "parse_action",
+    "register_ocr_backend",
+]
+
+__version__ = "0.1.0"

vco/adaptive_zoom.py

@@ -0,0 +1,245 @@
+"""Model-directed recursive zoom for dense-grid visual localization."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Tuple
+
+from PIL import Image
+
+from .geometry import GridMapper
+from .grid import render_numbered_grid
+from .models import Action, GridSpec, Region, parse_action
+from .zoom import neighborhood_crop_box
+
+
+Box = Tuple[int, int, int, int]
+
+
+@dataclass(frozen=True)
+class AdaptiveZoomLevel:
+    level: int
+    view_box: Box
+    decision: dict
+    next_view_box: Box | None
+    provider_metadata: dict | None
+
+    def as_dict(self) -> dict:
+        return {
+            "level": self.level,
+            "view_box_image_pixels": self.view_box,
+            "decision": self.decision,
+            "next_view_box_image_pixels": self.next_view_box,
+            "provider_metadata": self.provider_metadata,
+        }
+
+
+@dataclass(frozen=True)
+class AdaptiveZoomTrace:
+    levels: tuple[AdaptiveZoomLevel, ...]
+    final_action: dict
+
+    def as_dict(self) -> dict:
+        return {
+            "mode": "model-directed-adaptive-zoom",
+            "levels": [level.as_dict() for level in self.levels],
+            "final_action": self.final_action,
+        }
+
+
+def _map_crop_to_original(display_crop: Box, view_box: Box, display_size) -> Box:
+    display_width, display_height = display_size
+    view_left, view_top, view_right, view_bottom = view_box
+    view_width = view_right - view_left
+    view_height = view_bottom - view_top
+    left, top, right, bottom = display_crop
+    mapped = (
+        view_left + math.floor(left * view_width / display_width),
+        view_top + math.floor(top * view_height / display_height),
+        view_left + math.ceil(right * view_width / display_width),
+        view_top + math.ceil(bottom * view_height / display_height),
+    )
+    if mapped[2] <= mapped[0] or mapped[3] <= mapped[1]:
+        raise ValueError("adaptive zoom produced an empty crop")
+    return mapped
+
+
+def _map_point_to_original(point, view_box: Box, display_size):
+    x, y = point
+    display_width, display_height = display_size
+    left, top, right, bottom = view_box
+    view_width = right - left
+    view_height = bottom - top
+    original_x = left + round(x * max(0, view_width - 1) / max(1, display_width - 1))
+    original_y = top + round(y * max(0, view_height - 1) / max(1, display_height - 1))
+    return original_x, original_y
+
+
+class AdaptiveZoomProvider:
+    """Ask the model to click now or request another zoom level."""
+
+    def __init__(
+        self,
+        provider,
+        *,
+        max_levels: int = 3,
+        span_cells: float = 4.0,
+        force_initial_click: bool = False,
+        center_click_zoom_epsilon: float | None = 0.02,
+        always_refine: bool = False,
+    ):
+        if max_levels < 1:
+            raise ValueError("max_levels must be at least 1")
+        if span_cells <= 1:
+            raise ValueError("span_cells must be greater than 1")
+        if center_click_zoom_epsilon is not None and not (
+            0 <= center_click_zoom_epsilon < 0.5
+        ):
+            raise ValueError("center_click_zoom_epsilon must be in [0,0.5)")
+        self.provider = provider
+        self.max_levels = max_levels
+        self.span_cells = span_cells
+        self.force_initial_click = force_initial_click
+        self.center_click_zoom_epsilon = center_click_zoom_epsilon
+        self.always_refine = always_refine
+        self.last_trace: AdaptiveZoomTrace | None = None
+        self.last_zoom_images: list[tuple[Image.Image, Image.Image]] = []
+        self.last_metadata = None
+
+    def choose_action(
+        self,
+        *,
+        task: str,
+        clean: Image.Image,
+        gridded: Image.Image,
+        grid: GridSpec,
+        step: int,
+    ) -> Action:
+        original = clean
+        full_box = (0, 0, clean.width, clean.height)
+        view_box = full_box
+        view_clean = clean
+        view_grid = gridded
+        levels = []
+        call_metadata = []
+        self.last_zoom_images = []
+
+        for level in range(1, self.max_levels + 1):
+            level_task = task
+            if level > 1:
+                level_task += (
+                    f"\nAdaptive zoom level {level}: this is an enlarged crop from "
+                    "the previous requested zoom. Re-evaluate whether the target can "
+                    "now be clicked reliably."
+                )
+            chooser = (
+                self.provider.choose_action
+                if self.always_refine or level == self.max_levels
+                else self.provider.choose_action_or_zoom
+            )
+            decision = chooser(
+                task=level_task,
+                clean=view_clean,
+                gridded=view_grid,
+                grid=grid,
+                step=step,
+                force_click=self.force_initial_click or level > 1,
+            )
+            metadata = getattr(self.provider, "last_metadata", None)
+            call_metadata.append(metadata)
+
+            if (
+                not isinstance(decision, dict)
+                and decision.type == "click"
+                and level < self.max_levels
+                and not (metadata or {}).get("ocr_direct", False)
+                and (
+                    self.always_refine
+                    or (
+                        self.center_click_zoom_epsilon is not None
+                        and abs(decision.target.offset_x - 0.5)
+                        <= self.center_click_zoom_epsilon
+                        and abs(decision.target.offset_y - 0.5)
+                        <= self.center_click_zoom_epsilon
+                    )
+                )
+            ):
+                decision = {
+                    "type": "zoom",
+                    "cell": decision.target.cell,
+                    "reason": (
+                        "fixed-refinement"
+                        if self.always_refine
+                        else "model-returned-suspicious-cell-center"
+                    ),
+                }
+
+            if isinstance(decision, dict):
+                display_crop = neighborhood_crop_box(
+                    view_clean.size,
+                    grid,
+                    decision["cell"],
+                    span_cells=self.span_cells,
+                )
+                next_view_box = _map_crop_to_original(
+                    display_crop, view_box, view_clean.size
+                )
+                levels.append(
+                    AdaptiveZoomLevel(
+                        level,
+                        view_box,
+                        dict(decision),
+                        next_view_box,
+                        metadata,
+                    )
+                )
+                view_box = next_view_box
+                view_clean = original.crop(view_box).resize(
+                    original.size, Image.Resampling.LANCZOS
+                )
+                view_grid = render_numbered_grid(view_clean, grid)
+                self.last_zoom_images.append((view_clean, view_grid))
+                continue
+
+            dumped = decision.model_dump(mode="json")
+            if decision.type == "done":
+                levels.append(
+                    AdaptiveZoomLevel(level, view_box, dumped, None, metadata)
+                )
+                final = decision
+            elif decision.type == "click":
+                local_mapper = GridMapper(
+                    Region(width=view_clean.width, height=view_clean.height), grid
+                )
+                local_point = local_mapper.to_screen(decision.target)
+                full_x, full_y = _map_point_to_original(
+                    local_point, view_box, view_clean.size
+                )
+                full_mapper = GridMapper(
+                    Region(width=original.width, height=original.height), grid
+                )
+                final_point = full_mapper.from_screen(full_x, full_y)
+                final = parse_action(
+                    {"type": "click", "target": final_point.model_dump(mode="json")}
+                )
+                levels.append(
+                    AdaptiveZoomLevel(level, view_box, dumped, None, metadata)
+                )
+            else:
+                raise RuntimeError("adaptive zoom currently supports click or done")
+
+            self.last_trace = AdaptiveZoomTrace(
+                tuple(levels), final.model_dump(mode="json")
+            )
+            self.last_metadata = {
+                "adaptive_zoom_levels": level,
+                "calls": call_metadata,
+                "total_elapsed_seconds": round(
+                    sum((item or {}).get("elapsed_seconds", 0) for item in call_metadata),
+                    3,
+                ),
+            }
+            return final
+
+        raise AssertionError("adaptive zoom loop ended without a decision")

vco/backends/apple_vision_ocr.swift

@@ -0,0 +1,73 @@
+import Foundation
+import ImageIO
+import Vision
+
+guard CommandLine.arguments.count >= 3 else {
+    fputs("usage: apple_vision_ocr.swift IMAGE fast|accurate [WORDS] [LANGUAGES]\n", stderr)
+    exit(2)
+}
+
+let path = CommandLine.arguments[1]
+let mode = CommandLine.arguments[2]
+let customWords = CommandLine.arguments.count >= 4
+    ? CommandLine.arguments[3].split(separator: ",").map(String.init)
+    : []
+let languages = CommandLine.arguments.count >= 5
+    ? CommandLine.arguments[4].split(separator: ",").map(String.init)
+    : ["zh-Hans", "en-US"]
+
+guard mode == "fast" || mode == "accurate" else {
+    fputs("mode must be fast or accurate\n", stderr)
+    exit(2)
+}
+guard let source = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil),
+      let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else {
+    fputs("cannot load image\n", stderr)
+    exit(2)
+}
+
+let request = VNRecognizeTextRequest()
+request.recognitionLevel = mode == "accurate" ? .accurate : .fast
+request.recognitionLanguages = languages
+request.usesLanguageCorrection = false
+request.minimumTextHeight = 0.005
+request.customWords = customWords
+
+let started = CFAbsoluteTimeGetCurrent()
+do {
+    try VNImageRequestHandler(cgImage: image, options: [:]).perform([request])
+} catch {
+    fputs("Vision request failed: \(error)\n", stderr)
+    exit(1)
+}
+let elapsedMs = (CFAbsoluteTimeGetCurrent() - started) * 1000
+
+let width = CGFloat(image.width)
+let height = CGFloat(image.height)
+let rows: [[String: Any]] = (request.results ?? []).compactMap { observation in
+    guard let candidate = observation.topCandidates(1).first else { return nil }
+    let box = observation.boundingBox
+    return [
+        "text": candidate.string,
+        "confidence": Double(candidate.confidence),
+        "bbox": [
+            Double(box.minX * width),
+            Double((1 - box.maxY) * height),
+            Double(box.maxX * width),
+            Double((1 - box.minY) * height),
+        ],
+    ]
+}
+
+let output: [String: Any] = [
+    "mode": mode,
+    "elapsed_ms": elapsedMs,
+    "image_size": [image.width, image.height],
+    "results": rows,
+]
+let data = try JSONSerialization.data(
+    withJSONObject: output,
+    options: [.prettyPrinted, .sortedKeys]
+)
+FileHandle.standardOutput.write(data)
+FileHandle.standardOutput.write(Data("\n".utf8))

vco/browser.py

@@ -0,0 +1,341 @@
+"""Headless page rendering and DOM interaction via Playwright."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class _PageLog:
+    """Collect console errors, uncaught exceptions, and failed requests."""
+
+    def __init__(self, limit: int = 20):
+        self.console_errors: list[str] = []
+        self.page_errors: list[str] = []
+        self.failed_requests: list[str] = []
+        self._limit = limit
+
+    def attach(self, page) -> None:
+        def on_console(message):
+            if message.type == "error" and len(self.console_errors) < self._limit:
+                self.console_errors.append(message.text)
+
+        def on_page_error(error):
+            if len(self.page_errors) < self._limit:
+                self.page_errors.append(str(error))
+
+        def on_request_failed(request):
+            if len(self.failed_requests) < self._limit:
+                failure = request.failure or ""
+                self.failed_requests.append(f"{request.method} {request.url} {failure}")
+
+        page.on("console", on_console)
+        page.on("pageerror", on_page_error)
+        page.on("requestfailed", on_request_failed)
+
+    def as_dict(self) -> dict:
+        return {
+            "console_errors": self.console_errors,
+            "page_errors": self.page_errors,
+            "failed_requests": self.failed_requests,
+        }
+
+
+def _open(pw, url, *, width, height, timeout, profile=None, log=None, headless=True,
+          record_dir=None):
+    """Launch (persistent when ``profile`` is set) and open ``url``."""
+    video = {}
+    if record_dir:
+        video["record_video_dir"] = str(record_dir)
+        video["record_video_size"] = {"width": width, "height": height}
+    if profile:
+        context = pw.chromium.launch_persistent_context(
+            profile,
+            headless=headless,
+            viewport={"width": width, "height": height},
+            **video,
+        )
+        browser = None
+    else:
+        browser = pw.chromium.launch(headless=headless)
+        context = browser.new_context(
+            viewport={"width": width, "height": height}, **video
+        )
+    page = context.pages[0] if context.pages else context.new_page()
+    if log is not None:
+        log.attach(page)
+    page.goto(url, timeout=timeout * 1000)
+    page.wait_for_load_state("load", timeout=timeout * 1000)
+    return browser, context, page
+
+
+def _close(browser, context, page=None, video_path: str | None = None) -> str | None:
+    """Close and, when recording, move the video to ``video_path``."""
+    context.close()
+    saved = None
+    if page is not None and video_path is not None and page.video is not None:
+        original = page.video.path()
+        page.video.save_as(video_path)
+        Path(original).unlink(missing_ok=True)
+        saved = video_path
+    if browser is not None:
+        browser.close()
+    return saved
+
+
+def _load_pw():
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError as exc:
+        raise RuntimeError("requires: pip install playwright") from exc
+    return sync_playwright
+
+
+def screenshot(
+    url: str,
+    output: str,
+    *,
+    full_page: bool = False,
+    width: int = 1280,
+    height: int = 800,
+    timeout: float = 15.0,
+    settle: float = 0.0,
+    profile: str | None = None,
+    headless: bool = True,
+    hold: float = 0.0,
+    record: str | None = None,
+) -> dict:
+    """Render ``url`` in Chromium and save a screenshot."""
+    sync_playwright = _load_pw()
+    log = _PageLog()
+    with sync_playwright() as pw:
+        record_dir = None if record is None else str(Path(record).parent)
+        browser, context, page = _open(
+            pw, url, width=width, height=height, timeout=timeout, profile=profile,
+            log=log, headless=headless, record_dir=record_dir,
+        )
+        try:
+            if settle > 0:
+                page.wait_for_timeout(int(settle * 1000))
+            page.screenshot(path=output, full_page=full_page)
+            result = {
+                "path": output,
+                "url": page.url,
+                "title": page.title(),
+                "viewport": {"width": width, "height": height},
+                "full_page": full_page,
+                "headless": headless,
+            }
+            result.update(log.as_dict())
+            if hold > 0:
+                page.wait_for_timeout(int(hold * 1000))
+            return result
+        finally:
+            video = _close(browser, context, page=page, video_path=record)
+            if video:
+                result["video"] = video
+
+
+def _apply_fills(page, fills: list[str], visible: bool = False) -> list[str]:
+    """Fill inputs as ``placeholder=text`` pairs; returns error strings.
+
+    With ``visible=True`` types character by character so a human watching a
+    headed browser can see the text appear.
+    """
+    errors = []
+    for item in fills:
+        if "=" not in item:
+            errors.append(f"invalid --fill {item!r}; expected placeholder=text")
+            continue
+        placeholder, text = item.split("=", 1)
+        locator = page.get_by_placeholder(placeholder, exact=True)
+        if locator.count() == 0:
+            locator = page.get_by_placeholder(placeholder)
+        if locator.count() != 1:
+            errors.append(
+                f"--fill {placeholder!r}: expected 1 input, got {locator.count()}"
+            )
+            continue
+        if visible:
+            locator.first.click()
+            locator.first.press_sequentially(text, delay=90)
+        else:
+            locator.first.fill(text)
+    return errors
+
+
+def _flash_ring(page, cx: float, cy: float, radius: float) -> None:
+    """Draw a temporary orange halo at (cx, cy): transparent core, dense rim."""
+    page.evaluate(
+        """([x, y, r]) => {
+            const el = document.createElement('div');
+            el.style.cssText = 'position:fixed;pointer-events:none;z-index:2147483647;'
+                + 'left:' + (x - r) + 'px;top:' + (y - r) + 'px;'
+                + 'width:' + (2 * r) + 'px;height:' + (2 * r) + 'px;'
+                + 'border-radius:50%;'
+                + 'background:radial-gradient(circle,'
+                + ' rgba(255,140,0,0) 30%, rgba(255,140,0,0.85) 55%,'
+                + ' rgba(255,140,0,0.30) 75%, rgba(255,140,0,0) 100%);';
+            document.body.appendChild(el);
+            setTimeout(() => el.remove(), 1400);
+        }""",
+        [cx, cy, radius],
+    )
+
+
+def click(
+    url: str,
+    target: str | None,
+    *,
+    selector: str | None = None,
+    contains: bool = False,
+    fills: list[str] | None = None,
+    width: int = 1280,
+    height: int = 800,
+    timeout: float = 15.0,
+    settle: float = 0.5,
+    profile: str | None = None,
+    headless: bool = True,
+    hold: float = 0.0,
+    expect: str | None = None,
+    expect_timeout: float = 10.0,
+    record: str | None = None,
+    before_path: str | None = None,
+    after_path: str | None = None,
+) -> dict:
+    """Open ``url``, optionally fill inputs, DOM-click a unique ``target``."""
+    sync_playwright = _load_pw()
+    log = _PageLog()
+    with sync_playwright() as pw:
+        record_dir = None if record is None else str(Path(record).parent)
+        browser, context, page = _open(
+            pw, url, width=width, height=height, timeout=timeout, profile=profile,
+            log=log, headless=headless, record_dir=record_dir,
+        )
+        try:
+            if before_path:
+                page.screenshot(path=before_path, full_page=False)
+
+            demo = not headless or record is not None
+            fill_errors = _apply_fills(page, fills or [], visible=demo)
+
+            if selector:
+                locator = page.locator(selector)
+            else:
+                locator = page.get_by_text(target, exact=True)
+            count = locator.count()
+            if count == 0 and not selector and contains:
+                locator = page.get_by_text(target)
+                count = locator.count()
+            candidates = []
+            for i in range(min(count, 8)):
+                item = locator.nth(i)
+                box = item.bounding_box()
+                candidates.append(
+                    {
+                        "text": (item.text_content() or "").strip(),
+                        "bbox": (
+                            None
+                            if box is None
+                            else [
+                                box["x"],
+                                box["y"],
+                                box["x"] + box["width"],
+                                box["y"] + box["height"],
+                            ]
+                        ),
+                    }
+                )
+            result = {
+                "clicked": False,
+                "url": page.url,
+                "target": selector or target,
+                "candidate_count": count,
+                "candidates": candidates,
+                "fill_errors": fill_errors,
+                "before": before_path,
+                "after": None,
+                "x": None,
+                "y": None,
+            }
+            if count != 1:
+                result["error"] = "no match" if count == 0 else f"ambiguous: {count} matches"
+                result.update(log.as_dict())
+                return result
+            box = locator.first.bounding_box()
+            if demo and box is not None:
+                page.wait_for_timeout(400)
+                _flash_ring(
+                    page,
+                    box["x"] + box["width"] / 2,
+                    box["y"] + box["height"] / 2,
+                    max(14.0, min(32.0, float(min(box["width"], box["height"])))),
+                )
+                page.wait_for_timeout(900)
+            locator.first.click()
+            page.wait_for_timeout(int(settle * 1000))
+            verified = None
+            if expect:
+                try:
+                    page.get_by_text(expect).first.wait_for(
+                        state="visible", timeout=int(expect_timeout * 1000)
+                    )
+                    verified = True
+                except Exception:
+                    verified = False
+            if after_path:
+                page.screenshot(path=after_path, full_page=False)
+            result.update(
+                {
+                    "clicked": True,
+                    "x": None if box is None else round(box["x"] + box["width"] / 2),
+                    "y": None if box is None else round(box["y"] + box["height"] / 2),
+                    "after": after_path,
+                    "expect": expect,
+                    "verified": verified,
+                }
+            )
+            result.update(log.as_dict())
+            if hold > 0:
+                page.wait_for_timeout(int(hold * 1000))
+            return result
+        finally:
+            video = _close(browser, context, page=page, video_path=record)
+            if video:
+                result["video"] = video
+
+
+def snapshot(
+    url: str,
+    *,
+    width: int = 1280,
+    height: int = 800,
+    timeout: float = 15.0,
+    settle: float = 0.0,
+    profile: str | None = None,
+    headless: bool = True,
+    hold: float = 0.0,
+) -> dict:
+    """Return the page's accessibility tree as text (no vision model needed)."""
+    sync_playwright = _load_pw()
+    log = _PageLog()
+    with sync_playwright() as pw:
+        browser, context, page = _open(
+            pw, url, width=width, height=height, timeout=timeout, profile=profile,
+            log=log, headless=headless,
+        )
+        try:
+            if settle > 0:
+                page.wait_for_timeout(int(settle * 1000))
+            tree = page.locator("body").aria_snapshot()
+            result = {
+                "url": page.url,
+                "title": page.title(),
+                "snapshot": tree,
+                "headless": headless,
+            }
+            result.update(log.as_dict())
+            if hold > 0:
+                page.wait_for_timeout(int(hold * 1000))
+            return result
+        finally:
+            _close(browser, context)