pctr 0.1.0__tar.gz → 0.2.0__tar.gz

{pctr-0.1.0/src/pctr.egg-info → pctr-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pctr
-Version: 0.1.0
+Version: 0.2.0
 Summary: Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates.
 Author: Space-lab515
 License-Expression: MIT

{pctr-0.1.0 → pctr-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pctr"
-version = "0.1.0"
+version = "0.2.0"
 description = "Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -36,3 +36,6 @@ pctr = "pctr.cli:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]
+
+[tool.setuptools.package-data]
+pctr = ["data/*.md"]

{pctr-0.1.0 → pctr-0.2.0}/src/pctr/cli.py

@@ -314,6 +314,223 @@ def cmd_drag(args):
     print("dragged %d,%d -> %d,%d" % (x1, y1, x2, y2))
 
 
+_OCR_PS = r'''
+param([string]$Path)
+Add-Type -AssemblyName System.Runtime.WindowsRuntime
+$asTaskGeneric = ([System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object { $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1' })[0]
+function Await($op, $type) {
+  $m = $asTaskGeneric.MakeGenericMethod($type)
+  $t = $m.Invoke($null, @($op))
+  $t.Wait(-1) | Out-Null
+  $t.Result
+}
+$null = [Windows.Storage.StorageFile,Windows.Storage,ContentType=WindowsRuntime]
+$null = [Windows.Media.Ocr.OcrEngine,Windows.Foundation,ContentType=WindowsRuntime]
+$null = [Windows.Graphics.Imaging.BitmapDecoder,Windows.Foundation,ContentType=WindowsRuntime]
+$file = Await ([Windows.Storage.StorageFile]::GetFileFromPathAsync($Path)) ([Windows.Storage.StorageFile])
+$stream = Await ($file.OpenAsync([Windows.Storage.FileAccessMode]::Read)) ([Windows.Storage.Streams.IRandomAccessStream])
+$decoder = Await ([Windows.Graphics.Imaging.BitmapDecoder]::CreateAsync($stream)) ([Windows.Graphics.Imaging.BitmapDecoder])
+$bitmap = Await ($decoder.GetSoftwareBitmapAsync()) ([Windows.Graphics.Imaging.SoftwareBitmap])
+$engine = [Windows.Media.Ocr.OcrEngine]::TryCreateFromUserProfileLanguages()
+if (-not $engine) {
+  $null = [Windows.Globalization.Language,Windows.Foundation,ContentType=WindowsRuntime]
+  $engine = [Windows.Media.Ocr.OcrEngine]::TryCreateFromLanguage((New-Object Windows.Globalization.Language 'en-US'))
+}
+if (-not $engine) { Write-Error 'no OCR engine available'; exit 2 }
+$res = Await ($engine.RecognizeAsync($bitmap)) ([Windows.Media.Ocr.OcrResult])
+$out = foreach ($line in $res.Lines) { foreach ($w in $line.Words) { [pscustomobject]@{ text = $w.Text; x = [int]$w.BoundingRect.X; y = [int]$w.BoundingRect.Y; w = [int]$w.BoundingRect.Width; h = [int]$w.BoundingRect.Height } } }
+ConvertTo-Json -Compress -InputObject @($out)
+'''
+
+
+def _windows_ocr(image_path):
+    import json
+    import os
+    import subprocess
+    import tempfile
+    script = os.path.join(tempfile.gettempdir(), "pctr_ocr.ps1")
+    with open(script, "w", encoding="utf-8") as f:
+        f.write(_OCR_PS)
+    out = subprocess.run(
+        ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script, "-Path", image_path],
+        capture_output=True, text=True,
+    )
+    txt = (out.stdout or "").strip()
+    if not txt:
+        raise SystemExit("OCR failed: " + (out.stderr or "").strip()[:400])
+    data = json.loads(txt, strict=False)
+    if isinstance(data, dict):
+        data = [data]
+    return [w for w in data if w and w.get("text")]
+
+
+def _ocr_capture(args):
+    import os
+    import tempfile
+    import pyautogui
+    path = os.path.join(tempfile.gettempdir(), "pctr_ocr.png")
+    ox = oy = 0
+    title = getattr(args, "title", None)
+    if title:
+        win = _find_window(title, getattr(args, "process", None), getattr(args, "timeout", 5.0))
+        r = win.rectangle()
+        img = pyautogui.screenshot(region=(r.left, r.top, r.width(), r.height()))
+        ox, oy = r.left, r.top
+    else:
+        img = pyautogui.screenshot()
+    img.save(path)
+    return _windows_ocr(path), ox, oy
+
+
+def cmd_ocr(args):
+    words, ox, oy = _ocr_capture(args)
+    for w in words:
+        print("%s | %d,%d %dx%d" % (w["text"], ox + w["x"], oy + w["y"], w["w"], w["h"]))
+    print("(%d words)" % len(words))
+
+
+def cmd_ocrfind(args):
+    import re
+    words, ox, oy = _ocr_capture(args)
+    rx = re.compile(args.text, re.I)
+    hits = [w for w in words if rx.search(w["text"])]
+    for i, w in enumerate(hits):
+        print("[%d] %s | center=%d,%d" % (i, w["text"], ox + w["x"] + w["w"] // 2, oy + w["y"] + w["h"] // 2))
+    if not hits:
+        print("no OCR match for %r" % args.text)
+
+
+def cmd_ocrclick(args):
+    import re
+    import pyautogui
+    words, ox, oy = _ocr_capture(args)
+    rx = re.compile(args.text, re.I)
+    hits = [w for w in words if rx.search(w["text"])]
+    if not hits:
+        raise SystemExit("no OCR match for %r" % args.text)
+    idx = getattr(args, "nth", 0)
+    if idx >= len(hits):
+        raise SystemExit("matched %d word(s); nth=%d out of range" % (len(hits), idx))
+    w = hits[idx]
+    cx = ox + w["x"] + w["w"] // 2
+    cy = oy + w["y"] + w["h"] // 2
+    pyautogui.FAILSAFE = False
+    pyautogui.moveTo(cx, cy, duration=0.0)
+    pyautogui.click()
+    print("ocr-clicked %r at %d,%d" % (w["text"], cx, cy))
+
+
+def _look_detect(args):
+    import os
+    import tempfile
+    import pyautogui
+    try:
+        from ultralytics.models import YOLOWorld
+    except Exception as e:
+        raise SystemExit("YOLO-World needs ultralytics: pip install ultralytics (%s)" % e)
+    path = os.path.join(tempfile.gettempdir(), "pctr_look.png")
+    ox = oy = 0
+    title = getattr(args, "title", None)
+    if title:
+        win = _find_window(title, getattr(args, "process", None), getattr(args, "timeout", 5.0))
+        r = win.rectangle()
+        img = pyautogui.screenshot(region=(r.left, r.top, r.width(), r.height()))
+        ox, oy = r.left, r.top
+    else:
+        img = pyautogui.screenshot()
+    img.save(path)
+    model = YOLOWorld(getattr(args, "model", None) or "yolov8s-worldv2.pt")
+    model.set_classes([args.for_])
+    conf = args.conf if getattr(args, "conf", None) is not None else 0.35
+    results = model.predict(path, conf=conf, agnostic_nms=True, verbose=False)
+    r0 = results[0]
+    names = r0.names
+    boxes = r0.boxes
+    dets = []
+    if boxes is None:
+        return dets
+    for i in range(len(boxes)):
+        x1, y1, x2, y2 = [float(v) for v in boxes.xyxy[i].cpu().numpy()]
+        dets.append({
+            "label": names.get(int(boxes.cls[i]), str(int(boxes.cls[i]))) if isinstance(names, dict) else str(int(boxes.cls[i])),
+            "conf": float(boxes.conf[i].cpu().numpy()),
+            "x": int(ox + (x1 + x2) / 2),
+            "y": int(oy + (y1 + y2) / 2),
+            "box": (int(ox + x1), int(oy + y1), int(ox + x2), int(oy + y2)),
+        })
+    dets.sort(key=lambda d: -d["conf"])
+    return dets
+
+
+def cmd_look(args):
+    dets = _look_detect(args)
+    for i, d in enumerate(dets):
+        print("[%d] %s %.2f | center=%d,%d | box=%d,%d,%d,%d" % (i, d["label"], d["conf"], d["x"], d["y"], *d["box"]))
+    if not dets:
+        print("nothing found for %r" % args.for_)
+
+
+def cmd_lookclick(args):
+    import pyautogui
+    dets = _look_detect(args)
+    if not dets:
+        raise SystemExit("nothing found for %r" % args.for_)
+    idx = getattr(args, "nth", 0)
+    if idx >= len(dets):
+        raise SystemExit("found %d; nth=%d out of range" % (len(dets), idx))
+    d = dets[idx]
+    pyautogui.FAILSAFE = False
+    pyautogui.moveTo(d["x"], d["y"], duration=0.0)
+    pyautogui.click()
+    print("look-clicked %r (%.2f) at %d,%d" % (d["label"], d["conf"], d["x"], d["y"]))
+
+
+SETUP_TARGETS = {
+    "opencode": (".config/opencode/skills/pctr", "SKILL.md"),
+    "claude": (".claude/skills/pctr", "SKILL.md"),
+    "agents": (".agents/skills/pctr", "SKILL.md"),
+}
+
+
+def _data_text(name):
+    from importlib import resources
+    return resources.files("pctr").joinpath("data", name).read_text(encoding="utf-8")
+
+
+def cmd_skill(args):
+    if getattr(args, "install", False):
+        return cmd_setup(args)
+    print(_data_text("SKILL.md"))
+
+
+def cmd_setup(args):
+    import os
+    from pathlib import Path
+    skill = _data_text("SKILL.md")
+    frag = _data_text("AGENTS.md")
+    home = Path.home()
+    targets = list(SETUP_TARGETS) if getattr(args, "target", "all") in (None, "all") else [args.target]
+    for t in targets:
+        rel, fname = SETUP_TARGETS[t]
+        d = home / rel
+        d.mkdir(parents=True, exist_ok=True)
+        (d / fname).write_text(skill, encoding="utf-8")
+        print("installed skill: %s" % (d / fname))
+    proj = Path(getattr(args, "project", None) or os.getcwd())
+    ag = proj / "AGENTS.md"
+    existing = ag.read_text(encoding="utf-8", errors="replace") if ag.exists() else ""
+    marker = "<!-- pctr -->"
+    if marker in existing:
+        print("AGENTS.md already has a pctr section: %s" % ag)
+    else:
+        with open(ag, "a", encoding="utf-8") as f:
+            if existing and not existing.endswith("\n"):
+                f.write("\n")
+            f.write("\n" + frag + "\n")
+        print("appended pctr section to: %s" % ag)
+    print("done. restart your agent so it loads the skill.")
+
+
 def add_target(p, require_name=False):
     p.add_argument("--title", required=require_name, help="Window title regex")
     p.add_argument("--process", type=int, default=None, help="Filter by process id")
@@ -445,6 +662,57 @@ def build_parser():
     dr.add_argument("--direct", action="store_true")
     dr.set_defaults(func=cmd_drag)
 
+    oc = sub.add_parser("ocr", help="OCR the screen (or a window) and list words with boxes")
+    oc.add_argument("--title", default=None)
+    oc.add_argument("--process", type=int, default=None)
+    oc.add_argument("--timeout", type=float, default=5.0)
+    oc.set_defaults(func=cmd_ocr)
+
+    ocf = sub.add_parser("ocrfind", help="OCR then list words matching --text (with click centers)")
+    ocf.add_argument("--title", default=None)
+    ocf.add_argument("--process", type=int, default=None)
+    ocf.add_argument("--timeout", type=float, default=5.0)
+    ocf.add_argument("--text", required=True)
+    ocf.set_defaults(func=cmd_ocrfind)
+
+    occ = sub.add_parser("ocrclick", help="OCR then click the word matching --text")
+    occ.add_argument("--title", default=None)
+    occ.add_argument("--process", type=int, default=None)
+    occ.add_argument("--timeout", type=float, default=5.0)
+    occ.add_argument("--text", required=True)
+    occ.add_argument("--nth", type=int, default=0)
+    occ.set_defaults(func=cmd_ocrclick)
+
+    lk = sub.add_parser("look", help="Open-vocab detect objects by text prompt (YOLO-World)")
+    lk.add_argument("--for", dest="for_", required=True, help="object to look for, e.g. 'save button'")
+    lk.add_argument("--title", default=None)
+    lk.add_argument("--process", type=int, default=None)
+    lk.add_argument("--timeout", type=float, default=5.0)
+    lk.add_argument("--conf", type=float, default=None)
+    lk.add_argument("--model", default=None)
+    lk.set_defaults(func=cmd_look)
+
+    lkc = sub.add_parser("lookclick", help="Open-vocab detect then click the top match")
+    lkc.add_argument("--for", dest="for_", required=True, help="object to look for")
+    lkc.add_argument("--title", default=None)
+    lkc.add_argument("--process", type=int, default=None)
+    lkc.add_argument("--timeout", type=float, default=5.0)
+    lkc.add_argument("--conf", type=float, default=None)
+    lkc.add_argument("--model", default=None)
+    lkc.add_argument("--nth", type=int, default=0)
+    lkc.set_defaults(func=cmd_lookclick)
+
+    sk = sub.add_parser("skill", help="Print the agent skill (SKILL.md), or --install it")
+    sk.add_argument("--install", action="store_true")
+    sk.add_argument("--target", default="all", choices=["all", "opencode", "claude", "agents"])
+    sk.add_argument("--project", default=None)
+    sk.set_defaults(func=cmd_skill)
+
+    st = sub.add_parser("setup", help="Install the pctr skill for your agents + append to AGENTS.md")
+    st.add_argument("--target", default="all", choices=["all", "opencode", "claude", "agents"])
+    st.add_argument("--project", default=None)
+    st.set_defaults(func=cmd_setup)
+
     return ap
 
 

pctr-0.2.0/src/pctr/data/AGENTS.md

@@ -0,0 +1,17 @@
+<!-- pctr -->
+## Desktop automation (pctr)
+
+`pctr` is installed. Use it to drive Windows GUI apps by **element name**, not
+pixel coordinates:
+
+```bash
+pctr windows                         # list windows
+pctr tree  --title "App" --limit 100 # inspect controls
+pctr click --title "App" --name "Save"
+pctr set   --title "App" --name "Search" --text "hello"
+pctr ocrfind --text "Sign in"        # fallback: read text off the screen
+pctr look --for "a red error icon"   # fallback: open-vocab object detection
+```
+
+Prefer `set` over `type` for fields. Global `type`/`keys` need `pctr focus` first.
+Run `pctr -h` for all commands. Full skill: `pctr skill`.

pctr-0.2.0/src/pctr/data/SKILL.md

@@ -0,0 +1,81 @@
+---
+name: pctr
+description: Drive native Windows desktop apps (not browsers) with the pctr CLI powered by pywinauto/UIA, with OCR and YOLO-World fallbacks. Use whenever you must control or inspect a desktop GUI - list windows, dump a control tree, click buttons, type into fields, read on-screen text by OCR, find objects by description, send hotkeys, hold keys, drag, or screenshot an app. Element-based, no pixel hunting. Triggers: pctr, UIA, UI automation, pywinauto, desktop automation, click a button in an app, type into an app, drive a GUI, control a Windows application, usecomputer.
+---
+
+# pctr - desktop control
+
+Element-based Windows automation. Find a control by its name / control type /
+automation id and act on it, so window moves, DPI, and layout changes don't break
+you. Windows only (UI Automation). Built on `pywinauto` + `uiautomation`, with
+`pyautogui` / `pydirectinput` for raw mouse and keyboard.
+
+## Three ways to find a target
+
+1. **UIA elements** (best) - `tree` / `find` / `click` by control name/id.
+2. **OCR** - `ocr` / `ocrfind` / `ocrclick` (Windows built-in `Windows.Media.Ocr`, no deps) reads on-screen text that isn't exposed as a control.
+3. **YOLO-World** - `look --for "..."` / `lookclick --for "..."` open-vocabulary object detection by natural-language prompt (canvas, games, custom-drawn UI).
+
+## The loop
+
+```bash
+pctr windows                                   # top-level windows + pids
+pctr tree   --title "Notepad" --limit 120      # dump the control tree
+pctr find   --title "Notepad" --name "Submit"
+pctr click  --title "Notepad" --name "^File$" --control-type MenuItem
+pctr shot   --title "Notepad" --out "C:\temp\np.png"
+```
+
+Look at `tree`/`find` output first - the name/type/id it prints is what you pass
+back as `--name`, `--control-type`, `--auto-id`.
+
+## Commands
+
+| Command | What it does |
+|---------|--------------|
+| `windows [--filter RE]` | List top-level windows. |
+| `tree --title RE [--limit N]` | Dump the UIA control tree. |
+| `find --title RE --name RE [--control-type T] [--auto-id ID]` | List matching elements. |
+| `click --title RE --name RE [--method auto\|invoke\|mouse] [--dbl]` | Click an element. |
+| `set --title RE --name RE --text S` | Set a field via the UIA ValuePattern (exact, instant). |
+| `type [--title RE --name RE] --text S [--delay 0.03] [--chunk 1]` | Type literal text. |
+| `keys --keys "{ENTER}"` | Global key combo (pywinauto send_keys syntax). |
+| `hotkey --keys win+shift+s` | Modifier combo. |
+| `focus --title RE` | Bring a window to the foreground. |
+| `wait --title RE --name RE [--timeout S]` | Wait for an element. |
+| `shot [--title RE] --out PATH` | Screenshot the screen or a window. |
+| `size` | Print primary screen size as WxH. |
+| `move --x N --y N` / `down` / `up` / `hold --ms N` | Raw mouse control. |
+| `drag --start x,y --end x,y [--duration S]` | Drag between points. |
+| `keydown / keyup --key a` / `keyhold --key w --ms 1500` | Hold keyboard keys. |
+| `ocr [--title RE]` | OCR the screen (or window); list words with boxes. |
+| `ocrfind --text RE` | OCR then list matches with click centers. |
+| `ocrclick --text RE [--nth N]` | OCR then click a word. |
+| `look --for "a dog" [--title RE] [--conf X]` | YOLO-World detect objects by text prompt. |
+| `lookclick --for "a dog" [--nth N]` | Detect then click the top match. |
+| `skill [--install]` / `setup` | Print/install this skill + an AGENTS.md section. |
+
+Filters: `--title` (regex), `--name` (regex), `--control-type`, `--auto-id`,
+`--nth`, `--process`, `--timeout`.
+
+## Patterns
+
+- **Text fields -> use `set`, not `type`.** `set` uses the ValuePattern: atomic, can't drop chars.
+- **`type` is char-by-char** (`--chunk 1 --delay 0.03`); raise `--delay` for Qt/Electron.
+- **Global `type`/`keys` go to the OS-focused window** - `pctr focus --title ...` first.
+- Click falls back to a real mouse click when a control has no invoke pattern.
+- Disambiguate with `--control-type` / `--auto-id` / `--nth`.
+
+## App notes
+
+- **Qt apps** expose a rich tree including embedded webviews.
+- **Electron apps** (Discord, VS Code, OpenCode) hide web content - use `focus` +
+  global `type`/`keys`, verify with `shot` (and fall back to `ocr`/`look`).
+- **Canvas / games / custom UI**: no tree - use `ocr` for text and `look` for objects.
+
+## Setup
+
+```bash
+pip install pctr
+pctr setup        # installs this skill for opencode / claude / agents + appends to ./AGENTS.md
+```

{pctr-0.1.0 → pctr-0.2.0/src/pctr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pctr
-Version: 0.1.0
+Version: 0.2.0
 Summary: Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates.
 Author: Space-lab515
 License-Expression: MIT

{pctr-0.1.0 → pctr-0.2.0}/src/pctr.egg-info/SOURCES.txt

@@ -9,4 +9,6 @@ src/pctr.egg-info/SOURCES.txt
 src/pctr.egg-info/dependency_links.txt
 src/pctr.egg-info/entry_points.txt
 src/pctr.egg-info/requires.txt
-src/pctr.egg-info/top_level.txt
+src/pctr.egg-info/top_level.txt
+src/pctr/data/AGENTS.md
+src/pctr/data/SKILL.md