pctr 0.2.1__tar.gz → 0.2.2__tar.gz

{pctr-0.2.1/src/pctr.egg-info → pctr-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pctr
-Version: 0.2.1
+Version: 0.2.2
 Summary: Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates.
 Author: Space-lab515
 License-Expression: MIT

{pctr-0.2.1 → pctr-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pctr"
-version = "0.2.1"
+version = "0.2.2"
 description = "Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates."
 readme = "README.md"
 requires-python = ">=3.10"

pctr-0.2.2/src/pctr/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.2"

{pctr-0.2.1 → pctr-0.2.2}/src/pctr/cli.py

@@ -2,6 +2,7 @@ import argparse
 import re
 import sys
 import time
+from typing import NoReturn
 
 for _stream in (sys.stdout, sys.stderr):
     _reconf = getattr(_stream, "reconfigure", None)
@@ -16,11 +17,26 @@ from pywinauto import Desktop
 DESCRIPTION = "Element-based Windows UI Automation CLI for AI agents."
 
 
+def _fail(msg, code=2) -> NoReturn:
+    """Print a clean one-line error and exit non-zero (no traceback)."""
+    print("error: %s" % msg, file=sys.stderr)
+    raise SystemExit(code)
+
+
+def _compile(pattern, what, flags=re.I):
+    """Compile a user-supplied regex, failing cleanly on a bad pattern."""
+    try:
+        return re.compile(pattern, flags)
+    except re.error as e:
+        _fail("invalid %s regex %r: %s" % (what, pattern, e))
+
+
 def _iter_windows():
     return Desktop(backend="uia").windows()
 
 
 def _find_window(title_re, process=None, timeout=5.0, index=0):
+    rx_title = _compile(title_re, "title") if title_re else None
     desktop = Desktop(backend="uia")
     deadline = time.time() + timeout
     last = None
@@ -31,7 +47,7 @@ def _find_window(title_re, process=None, timeout=5.0, index=0):
                 t = w.window_text()
             except Exception:
                 continue
-            if title_re and not re.search(title_re, t, re.I):
+            if rx_title and not rx_title.search(t):
                 continue
             if process is not None and w.process_id() != process:
                 continue
@@ -47,15 +63,16 @@ def _find_window(title_re, process=None, timeout=5.0, index=0):
 
 
 def _matches(win, name=None, control_type=None, auto_id=None):
+    rx_name = _compile(name, "name") if name is not None else None
     out = []
     for el in win.descendants():
         try:
             info = el.element_info
         except Exception:
             continue
-        if name is not None:
+        if rx_name is not None:
             n = info.name or ""
-            if re.search(name, n, re.I) is None:
+            if rx_name.search(n) is None:
                 continue
         if control_type is not None:
             c = (info.control_type or "")
@@ -99,27 +116,56 @@ def _get_element(args):
 
 
 def cmd_windows(args):
+    rx = _compile(args.filter, "filter") if args.filter else None
+    proc = getattr(args, "process", None)
     for w in _iter_windows():
         t = ""
         try:
             t = w.window_text()
         except Exception:
             pass
-        if args.filter and re.search(args.filter, t, re.I) is None:
+        if rx and rx.search(t) is None:
+            continue
+        if proc is not None and w.process_id() != proc:
             continue
         print("pid=%s | %s" % (w.process_id(), t))
 
 
+def _walk(root, max_depth):
+    """Yield (element, depth) for descendants of root, bounded to max_depth."""
+    stack = []
+    try:
+        kids = root.children()
+    except Exception:
+        kids = []
+    for k in reversed(kids):
+        stack.append((k, 1))
+    while stack:
+        node, d = stack.pop()
+        yield node, d
+        if d >= max_depth:
+            continue
+        try:
+            kids = node.children()
+        except Exception:
+            kids = []
+        for k in reversed(kids):
+            stack.append((k, d + 1))
+
+
 def cmd_tree(args):
     win = _find_window(args.title, args.process, args.timeout)
     print("WINDOW: %s (pid=%s)" % (win.window_text(), win.process_id()))
     count = 0
-    for el in win.descendants():
+    depth = getattr(args, "depth", None)
+    depth_mode = depth is not None and 0 <= depth < 99
+    it = _walk(win, depth) if depth_mode else ((el, 1) for el in win.descendants())
+    for el, d in it:
         try:
             el.element_info
         except Exception:
             continue
-        print("  %s" % _describe(el))
+        print("  %s%s" % ("  " * (d - 1) if depth_mode else "", _describe(el)))
         count += 1
         if count >= args.limit:
             print("... (limit %d reached)" % args.limit)
@@ -131,10 +177,19 @@ def cmd_find(args):
     els = _matches(win, args.name, args.control_type, args.auto_id)
     if not els:
         print("no matches")
-        return
+        raise SystemExit(1)
+    nth = getattr(args, "nth", 0) or 0
+    if nth > 0:
+        if nth >= len(els):
+            _fail("matched %d element(s); nth=%d out of range" % (len(els), nth))
+        els = [els[nth]]
     print("WINDOW: %s" % win.window_text())
-    for i, el in enumerate(els):
+    limit = getattr(args, "limit", None)
+    shown = els[:limit] if limit else els
+    for i, el in enumerate(shown):
         print("[%d] %s" % (i, _describe(el)))
+    if limit and len(els) > limit:
+        print("... (%d more)" % (len(els) - limit))
 
 
 def cmd_click(args):
@@ -160,10 +215,13 @@ def cmd_set(args):
         el.set_edit_text(args.text)
         print("set via ValuePattern")
     except Exception:
-        el.click_input()
-        el.type_keys("^a{BACKSPACE}")
-        el.type_keys(args.text, with_spaces=True, with_newlines=True)
-        print("set via type_keys")
+        try:
+            el.click_input()
+            el.type_keys("^a{BACKSPACE}")
+            el.type_keys(args.text, with_spaces=True, with_newlines=True)
+            print("set via type_keys")
+        except Exception as e:
+            _fail("set failed (no ValuePattern and text has pywinauto key syntax): %s" % e)
 
 
 def _chunks(text, size):
@@ -171,13 +229,16 @@ def _chunks(text, size):
 
 
 def cmd_type(args):
-    delay = getattr(args, "delay", 0.02)
-    chunk = max(1, getattr(args, "chunk", 10))
+    delay = args.delay
+    chunk = max(1, args.chunk)
     if args.title:
         win, el, n = _get_element(args)
-        for part in _chunks(args.text, chunk):
-            el.type_keys(part, with_spaces=True, with_newlines=True)
-            time.sleep(delay)
+        try:
+            for part in _chunks(args.text, chunk):
+                el.type_keys(part, with_spaces=True, with_newlines=True)
+                time.sleep(delay)
+        except Exception as e:
+            _fail("type failed (text has pywinauto key syntax like { } + ^ %% ~ ()): %s" % e)
     else:
         import pyautogui
         pyautogui.write(args.text, interval=delay)
@@ -218,7 +279,10 @@ def cmd_shot(args):
         img = pyautogui.screenshot(region=(r.left, r.top, r.width(), r.height()))
     else:
         img = pyautogui.screenshot()
-    img.save(args.out)
+    try:
+        img.save(args.out)
+    except (OSError, ValueError) as e:
+        _fail("cannot write screenshot to %r: %s" % (args.out, e))
     print("saved %s (%dx%d)" % (args.out, img.width, img.height))
 
 
@@ -233,8 +297,11 @@ def _mouse_backend(args):
 
 
 def _parse_pair(text):
-    x, y = text.split(",")
-    return int(float(x)), int(float(y))
+    try:
+        x, y = text.split(",")
+        return int(float(x)), int(float(y))
+    except (ValueError, AttributeError):
+        _fail("bad coordinate pair %r (expected 'x,y')" % text)
 
 
 def cmd_move(args):
@@ -343,17 +410,29 @@ ConvertTo-Json -Compress -InputObject @($out)
 '''
 
 
+def _pctr_tempdir():
+    import os
+    import tempfile
+    d = os.path.join(tempfile.gettempdir(), "pctr")
+    os.makedirs(d, exist_ok=True)
+    return d
+
+
 def _windows_ocr(image_path):
+    import base64
     import json
     import os
     import subprocess
-    import tempfile
-    script = os.path.join(tempfile.gettempdir(), "pctr_ocr.ps1")
-    with open(script, "w", encoding="utf-8") as f:
-        f.write(_OCR_PS)
+    body = _OCR_PS.strip()
+    if body.startswith("param"):  # drop the param() line; path comes via env
+        body = body.split("\n", 1)[1]
+    ps = "$Path = $env:PCTR_OCR_PATH\n" + body
+    enc = base64.b64encode(ps.encode("utf-16-le")).decode("ascii")
+    env = dict(os.environ)
+    env["PCTR_OCR_PATH"] = image_path
     out = subprocess.run(
-        ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script, "-Path", image_path],
-        capture_output=True, text=True,
+        ["powershell", "-NoProfile", "-NonInteractive", "-EncodedCommand", enc],
+        capture_output=True, text=True, env=env,
     )
     txt = (out.stdout or "").strip()
     if not txt:
@@ -366,9 +445,8 @@ def _windows_ocr(image_path):
 
 def _ocr_capture(args):
     import os
-    import tempfile
     import pyautogui
-    path = os.path.join(tempfile.gettempdir(), "pctr_ocr.png")
+    path = os.path.join(_pctr_tempdir(), "ocr_%d.png" % os.getpid())
     ox = oy = 0
     title = getattr(args, "title", None)
     if title:
@@ -378,8 +456,15 @@ def _ocr_capture(args):
         ox, oy = r.left, r.top
     else:
         img = pyautogui.screenshot()
-    img.save(path)
-    return _windows_ocr(path), ox, oy
+    try:
+        img.save(path)
+        words = _windows_ocr(path)
+    finally:
+        try:
+            os.remove(path)
+        except OSError:
+            pass
+    return words, ox, oy
 
 
 def cmd_ocr(args):
@@ -420,15 +505,18 @@ def cmd_ocrclick(args):
     print("ocr-clicked %r at %d,%d" % (w["text"], cx, cy))
 
 
+_MODEL_CACHE = {}
+
+
 def _look_detect(args):
+    import contextlib
     import os
-    import tempfile
     import pyautogui
     try:
         from ultralytics.models import YOLOWorld
     except Exception as e:
         raise SystemExit("YOLO-World needs ultralytics: pip install ultralytics (%s)" % e)
-    path = os.path.join(tempfile.gettempdir(), "pctr_look.png")
+    path = os.path.join(_pctr_tempdir(), "look_%d.png" % os.getpid())
     ox = oy = 0
     title = getattr(args, "title", None)
     if title:
@@ -438,11 +526,23 @@ def _look_detect(args):
         ox, oy = r.left, r.top
     else:
         img = pyautogui.screenshot()
-    img.save(path)
-    model = YOLOWorld(getattr(args, "model", None) or "yolov8s-worldv2.pt")
-    model.set_classes([args.for_])
+    weights = getattr(args, "model", None) or os.environ.get("PCTR_YOLO_MODEL") or "yolov8s-worldv2.pt"
     conf = args.conf if getattr(args, "conf", None) is not None else 0.35
-    results = model.predict(path, conf=conf, agnostic_nms=True, verbose=False)
+    try:
+        img.save(path)
+        # keep stdout machine-readable: route ultralytics/tqdm chatter to stderr
+        with contextlib.redirect_stdout(sys.stderr):
+            model = _MODEL_CACHE.get(weights)
+            if model is None:
+                model = YOLOWorld(weights)
+                _MODEL_CACHE[weights] = model
+            model.set_classes([args.for_])
+            results = model.predict(path, conf=conf, agnostic_nms=True, verbose=False)
+    finally:
+        try:
+            os.remove(path)
+        except OSError:
+            pass
     r0 = results[0]
     names = r0.names
     boxes = r0.boxes
@@ -531,14 +631,33 @@ def cmd_setup(args):
     print("done. restart your agent so it loads the skill.")
 
 
-def add_target(p, require_name=False):
-    p.add_argument("--title", required=require_name, help="Window title regex")
+def add_target(p, require_selector=False, require_window=False):
+    p.add_argument("--title", default=None, help="Window title regex")
     p.add_argument("--process", type=int, default=None, help="Filter by process id")
     p.add_argument("--timeout", type=float, default=5.0)
-    p.add_argument("--name", required=require_name, help="Element name regex")
+    p.add_argument("--name", default=None, help="Element name regex")
     p.add_argument("--control-type", default=None, help="Exact UIA control type (Button, Edit, ...)")
     p.add_argument("--auto-id", default=None)
     p.add_argument("--nth", type=int, default=0)
+    if require_window:
+        p.set_defaults(_need_window=True)
+    if require_selector:
+        p.set_defaults(_need_selector=True)
+
+
+def _validate(args):
+    if getattr(args, "_need_window", False) and not (getattr(args, "title", None) or getattr(args, "process", None) is not None):
+        _fail("provide --title (or --process) to choose a window")
+    if getattr(args, "_need_selector", False):
+        have = any([
+            getattr(args, "title", None),
+            getattr(args, "process", None) is not None,
+            getattr(args, "name", None),
+            getattr(args, "control_type", None),
+            getattr(args, "auto_id", None),
+        ])
+        if not have:
+            _fail("provide at least one selector: --title / --process / --name / --control-type / --auto-id")
 
 
 def build_parser():
@@ -547,27 +666,29 @@ def build_parser():
 
     w = sub.add_parser("windows", help="List top-level windows")
     w.add_argument("--filter", default=None)
+    w.add_argument("--process", type=int, default=None, help="Only this process id")
     w.set_defaults(func=cmd_windows)
 
     t = sub.add_parser("tree", help="Dump UIA tree")
-    add_target(t)
+    add_target(t, require_window=True)
     t.add_argument("--depth", type=int, default=99)
     t.add_argument("--limit", type=int, default=400)
     t.set_defaults(func=cmd_tree)
 
     f = sub.add_parser("find", help="Find elements")
-    add_target(f)
+    add_target(f, require_window=True)
+    f.add_argument("--limit", type=int, default=None, help="Show at most N matches")
     f.set_defaults(func=cmd_find)
 
     c = sub.add_parser("click", help="Click an element")
-    add_target(c, require_name=True)
+    add_target(c, require_selector=True)
     c.add_argument("--method", choices=["auto", "invoke", "mouse"], default="auto")
     c.add_argument("--button", default="left")
     c.add_argument("--dbl", action="store_true")
     c.set_defaults(func=cmd_click)
 
     s = sub.add_parser("set", help="Set text on an element")
-    add_target(s, require_name=True)
+    add_target(s, require_selector=True)
     s.add_argument("--text", required=True)
     s.set_defaults(func=cmd_set)
 
@@ -595,7 +716,7 @@ def build_parser():
     fo.set_defaults(func=cmd_focus)
 
     wa = sub.add_parser("wait", help="Wait for an element")
-    add_target(wa, require_name=True)
+    add_target(wa, require_selector=True)
     wa.set_defaults(func=cmd_wait)
 
     sh = sub.add_parser("shot", help="Screenshot")
@@ -718,4 +839,12 @@ def build_parser():
 
 def main():
     args = build_parser().parse_args()
-    args.func(args)
+    _validate(args)
+    try:
+        args.func(args)
+    except SystemExit:
+        raise
+    except KeyboardInterrupt:
+        raise SystemExit(130)
+    except Exception as e:
+        _fail("%s: %s" % (type(e).__name__, e))

pctr-0.2.2/src/pctr/data/AGENTS.md

@@ -0,0 +1,23 @@
+<!-- pctr -->
+## Desktop automation (pctr)
+
+`pctr` is installed. Drive Windows GUI apps by **element name**, not pixel
+coordinates:
+
+```bash
+pctr windows                          # list windows
+pctr tree  --title "App" --limit 100  # inspect controls
+pctr click --title "App" --name "Save"
+pctr set   --title "App" --name "Search" --text "hello"   # prefer set for fields
+pctr ocrfind --text "Sign in"         # fallback: read text off the screen (single words)
+pctr look --for "a red error icon"    # fallback: open-vocab object detection
+```
+
+- Prefer `set` (ValuePattern) over `type` for fields - `type` interprets pywinauto key syntax (`{ } + ^ % ~ ( )`).
+- Global `type`/`keys` need `pctr focus` first. Prefer `pctr hotkey --keys ctrl+s` over `keys "^s"`.
+- `click`/`set`/`wait` accept any selector - `--name` is not required.
+- Exit codes: `0` ok, `1` no match, `2` usage error (clean `error:` line, no tracebacks).
+- Games ignore synthetic input: add `--direct` to mouse/key commands.
+- Electron apps expose only the frame - use `focus` + global keys, or CDP.
+
+Run `pctr -h` for all commands. Full skill: `pctr skill`.

{pctr-0.2.1 → pctr-0.2.2}/src/pctr/data/SKILL.md

@@ -33,23 +33,23 @@ back as `--name`, `--control-type`, `--auto-id`.
 
 | Command | What it does |
 |---------|--------------|
-| `windows [--filter RE]` | List top-level windows. |
-| `tree --title RE [--limit N]` | Dump the UIA control tree. |
-| `find --title RE --name RE [--control-type T] [--auto-id ID]` | List matching elements. |
+| `windows [--filter RE] [--process PID]` | List top-level windows. |
+| `tree --title RE [--depth N] [--limit N]` | Dump the UIA control tree. |
+| `find --title RE [--name RE] [--limit N] [--nth N]` | List matching elements. |
 | `click --title RE --name RE [--method auto\|invoke\|mouse] [--dbl]` | Click an element. |
 | `set --title RE --name RE --text S` | Set a field via the UIA ValuePattern (exact, instant). |
-| `type [--title RE --name RE] --text S [--delay 0.03] [--chunk 1]` | Type literal text. |
-| `keys --keys "{ENTER}"` | Global key combo (pywinauto send_keys syntax). |
-| `hotkey --keys win+shift+s` | Modifier combo. |
+| `type [--title RE --name RE] --text S [--delay 0.03] [--chunk 1]` | Send keystrokes (see caveats). |
+| `keys --keys "{ENTER}"` | Global combo via pywinauto send_keys syntax. |
+| `hotkey --keys ctrl+s` | Modifier combo (prefer this over `keys "^s"`). |
 | `focus --title RE` | Bring a window to the foreground. |
 | `wait --title RE --name RE [--timeout S]` | Wait for an element. |
 | `shot [--title RE] --out PATH` | Screenshot the screen or a window. |
 | `size` | Print primary screen size as WxH. |
-| `move --x N --y N` / `down` / `up` / `hold --ms N` | Raw mouse control. |
+| `move --x N --y N` / `down` / `up` / `hold --ms N` | Raw mouse control (`--direct` for games). |
 | `drag --start x,y --end x,y [--duration S]` | Drag between points. |
-| `keydown / keyup --key a` / `keyhold --key w --ms 1500` | Hold keyboard keys. |
+| `keydown / keyup --key a` / `keyhold --key w --ms 1500` | Hold keyboard keys (`--direct` for games). |
 | `ocr [--title RE]` | OCR the screen (or window); list words with boxes. |
-| `ocrfind --text RE` | OCR then list matches with click centers. |
+| `ocrfind --text RE` | OCR then list matches with click centers. Matches **single words**. |
 | `ocrclick --text RE [--nth N]` | OCR then click a word. |
 | `look --for "a dog" [--title RE] [--conf X]` | YOLO-World detect objects by text prompt. |
 | `lookclick --for "a dog" [--nth N]` | Detect then click the top match. |
@@ -58,20 +58,44 @@ back as `--name`, `--control-type`, `--auto-id`.
 Filters: `--title` (regex), `--name` (regex), `--control-type`, `--auto-id`,
 `--nth`, `--process`, `--timeout`.
 
+### Selectors
+
+`click`, `set`, `wait` accept **any** selector - you do **not** have to pass
+`--name`. Give at least one of `--title` / `--process` / `--name` /
+`--control-type` / `--auto-id`. `tree` and `find` require a *window* selector
+(`--title` or `--process`) so they never silently grab the wrong window.
+
+### Exit codes
+
+- `0` - success (and for `ocr`).
+- `1` - no match: `find`/`ocrfind` found nothing, or the window wasn't found.
+- `2` - usage/validation error: bad regex, bad `x,y` pair, unwritable `--out`, missing selector.
+
+Errors print a single `error: ...` line (no tracebacks).
+
 ## Patterns
 
-- **Text fields -> use `set`, not `type`.** `set` uses the ValuePattern: atomic, can't drop chars.
-- **`type` is char-by-char** (`--chunk 1 --delay 0.03`); raise `--delay` for Qt/Electron.
+- **Text fields -> use `set`, not `type`.** `set` uses the ValuePattern: atomic, exact, can't drop chars. `type`'s element path goes through pywinauto `type_keys`, which interprets `{ } + ^ % ~ ( )` as key syntax - so `--text "a{b}"` is **not** literal and an unmatched `{` fails. Prefer `set`; use `type` only for plain text on fields with no ValuePattern.
 - **Global `type`/`keys` go to the OS-focused window** - `pctr focus --title ...` first.
+- **Prefer `hotkey ctrl+s` over `keys "^s"`.** The `^` modifier in `keys` is fragile through shells; `hotkey` takes `ctrl+alt+x` form and is reliable.
 - Click falls back to a real mouse click when a control has no invoke pattern.
 - Disambiguate with `--control-type` / `--auto-id` / `--nth`.
+- **Quote regex args.** `$`, `^`, `|` are regex metachars and can be mangled by your shell; single-quote them, or use plain substrings.
 
 ## App notes
 
 - **Qt apps** expose a rich tree including embedded webviews.
-- **Electron apps** (Discord, VS Code, OpenCode) hide web content - use `focus` +
-  global `type`/`keys`, verify with `shot` (and fall back to `ocr`/`look`).
+- **Electron apps** (Discord, VS Code, OpenCode) hide web content behind the frame - the tree shows only window buttons. Use `focus` + global `type`/`keys`, verify with `shot`, and fall back to `ocr`/`look`. For deep control, drive them over CDP instead.
 - **Canvas / games / custom UI**: no tree - use `ocr` for text and `look` for objects.
+- **Games** (Minecraft, Trailmakers, ...) ignore synthetic input from `pyautogui`; pass `--direct` to mouse/key commands to use `pydirectinput`, and `pctr focus` the game first.
+
+## OCR / YOLO notes
+
+- `ocrfind`/`ocrclick` match **one word at a time** (Windows OCR tokenizes into words) - match a single distinctive word, not a phrase.
+- OCR needs the window visible/foreground; accuracy varies with font/scale.
+- `look` needs `pip install "pctr[vision]"`. Point at a local model with the
+  `PCTR_YOLO_MODEL` env var to avoid the first-run download. Progress bars are
+  sent to stderr, so `look`'s stdout stays parseable.
 
 ## Setup
 

{pctr-0.2.1 → pctr-0.2.2/src/pctr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pctr
-Version: 0.2.1
+Version: 0.2.2
 Summary: Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates.
 Author: Space-lab515
 License-Expression: MIT

pctr-0.2.1/src/pctr/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.0"

pctr-0.2.1/src/pctr/data/AGENTS.md

@@ -1,17 +0,0 @@
-<!-- pctr -->
-## Desktop automation (pctr)
-
-`pctr` is installed. Use it to drive Windows GUI apps by **element name**, not
-pixel coordinates:
-
-```bash
-pctr windows                         # list windows
-pctr tree  --title "App" --limit 100 # inspect controls
-pctr click --title "App" --name "Save"
-pctr set   --title "App" --name "Search" --text "hello"
-pctr ocrfind --text "Sign in"        # fallback: read text off the screen
-pctr look --for "a red error icon"   # fallback: open-vocab object detection
-```
-
-Prefer `set` over `type` for fields. Global `type`/`keys` need `pctr focus` first.
-Run `pctr -h` for all commands. Full skill: `pctr skill`.