pctr 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pctr-0.2.0/src/pctr.egg-info → pctr-0.2.2}/PKG-INFO +4 -1
- {pctr-0.2.0 → pctr-0.2.2}/pyproject.toml +5 -1
- pctr-0.2.2/src/pctr/__init__.py +1 -0
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr/cli.py +174 -45
- pctr-0.2.2/src/pctr/data/AGENTS.md +23 -0
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr/data/SKILL.md +37 -13
- {pctr-0.2.0 → pctr-0.2.2/src/pctr.egg-info}/PKG-INFO +4 -1
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr.egg-info/requires.txt +4 -0
- pctr-0.2.0/src/pctr/__init__.py +0 -1
- pctr-0.2.0/src/pctr/data/AGENTS.md +0 -17
- {pctr-0.2.0 → pctr-0.2.2}/LICENSE +0 -0
- {pctr-0.2.0 → pctr-0.2.2}/README.md +0 -0
- {pctr-0.2.0 → pctr-0.2.2}/setup.cfg +0 -0
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr/__main__.py +0 -0
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr.egg-info/SOURCES.txt +0 -0
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr.egg-info/dependency_links.txt +0 -0
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr.egg-info/entry_points.txt +0 -0
- {pctr-0.2.0 → pctr-0.2.2}/src/pctr.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pctr
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates.
|
|
5
5
|
Author: Space-lab515
|
|
6
6
|
License-Expression: MIT
|
|
@@ -23,6 +23,9 @@ Requires-Dist: pywinauto>=0.6.8; platform_system == "Windows"
|
|
|
23
23
|
Requires-Dist: uiautomation>=2.0; platform_system == "Windows"
|
|
24
24
|
Requires-Dist: pyautogui>=0.9.54; platform_system == "Windows"
|
|
25
25
|
Requires-Dist: pydirectinput>=1.0.4; platform_system == "Windows"
|
|
26
|
+
Requires-Dist: pillow>=9.0; platform_system == "Windows"
|
|
27
|
+
Provides-Extra: vision
|
|
28
|
+
Requires-Dist: ultralytics>=8.2; extra == "vision"
|
|
26
29
|
Dynamic: license-file
|
|
27
30
|
|
|
28
31
|
# pctr
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pctr"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -29,8 +29,12 @@ dependencies = [
|
|
|
29
29
|
"uiautomation>=2.0; platform_system == 'Windows'",
|
|
30
30
|
"pyautogui>=0.9.54; platform_system == 'Windows'",
|
|
31
31
|
"pydirectinput>=1.0.4; platform_system == 'Windows'",
|
|
32
|
+
"pillow>=9.0; platform_system == 'Windows'",
|
|
32
33
|
]
|
|
33
34
|
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
vision = ["ultralytics>=8.2"]
|
|
37
|
+
|
|
34
38
|
[project.scripts]
|
|
35
39
|
pctr = "pctr.cli:main"
|
|
36
40
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.2"
|
|
@@ -2,6 +2,7 @@ import argparse
|
|
|
2
2
|
import re
|
|
3
3
|
import sys
|
|
4
4
|
import time
|
|
5
|
+
from typing import NoReturn
|
|
5
6
|
|
|
6
7
|
for _stream in (sys.stdout, sys.stderr):
|
|
7
8
|
_reconf = getattr(_stream, "reconfigure", None)
|
|
@@ -16,11 +17,26 @@ from pywinauto import Desktop
|
|
|
16
17
|
DESCRIPTION = "Element-based Windows UI Automation CLI for AI agents."
|
|
17
18
|
|
|
18
19
|
|
|
20
|
+
def _fail(msg, code=2) -> NoReturn:
|
|
21
|
+
"""Print a clean one-line error and exit non-zero (no traceback)."""
|
|
22
|
+
print("error: %s" % msg, file=sys.stderr)
|
|
23
|
+
raise SystemExit(code)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _compile(pattern, what, flags=re.I):
|
|
27
|
+
"""Compile a user-supplied regex, failing cleanly on a bad pattern."""
|
|
28
|
+
try:
|
|
29
|
+
return re.compile(pattern, flags)
|
|
30
|
+
except re.error as e:
|
|
31
|
+
_fail("invalid %s regex %r: %s" % (what, pattern, e))
|
|
32
|
+
|
|
33
|
+
|
|
19
34
|
def _iter_windows():
|
|
20
35
|
return Desktop(backend="uia").windows()
|
|
21
36
|
|
|
22
37
|
|
|
23
38
|
def _find_window(title_re, process=None, timeout=5.0, index=0):
|
|
39
|
+
rx_title = _compile(title_re, "title") if title_re else None
|
|
24
40
|
desktop = Desktop(backend="uia")
|
|
25
41
|
deadline = time.time() + timeout
|
|
26
42
|
last = None
|
|
@@ -31,7 +47,7 @@ def _find_window(title_re, process=None, timeout=5.0, index=0):
|
|
|
31
47
|
t = w.window_text()
|
|
32
48
|
except Exception:
|
|
33
49
|
continue
|
|
34
|
-
if
|
|
50
|
+
if rx_title and not rx_title.search(t):
|
|
35
51
|
continue
|
|
36
52
|
if process is not None and w.process_id() != process:
|
|
37
53
|
continue
|
|
@@ -47,15 +63,16 @@ def _find_window(title_re, process=None, timeout=5.0, index=0):
|
|
|
47
63
|
|
|
48
64
|
|
|
49
65
|
def _matches(win, name=None, control_type=None, auto_id=None):
|
|
66
|
+
rx_name = _compile(name, "name") if name is not None else None
|
|
50
67
|
out = []
|
|
51
68
|
for el in win.descendants():
|
|
52
69
|
try:
|
|
53
70
|
info = el.element_info
|
|
54
71
|
except Exception:
|
|
55
72
|
continue
|
|
56
|
-
if
|
|
73
|
+
if rx_name is not None:
|
|
57
74
|
n = info.name or ""
|
|
58
|
-
if
|
|
75
|
+
if rx_name.search(n) is None:
|
|
59
76
|
continue
|
|
60
77
|
if control_type is not None:
|
|
61
78
|
c = (info.control_type or "")
|
|
@@ -99,27 +116,56 @@ def _get_element(args):
|
|
|
99
116
|
|
|
100
117
|
|
|
101
118
|
def cmd_windows(args):
|
|
119
|
+
rx = _compile(args.filter, "filter") if args.filter else None
|
|
120
|
+
proc = getattr(args, "process", None)
|
|
102
121
|
for w in _iter_windows():
|
|
103
122
|
t = ""
|
|
104
123
|
try:
|
|
105
124
|
t = w.window_text()
|
|
106
125
|
except Exception:
|
|
107
126
|
pass
|
|
108
|
-
if
|
|
127
|
+
if rx and rx.search(t) is None:
|
|
128
|
+
continue
|
|
129
|
+
if proc is not None and w.process_id() != proc:
|
|
109
130
|
continue
|
|
110
131
|
print("pid=%s | %s" % (w.process_id(), t))
|
|
111
132
|
|
|
112
133
|
|
|
134
|
+
def _walk(root, max_depth):
|
|
135
|
+
"""Yield (element, depth) for descendants of root, bounded to max_depth."""
|
|
136
|
+
stack = []
|
|
137
|
+
try:
|
|
138
|
+
kids = root.children()
|
|
139
|
+
except Exception:
|
|
140
|
+
kids = []
|
|
141
|
+
for k in reversed(kids):
|
|
142
|
+
stack.append((k, 1))
|
|
143
|
+
while stack:
|
|
144
|
+
node, d = stack.pop()
|
|
145
|
+
yield node, d
|
|
146
|
+
if d >= max_depth:
|
|
147
|
+
continue
|
|
148
|
+
try:
|
|
149
|
+
kids = node.children()
|
|
150
|
+
except Exception:
|
|
151
|
+
kids = []
|
|
152
|
+
for k in reversed(kids):
|
|
153
|
+
stack.append((k, d + 1))
|
|
154
|
+
|
|
155
|
+
|
|
113
156
|
def cmd_tree(args):
|
|
114
157
|
win = _find_window(args.title, args.process, args.timeout)
|
|
115
158
|
print("WINDOW: %s (pid=%s)" % (win.window_text(), win.process_id()))
|
|
116
159
|
count = 0
|
|
117
|
-
|
|
160
|
+
depth = getattr(args, "depth", None)
|
|
161
|
+
depth_mode = depth is not None and 0 <= depth < 99
|
|
162
|
+
it = _walk(win, depth) if depth_mode else ((el, 1) for el in win.descendants())
|
|
163
|
+
for el, d in it:
|
|
118
164
|
try:
|
|
119
165
|
el.element_info
|
|
120
166
|
except Exception:
|
|
121
167
|
continue
|
|
122
|
-
print(" %s" % _describe(el))
|
|
168
|
+
print(" %s%s" % (" " * (d - 1) if depth_mode else "", _describe(el)))
|
|
123
169
|
count += 1
|
|
124
170
|
if count >= args.limit:
|
|
125
171
|
print("... (limit %d reached)" % args.limit)
|
|
@@ -131,10 +177,19 @@ def cmd_find(args):
|
|
|
131
177
|
els = _matches(win, args.name, args.control_type, args.auto_id)
|
|
132
178
|
if not els:
|
|
133
179
|
print("no matches")
|
|
134
|
-
|
|
180
|
+
raise SystemExit(1)
|
|
181
|
+
nth = getattr(args, "nth", 0) or 0
|
|
182
|
+
if nth > 0:
|
|
183
|
+
if nth >= len(els):
|
|
184
|
+
_fail("matched %d element(s); nth=%d out of range" % (len(els), nth))
|
|
185
|
+
els = [els[nth]]
|
|
135
186
|
print("WINDOW: %s" % win.window_text())
|
|
136
|
-
|
|
187
|
+
limit = getattr(args, "limit", None)
|
|
188
|
+
shown = els[:limit] if limit else els
|
|
189
|
+
for i, el in enumerate(shown):
|
|
137
190
|
print("[%d] %s" % (i, _describe(el)))
|
|
191
|
+
if limit and len(els) > limit:
|
|
192
|
+
print("... (%d more)" % (len(els) - limit))
|
|
138
193
|
|
|
139
194
|
|
|
140
195
|
def cmd_click(args):
|
|
@@ -160,10 +215,13 @@ def cmd_set(args):
|
|
|
160
215
|
el.set_edit_text(args.text)
|
|
161
216
|
print("set via ValuePattern")
|
|
162
217
|
except Exception:
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
218
|
+
try:
|
|
219
|
+
el.click_input()
|
|
220
|
+
el.type_keys("^a{BACKSPACE}")
|
|
221
|
+
el.type_keys(args.text, with_spaces=True, with_newlines=True)
|
|
222
|
+
print("set via type_keys")
|
|
223
|
+
except Exception as e:
|
|
224
|
+
_fail("set failed (no ValuePattern and text has pywinauto key syntax): %s" % e)
|
|
167
225
|
|
|
168
226
|
|
|
169
227
|
def _chunks(text, size):
|
|
@@ -171,13 +229,16 @@ def _chunks(text, size):
|
|
|
171
229
|
|
|
172
230
|
|
|
173
231
|
def cmd_type(args):
|
|
174
|
-
delay =
|
|
175
|
-
chunk = max(1,
|
|
232
|
+
delay = args.delay
|
|
233
|
+
chunk = max(1, args.chunk)
|
|
176
234
|
if args.title:
|
|
177
235
|
win, el, n = _get_element(args)
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
236
|
+
try:
|
|
237
|
+
for part in _chunks(args.text, chunk):
|
|
238
|
+
el.type_keys(part, with_spaces=True, with_newlines=True)
|
|
239
|
+
time.sleep(delay)
|
|
240
|
+
except Exception as e:
|
|
241
|
+
_fail("type failed (text has pywinauto key syntax like { } + ^ %% ~ ()): %s" % e)
|
|
181
242
|
else:
|
|
182
243
|
import pyautogui
|
|
183
244
|
pyautogui.write(args.text, interval=delay)
|
|
@@ -218,7 +279,10 @@ def cmd_shot(args):
|
|
|
218
279
|
img = pyautogui.screenshot(region=(r.left, r.top, r.width(), r.height()))
|
|
219
280
|
else:
|
|
220
281
|
img = pyautogui.screenshot()
|
|
221
|
-
|
|
282
|
+
try:
|
|
283
|
+
img.save(args.out)
|
|
284
|
+
except (OSError, ValueError) as e:
|
|
285
|
+
_fail("cannot write screenshot to %r: %s" % (args.out, e))
|
|
222
286
|
print("saved %s (%dx%d)" % (args.out, img.width, img.height))
|
|
223
287
|
|
|
224
288
|
|
|
@@ -233,8 +297,11 @@ def _mouse_backend(args):
|
|
|
233
297
|
|
|
234
298
|
|
|
235
299
|
def _parse_pair(text):
|
|
236
|
-
|
|
237
|
-
|
|
300
|
+
try:
|
|
301
|
+
x, y = text.split(",")
|
|
302
|
+
return int(float(x)), int(float(y))
|
|
303
|
+
except (ValueError, AttributeError):
|
|
304
|
+
_fail("bad coordinate pair %r (expected 'x,y')" % text)
|
|
238
305
|
|
|
239
306
|
|
|
240
307
|
def cmd_move(args):
|
|
@@ -343,17 +410,29 @@ ConvertTo-Json -Compress -InputObject @($out)
|
|
|
343
410
|
'''
|
|
344
411
|
|
|
345
412
|
|
|
413
|
+
def _pctr_tempdir():
|
|
414
|
+
import os
|
|
415
|
+
import tempfile
|
|
416
|
+
d = os.path.join(tempfile.gettempdir(), "pctr")
|
|
417
|
+
os.makedirs(d, exist_ok=True)
|
|
418
|
+
return d
|
|
419
|
+
|
|
420
|
+
|
|
346
421
|
def _windows_ocr(image_path):
|
|
422
|
+
import base64
|
|
347
423
|
import json
|
|
348
424
|
import os
|
|
349
425
|
import subprocess
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
426
|
+
body = _OCR_PS.strip()
|
|
427
|
+
if body.startswith("param"): # drop the param() line; path comes via env
|
|
428
|
+
body = body.split("\n", 1)[1]
|
|
429
|
+
ps = "$Path = $env:PCTR_OCR_PATH\n" + body
|
|
430
|
+
enc = base64.b64encode(ps.encode("utf-16-le")).decode("ascii")
|
|
431
|
+
env = dict(os.environ)
|
|
432
|
+
env["PCTR_OCR_PATH"] = image_path
|
|
354
433
|
out = subprocess.run(
|
|
355
|
-
["powershell", "-NoProfile", "-
|
|
356
|
-
capture_output=True, text=True,
|
|
434
|
+
["powershell", "-NoProfile", "-NonInteractive", "-EncodedCommand", enc],
|
|
435
|
+
capture_output=True, text=True, env=env,
|
|
357
436
|
)
|
|
358
437
|
txt = (out.stdout or "").strip()
|
|
359
438
|
if not txt:
|
|
@@ -366,9 +445,8 @@ def _windows_ocr(image_path):
|
|
|
366
445
|
|
|
367
446
|
def _ocr_capture(args):
|
|
368
447
|
import os
|
|
369
|
-
import tempfile
|
|
370
448
|
import pyautogui
|
|
371
|
-
path = os.path.join(
|
|
449
|
+
path = os.path.join(_pctr_tempdir(), "ocr_%d.png" % os.getpid())
|
|
372
450
|
ox = oy = 0
|
|
373
451
|
title = getattr(args, "title", None)
|
|
374
452
|
if title:
|
|
@@ -378,8 +456,15 @@ def _ocr_capture(args):
|
|
|
378
456
|
ox, oy = r.left, r.top
|
|
379
457
|
else:
|
|
380
458
|
img = pyautogui.screenshot()
|
|
381
|
-
|
|
382
|
-
|
|
459
|
+
try:
|
|
460
|
+
img.save(path)
|
|
461
|
+
words = _windows_ocr(path)
|
|
462
|
+
finally:
|
|
463
|
+
try:
|
|
464
|
+
os.remove(path)
|
|
465
|
+
except OSError:
|
|
466
|
+
pass
|
|
467
|
+
return words, ox, oy
|
|
383
468
|
|
|
384
469
|
|
|
385
470
|
def cmd_ocr(args):
|
|
@@ -420,15 +505,18 @@ def cmd_ocrclick(args):
|
|
|
420
505
|
print("ocr-clicked %r at %d,%d" % (w["text"], cx, cy))
|
|
421
506
|
|
|
422
507
|
|
|
508
|
+
_MODEL_CACHE = {}
|
|
509
|
+
|
|
510
|
+
|
|
423
511
|
def _look_detect(args):
|
|
512
|
+
import contextlib
|
|
424
513
|
import os
|
|
425
|
-
import tempfile
|
|
426
514
|
import pyautogui
|
|
427
515
|
try:
|
|
428
516
|
from ultralytics.models import YOLOWorld
|
|
429
517
|
except Exception as e:
|
|
430
518
|
raise SystemExit("YOLO-World needs ultralytics: pip install ultralytics (%s)" % e)
|
|
431
|
-
path = os.path.join(
|
|
519
|
+
path = os.path.join(_pctr_tempdir(), "look_%d.png" % os.getpid())
|
|
432
520
|
ox = oy = 0
|
|
433
521
|
title = getattr(args, "title", None)
|
|
434
522
|
if title:
|
|
@@ -438,11 +526,23 @@ def _look_detect(args):
|
|
|
438
526
|
ox, oy = r.left, r.top
|
|
439
527
|
else:
|
|
440
528
|
img = pyautogui.screenshot()
|
|
441
|
-
|
|
442
|
-
model = YOLOWorld(getattr(args, "model", None) or "yolov8s-worldv2.pt")
|
|
443
|
-
model.set_classes([args.for_])
|
|
529
|
+
weights = getattr(args, "model", None) or os.environ.get("PCTR_YOLO_MODEL") or "yolov8s-worldv2.pt"
|
|
444
530
|
conf = args.conf if getattr(args, "conf", None) is not None else 0.35
|
|
445
|
-
|
|
531
|
+
try:
|
|
532
|
+
img.save(path)
|
|
533
|
+
# keep stdout machine-readable: route ultralytics/tqdm chatter to stderr
|
|
534
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
535
|
+
model = _MODEL_CACHE.get(weights)
|
|
536
|
+
if model is None:
|
|
537
|
+
model = YOLOWorld(weights)
|
|
538
|
+
_MODEL_CACHE[weights] = model
|
|
539
|
+
model.set_classes([args.for_])
|
|
540
|
+
results = model.predict(path, conf=conf, agnostic_nms=True, verbose=False)
|
|
541
|
+
finally:
|
|
542
|
+
try:
|
|
543
|
+
os.remove(path)
|
|
544
|
+
except OSError:
|
|
545
|
+
pass
|
|
446
546
|
r0 = results[0]
|
|
447
547
|
names = r0.names
|
|
448
548
|
boxes = r0.boxes
|
|
@@ -531,14 +631,33 @@ def cmd_setup(args):
|
|
|
531
631
|
print("done. restart your agent so it loads the skill.")
|
|
532
632
|
|
|
533
633
|
|
|
534
|
-
def add_target(p,
|
|
535
|
-
p.add_argument("--title",
|
|
634
|
+
def add_target(p, require_selector=False, require_window=False):
|
|
635
|
+
p.add_argument("--title", default=None, help="Window title regex")
|
|
536
636
|
p.add_argument("--process", type=int, default=None, help="Filter by process id")
|
|
537
637
|
p.add_argument("--timeout", type=float, default=5.0)
|
|
538
|
-
p.add_argument("--name",
|
|
638
|
+
p.add_argument("--name", default=None, help="Element name regex")
|
|
539
639
|
p.add_argument("--control-type", default=None, help="Exact UIA control type (Button, Edit, ...)")
|
|
540
640
|
p.add_argument("--auto-id", default=None)
|
|
541
641
|
p.add_argument("--nth", type=int, default=0)
|
|
642
|
+
if require_window:
|
|
643
|
+
p.set_defaults(_need_window=True)
|
|
644
|
+
if require_selector:
|
|
645
|
+
p.set_defaults(_need_selector=True)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _validate(args):
|
|
649
|
+
if getattr(args, "_need_window", False) and not (getattr(args, "title", None) or getattr(args, "process", None) is not None):
|
|
650
|
+
_fail("provide --title (or --process) to choose a window")
|
|
651
|
+
if getattr(args, "_need_selector", False):
|
|
652
|
+
have = any([
|
|
653
|
+
getattr(args, "title", None),
|
|
654
|
+
getattr(args, "process", None) is not None,
|
|
655
|
+
getattr(args, "name", None),
|
|
656
|
+
getattr(args, "control_type", None),
|
|
657
|
+
getattr(args, "auto_id", None),
|
|
658
|
+
])
|
|
659
|
+
if not have:
|
|
660
|
+
_fail("provide at least one selector: --title / --process / --name / --control-type / --auto-id")
|
|
542
661
|
|
|
543
662
|
|
|
544
663
|
def build_parser():
|
|
@@ -547,27 +666,29 @@ def build_parser():
|
|
|
547
666
|
|
|
548
667
|
w = sub.add_parser("windows", help="List top-level windows")
|
|
549
668
|
w.add_argument("--filter", default=None)
|
|
669
|
+
w.add_argument("--process", type=int, default=None, help="Only this process id")
|
|
550
670
|
w.set_defaults(func=cmd_windows)
|
|
551
671
|
|
|
552
672
|
t = sub.add_parser("tree", help="Dump UIA tree")
|
|
553
|
-
add_target(t)
|
|
673
|
+
add_target(t, require_window=True)
|
|
554
674
|
t.add_argument("--depth", type=int, default=99)
|
|
555
675
|
t.add_argument("--limit", type=int, default=400)
|
|
556
676
|
t.set_defaults(func=cmd_tree)
|
|
557
677
|
|
|
558
678
|
f = sub.add_parser("find", help="Find elements")
|
|
559
|
-
add_target(f)
|
|
679
|
+
add_target(f, require_window=True)
|
|
680
|
+
f.add_argument("--limit", type=int, default=None, help="Show at most N matches")
|
|
560
681
|
f.set_defaults(func=cmd_find)
|
|
561
682
|
|
|
562
683
|
c = sub.add_parser("click", help="Click an element")
|
|
563
|
-
add_target(c,
|
|
684
|
+
add_target(c, require_selector=True)
|
|
564
685
|
c.add_argument("--method", choices=["auto", "invoke", "mouse"], default="auto")
|
|
565
686
|
c.add_argument("--button", default="left")
|
|
566
687
|
c.add_argument("--dbl", action="store_true")
|
|
567
688
|
c.set_defaults(func=cmd_click)
|
|
568
689
|
|
|
569
690
|
s = sub.add_parser("set", help="Set text on an element")
|
|
570
|
-
add_target(s,
|
|
691
|
+
add_target(s, require_selector=True)
|
|
571
692
|
s.add_argument("--text", required=True)
|
|
572
693
|
s.set_defaults(func=cmd_set)
|
|
573
694
|
|
|
@@ -595,7 +716,7 @@ def build_parser():
|
|
|
595
716
|
fo.set_defaults(func=cmd_focus)
|
|
596
717
|
|
|
597
718
|
wa = sub.add_parser("wait", help="Wait for an element")
|
|
598
|
-
add_target(wa,
|
|
719
|
+
add_target(wa, require_selector=True)
|
|
599
720
|
wa.set_defaults(func=cmd_wait)
|
|
600
721
|
|
|
601
722
|
sh = sub.add_parser("shot", help="Screenshot")
|
|
@@ -718,4 +839,12 @@ def build_parser():
|
|
|
718
839
|
|
|
719
840
|
def main():
|
|
720
841
|
args = build_parser().parse_args()
|
|
721
|
-
|
|
842
|
+
_validate(args)
|
|
843
|
+
try:
|
|
844
|
+
args.func(args)
|
|
845
|
+
except SystemExit:
|
|
846
|
+
raise
|
|
847
|
+
except KeyboardInterrupt:
|
|
848
|
+
raise SystemExit(130)
|
|
849
|
+
except Exception as e:
|
|
850
|
+
_fail("%s: %s" % (type(e).__name__, e))
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
<!-- pctr -->
|
|
2
|
+
## Desktop automation (pctr)
|
|
3
|
+
|
|
4
|
+
`pctr` is installed. Drive Windows GUI apps by **element name**, not pixel
|
|
5
|
+
coordinates:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pctr windows # list windows
|
|
9
|
+
pctr tree --title "App" --limit 100 # inspect controls
|
|
10
|
+
pctr click --title "App" --name "Save"
|
|
11
|
+
pctr set --title "App" --name "Search" --text "hello" # prefer set for fields
|
|
12
|
+
pctr ocrfind --text "Sign in" # fallback: read text off the screen (single words)
|
|
13
|
+
pctr look --for "a red error icon" # fallback: open-vocab object detection
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
- Prefer `set` (ValuePattern) over `type` for fields - `type` interprets pywinauto key syntax (`{ } + ^ % ~ ( )`).
|
|
17
|
+
- Global `type`/`keys` need `pctr focus` first. Prefer `pctr hotkey --keys ctrl+s` over `keys "^s"`.
|
|
18
|
+
- `click`/`set`/`wait` accept any selector - `--name` is not required.
|
|
19
|
+
- Exit codes: `0` ok, `1` no match, `2` usage error (clean `error:` line, no tracebacks).
|
|
20
|
+
- Games ignore synthetic input: add `--direct` to mouse/key commands.
|
|
21
|
+
- Electron apps expose only the frame - use `focus` + global keys, or CDP.
|
|
22
|
+
|
|
23
|
+
Run `pctr -h` for all commands. Full skill: `pctr skill`.
|
|
@@ -33,23 +33,23 @@ back as `--name`, `--control-type`, `--auto-id`.
|
|
|
33
33
|
|
|
34
34
|
| Command | What it does |
|
|
35
35
|
|---------|--------------|
|
|
36
|
-
| `windows [--filter RE]` | List top-level windows. |
|
|
37
|
-
| `tree --title RE [--limit N]` | Dump the UIA control tree. |
|
|
38
|
-
| `find --title RE --name RE [--
|
|
36
|
+
| `windows [--filter RE] [--process PID]` | List top-level windows. |
|
|
37
|
+
| `tree --title RE [--depth N] [--limit N]` | Dump the UIA control tree. |
|
|
38
|
+
| `find --title RE [--name RE] [--limit N] [--nth N]` | List matching elements. |
|
|
39
39
|
| `click --title RE --name RE [--method auto\|invoke\|mouse] [--dbl]` | Click an element. |
|
|
40
40
|
| `set --title RE --name RE --text S` | Set a field via the UIA ValuePattern (exact, instant). |
|
|
41
|
-
| `type [--title RE --name RE] --text S [--delay 0.03] [--chunk 1]` |
|
|
42
|
-
| `keys --keys "{ENTER}"` | Global
|
|
43
|
-
| `hotkey --keys
|
|
41
|
+
| `type [--title RE --name RE] --text S [--delay 0.03] [--chunk 1]` | Send keystrokes (see caveats). |
|
|
42
|
+
| `keys --keys "{ENTER}"` | Global combo via pywinauto send_keys syntax. |
|
|
43
|
+
| `hotkey --keys ctrl+s` | Modifier combo (prefer this over `keys "^s"`). |
|
|
44
44
|
| `focus --title RE` | Bring a window to the foreground. |
|
|
45
45
|
| `wait --title RE --name RE [--timeout S]` | Wait for an element. |
|
|
46
46
|
| `shot [--title RE] --out PATH` | Screenshot the screen or a window. |
|
|
47
47
|
| `size` | Print primary screen size as WxH. |
|
|
48
|
-
| `move --x N --y N` / `down` / `up` / `hold --ms N` | Raw mouse control. |
|
|
48
|
+
| `move --x N --y N` / `down` / `up` / `hold --ms N` | Raw mouse control (`--direct` for games). |
|
|
49
49
|
| `drag --start x,y --end x,y [--duration S]` | Drag between points. |
|
|
50
|
-
| `keydown / keyup --key a` / `keyhold --key w --ms 1500` | Hold keyboard keys. |
|
|
50
|
+
| `keydown / keyup --key a` / `keyhold --key w --ms 1500` | Hold keyboard keys (`--direct` for games). |
|
|
51
51
|
| `ocr [--title RE]` | OCR the screen (or window); list words with boxes. |
|
|
52
|
-
| `ocrfind --text RE` | OCR then list matches with click centers. |
|
|
52
|
+
| `ocrfind --text RE` | OCR then list matches with click centers. Matches **single words**. |
|
|
53
53
|
| `ocrclick --text RE [--nth N]` | OCR then click a word. |
|
|
54
54
|
| `look --for "a dog" [--title RE] [--conf X]` | YOLO-World detect objects by text prompt. |
|
|
55
55
|
| `lookclick --for "a dog" [--nth N]` | Detect then click the top match. |
|
|
@@ -58,20 +58,44 @@ back as `--name`, `--control-type`, `--auto-id`.
|
|
|
58
58
|
Filters: `--title` (regex), `--name` (regex), `--control-type`, `--auto-id`,
|
|
59
59
|
`--nth`, `--process`, `--timeout`.
|
|
60
60
|
|
|
61
|
+
### Selectors
|
|
62
|
+
|
|
63
|
+
`click`, `set`, `wait` accept **any** selector - you do **not** have to pass
|
|
64
|
+
`--name`. Give at least one of `--title` / `--process` / `--name` /
|
|
65
|
+
`--control-type` / `--auto-id`. `tree` and `find` require a *window* selector
|
|
66
|
+
(`--title` or `--process`) so they never silently grab the wrong window.
|
|
67
|
+
|
|
68
|
+
### Exit codes
|
|
69
|
+
|
|
70
|
+
- `0` - success (and for `ocr`).
|
|
71
|
+
- `1` - no match: `find`/`ocrfind` found nothing, or the window wasn't found.
|
|
72
|
+
- `2` - usage/validation error: bad regex, bad `x,y` pair, unwritable `--out`, missing selector.
|
|
73
|
+
|
|
74
|
+
Errors print a single `error: ...` line (no tracebacks).
|
|
75
|
+
|
|
61
76
|
## Patterns
|
|
62
77
|
|
|
63
|
-
- **Text fields -> use `set`, not `type`.** `set` uses the ValuePattern: atomic, can't drop chars.
|
|
64
|
-
- **`type` is char-by-char** (`--chunk 1 --delay 0.03`); raise `--delay` for Qt/Electron.
|
|
78
|
+
- **Text fields -> use `set`, not `type`.** `set` uses the ValuePattern: atomic, exact, can't drop chars. `type`'s element path goes through pywinauto `type_keys`, which interprets `{ } + ^ % ~ ( )` as key syntax - so `--text "a{b}"` is **not** literal and an unmatched `{` fails. Prefer `set`; use `type` only for plain text on fields with no ValuePattern.
|
|
65
79
|
- **Global `type`/`keys` go to the OS-focused window** - `pctr focus --title ...` first.
|
|
80
|
+
- **Prefer `hotkey ctrl+s` over `keys "^s"`.** The `^` modifier in `keys` is fragile through shells; `hotkey` takes `ctrl+alt+x` form and is reliable.
|
|
66
81
|
- Click falls back to a real mouse click when a control has no invoke pattern.
|
|
67
82
|
- Disambiguate with `--control-type` / `--auto-id` / `--nth`.
|
|
83
|
+
- **Quote regex args.** `$`, `^`, `|` are regex metachars and can be mangled by your shell; single-quote them, or use plain substrings.
|
|
68
84
|
|
|
69
85
|
## App notes
|
|
70
86
|
|
|
71
87
|
- **Qt apps** expose a rich tree including embedded webviews.
|
|
72
|
-
- **Electron apps** (Discord, VS Code, OpenCode) hide web content -
|
|
73
|
-
global `type`/`keys`, verify with `shot` (and fall back to `ocr`/`look`).
|
|
88
|
+
- **Electron apps** (Discord, VS Code, OpenCode) hide web content behind the frame - the tree shows only window buttons. Use `focus` + global `type`/`keys`, verify with `shot`, and fall back to `ocr`/`look`. For deep control, drive them over CDP instead.
|
|
74
89
|
- **Canvas / games / custom UI**: no tree - use `ocr` for text and `look` for objects.
|
|
90
|
+
- **Games** (Minecraft, Trailmakers, ...) ignore synthetic input from `pyautogui`; pass `--direct` to mouse/key commands to use `pydirectinput`, and `pctr focus` the game first.
|
|
91
|
+
|
|
92
|
+
## OCR / YOLO notes
|
|
93
|
+
|
|
94
|
+
- `ocrfind`/`ocrclick` match **one word at a time** (Windows OCR tokenizes into words) - match a single distinctive word, not a phrase.
|
|
95
|
+
- OCR needs the window visible/foreground; accuracy varies with font/scale.
|
|
96
|
+
- `look` needs `pip install "pctr[vision]"`. Point at a local model with the
|
|
97
|
+
`PCTR_YOLO_MODEL` env var to avoid the first-run download. Progress bars are
|
|
98
|
+
sent to stderr, so `look`'s stdout stays parseable.
|
|
75
99
|
|
|
76
100
|
## Setup
|
|
77
101
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pctr
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Element-based Windows UI Automation CLI for AI agents. Find controls by name/type/id and click, type, hold, drag, and screenshot - no pixel coordinates.
|
|
5
5
|
Author: Space-lab515
|
|
6
6
|
License-Expression: MIT
|
|
@@ -23,6 +23,9 @@ Requires-Dist: pywinauto>=0.6.8; platform_system == "Windows"
|
|
|
23
23
|
Requires-Dist: uiautomation>=2.0; platform_system == "Windows"
|
|
24
24
|
Requires-Dist: pyautogui>=0.9.54; platform_system == "Windows"
|
|
25
25
|
Requires-Dist: pydirectinput>=1.0.4; platform_system == "Windows"
|
|
26
|
+
Requires-Dist: pillow>=9.0; platform_system == "Windows"
|
|
27
|
+
Provides-Extra: vision
|
|
28
|
+
Requires-Dist: ultralytics>=8.2; extra == "vision"
|
|
26
29
|
Dynamic: license-file
|
|
27
30
|
|
|
28
31
|
# pctr
|
pctr-0.2.0/src/pctr/__init__.py
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.1.0"
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
<!-- pctr -->
|
|
2
|
-
## Desktop automation (pctr)
|
|
3
|
-
|
|
4
|
-
`pctr` is installed. Use it to drive Windows GUI apps by **element name**, not
|
|
5
|
-
pixel coordinates:
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
pctr windows # list windows
|
|
9
|
-
pctr tree --title "App" --limit 100 # inspect controls
|
|
10
|
-
pctr click --title "App" --name "Save"
|
|
11
|
-
pctr set --title "App" --name "Search" --text "hello"
|
|
12
|
-
pctr ocrfind --text "Sign in" # fallback: read text off the screen
|
|
13
|
-
pctr look --for "a red error icon" # fallback: open-vocab object detection
|
|
14
|
-
```
|
|
15
|
-
|
|
16
|
-
Prefer `set` over `type` for fields. Global `type`/`keys` need `pctr focus` first.
|
|
17
|
-
Run `pctr -h` for all commands. Full skill: `pctr skill`.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|