use-computer-cli 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/PKG-INFO +2 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/pyproject.toml +7 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/accessibility/atspi.py +49 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/accessibility/ax.py +12 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/accessibility/base.py +10 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/accessibility/uia.py +12 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/cli.py +34 -7
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/render.py +12 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/runner.py +14 -5
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/selectors.py +32 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/skill/SKILL.md +9 -5
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/tree.py +14 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/fake_provider.py +6 -1
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_cli.py +15 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_skill.py +6 -6
- use_computer_cli-0.3.1/tests/test_which_window.py +113 -0
- use_computer_cli-0.3.0/tests/test_which_window.py +0 -59
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/.gitignore +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/LICENSE +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/README.md +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/__init__.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/accessibility/__init__.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/accessibility/roles.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/actions.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/backends/__init__.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/backends/base.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/backends/local.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/backends/vnc.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/compare.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/config.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/coordinates.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/errors.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/keys.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/prune.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/skill/__init__.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/__init__.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/conftest.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/fake_backend.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_atspi_hint.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_backends.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_blind_spots.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_compare.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_config.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_config_init.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_coordinates.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_elements.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_keys.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_prune.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_render.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_runner.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_selectors.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_skill_is_true.py +0 -0
- {use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/tests/test_trusting_a_result.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: use-computer-cli
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot.
|
|
5
5
|
Project-URL: Homepage, https://github.com/applica-software-guru/use-computer
|
|
6
6
|
Author: Bruno Fortunato
|
|
@@ -52,6 +52,7 @@ Requires-Dist: mss>=10; extra == 'local'
|
|
|
52
52
|
Requires-Dist: pynput>=1.8; extra == 'local'
|
|
53
53
|
Provides-Extra: tree
|
|
54
54
|
Requires-Dist: pyobjc-framework-applicationservices>=10; (sys_platform == 'darwin') and extra == 'tree'
|
|
55
|
+
Requires-Dist: python-xlib>=0.33; (sys_platform == 'linux') and extra == 'tree'
|
|
55
56
|
Requires-Dist: uiautomation>=2.0; (sys_platform == 'win32') and extra == 'tree'
|
|
56
57
|
Provides-Extra: vnc
|
|
57
58
|
Requires-Dist: vncdotool>=1.3; extra == 'vnc'
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "use-computer-cli"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -52,6 +52,11 @@ vnc = ["vncdotool>=1.3"]
|
|
|
52
52
|
# bindings come from the distro (python3-gi, gir1.2-atspi-2.0) and are reached with a
|
|
53
53
|
# --system-site-packages virtualenv; the error message says so when they are absent.
|
|
54
54
|
tree = [
|
|
55
|
+
# python-xlib reads _NET_ACTIVE_WINDOW, the only reliable answer to "which window is in
|
|
56
|
+
# front" -- AT-SPI reports `active` per application and marks several at once. A pure-Python
|
|
57
|
+
# wheel with no build step, which is exactly what PyGObject was not, so it does not
|
|
58
|
+
# reintroduce BUG-003. Imported softly: absent, or on Wayland, the tool falls back.
|
|
59
|
+
"python-xlib>=0.33; sys_platform == 'linux'",
|
|
55
60
|
"uiautomation>=2.0; sys_platform == 'win32'",
|
|
56
61
|
"pyobjc-framework-ApplicationServices>=10; sys_platform == 'darwin'",
|
|
57
62
|
]
|
|
@@ -102,6 +107,7 @@ module = [
|
|
|
102
107
|
"ApplicationServices.*",
|
|
103
108
|
"Quartz.*",
|
|
104
109
|
"gi.*",
|
|
110
|
+
"Xlib.*",
|
|
105
111
|
"uiautomation.*",
|
|
106
112
|
"tomli",
|
|
107
113
|
]
|
|
@@ -17,7 +17,14 @@ from typing import Any
|
|
|
17
17
|
from use_computer.accessibility import roles
|
|
18
18
|
from use_computer.accessibility.base import require
|
|
19
19
|
from use_computer.errors import UITreeUnavailableError, UseComputerError
|
|
20
|
-
from use_computer.tree import
|
|
20
|
+
from use_computer.tree import (
|
|
21
|
+
ActiveWindow,
|
|
22
|
+
Box,
|
|
23
|
+
TreeScope,
|
|
24
|
+
TreeScopeKind,
|
|
25
|
+
UINode,
|
|
26
|
+
WindowInfo,
|
|
27
|
+
)
|
|
21
28
|
|
|
22
29
|
#: Where a distro puts PyGObject. The compiled part carries the Python version it was built for,
|
|
23
30
|
#: which is the fact that decides whether any of the advice below will work.
|
|
@@ -402,6 +409,47 @@ class AtspiProvider:
|
|
|
402
409
|
return bool(interface.do_action(index))
|
|
403
410
|
return False
|
|
404
411
|
|
|
412
|
+
def active_window(self) -> ActiveWindow | None:
|
|
413
|
+
"""Ask the window manager, because AT-SPI cannot answer this.
|
|
414
|
+
|
|
415
|
+
`_NET_ACTIVE_WINDOW` on the root window names exactly one window; AT-SPI's `active` state
|
|
416
|
+
is per application and marked three at once on the desktop that produced the bug. Imported
|
|
417
|
+
softly on purpose: no python-xlib, or a Wayland session that publishes no such property,
|
|
418
|
+
and the caller falls back to the flags.
|
|
419
|
+
"""
|
|
420
|
+
try:
|
|
421
|
+
from Xlib import X, display
|
|
422
|
+
except ImportError:
|
|
423
|
+
return None
|
|
424
|
+
try:
|
|
425
|
+
connection = display.Display()
|
|
426
|
+
except Exception:
|
|
427
|
+
return None # no X server: Wayland, or no session at all
|
|
428
|
+
try:
|
|
429
|
+
root = connection.screen().root
|
|
430
|
+
active = root.get_full_property(
|
|
431
|
+
connection.intern_atom("_NET_ACTIVE_WINDOW"), X.AnyPropertyType
|
|
432
|
+
)
|
|
433
|
+
if active is None or not active.value:
|
|
434
|
+
return None
|
|
435
|
+
window = connection.create_resource_object("window", active.value[0])
|
|
436
|
+
pid = window.get_full_property(
|
|
437
|
+
connection.intern_atom("_NET_WM_PID"), X.AnyPropertyType
|
|
438
|
+
)
|
|
439
|
+
name = window.get_full_property(
|
|
440
|
+
connection.intern_atom("_NET_WM_NAME"), connection.intern_atom("UTF8_STRING")
|
|
441
|
+
)
|
|
442
|
+
return ActiveWindow(
|
|
443
|
+
pid=int(pid.value[0]) if pid is not None and pid.value else None,
|
|
444
|
+
title=name.value.decode("utf-8", "replace") if name is not None else None,
|
|
445
|
+
)
|
|
446
|
+
except Exception:
|
|
447
|
+
# A hint that fails is not an error: the caller has a working fallback.
|
|
448
|
+
return None
|
|
449
|
+
finally:
|
|
450
|
+
with suppress(Exception):
|
|
451
|
+
connection.close()
|
|
452
|
+
|
|
405
453
|
def activate(self, window_id: str) -> bool:
|
|
406
454
|
"""AT-SPI offers no raise for a window.
|
|
407
455
|
|
|
@@ -15,7 +15,14 @@ from typing import Any
|
|
|
15
15
|
from use_computer.accessibility import roles
|
|
16
16
|
from use_computer.accessibility.base import require
|
|
17
17
|
from use_computer.errors import PermissionDeniedError
|
|
18
|
-
from use_computer.tree import
|
|
18
|
+
from use_computer.tree import (
|
|
19
|
+
ActiveWindow,
|
|
20
|
+
Box,
|
|
21
|
+
TreeScope,
|
|
22
|
+
TreeScopeKind,
|
|
23
|
+
UINode,
|
|
24
|
+
WindowInfo,
|
|
25
|
+
)
|
|
19
26
|
|
|
20
27
|
#: kAXErrorAPIDisabled -- the process is not trusted for accessibility.
|
|
21
28
|
API_DISABLED = -25211
|
|
@@ -197,6 +204,10 @@ class AxProvider:
|
|
|
197
204
|
roles.COLLAPSE: "AXPress",
|
|
198
205
|
}
|
|
199
206
|
|
|
207
|
+
def active_window(self) -> ActiveWindow | None:
|
|
208
|
+
"""This platform's own `active` flag is already per window, so there is nothing to add."""
|
|
209
|
+
return None
|
|
210
|
+
|
|
200
211
|
def activate(self, window_id: str) -> bool:
|
|
201
212
|
"""macOS windows do accept a raise of their own: AXRaise."""
|
|
202
213
|
element = self._index.get(window_id)
|
|
@@ -15,7 +15,7 @@ from types import ModuleType
|
|
|
15
15
|
from typing import Protocol, runtime_checkable
|
|
16
16
|
|
|
17
17
|
from use_computer.errors import UITreeUnavailableError
|
|
18
|
-
from use_computer.tree import TreeScope, UINode, WindowInfo
|
|
18
|
+
from use_computer.tree import ActiveWindow, TreeScope, UINode, WindowInfo
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
@runtime_checkable
|
|
@@ -59,6 +59,15 @@ class AccessibilityProvider(Protocol):
|
|
|
59
59
|
accessible there reports no actions at all.
|
|
60
60
|
"""
|
|
61
61
|
|
|
62
|
+
def active_window(self) -> ActiveWindow | None:
|
|
63
|
+
"""What the window manager says is in front, as a pid and a title.
|
|
64
|
+
|
|
65
|
+
A *hint*. ``None`` means this platform has nothing better than the per-window flags, which
|
|
66
|
+
is the honest answer on Wayland or without the optional binding. Matching it to a window is
|
|
67
|
+
policy and lives in ``selectors.mark_active``: a rule that lives in a provider is a rule
|
|
68
|
+
the other two platforms drift from, which has already happened once here.
|
|
69
|
+
"""
|
|
70
|
+
|
|
62
71
|
def close(self) -> None:
|
|
63
72
|
"""Release whatever the provider holds."""
|
|
64
73
|
|
|
@@ -11,7 +11,14 @@ from typing import Any
|
|
|
11
11
|
from use_computer.accessibility import roles
|
|
12
12
|
from use_computer.accessibility.base import require
|
|
13
13
|
from use_computer.errors import UITreeUnavailableError
|
|
14
|
-
from use_computer.tree import
|
|
14
|
+
from use_computer.tree import (
|
|
15
|
+
ActiveWindow,
|
|
16
|
+
Box,
|
|
17
|
+
TreeScope,
|
|
18
|
+
TreeScopeKind,
|
|
19
|
+
UINode,
|
|
20
|
+
WindowInfo,
|
|
21
|
+
)
|
|
15
22
|
|
|
16
23
|
|
|
17
24
|
class UiaProvider:
|
|
@@ -167,6 +174,10 @@ class UiaProvider:
|
|
|
167
174
|
|
|
168
175
|
# --- acting ------------------------------------------------------------------------------
|
|
169
176
|
|
|
177
|
+
def active_window(self) -> ActiveWindow | None:
|
|
178
|
+
"""This platform's own `active` flag is already per window, so there is nothing to add."""
|
|
179
|
+
return None
|
|
180
|
+
|
|
170
181
|
def activate(self, window_id: str) -> bool:
|
|
171
182
|
"""Windows has a native raise: `SetActive` on the top-level control."""
|
|
172
183
|
control = self._index.get(window_id)
|
|
@@ -1210,40 +1210,67 @@ def skill_install_command(
|
|
|
1210
1210
|
scope: ScopeOption = Scope.PROJECT,
|
|
1211
1211
|
dir: DirOption = None,
|
|
1212
1212
|
force: Annotated[bool, typer.Option("--force", help="Overwrite an existing copy.")] = False,
|
|
1213
|
+
format: FormatOption = OutputFormat.TEXT,
|
|
1213
1214
|
) -> None:
|
|
1214
1215
|
"""Install the bundled skill."""
|
|
1215
1216
|
try:
|
|
1216
1217
|
state = skill_install(scope, override=dir, force=force)
|
|
1217
1218
|
except UseComputerError as exc:
|
|
1218
1219
|
_fail(exc)
|
|
1219
|
-
|
|
1220
|
+
_skill_say("install", state, format)
|
|
1220
1221
|
|
|
1221
1222
|
|
|
1222
1223
|
@skill_app.command("update")
|
|
1223
|
-
def skill_update_command(
|
|
1224
|
+
def skill_update_command(
|
|
1225
|
+
scope: ScopeOption = Scope.PROJECT,
|
|
1226
|
+
dir: DirOption = None,
|
|
1227
|
+
format: FormatOption = OutputFormat.TEXT,
|
|
1228
|
+
) -> None:
|
|
1224
1229
|
"""Refresh an installed skill from the bundled copy."""
|
|
1225
1230
|
try:
|
|
1226
1231
|
state = skill_update(scope, override=dir)
|
|
1227
1232
|
except UseComputerError as exc:
|
|
1228
1233
|
_fail(exc)
|
|
1229
|
-
|
|
1234
|
+
_skill_say("update", state, format)
|
|
1230
1235
|
|
|
1231
1236
|
|
|
1232
1237
|
@skill_app.command("remove")
|
|
1233
|
-
def skill_remove_command(
|
|
1238
|
+
def skill_remove_command(
|
|
1239
|
+
scope: ScopeOption = Scope.PROJECT,
|
|
1240
|
+
dir: DirOption = None,
|
|
1241
|
+
format: FormatOption = OutputFormat.TEXT,
|
|
1242
|
+
) -> None:
|
|
1234
1243
|
"""Remove an installed skill."""
|
|
1235
1244
|
try:
|
|
1236
1245
|
state = skill_remove(scope, override=dir)
|
|
1237
1246
|
except UseComputerError as exc:
|
|
1238
1247
|
_fail(exc)
|
|
1239
|
-
|
|
1248
|
+
_skill_say("remove", state, format)
|
|
1240
1249
|
|
|
1241
1250
|
|
|
1242
1251
|
@skill_app.command("status")
|
|
1243
|
-
def skill_status_command(
|
|
1252
|
+
def skill_status_command(
|
|
1253
|
+
scope: ScopeOption = Scope.PROJECT,
|
|
1254
|
+
dir: DirOption = None,
|
|
1255
|
+
format: FormatOption = OutputFormat.TEXT,
|
|
1256
|
+
) -> None:
|
|
1244
1257
|
"""Report whether the skill is installed and current."""
|
|
1245
1258
|
state = skill_status(scope, override=dir)
|
|
1246
|
-
|
|
1259
|
+
_skill_say("status", state, format)
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def _skill_say(action: str, state: Any, format: OutputFormat) -> None:
|
|
1263
|
+
"""The skill commands answer in text like everything else.
|
|
1264
|
+
|
|
1265
|
+
This is the command an agent runs to find out whether its own instructions are current, and it
|
|
1266
|
+
was the last one still replying with a JSON object. A contract with three exceptions is not a
|
|
1267
|
+
contract.
|
|
1268
|
+
"""
|
|
1269
|
+
payload = {"action": action, **_skill_json(state)}
|
|
1270
|
+
if format is OutputFormat.JSON:
|
|
1271
|
+
_emit(payload)
|
|
1272
|
+
return
|
|
1273
|
+
_write(render.skill(payload))
|
|
1247
1274
|
|
|
1248
1275
|
|
|
1249
1276
|
def _skill_json(state: Any) -> dict[str, Any]:
|
|
@@ -159,6 +159,17 @@ def config(payload: Mapping[str, Any]) -> str:
|
|
|
159
159
|
return "\n".join(lines)
|
|
160
160
|
|
|
161
161
|
|
|
162
|
+
def skill(payload: Mapping[str, Any]) -> str:
|
|
163
|
+
"""What a `skill` command did, on one line: action, scope, state, path."""
|
|
164
|
+
parts = [
|
|
165
|
+
str(payload.get("action", "")),
|
|
166
|
+
str(payload.get("scope", "")),
|
|
167
|
+
str(payload.get("status", "")),
|
|
168
|
+
str(payload.get("path", "")),
|
|
169
|
+
]
|
|
170
|
+
return " ".join(part for part in parts if part)
|
|
171
|
+
|
|
172
|
+
|
|
162
173
|
__all__ = [
|
|
163
174
|
"TREE_LEGEND",
|
|
164
175
|
"WINDOWS_LEGEND",
|
|
@@ -166,4 +177,5 @@ __all__ = [
|
|
|
166
177
|
"windows",
|
|
167
178
|
"windows_for_a_reader",
|
|
168
179
|
"config",
|
|
180
|
+
"skill",
|
|
169
181
|
]
|
|
@@ -68,12 +68,12 @@ from use_computer.selectors import (
|
|
|
68
68
|
clamp_text,
|
|
69
69
|
count,
|
|
70
70
|
find,
|
|
71
|
+
mark_active,
|
|
71
72
|
mark_unexposed,
|
|
72
73
|
notable_states,
|
|
73
74
|
prune,
|
|
74
75
|
resolve_one,
|
|
75
76
|
resolve_window,
|
|
76
|
-
sole_active,
|
|
77
77
|
subtree,
|
|
78
78
|
summarise_offscreen,
|
|
79
79
|
walk,
|
|
@@ -428,7 +428,7 @@ class Session:
|
|
|
428
428
|
return _Outcome(tree=self._tree(action))
|
|
429
429
|
|
|
430
430
|
if isinstance(action, WindowsAction):
|
|
431
|
-
found = tuple(
|
|
431
|
+
found = tuple(self._window_list())
|
|
432
432
|
if action.format is OutputFormat.JSON:
|
|
433
433
|
return _Outcome(windows=WindowsResult(windows=found))
|
|
434
434
|
return _Outcome(windows=WindowsResult(text=render.windows(found)))
|
|
@@ -540,9 +540,19 @@ class Session:
|
|
|
540
540
|
again = self._window_for(resolved)
|
|
541
541
|
return self._name_window(again) if again is not None else root.describe()
|
|
542
542
|
|
|
543
|
+
def _window_list(self) -> list[WindowInfo]:
|
|
544
|
+
"""The windows, with the active mark settled once.
|
|
545
|
+
|
|
546
|
+
Settled here rather than in each provider, and with the window manager's answer preferred
|
|
547
|
+
over the per-window flags where a provider can offer one.
|
|
548
|
+
"""
|
|
549
|
+
provider = self._provider()
|
|
550
|
+
entries: list[WindowInfo] = mark_active(provider.windows(), provider.active_window())
|
|
551
|
+
return entries
|
|
552
|
+
|
|
543
553
|
def _window_for(self, scope: TreeScope) -> WindowInfo | None:
|
|
544
554
|
"""The entry in `windows` this scope names, when it names exactly one."""
|
|
545
|
-
entries =
|
|
555
|
+
entries = self._window_list()
|
|
546
556
|
if scope.kind is TreeScopeKind.FOCUSED:
|
|
547
557
|
return next((entry for entry in entries if entry.active), None)
|
|
548
558
|
if scope.kind is TreeScopeKind.ID:
|
|
@@ -632,8 +642,7 @@ class Session:
|
|
|
632
642
|
# windows claiming to be active is not an answer, and picking the first is how an
|
|
633
643
|
# agent reads, clicks and verifies inside the wrong application, consistently.
|
|
634
644
|
return TreeScope(
|
|
635
|
-
kind=TreeScopeKind.ID,
|
|
636
|
-
value=active_window(sole_active(self._provider().windows())).id,
|
|
645
|
+
kind=TreeScopeKind.ID, value=active_window(self._window_list()).id
|
|
637
646
|
)
|
|
638
647
|
if scope.kind is not TreeScopeKind.TITLE or not scope.value:
|
|
639
648
|
return scope
|
|
@@ -15,7 +15,7 @@ from use_computer.errors import (
|
|
|
15
15
|
NodeNotFoundError,
|
|
16
16
|
UITreeUnavailableError,
|
|
17
17
|
)
|
|
18
|
-
from use_computer.tree import Box, NodeSelector, UINode, WindowInfo
|
|
18
|
+
from use_computer.tree import ActiveWindow, Box, NodeSelector, UINode, WindowInfo
|
|
19
19
|
|
|
20
20
|
#: Roles that are worth keeping even when the platform reports no actions and no name -- an
|
|
21
21
|
#: empty text field has nothing to say about itself and is still the thing an agent came for.
|
|
@@ -111,6 +111,37 @@ def is_interesting(node: UINode) -> bool:
|
|
|
111
111
|
return is_on_screen(node) and (is_interactable(node) or carries_text(node))
|
|
112
112
|
|
|
113
113
|
|
|
114
|
+
def mark_active(
|
|
115
|
+
entries: Sequence[WindowInfo], hint: ActiveWindow | None = None
|
|
116
|
+
) -> list[WindowInfo]:
|
|
117
|
+
"""Decide which single window is the active one, preferring what the window manager says.
|
|
118
|
+
|
|
119
|
+
The flags cannot settle it. AT-SPI reports `active` per *application*, so on an ordinary
|
|
120
|
+
desktop three windows claimed it at once. A window manager can settle it -- on X11
|
|
121
|
+
`_NET_ACTIVE_WINDOW` names one window, by pid and title -- and where that answer exists it wins
|
|
122
|
+
over the flags.
|
|
123
|
+
|
|
124
|
+
The hint is a hint: absent on Wayland, absent without the binding, and wrong if an application
|
|
125
|
+
renamed its window between the two reads. It never invents a mark, and when it matches nothing
|
|
126
|
+
the flags decide exactly as before.
|
|
127
|
+
"""
|
|
128
|
+
matched = _by_hint(entries, hint) if hint is not None else None
|
|
129
|
+
if matched is None:
|
|
130
|
+
return sole_active(entries)
|
|
131
|
+
return [entry.model_copy(update={"active": entry.id == matched.id}) for entry in entries]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _by_hint(entries: Sequence[WindowInfo], hint: ActiveWindow) -> WindowInfo | None:
|
|
135
|
+
"""The one window this hint names, or nothing -- never a first match."""
|
|
136
|
+
candidates = [entry for entry in entries if hint.pid is not None and entry.pid == hint.pid]
|
|
137
|
+
if len(candidates) == 1:
|
|
138
|
+
return candidates[0]
|
|
139
|
+
if not candidates:
|
|
140
|
+
return None
|
|
141
|
+
named = [entry for entry in candidates if entry.title == hint.title]
|
|
142
|
+
return named[0] if len(named) == 1 else None
|
|
143
|
+
|
|
144
|
+
|
|
114
145
|
def sole_active(entries: Sequence[WindowInfo]) -> list[WindowInfo]:
|
|
115
146
|
"""Leave the active mark on at most one window.
|
|
116
147
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: use-computer
|
|
3
3
|
description: Read and act on a GUI — the accessibility tree of what is on screen (roles, names, clickable boxes), then click, focus, toggle, expand, select, set a value, type, press keys, drag, scroll, screenshot. Locally or over VNC. Ask the tree first and use ui-locator's pixel coordinates only when the tree cannot see the element. Bring a window forward before aiming at it, and confirm what happened by re-reading, not by trusting the line.
|
|
4
4
|
x-skill-id: use-computer
|
|
5
|
-
x-skill-version: "
|
|
5
|
+
x-skill-version: "6"
|
|
6
6
|
---
|
|
7
7
|
|
|
8
8
|
# use-computer
|
|
@@ -136,6 +136,9 @@ id app role title pid box
|
|
|
136
136
|
0/34/0 Codex window ChatGPT 144775 0,0 1920x1038 *
|
|
137
137
|
```
|
|
138
138
|
|
|
139
|
+
`*` marks the window that is actually in front — the window manager's answer where there is one,
|
|
140
|
+
not a flag the toolkit sets per application. At most one window ever carries it.
|
|
141
|
+
|
|
139
142
|
Use the `title` as `--window` for everything that follows, and **`app` to tell windows apart** — a
|
|
140
143
|
title alone will not tell you which one is Telegram. The `*` marks the one `--window focused`
|
|
141
144
|
resolves to.
|
|
@@ -450,10 +453,11 @@ of an element action is also the cheapest way to check a selector is unambiguous
|
|
|
450
453
|
screenshots and coordinates. Otherwise the message names exactly what to install: an extra on
|
|
451
454
|
Windows and macOS, and on Linux the distro packages plus a `--system-site-packages` virtualenv,
|
|
452
455
|
because the extra does not help there.
|
|
453
|
-
- **`AmbiguousWindowError` on `focused`** —
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
guess — an empty column is the
|
|
456
|
+
- **`AmbiguousWindowError` on `focused`** — nothing could say which window is on top. Where a
|
|
457
|
+
window manager can be asked it settles this, so seeing it means it could not be (Wayland, no
|
|
458
|
+
session, a window renamed between two reads). Read `windows` and pass `--window ID`. When it is
|
|
459
|
+
ambiguous, `windows` shows **no** `*` at all rather than a guess — an empty column is the
|
|
460
|
+
answer, not a missing one.
|
|
457
461
|
- **`BackendNotAvailableError`** — the extra is not installed. The message names it.
|
|
458
462
|
- **Local backend not enabled** — the `local` backend controls the user's own machine and needs
|
|
459
463
|
an explicit opt-in. Tell the user to set `allow-local = true` in the profile; do not work
|
|
@@ -183,6 +183,20 @@ class WindowsResult(BaseModel):
|
|
|
183
183
|
windows: tuple[WindowInfo, ...] = ()
|
|
184
184
|
|
|
185
185
|
|
|
186
|
+
class ActiveWindow(BaseModel):
|
|
187
|
+
"""What a window manager says is in front: a hint, never a decision.
|
|
188
|
+
|
|
189
|
+
AT-SPI cannot answer this -- it reports `active` per application, so several windows claim it
|
|
190
|
+
at once. A window manager can: on X11 `_NET_ACTIVE_WINDOW` names exactly one. Matching the hint
|
|
191
|
+
to a window is policy and lives in `selectors.py`, so the three platforms cannot drift on it.
|
|
192
|
+
"""
|
|
193
|
+
|
|
194
|
+
model_config = ConfigDict(frozen=True)
|
|
195
|
+
|
|
196
|
+
pid: int | None = None
|
|
197
|
+
title: str | None = None
|
|
198
|
+
|
|
199
|
+
|
|
186
200
|
class TreeScope(BaseModel):
|
|
187
201
|
"""What to snapshot. The focused window by default, never the whole desktop."""
|
|
188
202
|
|
|
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|
|
11
11
|
from dataclasses import dataclass, field
|
|
12
12
|
|
|
13
13
|
from use_computer.errors import PermissionDeniedError, UITreeUnavailableError
|
|
14
|
-
from use_computer.tree import Box, TreeScope, UINode, WindowInfo
|
|
14
|
+
from use_computer.tree import ActiveWindow, Box, TreeScope, UINode, WindowInfo
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def node(
|
|
@@ -100,6 +100,8 @@ class FakeProvider:
|
|
|
100
100
|
window_list: tuple[WindowInfo, ...] | None = None
|
|
101
101
|
#: Whether this platform has a native raise. AT-SPI does not; Windows and macOS do.
|
|
102
102
|
native_raise: bool = False
|
|
103
|
+
#: What a window manager would say is in front, when the test is about that.
|
|
104
|
+
hint: ActiveWindow | None = None
|
|
103
105
|
activated: list[str] = field(default_factory=list)
|
|
104
106
|
|
|
105
107
|
def windows(self) -> list[WindowInfo]:
|
|
@@ -131,6 +133,9 @@ class FakeProvider:
|
|
|
131
133
|
self.calls.append((node_id, action, value))
|
|
132
134
|
return action not in self.refuse
|
|
133
135
|
|
|
136
|
+
def active_window(self) -> ActiveWindow | None:
|
|
137
|
+
return self.hint
|
|
138
|
+
|
|
134
139
|
def activate(self, window_id: str) -> bool:
|
|
135
140
|
self.activated.append(window_id)
|
|
136
141
|
return self.native_raise
|
|
@@ -592,3 +592,18 @@ def test_a_single_screenshot_keeps_its_whole_path(
|
|
|
592
592
|
result = invoke(runner, "screenshot")
|
|
593
593
|
assert strip_ansi(result.stdout).split("\n")[0].startswith("/")
|
|
594
594
|
assert "screenshots in " not in strip_ansi(result.stdout)
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def test_the_skill_command_answers_in_text(runner: CliRunner) -> None:
|
|
598
|
+
# The command an agent runs to find out whether its own instructions are current, and the last
|
|
599
|
+
# one still replying with a JSON object after the contract was inverted.
|
|
600
|
+
result = invoke(runner, "skill", "status")
|
|
601
|
+
assert result.exit_code == EXIT_OK
|
|
602
|
+
assert not result.stdout.lstrip().startswith("{")
|
|
603
|
+
assert result.stdout.split()[0] == "status"
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def test_the_skill_command_still_has_a_json_form(runner: CliRunner) -> None:
|
|
607
|
+
result = invoke(runner, "skill", "status", "--format", "json")
|
|
608
|
+
assert result.exit_code == EXIT_OK
|
|
609
|
+
assert json.loads(result.stdout)["action"] == "status"
|
|
@@ -111,19 +111,19 @@ def test_the_scope_flag_accepts_every_documented_value() -> None:
|
|
|
111
111
|
assert {scope.value for scope in Scope} == {"user", "project", "agents", "claude"}
|
|
112
112
|
|
|
113
113
|
|
|
114
|
-
def
|
|
114
|
+
def test_the_cli_reports_what_it_did(project: Path) -> None:
|
|
115
115
|
runner = CliRunner()
|
|
116
116
|
result = runner.invoke(app, ["skill", "install"], catch_exceptions=False)
|
|
117
117
|
assert result.exit_code == EXIT_OK
|
|
118
|
-
|
|
119
|
-
assert payload["status"] == "up-to-date"
|
|
120
|
-
assert payload["scope"] == "project"
|
|
118
|
+
assert result.stdout.split()[:3] == ["install", "project", "up-to-date"]
|
|
121
119
|
|
|
122
120
|
again = runner.invoke(app, ["skill", "install"], catch_exceptions=False)
|
|
123
121
|
assert again.exit_code == EXIT_FAILURE
|
|
124
122
|
|
|
125
|
-
status_result = runner.invoke(
|
|
123
|
+
status_result = runner.invoke(
|
|
124
|
+
app, ["skill", "status", "--format", "json"], catch_exceptions=False
|
|
125
|
+
)
|
|
126
126
|
assert json.loads(status_result.stdout)["installed"] is True
|
|
127
127
|
|
|
128
128
|
removed = runner.invoke(app, ["skill", "remove"], catch_exceptions=False)
|
|
129
|
-
assert
|
|
129
|
+
assert removed.stdout.split()[:3] == ["remove", "project", "missing"]
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Which window is the active one, and what happens when the platform cannot say.
|
|
2
|
+
|
|
3
|
+
Measured on a real desktop: three windows carried the mark at once, because AT-SPI reports
|
|
4
|
+
`active` per *application*. Neither that flag, nor `focused`, nor looking for a focused
|
|
5
|
+
descendant identified the window actually on top.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from tests.fake_provider import window
|
|
13
|
+
from use_computer.errors import AmbiguousWindowError, UITreeUnavailableError
|
|
14
|
+
from use_computer.render import windows_for_a_reader
|
|
15
|
+
from use_computer.selectors import active_window, mark_active, sole_active
|
|
16
|
+
from use_computer.tree import ActiveWindow
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_one_claim_is_left_alone() -> None:
|
|
20
|
+
marked = sole_active([window("0/1", "A", active=True), window("0/2", "B")])
|
|
21
|
+
assert [entry.active for entry in marked] == [True, False]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_several_claims_leave_no_mark_at_all() -> None:
|
|
25
|
+
# Saying nothing is worth more than a mark that is wrong two times in three, and this column
|
|
26
|
+
# is the first thing an agent is told to read.
|
|
27
|
+
marked = sole_active(
|
|
28
|
+
[
|
|
29
|
+
window("0/19", "", active=True),
|
|
30
|
+
window("0/34", "ChatGPT", active=True),
|
|
31
|
+
window("0/37", "Drawing", active=True),
|
|
32
|
+
]
|
|
33
|
+
)
|
|
34
|
+
assert [entry.active for entry in marked] == [False, False, False]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_the_column_then_shows_no_star() -> None:
|
|
38
|
+
text = windows_for_a_reader(
|
|
39
|
+
sole_active([window("0/1", "A", active=True), window("0/2", "B", active=True)])
|
|
40
|
+
)
|
|
41
|
+
assert "*" not in text.replace("active", "")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_focused_refuses_rather_than_taking_the_first() -> None:
|
|
45
|
+
# Picking the first match is how an agent reads, clicks and verifies inside the wrong
|
|
46
|
+
# application, consistently and with no sign anything went wrong.
|
|
47
|
+
with pytest.raises(AmbiguousWindowError) as excinfo:
|
|
48
|
+
active_window([window("0/34", "ChatGPT", active=True), window("0/37", "Draw", active=True)])
|
|
49
|
+
assert {entry.id for entry in excinfo.value.candidates} == {"0/34", "0/37"}
|
|
50
|
+
assert "--window" in str(excinfo.value)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_no_claim_at_all_says_how_to_name_one() -> None:
|
|
54
|
+
with pytest.raises(UITreeUnavailableError) as excinfo:
|
|
55
|
+
active_window([window("0/1", "A"), window("0/2", "B")])
|
|
56
|
+
assert "--window" in str(excinfo.value)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_one_claim_resolves() -> None:
|
|
60
|
+
assert active_window([window("0/1", "A"), window("0/2", "B", active=True)]).id == "0/2"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# --- what the window manager says --------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_the_window_manager_settles_what_the_flags_could_not() -> None:
|
|
67
|
+
# Three windows claimed `active`; `_NET_ACTIVE_WINDOW` names one, by pid and title.
|
|
68
|
+
entries = [
|
|
69
|
+
window("0/19", "", active=True),
|
|
70
|
+
window("0/29", "a terminal", active=True),
|
|
71
|
+
window("0/37", "Drawing", active=True),
|
|
72
|
+
]
|
|
73
|
+
entries[1] = entries[1].model_copy(update={"pid": 2379})
|
|
74
|
+
marked = mark_active(entries, ActiveWindow(pid=2379, title="a terminal"))
|
|
75
|
+
assert [entry.active for entry in marked] == [False, True, False]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_the_hint_wins_over_a_flag_that_disagrees() -> None:
|
|
79
|
+
entries = [
|
|
80
|
+
window("0/29", "a terminal", active=True).model_copy(update={"pid": 1}),
|
|
81
|
+
window("0/37", "Drawing").model_copy(update={"pid": 2}),
|
|
82
|
+
]
|
|
83
|
+
marked = mark_active(entries, ActiveWindow(pid=2, title="Drawing"))
|
|
84
|
+
assert [entry.active for entry in marked] == [False, True]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_one_pid_with_several_windows_is_settled_by_the_title() -> None:
|
|
88
|
+
entries = [
|
|
89
|
+
window("0/34/0", "ChatGPT").model_copy(update={"pid": 9}),
|
|
90
|
+
window("0/34/1", "Notes").model_copy(update={"pid": 9}),
|
|
91
|
+
]
|
|
92
|
+
marked = mark_active(entries, ActiveWindow(pid=9, title="Notes"))
|
|
93
|
+
assert [entry.active for entry in marked] == [False, True]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_a_hint_that_matches_nothing_leaves_the_flags_alone() -> None:
|
|
97
|
+
# Wayland, a renamed window, a pid the tree does not know: never invent a mark.
|
|
98
|
+
entries = [window("0/1", "A", active=True), window("0/2", "B")]
|
|
99
|
+
assert [e.active for e in mark_active(entries, ActiveWindow(pid=999))] == [True, False]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_no_hint_at_all_behaves_exactly_as_before() -> None:
|
|
103
|
+
entries = [window("0/1", "A", active=True), window("0/2", "B", active=True)]
|
|
104
|
+
assert [entry.active for entry in mark_active(entries, None)] == [False, False]
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_an_ambiguous_hint_does_not_guess() -> None:
|
|
108
|
+
entries = [
|
|
109
|
+
window("0/34/0", "ChatGPT").model_copy(update={"pid": 9}),
|
|
110
|
+
window("0/34/1", "ChatGPT").model_copy(update={"pid": 9}),
|
|
111
|
+
]
|
|
112
|
+
marked = mark_active(entries, ActiveWindow(pid=9, title="ChatGPT"))
|
|
113
|
+
assert [entry.active for entry in marked] == [False, False]
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
"""Which window is the active one, and what happens when the platform cannot say.
|
|
2
|
-
|
|
3
|
-
Measured on a real desktop: three windows carried the mark at once, because AT-SPI reports
|
|
4
|
-
`active` per *application*. Neither that flag, nor `focused`, nor looking for a focused
|
|
5
|
-
descendant identified the window actually on top.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from __future__ import annotations
|
|
9
|
-
|
|
10
|
-
import pytest
|
|
11
|
-
|
|
12
|
-
from tests.fake_provider import window
|
|
13
|
-
from use_computer.errors import AmbiguousWindowError, UITreeUnavailableError
|
|
14
|
-
from use_computer.render import windows_for_a_reader
|
|
15
|
-
from use_computer.selectors import active_window, sole_active
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def test_one_claim_is_left_alone() -> None:
|
|
19
|
-
marked = sole_active([window("0/1", "A", active=True), window("0/2", "B")])
|
|
20
|
-
assert [entry.active for entry in marked] == [True, False]
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def test_several_claims_leave_no_mark_at_all() -> None:
|
|
24
|
-
# Saying nothing is worth more than a mark that is wrong two times in three, and this column
|
|
25
|
-
# is the first thing an agent is told to read.
|
|
26
|
-
marked = sole_active(
|
|
27
|
-
[
|
|
28
|
-
window("0/19", "", active=True),
|
|
29
|
-
window("0/34", "ChatGPT", active=True),
|
|
30
|
-
window("0/37", "Drawing", active=True),
|
|
31
|
-
]
|
|
32
|
-
)
|
|
33
|
-
assert [entry.active for entry in marked] == [False, False, False]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def test_the_column_then_shows_no_star() -> None:
|
|
37
|
-
text = windows_for_a_reader(
|
|
38
|
-
sole_active([window("0/1", "A", active=True), window("0/2", "B", active=True)])
|
|
39
|
-
)
|
|
40
|
-
assert "*" not in text.replace("active", "")
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def test_focused_refuses_rather_than_taking_the_first() -> None:
|
|
44
|
-
# Picking the first match is how an agent reads, clicks and verifies inside the wrong
|
|
45
|
-
# application, consistently and with no sign anything went wrong.
|
|
46
|
-
with pytest.raises(AmbiguousWindowError) as excinfo:
|
|
47
|
-
active_window([window("0/34", "ChatGPT", active=True), window("0/37", "Draw", active=True)])
|
|
48
|
-
assert {entry.id for entry in excinfo.value.candidates} == {"0/34", "0/37"}
|
|
49
|
-
assert "--window" in str(excinfo.value)
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def test_no_claim_at_all_says_how_to_name_one() -> None:
|
|
53
|
-
with pytest.raises(UITreeUnavailableError) as excinfo:
|
|
54
|
-
active_window([window("0/1", "A"), window("0/2", "B")])
|
|
55
|
-
assert "--window" in str(excinfo.value)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def test_one_claim_resolves() -> None:
|
|
59
|
-
assert active_window([window("0/1", "A"), window("0/2", "B", active=True)]).id == "0/2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{use_computer_cli-0.3.0 → use_computer_cli-0.3.1}/src/use_computer/accessibility/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|