use-computer-cli 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/PKG-INFO +1 -1
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/pyproject.toml +1 -1
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/cli.py +83 -3
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/config.py +37 -2
- use_computer_cli-0.2.2/src/use_computer/prune.py +95 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/runner.py +5 -1
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/skill/SKILL.md +67 -4
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_cli.py +98 -9
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_config.py +9 -2
- use_computer_cli-0.2.2/tests/test_prune.py +76 -0
- use_computer_cli-0.2.2/tests/test_skill_is_true.py +90 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/.gitignore +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/LICENSE +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/README.md +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/__init__.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/__init__.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/atspi.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/ax.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/base.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/roles.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/uia.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/actions.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/__init__.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/base.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/local.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/vnc.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/compare.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/coordinates.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/errors.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/keys.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/render.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/selectors.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/skill/__init__.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/tree.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/__init__.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/conftest.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/fake_backend.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/fake_provider.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_atspi_hint.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_backends.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_compare.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_config_init.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_coordinates.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_elements.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_keys.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_render.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_runner.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_selectors.py +0 -0
- {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_skill.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: use-computer-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot.
|
|
5
5
|
Project-URL: Homepage, https://github.com/applica-software-guru/use-computer
|
|
6
6
|
Author: Bruno Fortunato
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "use-computer-cli"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -48,10 +48,16 @@ from use_computer.actions import (
|
|
|
48
48
|
TypeAction,
|
|
49
49
|
WindowsAction,
|
|
50
50
|
)
|
|
51
|
-
from use_computer.config import
|
|
51
|
+
from use_computer.config import (
|
|
52
|
+
ResolvedConfig,
|
|
53
|
+
default_screenshot_dir,
|
|
54
|
+
profile_env_var,
|
|
55
|
+
write_initial_config,
|
|
56
|
+
)
|
|
52
57
|
from use_computer.config import load as load_config
|
|
53
58
|
from use_computer.coordinates import CoordinateSpace
|
|
54
59
|
from use_computer.errors import UseComputerError
|
|
60
|
+
from use_computer.prune import prune as prune_screenshots
|
|
55
61
|
from use_computer.runner import Session, as_json
|
|
56
62
|
from use_computer.skill import Scope
|
|
57
63
|
from use_computer.skill import install as skill_install
|
|
@@ -276,6 +282,22 @@ def _config(
|
|
|
276
282
|
return resolved
|
|
277
283
|
|
|
278
284
|
|
|
285
|
+
def _shared_directory(result: Any) -> Path | None:
|
|
286
|
+
"""The directory every screenshot in this run went to, if they share one.
|
|
287
|
+
|
|
288
|
+
A path is 23 tokens and a batch of five verified actions repeats the same directory in every
|
|
289
|
+
one of them -- 70 tokens of it, five times what the whole closing line costs. Naming it once
|
|
290
|
+
and printing filenames is the same information for a third of the price.
|
|
291
|
+
"""
|
|
292
|
+
paths = [
|
|
293
|
+
item.screenshot.path
|
|
294
|
+
for item in result.results
|
|
295
|
+
if item.screenshot is not None and item.screenshot.path is not None
|
|
296
|
+
]
|
|
297
|
+
parents = {path.parent for path in paths}
|
|
298
|
+
return parents.pop() if len(parents) == 1 and len(paths) > 1 else None
|
|
299
|
+
|
|
300
|
+
|
|
279
301
|
def _text_lines(result: Any) -> str:
|
|
280
302
|
"""What a run says, as prose: the read, or a line per action.
|
|
281
303
|
|
|
@@ -284,6 +306,7 @@ def _text_lines(result: Any) -> str:
|
|
|
284
306
|
and a model reads this at a third of the tokens the JSON costs.
|
|
285
307
|
"""
|
|
286
308
|
chunks: list[str] = []
|
|
309
|
+
folder = _shared_directory(result)
|
|
287
310
|
for item in result.results:
|
|
288
311
|
if item.tree is not None:
|
|
289
312
|
if item.tree.text:
|
|
@@ -305,7 +328,7 @@ def _text_lines(result: Any) -> str:
|
|
|
305
328
|
# The path is the answer, and the only part of it worth any tokens.
|
|
306
329
|
shot = item.screenshot
|
|
307
330
|
where = f" {shot.box[2]}x{shot.box[3]} of {shot.of}" if shot.box else ""
|
|
308
|
-
chunks.append(f"{shot.path}{where}")
|
|
331
|
+
chunks.append(f"{_shorten(shot.path, folder)}{where}")
|
|
309
332
|
continue
|
|
310
333
|
what = item.action.action
|
|
311
334
|
if item.matched is not None:
|
|
@@ -319,10 +342,48 @@ def _text_lines(result: Any) -> str:
|
|
|
319
342
|
elif item.resolved is not None:
|
|
320
343
|
how = f" at ({item.resolved.x}, {item.resolved.y})"
|
|
321
344
|
done = "would " if not item.performed else ""
|
|
322
|
-
|
|
345
|
+
# --verify exists to give feedback, so its answer belongs on the line. `unchanged` is the
|
|
346
|
+
# word an agent has to notice: it means the coordinate was stale, and it is what stops it
|
|
347
|
+
# clicking the same wrong pixel forever.
|
|
348
|
+
verified = ""
|
|
349
|
+
if item.change is not None:
|
|
350
|
+
verified = (
|
|
351
|
+
f" \u2014 changed {item.change.magnitude:.0%}"
|
|
352
|
+
if item.change.changed
|
|
353
|
+
else " \u2014 unchanged"
|
|
354
|
+
)
|
|
355
|
+
chunks.append(f"{done}{what}{how}{verified} \u2014 {item.duration_ms:.0f} ms")
|
|
356
|
+
if item.screenshot is not None and item.screenshot.path is not None:
|
|
357
|
+
# Already captured and already paid for. Saying where saves the agent asking again,
|
|
358
|
+
# which is the whole reason verify writes it down.
|
|
359
|
+
chunks.append(f" {_shorten(item.screenshot.path, folder)}")
|
|
360
|
+
|
|
361
|
+
chunks.append(_summary(result, folder))
|
|
323
362
|
return "\n".join(chunks)
|
|
324
363
|
|
|
325
364
|
|
|
365
|
+
def _shorten(path: Path | None, folder: Path | None) -> str:
|
|
366
|
+
"""The filename when the directory has already been named, the whole path otherwise."""
|
|
367
|
+
if path is None:
|
|
368
|
+
return ""
|
|
369
|
+
return path.name if folder is not None and path.parent == folder else str(path)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _summary(result: Any, folder: Path | None = None) -> str:
|
|
373
|
+
"""The closing line: everything the envelope carried, for about twenty tokens.
|
|
374
|
+
|
|
375
|
+
Cheap must not mean lossy. The envelope was dropped because 177 tokens of braces and repeated
|
|
376
|
+
keys is a bad price for it, not because the facts in it were worthless -- and the scale in
|
|
377
|
+
particular is what an agent needs the moment a coordinate lands somewhere surprising.
|
|
378
|
+
"""
|
|
379
|
+
screen = result.screen
|
|
380
|
+
scale = f"scale {screen.scale:g}" if screen.scale is not None else "scale unknown"
|
|
381
|
+
where = f"screen {screen.width}x{screen.height}, {scale}"
|
|
382
|
+
state = "ok" if result.ok else f"failed at action {(result.failed_index or 0) + 1}"
|
|
383
|
+
shots = f", screenshots in {folder}" if folder is not None else ""
|
|
384
|
+
return f"{state} — profile {result.profile}, backend {result.backend}, {where}{shots}"
|
|
385
|
+
|
|
386
|
+
|
|
326
387
|
def _run(
|
|
327
388
|
actions: Sequence[Action],
|
|
328
389
|
config: ResolvedConfig,
|
|
@@ -749,6 +810,24 @@ app.command("show-menu")(
|
|
|
749
810
|
)
|
|
750
811
|
|
|
751
812
|
|
|
813
|
+
@app.command()
|
|
814
|
+
def prune(
|
|
815
|
+
keep: Annotated[int, typer.Option("--keep", help="Leave the most recent N.")] = 0,
|
|
816
|
+
dry_run: Annotated[
|
|
817
|
+
bool, typer.Option("--dry-run", help="Say what would go; remove nothing.")
|
|
818
|
+
] = False,
|
|
819
|
+
use: UseOption = None,
|
|
820
|
+
verbose: VerboseOption = 0,
|
|
821
|
+
) -> None:
|
|
822
|
+
"""Remove the screenshots this tool wrote. Only those, and never the directory."""
|
|
823
|
+
config = _config(use, verbose=verbose)
|
|
824
|
+
settings = config.settings
|
|
825
|
+
directory = settings.screenshot_dir or default_screenshot_dir()
|
|
826
|
+
result = prune_screenshots(directory, keep=keep, dry_run=dry_run)
|
|
827
|
+
sys.stdout.write(result.describe() + "\n")
|
|
828
|
+
raise typer.Exit(EXIT_OK)
|
|
829
|
+
|
|
830
|
+
|
|
752
831
|
@app.command()
|
|
753
832
|
def batch(
|
|
754
833
|
source: Annotated[str, typer.Argument(metavar="PATH|-", help="JSON array of actions, or -.")],
|
|
@@ -1073,6 +1152,7 @@ _COMMANDS = frozenset(
|
|
|
1073
1152
|
"type",
|
|
1074
1153
|
"key",
|
|
1075
1154
|
"screenshot",
|
|
1155
|
+
"prune",
|
|
1076
1156
|
"tree",
|
|
1077
1157
|
"windows",
|
|
1078
1158
|
"focus",
|
|
@@ -215,15 +215,50 @@ def xdg_data_dir() -> Path:
|
|
|
215
215
|
return (Path(base) if base else Path.home() / ".local" / "share") / "use-computer"
|
|
216
216
|
|
|
217
217
|
|
|
218
|
+
#: Files the tool writes into its own directory that must not be committed. It has always been
|
|
219
|
+
#: documented that `.env` is gitignored and nothing ever made it so.
|
|
220
|
+
GITIGNORE_LINES = ("# written by use-computer", ".env", "screens/")
|
|
221
|
+
|
|
222
|
+
SCREENS_DIR = "screens"
|
|
223
|
+
|
|
224
|
+
|
|
218
225
|
def default_screenshot_dir() -> Path:
|
|
219
226
|
"""Where screenshots land when nothing said otherwise.
|
|
220
227
|
|
|
221
|
-
|
|
222
|
-
and
|
|
228
|
+
Beside the work that produced them when there is a project -- easy to open, easy to throw
|
|
229
|
+
away, and separate from another project's. Under `.use-computer/` rather than a second hidden
|
|
230
|
+
directory at the root, because the tool already owns that one.
|
|
231
|
+
|
|
232
|
+
Without a project, the XDG *data* directory: a screenshot of somebody's desktop is not
|
|
233
|
+
disposable like a cache.
|
|
223
234
|
"""
|
|
235
|
+
root = find_project_root()
|
|
236
|
+
if root is not None:
|
|
237
|
+
return root / PROJECT_DIR / SCREENS_DIR
|
|
224
238
|
return xdg_data_dir() / "screenshots"
|
|
225
239
|
|
|
226
240
|
|
|
241
|
+
def ensure_gitignore(project_dir: Path) -> None:
|
|
242
|
+
"""Keep the tool's own directory out of a commit.
|
|
243
|
+
|
|
244
|
+
A screenshot here is the whole desktop -- open conversations, mail, whatever is on it -- and in
|
|
245
|
+
a working tree one ``git add -A`` commits it. That objection is the reason the docs used to
|
|
246
|
+
forbid this outright, so the answer travels with the change rather than being left as a note.
|
|
247
|
+
|
|
248
|
+
Only ever this directory. Editing the project's own `.gitignore` is the user's business, not a
|
|
249
|
+
side effect of taking a picture.
|
|
250
|
+
"""
|
|
251
|
+
path = project_dir / ".gitignore"
|
|
252
|
+
if path.exists():
|
|
253
|
+
return
|
|
254
|
+
try:
|
|
255
|
+
project_dir.mkdir(parents=True, exist_ok=True)
|
|
256
|
+
path.write_text("\n".join(GITIGNORE_LINES) + "\n", encoding="utf-8")
|
|
257
|
+
except OSError:
|
|
258
|
+
# Not being able to write it must never stop a screenshot being taken.
|
|
259
|
+
return
|
|
260
|
+
|
|
261
|
+
|
|
227
262
|
def env_var_for(field: str) -> str:
|
|
228
263
|
return ENV_PREFIX + field.upper()
|
|
229
264
|
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Removing the screenshots this tool wrote.
|
|
2
|
+
|
|
3
|
+
CR-004 left it open -- "they are not pruned. The directory grows, and that is the user's to manage
|
|
4
|
+
for now" -- and with the pictures inside the project, for now has run out.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from pydantic import BaseModel, ConfigDict
|
|
13
|
+
|
|
14
|
+
#: Exactly the names :meth:`Session._screenshot_path` writes: a sortable UTC stamp, a label, .png.
|
|
15
|
+
#: A pattern rather than "every .png in there", because `screenshot-dir` is configurable and the
|
|
16
|
+
#: first person to point it at their Pictures folder must not lose anything. This tool refuses to
|
|
17
|
+
#: guess everywhere else; it will not guess about deleting.
|
|
18
|
+
OURS = re.compile(r"^\d{8}T\d{6}\.\d{3}Z-[a-z_]+\.png$")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PruneResult(BaseModel):
|
|
22
|
+
"""What a prune did, or would do."""
|
|
23
|
+
|
|
24
|
+
model_config = ConfigDict(frozen=True)
|
|
25
|
+
|
|
26
|
+
directory: Path
|
|
27
|
+
removed: tuple[Path, ...] = ()
|
|
28
|
+
kept: int = 0
|
|
29
|
+
skipped: int = 0
|
|
30
|
+
bytes_freed: int = 0
|
|
31
|
+
performed: bool = True
|
|
32
|
+
|
|
33
|
+
def describe(self) -> str:
|
|
34
|
+
if not self.directory.exists():
|
|
35
|
+
return f"nothing to remove: {self.directory} does not exist"
|
|
36
|
+
verb = "removed" if self.performed else "would remove"
|
|
37
|
+
megabytes = self.bytes_freed / 1_000_000
|
|
38
|
+
count = len(self.removed)
|
|
39
|
+
parts = [
|
|
40
|
+
f"{verb} {count} screenshot{'' if count == 1 else 's'} "
|
|
41
|
+
f"({megabytes:.1f} MB) from {self.directory}"
|
|
42
|
+
]
|
|
43
|
+
if self.kept:
|
|
44
|
+
parts.append(f"kept {self.kept}")
|
|
45
|
+
if self.skipped:
|
|
46
|
+
# Said out loud: silence here would look like the files had been deleted.
|
|
47
|
+
noun = "file" if self.skipped == 1 else "files"
|
|
48
|
+
parts.append(f"left {self.skipped} {noun} this tool did not write")
|
|
49
|
+
return ", ".join(parts)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def ours(path: Path) -> bool:
|
|
53
|
+
return bool(OURS.match(path.name))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def prune(directory: Path, *, keep: int = 0, dry_run: bool = False) -> PruneResult:
|
|
57
|
+
"""Delete the screenshots in ``directory``, newest ``keep`` survive.
|
|
58
|
+
|
|
59
|
+
Only files this tool wrote, and never the directory itself. Anything else in there belongs to
|
|
60
|
+
somebody else and is counted, not touched.
|
|
61
|
+
|
|
62
|
+
``keep`` uses the timestamp in the name rather than the filesystem: the name is the record,
|
|
63
|
+
which is why it was made to sort.
|
|
64
|
+
"""
|
|
65
|
+
if not directory.exists():
|
|
66
|
+
return PruneResult(directory=directory, performed=not dry_run)
|
|
67
|
+
|
|
68
|
+
entries = sorted(directory.iterdir())
|
|
69
|
+
mine = [path for path in entries if path.is_file() and ours(path)]
|
|
70
|
+
skipped = len([path for path in entries if path.is_file() and not ours(path)])
|
|
71
|
+
|
|
72
|
+
doomed = mine[: len(mine) - keep] if keep else mine
|
|
73
|
+
freed = 0
|
|
74
|
+
removed: list[Path] = []
|
|
75
|
+
for path in doomed:
|
|
76
|
+
size = path.stat().st_size
|
|
77
|
+
if not dry_run:
|
|
78
|
+
try:
|
|
79
|
+
path.unlink()
|
|
80
|
+
except OSError:
|
|
81
|
+
continue
|
|
82
|
+
removed.append(path)
|
|
83
|
+
freed += size
|
|
84
|
+
|
|
85
|
+
return PruneResult(
|
|
86
|
+
directory=directory,
|
|
87
|
+
removed=tuple(removed),
|
|
88
|
+
kept=len(mine) - len(removed),
|
|
89
|
+
skipped=skipped,
|
|
90
|
+
bytes_freed=freed,
|
|
91
|
+
performed=not dry_run,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
__all__ = ["PruneResult", "ours", "prune"]
|
|
@@ -41,10 +41,12 @@ from use_computer.actions import (
|
|
|
41
41
|
from use_computer.backends import Backend, create_backend
|
|
42
42
|
from use_computer.compare import ChangeReport, Screenshot, compare, crop
|
|
43
43
|
from use_computer.config import (
|
|
44
|
+
PROJECT_DIR,
|
|
44
45
|
BackendProfile,
|
|
45
46
|
ResolvedConfig,
|
|
46
47
|
Settings,
|
|
47
48
|
default_screenshot_dir,
|
|
49
|
+
ensure_gitignore,
|
|
48
50
|
)
|
|
49
51
|
from use_computer.config import load as load_config
|
|
50
52
|
from use_computer.coordinates import Coordinate, ScreenInfo
|
|
@@ -658,7 +660,9 @@ class Session:
|
|
|
658
660
|
return crop(shot, (left, top, width, height), node_id)
|
|
659
661
|
|
|
660
662
|
def _screenshot_path(self, label: str) -> Path:
|
|
661
|
-
"""A name that sorts and does not collide."""
|
|
663
|
+
"""A name that sorts and does not collide -- and which `prune` recognises as ours."""
|
|
664
|
+
if self._screenshot_dir.parent.name == PROJECT_DIR:
|
|
665
|
+
ensure_gitignore(self._screenshot_dir.parent)
|
|
662
666
|
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%f")[:-3] + "Z"
|
|
663
667
|
return self._screenshot_dir / f"{stamp}-{label}.png"
|
|
664
668
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: use-computer
|
|
3
3
|
description: Read and act on a GUI — the accessibility tree of what is on screen (roles, names, clickable boxes), then click, focus, toggle, expand, select, set a value, type, press keys, drag, scroll, screenshot. Locally or over VNC. Ask the tree first and use ui-locator's pixel coordinates only when the tree cannot see the element.
|
|
4
4
|
x-skill-id: use-computer
|
|
5
|
-
x-skill-version: "
|
|
5
|
+
x-skill-version: "4"
|
|
6
6
|
---
|
|
7
7
|
|
|
8
8
|
# use-computer
|
|
@@ -11,7 +11,37 @@ You read and act on a screen through the `use-computer` CLI. It reads the access
|
|
|
11
11
|
operating system already maintains, and it moves a real pointer and types real keystrokes. It does
|
|
12
12
|
not decide *what* to do — you do.
|
|
13
13
|
|
|
14
|
-
##
|
|
14
|
+
## Two ways to know what is on the screen
|
|
15
|
+
|
|
16
|
+
They fail in different places, and choosing between them is most of using this tool well.
|
|
17
|
+
|
|
18
|
+
| | **Structure** (`tree`) | **Pixels** (`screenshot`) |
|
|
19
|
+
| --- | --- | --- |
|
|
20
|
+
| Tells you | roles, names, states, boxes, what is operable | what is *drawn*: painted text, icons, colour, layout |
|
|
21
|
+
| Reaches | things not on screen — a closed menu's items | only what is visible |
|
|
22
|
+
| Costs | a few hundred tokens | an image, and a vision pass |
|
|
23
|
+
| Blind to | anything an app paints instead of exposing | node ids, `enabled`, everything off screen |
|
|
24
|
+
| Acts by | name or id, **with no coordinates at all** | a coordinate you have to aim |
|
|
25
|
+
|
|
26
|
+
Structure first, always: it is cheaper and it is exact. Go to pixels when structure cannot answer,
|
|
27
|
+
and when you do, **crop to the node** rather than photographing the screen.
|
|
28
|
+
|
|
29
|
+
### How to move between them
|
|
30
|
+
|
|
31
|
+
- `reason: unavailable` or `empty` — this application exposes nothing (Qt, Electron, canvas, games
|
|
32
|
+
do this). A screenshot is already attached to that answer. Go and look.
|
|
33
|
+
- A selector matched nothing — same, and for the same reason. Do not try more selectors.
|
|
34
|
+
- **The tree sees a node but cannot name it.** Two anonymous text fields, say. Do *not* photograph
|
|
35
|
+
the screen: `screenshot --of <id>` crops to that node. The tree knows exactly *where*; only
|
|
36
|
+
*what* is missing.
|
|
37
|
+
- Before reaching for pixels at all, two things are usually enough:
|
|
38
|
+
- **Geometry**, which is already in the tree. A message box is wide and at the bottom; a search
|
|
39
|
+
box is narrow and near the top. A label sitting to the left of a field, on the same line, names
|
|
40
|
+
it.
|
|
41
|
+
- **Focus.** Click one of two identical fields, re-read the tree, and see which now says
|
|
42
|
+
`!focused`.
|
|
43
|
+
|
|
44
|
+
## The ladder
|
|
15
45
|
|
|
16
46
|
Take the highest rung you can reach. Each one is cheaper, faster and more accurate than the one
|
|
17
47
|
below it.
|
|
@@ -40,6 +70,13 @@ that still resolves.
|
|
|
40
70
|
`--format json` returns a JSON envelope instead. You almost never want it: the same answers cost
|
|
41
71
|
three times the tokens, and 177 of those go on the envelope before anything is said.
|
|
42
72
|
|
|
73
|
+
Every run ends with one line saying whether it worked, which profile and backend answered, the
|
|
74
|
+
screen and the **scale** — `scale unknown` means coordinate conversion will refuse, and is worth
|
|
75
|
+
noticing.
|
|
76
|
+
|
|
77
|
+
`use-computer <command> --help` lists every flag. This document covers the ones that carry a
|
|
78
|
+
judgement; `--help` covers the rest.
|
|
79
|
+
|
|
43
80
|
Always pass `--use <profile>` unless a default profile is configured.
|
|
44
81
|
|
|
45
82
|
## Coordinates: the thing that goes wrong
|
|
@@ -168,8 +205,20 @@ use-computer select --role listitem --name "Italia"
|
|
|
168
205
|
use-computer show-menu --id 0/1/4
|
|
169
206
|
```
|
|
170
207
|
|
|
171
|
-
`--name` is a case-insensitive substring; add `--exact` for equality
|
|
172
|
-
`focused` (default), `all`, a window title,
|
|
208
|
+
`--name` is a case-insensitive substring; add `--exact` for equality, and `--nth N` to pick when
|
|
209
|
+
several match. `--window` takes `focused` (default), `all`, a window title, a window **id**, or
|
|
210
|
+
`@1234` for a pid. `collapse` closes what `expand` opened.
|
|
211
|
+
|
|
212
|
+
**`--via` chooses which rung**, and is the one flag worth understanding:
|
|
213
|
+
|
|
214
|
+
- `--via auto` (default) — the platform API if the node supports it, otherwise a click at its
|
|
215
|
+
centre. Almost always right.
|
|
216
|
+
- `--via action` — refuse rather than fall back. Use it when a coordinate click would be wrong,
|
|
217
|
+
and note it **works even when the scale is unknown**, because it involves no coordinates.
|
|
218
|
+
- `--via coordinate` — resolve the element, then click it with a real pointer. For interfaces that
|
|
219
|
+
only respond to genuine input: hover states, drag handles, canvases.
|
|
220
|
+
|
|
221
|
+
`--delay SECONDS` waits after each action, for an application that needs a moment to catch up.
|
|
173
222
|
|
|
174
223
|
**Pass `--id` together with `--role` and `--name`** as they came out of `tree`. The id alone is
|
|
175
224
|
just a path, and paths shift when a row is inserted above; with the role and name it is checked,
|
|
@@ -279,6 +328,20 @@ follow a verified action with a `screenshot` call. That is the round trip verify
|
|
|
279
328
|
Verification costs two screenshots per action, so use it on the actions whose effect you need
|
|
280
329
|
to confirm, not on every one.
|
|
281
330
|
|
|
331
|
+
## Where the pictures go, and getting rid of them
|
|
332
|
+
|
|
333
|
+
Inside the project, in `.use-computer/screens/`, which the tool keeps out of git for you. Without a
|
|
334
|
+
project, the XDG data directory. A run names the directory once and then just filenames.
|
|
335
|
+
|
|
336
|
+
```bash
|
|
337
|
+
use-computer prune # remove them
|
|
338
|
+
use-computer prune --dry-run # say what would go
|
|
339
|
+
use-computer prune --keep 20 # leave the most recent 20
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
`prune` only ever removes files this tool wrote, and never the directory. Anything else in there is
|
|
343
|
+
counted and left alone.
|
|
344
|
+
|
|
282
345
|
## Before you act on something risky
|
|
283
346
|
|
|
284
347
|
`--dry-run` resolves and logs everything — profile, scaled coordinates, normalised keys, and the
|
|
@@ -43,8 +43,8 @@ def runner() -> CliRunner:
|
|
|
43
43
|
return CliRunner()
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
def invoke(runner: CliRunner, *args: str) -> CliResult:
|
|
47
|
-
return runner.invoke(app, list(args), catch_exceptions=False)
|
|
46
|
+
def invoke(runner: CliRunner, *args: str, input: str | None = None) -> CliResult:
|
|
47
|
+
return runner.invoke(app, list(args), catch_exceptions=False, input=input)
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
def test_the_envelope_is_json_and_nothing_else(
|
|
@@ -446,20 +446,23 @@ def test_an_action_is_one_line(
|
|
|
446
446
|
write_config(CONFIG)
|
|
447
447
|
result = invoke(runner, "click", "--role", "button", "--name", "Invia")
|
|
448
448
|
assert result.exit_code == EXIT_OK
|
|
449
|
-
|
|
450
|
-
assert
|
|
451
|
-
assert "click button 'Invia' at 0/1/0" in
|
|
452
|
-
assert "via the platform API" in
|
|
449
|
+
lines = strip_ansi(result.stdout).strip().split("\n")
|
|
450
|
+
assert len(lines) == 2 # the action, then the closing line
|
|
451
|
+
assert "click button 'Invia' at 0/1/0" in lines[0]
|
|
452
|
+
assert "via the platform API" in lines[0]
|
|
453
453
|
|
|
454
454
|
|
|
455
|
-
def
|
|
455
|
+
def test_a_failure_splits_what_happened_from_why(
|
|
456
456
|
runner: CliRunner, backend: FakeBackend, provider: object, write_config: WriteConfig
|
|
457
457
|
) -> None:
|
|
458
|
-
#
|
|
458
|
+
# stdout says what happened, stderr says why. Neither repeats the other, so a reader
|
|
459
|
+
# scrolling back never has to work out which stream is which.
|
|
459
460
|
write_config(CONFIG)
|
|
460
461
|
result = invoke(runner, "click", "--role", "button")
|
|
461
462
|
assert result.exit_code == EXIT_FAILURE
|
|
462
|
-
|
|
463
|
+
out = strip_ansi(result.stdout).strip()
|
|
464
|
+
assert out.startswith("failed at action 1")
|
|
465
|
+
assert "AmbiguousNodeError" not in out
|
|
463
466
|
assert "AmbiguousNodeError" in strip_ansi(result.stderr)
|
|
464
467
|
|
|
465
468
|
|
|
@@ -471,3 +474,89 @@ def test_the_envelope_is_one_flag_away(
|
|
|
471
474
|
write_config(CONFIG)
|
|
472
475
|
result = invoke(runner, "windows", "--format", "json")
|
|
473
476
|
assert json.loads(result.stdout)["ok"] is True
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def test_verify_reports_its_answer_and_where_it_looked(
|
|
480
|
+
runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
|
|
481
|
+
) -> None:
|
|
482
|
+
# --verify exists to give feedback. A flag whose whole purpose is feedback must not be silent
|
|
483
|
+
# in the format everyone gets by default.
|
|
484
|
+
backend.colours = [(0, 0, 0), (255, 255, 255)]
|
|
485
|
+
write_config(CONFIG)
|
|
486
|
+
result = invoke(runner, "click", "--x", "10", "--y", "10", "--verify")
|
|
487
|
+
assert result.exit_code == EXIT_OK
|
|
488
|
+
out = strip_ansi(result.stdout)
|
|
489
|
+
assert "changed" in out
|
|
490
|
+
assert ".png" in out # the picture it already paid for, so nobody asks for it twice
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def test_an_unchanged_screen_says_so_in_a_word(
|
|
494
|
+
runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
|
|
495
|
+
) -> None:
|
|
496
|
+
# `unchanged` is the one an agent has to notice: it means the coordinate was stale.
|
|
497
|
+
backend.colours = [(0, 0, 0), (0, 0, 0)]
|
|
498
|
+
write_config(CONFIG)
|
|
499
|
+
result = invoke(runner, "click", "--x", "10", "--y", "10", "--verify")
|
|
500
|
+
assert "unchanged" in strip_ansi(result.stdout)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def test_every_run_ends_with_what_the_envelope_carried(
|
|
504
|
+
runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
|
|
505
|
+
) -> None:
|
|
506
|
+
# Cheap must not mean lossy. The envelope was dropped for costing 177 tokens, not because the
|
|
507
|
+
# facts in it were worthless -- the scale especially, which is what an agent needs the moment
|
|
508
|
+
# a coordinate lands somewhere surprising.
|
|
509
|
+
write_config(CONFIG)
|
|
510
|
+
result = invoke(runner, "click", "--x", "10", "--y", "10")
|
|
511
|
+
last = strip_ansi(result.stdout).strip().split("\n")[-1]
|
|
512
|
+
assert last.startswith("ok — ")
|
|
513
|
+
assert "profile fake" in last
|
|
514
|
+
assert "backend fake" in last
|
|
515
|
+
assert "screen 1280x800" in last
|
|
516
|
+
assert "scale 2" in last
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def test_a_failure_says_which_action_it_was(
|
|
520
|
+
runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
|
|
521
|
+
) -> None:
|
|
522
|
+
write_config(CONFIG)
|
|
523
|
+
backend.fail_on = "click"
|
|
524
|
+
result = invoke(runner, "click", "--x", "10", "--y", "10")
|
|
525
|
+
assert result.exit_code == EXIT_FAILURE
|
|
526
|
+
assert "failed at action 1" in strip_ansi(result.stdout)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def test_an_unknown_scale_is_named_not_omitted(
|
|
530
|
+
runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
|
|
531
|
+
) -> None:
|
|
532
|
+
# Refusing to guess a scale is only useful if the caller can see that it is unknown.
|
|
533
|
+
backend.scale = None
|
|
534
|
+
write_config(CONFIG)
|
|
535
|
+
result = invoke(runner, "screenshot")
|
|
536
|
+
assert "scale unknown" in strip_ansi(result.stdout)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def test_a_batch_names_the_screenshot_directory_once(
|
|
540
|
+
runner: CliRunner, backend: FakeBackend, write_config: WriteConfig, tmp_path: Path
|
|
541
|
+
) -> None:
|
|
542
|
+
# A path is 23 tokens, and a batch of five verified actions repeats the same directory in
|
|
543
|
+
# every one of them -- 70 tokens of it, five times what the whole closing line costs.
|
|
544
|
+
backend.colours = [(0, 0, 0), (255, 255, 255)]
|
|
545
|
+
write_config(CONFIG)
|
|
546
|
+
plan = '[{"action":"click","x":1,"y":1},{"action":"click","x":2,"y":2}]'
|
|
547
|
+
result = invoke(runner, "batch", "-", "--verify", input=plan)
|
|
548
|
+
out = strip_ansi(result.stdout)
|
|
549
|
+
assert "screenshots in " in out.split("\n")[-2]
|
|
550
|
+
for line in out.split("\n"):
|
|
551
|
+
if line.startswith(" ") and line.strip().endswith(".png"):
|
|
552
|
+
assert "/" not in line # a filename, because the directory was already named
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def test_a_single_screenshot_keeps_its_whole_path(
|
|
556
|
+
runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
|
|
557
|
+
) -> None:
|
|
558
|
+
# Nothing to save with one file, and an indirection to read would cost more than it returns.
|
|
559
|
+
write_config(CONFIG)
|
|
560
|
+
result = invoke(runner, "screenshot")
|
|
561
|
+
assert strip_ansi(result.stdout).split("\n")[0].startswith("/")
|
|
562
|
+
assert "screenshots in " not in strip_ansi(result.stdout)
|
|
@@ -173,14 +173,21 @@ def test_a_profile_field_default_is_reported_like_any_other(write_config: WriteC
|
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
|
|
176
|
-
def
|
|
177
|
-
monkeypatch: pytest.MonkeyPatch,
|
|
176
|
+
def test_without_a_project_the_screenshots_go_to_the_xdg_data_dir(
|
|
177
|
+
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
178
178
|
) -> None:
|
|
179
179
|
# Never a cache directory: a screenshot of somebody's desktop is not disposable.
|
|
180
|
+
monkeypatch.chdir(tmp_path)
|
|
180
181
|
monkeypatch.setenv("XDG_DATA_HOME", "/somewhere/data")
|
|
181
182
|
assert default_screenshot_dir() == Path("/somewhere/data/use-computer/screenshots")
|
|
182
183
|
|
|
183
184
|
|
|
185
|
+
def test_inside_a_project_they_go_beside_the_config(project: Path) -> None:
|
|
186
|
+
# Beside the work that produced them, and under the directory this tool already owns rather
|
|
187
|
+
# than a second hidden one at the root.
|
|
188
|
+
assert default_screenshot_dir() == project / ".use-computer" / "screens"
|
|
189
|
+
|
|
190
|
+
|
|
184
191
|
def test_the_screenshot_directory_is_configurable_like_anything_else(
|
|
185
192
|
write_config: WriteConfig,
|
|
186
193
|
) -> None:
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Removing screenshots: only the ones this tool wrote, and never the directory."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from use_computer.config import GITIGNORE_LINES, ensure_gitignore
|
|
8
|
+
from use_computer.prune import ours, prune
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def shot(directory: Path, stamp: str, label: str = "click") -> Path:
|
|
12
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
13
|
+
path = directory / f"{stamp}-{label}.png"
|
|
14
|
+
path.write_bytes(b"x" * 10)
|
|
15
|
+
return path
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_it_recognises_only_the_names_it_writes() -> None:
|
|
19
|
+
assert ours(Path("20260905T094734.680Z-click.png"))
|
|
20
|
+
assert ours(Path("20260905T094734.680Z-screenshot.png"))
|
|
21
|
+
assert not ours(Path("holiday.png"))
|
|
22
|
+
assert not ours(Path("screenshot.png"))
|
|
23
|
+
assert not ours(Path("20260905-click.png"))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_a_file_it_did_not_write_is_counted_and_left(tmp_path: Path) -> None:
|
|
27
|
+
# `screenshot-dir` is configurable, and the first person to point it at their Pictures folder
|
|
28
|
+
# must not lose anything. This tool refuses to guess everywhere else.
|
|
29
|
+
shot(tmp_path, "20260905T094734.680Z")
|
|
30
|
+
stranger = tmp_path / "holiday.png"
|
|
31
|
+
stranger.write_bytes(b"mine")
|
|
32
|
+
|
|
33
|
+
result = prune(tmp_path)
|
|
34
|
+
assert len(result.removed) == 1
|
|
35
|
+
assert result.skipped == 1
|
|
36
|
+
assert stranger.exists()
|
|
37
|
+
assert "left 1 file this tool did not write" in result.describe()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_dry_run_removes_nothing_and_says_so(tmp_path: Path) -> None:
|
|
41
|
+
kept = shot(tmp_path, "20260905T094734.680Z")
|
|
42
|
+
result = prune(tmp_path, dry_run=True)
|
|
43
|
+
assert kept.exists()
|
|
44
|
+
assert result.performed is False
|
|
45
|
+
assert result.describe().startswith("would remove 1 screenshot ")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_keep_leaves_the_most_recent_by_name(tmp_path: Path) -> None:
|
|
49
|
+
# The name is the record -- which is why it was made to sort -- not the filesystem's mtime.
|
|
50
|
+
old = shot(tmp_path, "20260905T090000.000Z")
|
|
51
|
+
new = shot(tmp_path, "20260905T100000.000Z")
|
|
52
|
+
prune(tmp_path, keep=1)
|
|
53
|
+
assert new.exists()
|
|
54
|
+
assert not old.exists()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_the_directory_itself_survives(tmp_path: Path) -> None:
|
|
58
|
+
shot(tmp_path, "20260905T094734.680Z")
|
|
59
|
+
prune(tmp_path)
|
|
60
|
+
assert tmp_path.exists()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_a_missing_directory_is_not_an_error(tmp_path: Path) -> None:
|
|
64
|
+
result = prune(tmp_path / "nowhere")
|
|
65
|
+
assert result.removed == ()
|
|
66
|
+
assert "does not exist" in result.describe()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_the_gitignore_is_written_once_and_never_overwritten(tmp_path: Path) -> None:
|
|
70
|
+
project = tmp_path / ".use-computer"
|
|
71
|
+
ensure_gitignore(project)
|
|
72
|
+
assert (project / ".gitignore").read_text().splitlines() == list(GITIGNORE_LINES)
|
|
73
|
+
|
|
74
|
+
(project / ".gitignore").write_text("mine\n")
|
|
75
|
+
ensure_gitignore(project)
|
|
76
|
+
assert (project / ".gitignore").read_text() == "mine\n" # somebody else's, left alone
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Everything the skill mentions must exist.
|
|
2
|
+
|
|
3
|
+
The obvious test is the opposite -- assert every command and flag appears in the skill -- and it
|
|
4
|
+
is the wrong direction: it would enforce a second `--help`, longer than the first, that still does
|
|
5
|
+
not say which of two ways to look or what an error means about which to try next.
|
|
6
|
+
|
|
7
|
+
Rot in the direction of *lying* is what needs catching. A skill that tells an agent to pass
|
|
8
|
+
`--human` after it was removed is actively harmful; one that does not mention `collapse` is merely
|
|
9
|
+
thin, and `--help` covers thin.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
import pytest
|
|
17
|
+
import typer.main
|
|
18
|
+
|
|
19
|
+
from use_computer.cli import app
|
|
20
|
+
from use_computer.skill import bundled_text
|
|
21
|
+
|
|
22
|
+
#: Flags belonging to other tools the skill legitimately shows in shell examples.
|
|
23
|
+
FOREIGN = frozenset(
|
|
24
|
+
{"--forms", "--add-entry", "--force-renderer-accessibility", "--system-site-packages"}
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
#: click adds this to every command and never declares it on one.
|
|
28
|
+
UNIVERSAL = frozenset({"--help"})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def skill() -> str:
|
|
32
|
+
return bundled_text()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _walk(group: object, found: set[str], names: set[str]) -> None:
|
|
36
|
+
for name, command in getattr(group, "commands", {}).items():
|
|
37
|
+
names.add(name)
|
|
38
|
+
for parameter in getattr(command, "params", []):
|
|
39
|
+
found.update(opt for opt in parameter.opts if opt.startswith("--"))
|
|
40
|
+
_walk(command, found, names)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _surface() -> tuple[set[str], set[str]]:
|
|
44
|
+
"""What the CLI actually declares, from the app itself rather than a hand-kept list.
|
|
45
|
+
|
|
46
|
+
A list would drift in exactly the way this test exists to catch.
|
|
47
|
+
"""
|
|
48
|
+
root = typer.main.get_command(app)
|
|
49
|
+
options: set[str] = {opt for p in getattr(root, "params", []) for opt in p.opts
|
|
50
|
+
if opt.startswith("--")}
|
|
51
|
+
names: set[str] = set()
|
|
52
|
+
_walk(root, options, names)
|
|
53
|
+
return options, names
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def declared_options() -> set[str]:
|
|
57
|
+
return _surface()[0]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def declared_commands() -> set[str]:
|
|
61
|
+
return _surface()[1]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_every_flag_the_skill_mentions_is_real() -> None:
|
|
65
|
+
mentioned = set(re.findall(r"--[a-z][a-z-]+", skill())) - FOREIGN - UNIVERSAL
|
|
66
|
+
unknown = mentioned - declared_options()
|
|
67
|
+
assert not unknown, f"the skill names flags that do not exist: {sorted(unknown)}"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_every_command_the_skill_shows_is_real() -> None:
|
|
71
|
+
invoked = set(re.findall(r"^use-computer ([a-z-]+)", skill(), re.M))
|
|
72
|
+
groups = {"config", "skill"} # sub-apps, whose own names are checked by their commands
|
|
73
|
+
unknown = invoked - declared_commands() - groups - {"-"}
|
|
74
|
+
assert not unknown, f"the skill invokes commands that do not exist: {sorted(unknown)}"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@pytest.mark.parametrize("flag", ["--via", "--of", "--format", "--window", "--id"])
|
|
78
|
+
def test_the_flags_that_carry_a_judgement_are_taught(flag: str) -> None:
|
|
79
|
+
"""Not coverage for its own sake: these are the ones `--help` cannot explain.
|
|
80
|
+
|
|
81
|
+
`--via` chooses which rung of the ladder an action takes -- the central idea of the tool --
|
|
82
|
+
and went unmentioned through four revisions of this document.
|
|
83
|
+
"""
|
|
84
|
+
assert flag in skill()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_it_teaches_the_two_ways_of_knowing() -> None:
|
|
88
|
+
text = skill()
|
|
89
|
+
assert "Structure" in text and "Pixels" in text
|
|
90
|
+
assert "screenshot --of" in text # the cheap way down to vision
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|