use-computer-cli 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/PKG-INFO +1 -1
  2. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/pyproject.toml +1 -1
  3. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/cli.py +83 -3
  4. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/config.py +37 -2
  5. use_computer_cli-0.2.2/src/use_computer/prune.py +95 -0
  6. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/runner.py +5 -1
  7. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/skill/SKILL.md +67 -4
  8. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_cli.py +98 -9
  9. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_config.py +9 -2
  10. use_computer_cli-0.2.2/tests/test_prune.py +76 -0
  11. use_computer_cli-0.2.2/tests/test_skill_is_true.py +90 -0
  12. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/.gitignore +0 -0
  13. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/LICENSE +0 -0
  14. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/README.md +0 -0
  15. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/__init__.py +0 -0
  16. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/__init__.py +0 -0
  17. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/atspi.py +0 -0
  18. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/ax.py +0 -0
  19. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/base.py +0 -0
  20. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/roles.py +0 -0
  21. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/accessibility/uia.py +0 -0
  22. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/actions.py +0 -0
  23. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/__init__.py +0 -0
  24. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/base.py +0 -0
  25. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/local.py +0 -0
  26. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/backends/vnc.py +0 -0
  27. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/compare.py +0 -0
  28. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/coordinates.py +0 -0
  29. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/errors.py +0 -0
  30. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/keys.py +0 -0
  31. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/render.py +0 -0
  32. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/selectors.py +0 -0
  33. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/skill/__init__.py +0 -0
  34. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/src/use_computer/tree.py +0 -0
  35. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/__init__.py +0 -0
  36. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/conftest.py +0 -0
  37. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/fake_backend.py +0 -0
  38. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/fake_provider.py +0 -0
  39. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_atspi_hint.py +0 -0
  40. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_backends.py +0 -0
  41. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_compare.py +0 -0
  42. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_config_init.py +0 -0
  43. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_coordinates.py +0 -0
  44. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_elements.py +0 -0
  45. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_keys.py +0 -0
  46. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_render.py +0 -0
  47. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_runner.py +0 -0
  48. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_selectors.py +0 -0
  49. {use_computer_cli-0.2.1 → use_computer_cli-0.2.2}/tests/test_skill.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: use-computer-cli
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot.
5
5
  Project-URL: Homepage, https://github.com/applica-software-guru/use-computer
6
6
  Author: Bruno Fortunato
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "use-computer-cli"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "Execute input on a screen for computer-use agents: move, click, drag, scroll, type, key, screenshot."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -48,10 +48,16 @@ from use_computer.actions import (
48
48
  TypeAction,
49
49
  WindowsAction,
50
50
  )
51
- from use_computer.config import ResolvedConfig, profile_env_var, write_initial_config
51
+ from use_computer.config import (
52
+ ResolvedConfig,
53
+ default_screenshot_dir,
54
+ profile_env_var,
55
+ write_initial_config,
56
+ )
52
57
  from use_computer.config import load as load_config
53
58
  from use_computer.coordinates import CoordinateSpace
54
59
  from use_computer.errors import UseComputerError
60
+ from use_computer.prune import prune as prune_screenshots
55
61
  from use_computer.runner import Session, as_json
56
62
  from use_computer.skill import Scope
57
63
  from use_computer.skill import install as skill_install
@@ -276,6 +282,22 @@ def _config(
276
282
  return resolved
277
283
 
278
284
 
285
+ def _shared_directory(result: Any) -> Path | None:
286
+ """The directory every screenshot in this run went to, if they share one.
287
+
288
+ A path is 23 tokens and a batch of five verified actions repeats the same directory in every
289
+ one of them -- 70 tokens of it, five times what the whole closing line costs. Naming it once
290
+ and printing filenames is the same information for a third of the price.
291
+ """
292
+ paths = [
293
+ item.screenshot.path
294
+ for item in result.results
295
+ if item.screenshot is not None and item.screenshot.path is not None
296
+ ]
297
+ parents = {path.parent for path in paths}
298
+ return parents.pop() if len(parents) == 1 and len(paths) > 1 else None
299
+
300
+
279
301
  def _text_lines(result: Any) -> str:
280
302
  """What a run says, as prose: the read, or a line per action.
281
303
 
@@ -284,6 +306,7 @@ def _text_lines(result: Any) -> str:
284
306
  and a model reads this at a third of the tokens the JSON costs.
285
307
  """
286
308
  chunks: list[str] = []
309
+ folder = _shared_directory(result)
287
310
  for item in result.results:
288
311
  if item.tree is not None:
289
312
  if item.tree.text:
@@ -305,7 +328,7 @@ def _text_lines(result: Any) -> str:
305
328
  # The path is the answer, and the only part of it worth any tokens.
306
329
  shot = item.screenshot
307
330
  where = f" {shot.box[2]}x{shot.box[3]} of {shot.of}" if shot.box else ""
308
- chunks.append(f"{shot.path}{where}")
331
+ chunks.append(f"{_shorten(shot.path, folder)}{where}")
309
332
  continue
310
333
  what = item.action.action
311
334
  if item.matched is not None:
@@ -319,10 +342,48 @@ def _text_lines(result: Any) -> str:
319
342
  elif item.resolved is not None:
320
343
  how = f" at ({item.resolved.x}, {item.resolved.y})"
321
344
  done = "would " if not item.performed else ""
322
- chunks.append(f"{done}{what}{how} \u2014 {item.duration_ms:.0f} ms")
345
+ # --verify exists to give feedback, so its answer belongs on the line. `unchanged` is the
346
+ # word an agent has to notice: it means the coordinate was stale, and it is what stops it
347
+ # clicking the same wrong pixel forever.
348
+ verified = ""
349
+ if item.change is not None:
350
+ verified = (
351
+ f" \u2014 changed {item.change.magnitude:.0%}"
352
+ if item.change.changed
353
+ else " \u2014 unchanged"
354
+ )
355
+ chunks.append(f"{done}{what}{how}{verified} \u2014 {item.duration_ms:.0f} ms")
356
+ if item.screenshot is not None and item.screenshot.path is not None:
357
+ # Already captured and already paid for. Saying where saves the agent asking again,
358
+ # which is the whole reason verify writes it down.
359
+ chunks.append(f" {_shorten(item.screenshot.path, folder)}")
360
+
361
+ chunks.append(_summary(result, folder))
323
362
  return "\n".join(chunks)
324
363
 
325
364
 
365
+ def _shorten(path: Path | None, folder: Path | None) -> str:
366
+ """The filename when the directory has already been named, the whole path otherwise."""
367
+ if path is None:
368
+ return ""
369
+ return path.name if folder is not None and path.parent == folder else str(path)
370
+
371
+
372
+ def _summary(result: Any, folder: Path | None = None) -> str:
373
+ """The closing line: everything the envelope carried, for about twenty tokens.
374
+
375
+ Cheap must not mean lossy. The envelope was dropped because 177 tokens of braces and repeated
376
+ keys is a bad price for it, not because the facts in it were worthless -- and the scale in
377
+ particular is what an agent needs the moment a coordinate lands somewhere surprising.
378
+ """
379
+ screen = result.screen
380
+ scale = f"scale {screen.scale:g}" if screen.scale is not None else "scale unknown"
381
+ where = f"screen {screen.width}x{screen.height}, {scale}"
382
+ state = "ok" if result.ok else f"failed at action {(result.failed_index or 0) + 1}"
383
+ shots = f", screenshots in {folder}" if folder is not None else ""
384
+ return f"{state} — profile {result.profile}, backend {result.backend}, {where}{shots}"
385
+
386
+
326
387
  def _run(
327
388
  actions: Sequence[Action],
328
389
  config: ResolvedConfig,
@@ -749,6 +810,24 @@ app.command("show-menu")(
749
810
  )
750
811
 
751
812
 
813
+ @app.command()
814
+ def prune(
815
+ keep: Annotated[int, typer.Option("--keep", help="Leave the most recent N.")] = 0,
816
+ dry_run: Annotated[
817
+ bool, typer.Option("--dry-run", help="Say what would go; remove nothing.")
818
+ ] = False,
819
+ use: UseOption = None,
820
+ verbose: VerboseOption = 0,
821
+ ) -> None:
822
+ """Remove the screenshots this tool wrote. Only those, and never the directory."""
823
+ config = _config(use, verbose=verbose)
824
+ settings = config.settings
825
+ directory = settings.screenshot_dir or default_screenshot_dir()
826
+ result = prune_screenshots(directory, keep=keep, dry_run=dry_run)
827
+ sys.stdout.write(result.describe() + "\n")
828
+ raise typer.Exit(EXIT_OK)
829
+
830
+
752
831
  @app.command()
753
832
  def batch(
754
833
  source: Annotated[str, typer.Argument(metavar="PATH|-", help="JSON array of actions, or -.")],
@@ -1073,6 +1152,7 @@ _COMMANDS = frozenset(
1073
1152
  "type",
1074
1153
  "key",
1075
1154
  "screenshot",
1155
+ "prune",
1076
1156
  "tree",
1077
1157
  "windows",
1078
1158
  "focus",
@@ -215,15 +215,50 @@ def xdg_data_dir() -> Path:
215
215
  return (Path(base) if base else Path.home() / ".local" / "share") / "use-computer"
216
216
 
217
217
 
218
+ #: Files the tool writes into its own directory that must not be committed. It has always been
219
+ #: documented that `.env` is gitignored and nothing ever made it so.
220
+ GITIGNORE_LINES = ("# written by use-computer", ".env", "screens/")
221
+
222
+ SCREENS_DIR = "screens"
223
+
224
+
218
225
  def default_screenshot_dir() -> Path:
219
226
  """Where screenshots land when nothing said otherwise.
220
227
 
221
- The XDG *data* directory: a screenshot of somebody's desktop is not disposable like a cache,
222
- and it does not belong in a working tree.
228
+ Beside the work that produced them when there is a project -- easy to open, easy to throw
229
+ away, and separate from another project's. Under `.use-computer/` rather than a second hidden
230
+ directory at the root, because the tool already owns that one.
231
+
232
+ Without a project, the XDG *data* directory: a screenshot of somebody's desktop is not
233
+ disposable like a cache.
223
234
  """
235
+ root = find_project_root()
236
+ if root is not None:
237
+ return root / PROJECT_DIR / SCREENS_DIR
224
238
  return xdg_data_dir() / "screenshots"
225
239
 
226
240
 
241
+ def ensure_gitignore(project_dir: Path) -> None:
242
+ """Keep the tool's own directory out of a commit.
243
+
244
+ A screenshot here is the whole desktop -- open conversations, mail, whatever is on it -- and in
245
+ a working tree one ``git add -A`` commits it. That objection is the reason the docs used to
246
+ forbid this outright, so the answer travels with the change rather than being left as a note.
247
+
248
+ Only ever this directory. Editing the project's own `.gitignore` is the user's business, not a
249
+ side effect of taking a picture.
250
+ """
251
+ path = project_dir / ".gitignore"
252
+ if path.exists():
253
+ return
254
+ try:
255
+ project_dir.mkdir(parents=True, exist_ok=True)
256
+ path.write_text("\n".join(GITIGNORE_LINES) + "\n", encoding="utf-8")
257
+ except OSError:
258
+ # Not being able to write it must never stop a screenshot being taken.
259
+ return
260
+
261
+
227
262
  def env_var_for(field: str) -> str:
228
263
  return ENV_PREFIX + field.upper()
229
264
 
@@ -0,0 +1,95 @@
1
+ """Removing the screenshots this tool wrote.
2
+
3
+ CR-004 left it open -- "they are not pruned. The directory grows, and that is the user's to manage
4
+ for now" -- and with the pictures inside the project, for now has run out.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import re
10
+ from pathlib import Path
11
+
12
+ from pydantic import BaseModel, ConfigDict
13
+
14
+ #: Exactly the names :meth:`Session._screenshot_path` writes: a sortable UTC stamp, a label, .png.
15
+ #: A pattern rather than "every .png in there", because `screenshot-dir` is configurable and the
16
+ #: first person to point it at their Pictures folder must not lose anything. This tool refuses to
17
+ #: guess everywhere else; it will not guess about deleting.
18
+ OURS = re.compile(r"^\d{8}T\d{6}\.\d{3}Z-[a-z_]+\.png$")
19
+
20
+
21
+ class PruneResult(BaseModel):
22
+ """What a prune did, or would do."""
23
+
24
+ model_config = ConfigDict(frozen=True)
25
+
26
+ directory: Path
27
+ removed: tuple[Path, ...] = ()
28
+ kept: int = 0
29
+ skipped: int = 0
30
+ bytes_freed: int = 0
31
+ performed: bool = True
32
+
33
+ def describe(self) -> str:
34
+ if not self.directory.exists():
35
+ return f"nothing to remove: {self.directory} does not exist"
36
+ verb = "removed" if self.performed else "would remove"
37
+ megabytes = self.bytes_freed / 1_000_000
38
+ count = len(self.removed)
39
+ parts = [
40
+ f"{verb} {count} screenshot{'' if count == 1 else 's'} "
41
+ f"({megabytes:.1f} MB) from {self.directory}"
42
+ ]
43
+ if self.kept:
44
+ parts.append(f"kept {self.kept}")
45
+ if self.skipped:
46
+ # Said out loud: silence here would look like the files had been deleted.
47
+ noun = "file" if self.skipped == 1 else "files"
48
+ parts.append(f"left {self.skipped} {noun} this tool did not write")
49
+ return ", ".join(parts)
50
+
51
+
52
+ def ours(path: Path) -> bool:
53
+ return bool(OURS.match(path.name))
54
+
55
+
56
+ def prune(directory: Path, *, keep: int = 0, dry_run: bool = False) -> PruneResult:
57
+ """Delete the screenshots in ``directory``, newest ``keep`` survive.
58
+
59
+ Only files this tool wrote, and never the directory itself. Anything else in there belongs to
60
+ somebody else and is counted, not touched.
61
+
62
+ ``keep`` uses the timestamp in the name rather than the filesystem: the name is the record,
63
+ which is why it was made to sort.
64
+ """
65
+ if not directory.exists():
66
+ return PruneResult(directory=directory, performed=not dry_run)
67
+
68
+ entries = sorted(directory.iterdir())
69
+ mine = [path for path in entries if path.is_file() and ours(path)]
70
+ skipped = len([path for path in entries if path.is_file() and not ours(path)])
71
+
72
+ doomed = mine[: len(mine) - keep] if keep else mine
73
+ freed = 0
74
+ removed: list[Path] = []
75
+ for path in doomed:
76
+ size = path.stat().st_size
77
+ if not dry_run:
78
+ try:
79
+ path.unlink()
80
+ except OSError:
81
+ continue
82
+ removed.append(path)
83
+ freed += size
84
+
85
+ return PruneResult(
86
+ directory=directory,
87
+ removed=tuple(removed),
88
+ kept=len(mine) - len(removed),
89
+ skipped=skipped,
90
+ bytes_freed=freed,
91
+ performed=not dry_run,
92
+ )
93
+
94
+
95
+ __all__ = ["PruneResult", "ours", "prune"]
@@ -41,10 +41,12 @@ from use_computer.actions import (
41
41
  from use_computer.backends import Backend, create_backend
42
42
  from use_computer.compare import ChangeReport, Screenshot, compare, crop
43
43
  from use_computer.config import (
44
+ PROJECT_DIR,
44
45
  BackendProfile,
45
46
  ResolvedConfig,
46
47
  Settings,
47
48
  default_screenshot_dir,
49
+ ensure_gitignore,
48
50
  )
49
51
  from use_computer.config import load as load_config
50
52
  from use_computer.coordinates import Coordinate, ScreenInfo
@@ -658,7 +660,9 @@ class Session:
658
660
  return crop(shot, (left, top, width, height), node_id)
659
661
 
660
662
  def _screenshot_path(self, label: str) -> Path:
661
- """A name that sorts and does not collide."""
663
+ """A name that sorts and does not collide -- and which `prune` recognises as ours."""
664
+ if self._screenshot_dir.parent.name == PROJECT_DIR:
665
+ ensure_gitignore(self._screenshot_dir.parent)
662
666
  stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%f")[:-3] + "Z"
663
667
  return self._screenshot_dir / f"{stamp}-{label}.png"
664
668
 
@@ -2,7 +2,7 @@
2
2
  name: use-computer
3
3
  description: Read and act on a GUI — the accessibility tree of what is on screen (roles, names, clickable boxes), then click, focus, toggle, expand, select, set a value, type, press keys, drag, scroll, screenshot. Locally or over VNC. Ask the tree first and use ui-locator's pixel coordinates only when the tree cannot see the element.
4
4
  x-skill-id: use-computer
5
- x-skill-version: "3"
5
+ x-skill-version: "4"
6
6
  ---
7
7
 
8
8
  # use-computer
@@ -11,7 +11,37 @@ You read and act on a screen through the `use-computer` CLI. It reads the access
11
11
  operating system already maintains, and it moves a real pointer and types real keystrokes. It does
12
12
  not decide *what* to do — you do.
13
13
 
14
- ## Start here: the ladder
14
+ ## Two ways to know what is on the screen
15
+
16
+ They fail in different places, and choosing between them is most of using this tool well.
17
+
18
+ | | **Structure** (`tree`) | **Pixels** (`screenshot`) |
19
+ | --- | --- | --- |
20
+ | Tells you | roles, names, states, boxes, what is operable | what is *drawn*: painted text, icons, colour, layout |
21
+ | Reaches | things not on screen — a closed menu's items | only what is visible |
22
+ | Costs | a few hundred tokens | an image, and a vision pass |
23
+ | Blind to | anything an app paints instead of exposing | node ids, `enabled`, everything off screen |
24
+ | Acts by | name or id, **with no coordinates at all** | a coordinate you have to aim |
25
+
26
+ Structure first, always: it is cheaper and it is exact. Go to pixels when structure cannot answer,
27
+ and when you do, **crop to the node** rather than photographing the screen.
28
+
29
+ ### How to move between them
30
+
31
+ - `reason: unavailable` or `empty` — this application exposes nothing (Qt, Electron, canvas, games
32
+ do this). A screenshot is already attached to that answer. Go and look.
33
+ - A selector matched nothing — same, and for the same reason. Do not try more selectors.
34
+ - **The tree sees a node but cannot name it.** Two anonymous text fields, say. Do *not* photograph
35
+ the screen: `screenshot --of <id>` crops to that node. The tree knows exactly *where*; only
36
+ *what* is missing.
37
+ - Before reaching for pixels at all, two things are usually enough:
38
+ - **Geometry**, which is already in the tree. A message box is wide and at the bottom; a search
39
+ box is narrow and near the top. A label sitting to the left of a field, on the same line, names
40
+ it.
41
+ - **Focus.** Click one of two identical fields, re-read the tree, and see which now says
42
+ `!focused`.
43
+
44
+ ## The ladder
15
45
 
16
46
  Take the highest rung you can reach. Each one is cheaper, faster and more accurate than the one
17
47
  below it.
@@ -40,6 +70,13 @@ that still resolves.
40
70
  `--format json` returns a JSON envelope instead. You almost never want it: the same answers cost
41
71
  three times the tokens, and 177 of those go on the envelope before anything is said.
42
72
 
73
+ Every run ends with one line saying whether it worked, which profile and backend answered, the
74
+ screen and the **scale** — `scale unknown` means coordinate conversion will refuse, and is worth
75
+ noticing.
76
+
77
+ `use-computer <command> --help` lists every flag. This document covers the ones that carry a
78
+ judgement; `--help` covers the rest.
79
+
43
80
  Always pass `--use <profile>` unless a default profile is configured.
44
81
 
45
82
  ## Coordinates: the thing that goes wrong
@@ -168,8 +205,20 @@ use-computer select --role listitem --name "Italia"
168
205
  use-computer show-menu --id 0/1/4
169
206
  ```
170
207
 
171
- `--name` is a case-insensitive substring; add `--exact` for equality. `--window` takes
172
- `focused` (default), `all`, a window title, or `@1234` for a pid.
208
+ `--name` is a case-insensitive substring; add `--exact` for equality, and `--nth N` to pick when
209
+ several match. `--window` takes `focused` (default), `all`, a window title, a window **id**, or
210
+ `@1234` for a pid. `collapse` closes what `expand` opened.
211
+
212
+ **`--via` chooses which rung**, and is the one flag worth understanding:
213
+
214
+ - `--via auto` (default) — the platform API if the node supports it, otherwise a click at its
215
+ centre. Almost always right.
216
+ - `--via action` — refuse rather than fall back. Use it when a coordinate click would be wrong,
217
+ and note it **works even when the scale is unknown**, because it involves no coordinates.
218
+ - `--via coordinate` — resolve the element, then click it with a real pointer. For interfaces that
219
+ only respond to genuine input: hover states, drag handles, canvases.
220
+
221
+ `--delay SECONDS` waits after each action, for an application that needs a moment to catch up.
173
222
 
174
223
  **Pass `--id` together with `--role` and `--name`** as they came out of `tree`. The id alone is
175
224
  just a path, and paths shift when a row is inserted above; with the role and name it is checked,
@@ -279,6 +328,20 @@ follow a verified action with a `screenshot` call. That is the round trip verify
279
328
  Verification costs two screenshots per action, so use it on the actions whose effect you need
280
329
  to confirm, not on every one.
281
330
 
331
+ ## Where the pictures go, and getting rid of them
332
+
333
+ Inside the project, in `.use-computer/screens/`, which the tool keeps out of git for you. Without a
334
+ project, the XDG data directory. A run names the directory once and then just filenames.
335
+
336
+ ```bash
337
+ use-computer prune # remove them
338
+ use-computer prune --dry-run # say what would go
339
+ use-computer prune --keep 20 # leave the most recent 20
340
+ ```
341
+
342
+ `prune` only ever removes files this tool wrote, and never the directory. Anything else in there is
343
+ counted and left alone.
344
+
282
345
  ## Before you act on something risky
283
346
 
284
347
  `--dry-run` resolves and logs everything — profile, scaled coordinates, normalised keys, and the
@@ -43,8 +43,8 @@ def runner() -> CliRunner:
43
43
  return CliRunner()
44
44
 
45
45
 
46
- def invoke(runner: CliRunner, *args: str) -> CliResult:
47
- return runner.invoke(app, list(args), catch_exceptions=False)
46
+ def invoke(runner: CliRunner, *args: str, input: str | None = None) -> CliResult:
47
+ return runner.invoke(app, list(args), catch_exceptions=False, input=input)
48
48
 
49
49
 
50
50
  def test_the_envelope_is_json_and_nothing_else(
@@ -446,20 +446,23 @@ def test_an_action_is_one_line(
446
446
  write_config(CONFIG)
447
447
  result = invoke(runner, "click", "--role", "button", "--name", "Invia")
448
448
  assert result.exit_code == EXIT_OK
449
- out = strip_ansi(result.stdout).strip()
450
- assert out.count("\n") == 0
451
- assert "click button 'Invia' at 0/1/0" in out
452
- assert "via the platform API" in out
449
+ lines = strip_ansi(result.stdout).strip().split("\n")
450
+ assert len(lines) == 2 # the action, then the closing line
451
+ assert "click button 'Invia' at 0/1/0" in lines[0]
452
+ assert "via the platform API" in lines[0]
453
453
 
454
454
 
455
- def test_an_error_leaves_stdout_empty(
455
+ def test_a_failure_splits_what_happened_from_why(
456
456
  runner: CliRunner, backend: FakeBackend, provider: object, write_config: WriteConfig
457
457
  ) -> None:
458
- # A reader scrolling back should not have to work out which stream said what.
458
+ # stdout says what happened, stderr says why. Neither repeats the other, so a reader
459
+ # scrolling back never has to work out which stream is which.
459
460
  write_config(CONFIG)
460
461
  result = invoke(runner, "click", "--role", "button")
461
462
  assert result.exit_code == EXIT_FAILURE
462
- assert strip_ansi(result.stdout).strip() == ""
463
+ out = strip_ansi(result.stdout).strip()
464
+ assert out.startswith("failed at action 1")
465
+ assert "AmbiguousNodeError" not in out
463
466
  assert "AmbiguousNodeError" in strip_ansi(result.stderr)
464
467
 
465
468
 
@@ -471,3 +474,89 @@ def test_the_envelope_is_one_flag_away(
471
474
  write_config(CONFIG)
472
475
  result = invoke(runner, "windows", "--format", "json")
473
476
  assert json.loads(result.stdout)["ok"] is True
477
+
478
+
479
+ def test_verify_reports_its_answer_and_where_it_looked(
480
+ runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
481
+ ) -> None:
482
+ # --verify exists to give feedback. A flag whose whole purpose is feedback must not be silent
483
+ # in the format everyone gets by default.
484
+ backend.colours = [(0, 0, 0), (255, 255, 255)]
485
+ write_config(CONFIG)
486
+ result = invoke(runner, "click", "--x", "10", "--y", "10", "--verify")
487
+ assert result.exit_code == EXIT_OK
488
+ out = strip_ansi(result.stdout)
489
+ assert "changed" in out
490
+ assert ".png" in out # the picture it already paid for, so nobody asks for it twice
491
+
492
+
493
+ def test_an_unchanged_screen_says_so_in_a_word(
494
+ runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
495
+ ) -> None:
496
+ # `unchanged` is the one an agent has to notice: it means the coordinate was stale.
497
+ backend.colours = [(0, 0, 0), (0, 0, 0)]
498
+ write_config(CONFIG)
499
+ result = invoke(runner, "click", "--x", "10", "--y", "10", "--verify")
500
+ assert "unchanged" in strip_ansi(result.stdout)
501
+
502
+
503
+ def test_every_run_ends_with_what_the_envelope_carried(
504
+ runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
505
+ ) -> None:
506
+ # Cheap must not mean lossy. The envelope was dropped for costing 177 tokens, not because the
507
+ # facts in it were worthless -- the scale especially, which is what an agent needs the moment
508
+ # a coordinate lands somewhere surprising.
509
+ write_config(CONFIG)
510
+ result = invoke(runner, "click", "--x", "10", "--y", "10")
511
+ last = strip_ansi(result.stdout).strip().split("\n")[-1]
512
+ assert last.startswith("ok — ")
513
+ assert "profile fake" in last
514
+ assert "backend fake" in last
515
+ assert "screen 1280x800" in last
516
+ assert "scale 2" in last
517
+
518
+
519
+ def test_a_failure_says_which_action_it_was(
520
+ runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
521
+ ) -> None:
522
+ write_config(CONFIG)
523
+ backend.fail_on = "click"
524
+ result = invoke(runner, "click", "--x", "10", "--y", "10")
525
+ assert result.exit_code == EXIT_FAILURE
526
+ assert "failed at action 1" in strip_ansi(result.stdout)
527
+
528
+
529
+ def test_an_unknown_scale_is_named_not_omitted(
530
+ runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
531
+ ) -> None:
532
+ # Refusing to guess a scale is only useful if the caller can see that it is unknown.
533
+ backend.scale = None
534
+ write_config(CONFIG)
535
+ result = invoke(runner, "screenshot")
536
+ assert "scale unknown" in strip_ansi(result.stdout)
537
+
538
+
539
+ def test_a_batch_names_the_screenshot_directory_once(
540
+ runner: CliRunner, backend: FakeBackend, write_config: WriteConfig, tmp_path: Path
541
+ ) -> None:
542
+ # A path is 23 tokens, and a batch of five verified actions repeats the same directory in
543
+ # every one of them -- 70 tokens of it, five times what the whole closing line costs.
544
+ backend.colours = [(0, 0, 0), (255, 255, 255)]
545
+ write_config(CONFIG)
546
+ plan = '[{"action":"click","x":1,"y":1},{"action":"click","x":2,"y":2}]'
547
+ result = invoke(runner, "batch", "-", "--verify", input=plan)
548
+ out = strip_ansi(result.stdout)
549
+ assert "screenshots in " in out.split("\n")[-2]
550
+ for line in out.split("\n"):
551
+ if line.startswith(" ") and line.strip().endswith(".png"):
552
+ assert "/" not in line # a filename, because the directory was already named
553
+
554
+
555
+ def test_a_single_screenshot_keeps_its_whole_path(
556
+ runner: CliRunner, backend: FakeBackend, write_config: WriteConfig
557
+ ) -> None:
558
+ # Nothing to save with one file, and an indirection to read would cost more than it returns.
559
+ write_config(CONFIG)
560
+ result = invoke(runner, "screenshot")
561
+ assert strip_ansi(result.stdout).split("\n")[0].startswith("/")
562
+ assert "screenshots in " not in strip_ansi(result.stdout)
@@ -173,14 +173,21 @@ def test_a_profile_field_default_is_reported_like_any_other(write_config: WriteC
173
173
  }
174
174
 
175
175
 
176
- def test_the_screenshot_directory_defaults_to_the_xdg_data_dir(
177
- monkeypatch: pytest.MonkeyPatch,
176
+ def test_without_a_project_the_screenshots_go_to_the_xdg_data_dir(
177
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
178
178
  ) -> None:
179
179
  # Never a cache directory: a screenshot of somebody's desktop is not disposable.
180
+ monkeypatch.chdir(tmp_path)
180
181
  monkeypatch.setenv("XDG_DATA_HOME", "/somewhere/data")
181
182
  assert default_screenshot_dir() == Path("/somewhere/data/use-computer/screenshots")
182
183
 
183
184
 
185
+ def test_inside_a_project_they_go_beside_the_config(project: Path) -> None:
186
+ # Beside the work that produced them, and under the directory this tool already owns rather
187
+ # than a second hidden one at the root.
188
+ assert default_screenshot_dir() == project / ".use-computer" / "screens"
189
+
190
+
184
191
  def test_the_screenshot_directory_is_configurable_like_anything_else(
185
192
  write_config: WriteConfig,
186
193
  ) -> None:
@@ -0,0 +1,76 @@
1
+ """Removing screenshots: only the ones this tool wrote, and never the directory."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+
7
+ from use_computer.config import GITIGNORE_LINES, ensure_gitignore
8
+ from use_computer.prune import ours, prune
9
+
10
+
11
+ def shot(directory: Path, stamp: str, label: str = "click") -> Path:
12
+ directory.mkdir(parents=True, exist_ok=True)
13
+ path = directory / f"{stamp}-{label}.png"
14
+ path.write_bytes(b"x" * 10)
15
+ return path
16
+
17
+
18
+ def test_it_recognises_only_the_names_it_writes() -> None:
19
+ assert ours(Path("20260905T094734.680Z-click.png"))
20
+ assert ours(Path("20260905T094734.680Z-screenshot.png"))
21
+ assert not ours(Path("holiday.png"))
22
+ assert not ours(Path("screenshot.png"))
23
+ assert not ours(Path("20260905-click.png"))
24
+
25
+
26
+ def test_a_file_it_did_not_write_is_counted_and_left(tmp_path: Path) -> None:
27
+ # `screenshot-dir` is configurable, and the first person to point it at their Pictures folder
28
+ # must not lose anything. This tool refuses to guess everywhere else.
29
+ shot(tmp_path, "20260905T094734.680Z")
30
+ stranger = tmp_path / "holiday.png"
31
+ stranger.write_bytes(b"mine")
32
+
33
+ result = prune(tmp_path)
34
+ assert len(result.removed) == 1
35
+ assert result.skipped == 1
36
+ assert stranger.exists()
37
+ assert "left 1 file this tool did not write" in result.describe()
38
+
39
+
40
+ def test_dry_run_removes_nothing_and_says_so(tmp_path: Path) -> None:
41
+ kept = shot(tmp_path, "20260905T094734.680Z")
42
+ result = prune(tmp_path, dry_run=True)
43
+ assert kept.exists()
44
+ assert result.performed is False
45
+ assert result.describe().startswith("would remove 1 screenshot ")
46
+
47
+
48
+ def test_keep_leaves_the_most_recent_by_name(tmp_path: Path) -> None:
49
+ # The name is the record -- which is why it was made to sort -- not the filesystem's mtime.
50
+ old = shot(tmp_path, "20260905T090000.000Z")
51
+ new = shot(tmp_path, "20260905T100000.000Z")
52
+ prune(tmp_path, keep=1)
53
+ assert new.exists()
54
+ assert not old.exists()
55
+
56
+
57
+ def test_the_directory_itself_survives(tmp_path: Path) -> None:
58
+ shot(tmp_path, "20260905T094734.680Z")
59
+ prune(tmp_path)
60
+ assert tmp_path.exists()
61
+
62
+
63
+ def test_a_missing_directory_is_not_an_error(tmp_path: Path) -> None:
64
+ result = prune(tmp_path / "nowhere")
65
+ assert result.removed == ()
66
+ assert "does not exist" in result.describe()
67
+
68
+
69
+ def test_the_gitignore_is_written_once_and_never_overwritten(tmp_path: Path) -> None:
70
+ project = tmp_path / ".use-computer"
71
+ ensure_gitignore(project)
72
+ assert (project / ".gitignore").read_text().splitlines() == list(GITIGNORE_LINES)
73
+
74
+ (project / ".gitignore").write_text("mine\n")
75
+ ensure_gitignore(project)
76
+ assert (project / ".gitignore").read_text() == "mine\n" # somebody else's, left alone
@@ -0,0 +1,90 @@
1
+ """Everything the skill mentions must exist.
2
+
3
+ The obvious test is the opposite -- assert every command and flag appears in the skill -- and it
4
+ is the wrong direction: it would enforce a second `--help`, longer than the first, that still does
5
+ not say which of two ways to look or what an error means about which to try next.
6
+
7
+ Rot in the direction of *lying* is what needs catching. A skill that tells an agent to pass
8
+ `--human` after it was removed is actively harmful; one that does not mention `collapse` is merely
9
+ thin, and `--help` covers thin.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import re
15
+
16
+ import pytest
17
+ import typer.main
18
+
19
+ from use_computer.cli import app
20
+ from use_computer.skill import bundled_text
21
+
22
+ #: Flags belonging to other tools the skill legitimately shows in shell examples.
23
+ FOREIGN = frozenset(
24
+ {"--forms", "--add-entry", "--force-renderer-accessibility", "--system-site-packages"}
25
+ )
26
+
27
+ #: click adds this to every command and never declares it on one.
28
+ UNIVERSAL = frozenset({"--help"})
29
+
30
+
31
+ def skill() -> str:
32
+ return bundled_text()
33
+
34
+
35
+ def _walk(group: object, found: set[str], names: set[str]) -> None:
36
+ for name, command in getattr(group, "commands", {}).items():
37
+ names.add(name)
38
+ for parameter in getattr(command, "params", []):
39
+ found.update(opt for opt in parameter.opts if opt.startswith("--"))
40
+ _walk(command, found, names)
41
+
42
+
43
+ def _surface() -> tuple[set[str], set[str]]:
44
+ """What the CLI actually declares, from the app itself rather than a hand-kept list.
45
+
46
+ A list would drift in exactly the way this test exists to catch.
47
+ """
48
+ root = typer.main.get_command(app)
49
+ options: set[str] = {opt for p in getattr(root, "params", []) for opt in p.opts
50
+ if opt.startswith("--")}
51
+ names: set[str] = set()
52
+ _walk(root, options, names)
53
+ return options, names
54
+
55
+
56
+ def declared_options() -> set[str]:
57
+ return _surface()[0]
58
+
59
+
60
+ def declared_commands() -> set[str]:
61
+ return _surface()[1]
62
+
63
+
64
+ def test_every_flag_the_skill_mentions_is_real() -> None:
65
+ mentioned = set(re.findall(r"--[a-z][a-z-]+", skill())) - FOREIGN - UNIVERSAL
66
+ unknown = mentioned - declared_options()
67
+ assert not unknown, f"the skill names flags that do not exist: {sorted(unknown)}"
68
+
69
+
70
+ def test_every_command_the_skill_shows_is_real() -> None:
71
+ invoked = set(re.findall(r"^use-computer ([a-z-]+)", skill(), re.M))
72
+ groups = {"config", "skill"} # sub-apps, whose own names are checked by their commands
73
+ unknown = invoked - declared_commands() - groups - {"-"}
74
+ assert not unknown, f"the skill invokes commands that do not exist: {sorted(unknown)}"
75
+
76
+
77
+ @pytest.mark.parametrize("flag", ["--via", "--of", "--format", "--window", "--id"])
78
+ def test_the_flags_that_carry_a_judgement_are_taught(flag: str) -> None:
79
+ """Not coverage for its own sake: these are the ones `--help` cannot explain.
80
+
81
+ `--via` chooses which rung of the ladder an action takes -- the central idea of the tool --
82
+ and went unmentioned through four revisions of this document.
83
+ """
84
+ assert flag in skill()
85
+
86
+
87
+ def test_it_teaches_the_two_ways_of_knowing() -> None:
88
+ text = skill()
89
+ assert "Structure" in text and "Pixels" in text
90
+ assert "screenshot --of" in text # the cheap way down to vision