opencommand 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {opencommand-0.1.2 → opencommand-0.1.4}/PKG-INFO +2 -1
  2. {opencommand-0.1.2 → opencommand-0.1.4}/pyproject.toml +2 -1
  3. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/cli.py +70 -5
  4. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/core/config.py +33 -1
  5. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/engine/__init__.py +25 -4
  6. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/engine/runner.py +29 -1
  7. opencommand-0.1.4/src/opencommand/modules/skills/desktop_linux.md +41 -0
  8. opencommand-0.1.4/src/opencommand/modules/skills/desktop_macos.md +42 -0
  9. opencommand-0.1.4/src/opencommand/modules/skills/desktop_windows.md +44 -0
  10. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/system/automatic.py +23 -0
  11. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/system/cron.py +19 -0
  12. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/system/platform.py +14 -2
  13. opencommand-0.1.4/src/opencommand/modules/tools/desktop.py +202 -0
  14. opencommand-0.1.4/src/opencommand/modules/tools/desktop_safety.py +162 -0
  15. opencommand-0.1.4/src/opencommand/modules/tools/system.py +285 -0
  16. opencommand-0.1.4/src/opencommand/modules/tools/yara_rules/default.yar +61 -0
  17. opencommand-0.1.2/src/opencommand/modules/tools/desktop.py +0 -127
  18. opencommand-0.1.2/src/opencommand/modules/tools/system.py +0 -163
  19. {opencommand-0.1.2 → opencommand-0.1.4}/README.md +0 -0
  20. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/__init__.py +0 -0
  21. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/core/agent.py +0 -0
  22. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/core/errors.py +0 -0
  23. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/core/logging.py +0 -0
  24. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/core/supervisor.py +0 -0
  25. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/core/tools.py +0 -0
  26. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/knowledge.py +0 -0
  27. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/memory/MEMORY.md +0 -0
  28. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/skills/bash.md +0 -0
  29. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/skills/powershell.md +0 -0
  30. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/skills/python.md +0 -0
  31. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/skills/system.md +0 -0
  32. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/system/notify.py +0 -0
  33. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/templates.py +0 -0
  34. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/tools/files.py +0 -0
  35. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/tools/instructions.py +0 -0
  36. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/tools/memory.py +0 -0
  37. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/tools/playwright.py +0 -0
  38. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/tools/research.py +0 -0
  39. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/modules/tools/terminal.py +0 -0
  40. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/providers/provider.py +0 -0
  41. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/swarm/agents/commander.py +0 -0
  42. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/swarm/agents/worker.py +0 -0
  43. {opencommand-0.1.2 → opencommand-0.1.4}/src/opencommand/tui/dashboard.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: opencommand
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: Local-first swarm agent framework: a Commander plans and dispatches a swarm of Workers that research, code, test, and interactively drive the computer to complete long-horizon codebase goals.
5
5
  Author: DSiENT
6
6
  Author-email: DSiENT <w.caskey7@gmail.com>
@@ -16,6 +16,7 @@ Requires-Dist: llama-cpp-python>=0.3.33
16
16
  Requires-Dist: huggingface-hub>=0.30.0
17
17
  Requires-Dist: watchdog>=6.0.0
18
18
  Requires-Dist: ntfy>=2.7.1
19
+ Requires-Dist: yara-python>=4.5.0
19
20
  Requires-Python: >=3.14
20
21
  Project-URL: Homepage, https://github.com/D5-Interactive/OpenCommand
21
22
  Project-URL: Repository, https://github.com/D5-Interactive/OpenCommand
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "opencommand"
3
- version = "0.1.2"
3
+ version = "0.1.4"
4
4
  description = "Local-first swarm agent framework: a Commander plans and dispatches a swarm of Workers that research, code, test, and interactively drive the computer to complete long-horizon codebase goals."
5
5
  readme = "README.md"
6
6
  authors = [
@@ -20,6 +20,7 @@ dependencies = [
20
20
  "huggingface_hub>=0.30.0",
21
21
  "watchdog>=6.0.0",
22
22
  "ntfy>=2.7.1",
23
+ "yara-python>=4.5.0",
23
24
  ]
24
25
 
25
26
  [project.scripts]
@@ -47,7 +47,10 @@ def _build_tools(engine: Engine, cfg: OpenCommandConfig) -> ToolRegistry:
47
47
  reg.register(PlaywrightTool())
48
48
  reg.register(SystemTool())
49
49
  vision_role = cfg.vision_model if cfg.vision else None
50
- reg.register(DesktopTool(engine.runner(vision_role) if vision_role else None))
50
+ from .modules.tools.desktop_safety import DesktopPolicy
51
+ desktop_policy = DesktopPolicy.from_dict(cfg.desktop_policy)
52
+ reg.register(DesktopTool(engine.runner(vision_role) if vision_role else None,
53
+ policy=desktop_policy))
51
54
  return reg
52
55
 
53
56
 
@@ -57,7 +60,7 @@ def _load_cfg(path: str | None) -> OpenCommandConfig:
57
60
 
58
61
 
59
62
  @click.group()
60
- @click.version_option(version="0.1.1")
63
+ @click.version_option(version="0.1.4")
61
64
  @click.option("-v", "--verbose", is_flag=True, help="Enable verbose (DEBUG) logging.")
62
65
  def cli(verbose: bool) -> None:
63
66
  """OpenCommand - all-in-one local model runner & command center."""
@@ -70,7 +73,7 @@ def cli(verbose: bool) -> None:
70
73
  def workspace(path: str | None) -> None:
71
74
  """Show workspace info and embedded engine status."""
72
75
  cfg = _load_cfg(path)
73
- engine = Engine(model_dir=cfg.workspace / "models")
76
+ engine = Engine(model_dir=cfg.workspace / "models", device_map=cfg.device_map)
74
77
  click.echo(f"Workspace: {cfg.workspace}")
75
78
  click.echo(f"Platform: {describe()}")
76
79
  click.echo(engine.status())
@@ -149,6 +152,8 @@ def _build_rebuild_command(cmake_args: str) -> tuple:
149
152
  @click.argument("path", required=False)
150
153
  def config(path: str | None) -> None:
151
154
  """Interactive menu to set advanced swarm settings (persisted to config.json)."""
155
+ from .modules.tools.desktop_safety import DesktopPolicy
156
+
152
157
  cfg = _load_cfg(path)
153
158
  click.echo(click.style("OpenCommand configuration", fg="cyan", bold=True))
154
159
  click.echo(f"Workspace: {cfg.workspace}\n")
@@ -175,10 +180,42 @@ def config(path: str | None) -> None:
175
180
  cfg.max_test_loops = click.prompt(
176
181
  "Max test+debug fix loops / phase", type=int, default=cfg.max_test_loops)
177
182
  cfg.n_gpu_layers = click.prompt(
178
- "GPU layers to offload (0=CPU, -1=all)", type=int, default=cfg.n_gpu_layers)
183
+ "GPU layers to offload (0=CPU, -1=all) [legacy default for unpinned roles]",
184
+ type=int, default=cfg.n_gpu_layers)
179
185
  cfg.goal_file = click.prompt(
180
186
  "Session goal spec file name", default=cfg.goal_file)
181
187
 
188
+ # --- Per-role device placement ---------------------------------------
189
+ click.echo(click.style("\nModel device placement", fg="cyan", bold=True))
190
+ click.echo("Pin each model role to GPU (offload all layers) or CPU+RAM.")
191
+ click.echo("Example: Commander=GPU, Workers=CPU, VL model=GPU (low VRAM host).")
192
+ dm = dict(cfg.device_map)
193
+ for role in ("commander", "worker", "vision", "vision_small"):
194
+ cur = dm.get(role, "default")
195
+ choice = click.prompt(
196
+ f" {role} device", type=click.Choice(["gpu", "cpu", "default"]),
197
+ default=cur)
198
+ if choice == "default":
199
+ dm.pop(role, None)
200
+ else:
201
+ dm[role] = choice
202
+ cfg.device_map = dm
203
+
204
+ # --- Desktop takeover safety policy ------------------------------------
205
+ click.echo(click.style("\nDesktop takeover safety policy", fg="cyan", bold=True))
206
+ click.echo("Defines exactly what the desktop agent is allowed to do.")
207
+ pol = DesktopPolicy.from_dict(cfg.desktop_policy)
208
+ pol.allow_mouse = click.confirm(" Allow mouse control (click)?", default=pol.allow_mouse)
209
+ pol.allow_keyboard = click.confirm(" Allow keyboard control (key)?", default=pol.allow_keyboard)
210
+ pol.allow_type = click.confirm(" Allow typing text?", default=pol.allow_type)
211
+ pol.require_confirm = click.confirm(
212
+ " Require confirmation for risky-but-permitted actions?", default=pol.require_confirm)
213
+ pol.max_steps = click.prompt(
214
+ " Max desktop interaction steps per run", type=int, default=pol.max_steps)
215
+ pol.pause_seconds = click.prompt(
216
+ " Pause (seconds) between actions", type=float, default=pol.pause_seconds)
217
+ cfg.desktop_policy = pol.to_dict()
218
+
182
219
  cfg.save()
183
220
  click.echo(click.style(f"\nSaved config -> {cfg._path()}", fg="green"))
184
221
 
@@ -267,6 +304,33 @@ def cron_remove(name: str, path: str | None) -> None:
267
304
  click.echo(f"Removed cron job '{name}'.")
268
305
 
269
306
 
307
+ @cron.command("edit")
308
+ @click.argument("name")
309
+ @click.option("--schedule", default=None, help="New schedule expression.")
310
+ @click.option("--command", default=None, help="New shell command.")
311
+ @click.option("--goal", default=None, help="New swarm goal (replaces command).")
312
+ @click.option("--path", default=None, help="Workspace directory.")
313
+ def cron_edit(name: str, schedule: str | None, command: str | None,
314
+ goal: str | None, path: str | None) -> None:
315
+ """Edit an existing cron job NAME (change schedule/command/goal)."""
316
+ from .modules.system.cron import CronScheduler
317
+
318
+ if schedule is None and command is None and goal is None:
319
+ click.echo("Nothing to change: pass --schedule, --command, or --goal.")
320
+ return
321
+ cfg = _load_cfg(path)
322
+ cron_file = cfg.workspace / ".opencommand" / "cron.json"
323
+ sched = CronScheduler.load(cron_file)
324
+ try:
325
+ sched.edit(name, schedule=schedule, command=command, goal=goal)
326
+ except KeyError:
327
+ click.echo(f"No cron job named '{name}'.")
328
+ return
329
+ sched.save(cron_file)
330
+ click.echo(f"Updated cron job '{name}'.")
331
+ click.echo(f"Persisted to {cron_file}")
332
+
333
+
270
334
  @cron.command("status")
271
335
  @click.option("--path", default=None, help="Workspace directory.")
272
336
  def cron_status(path: str | None) -> None:
@@ -374,7 +438,8 @@ def run(goal: str | None, path: str | None, goal_file: str | None, max_iter: int
374
438
  click.echo(click.style(f"Loaded goal spec: {goal_path}", fg="green"))
375
439
 
376
440
  engine = Engine(model_dir=cfg.workspace / "models",
377
- pool_size=pool or cfg.max_workers)
441
+ pool_size=pool or cfg.max_workers,
442
+ device_map=cfg.device_map)
378
443
  sup = Supervisor(max_retries=cfg.max_retries)
379
444
  tools = _build_tools(engine, cfg)
380
445
  commander = Commander(engine, sup, tools, cfg.workspace)
@@ -22,8 +22,17 @@ class OpenCommandConfig:
22
22
  max_research_loops: int = 3 # research refinement loops per phase
23
23
  max_test_loops: int = 5 # test+debug fix loops per phase
24
24
  batch_size: int = 4 # workers spawned per swarm batch
25
- n_gpu_layers: int = 0 # 0 = CPU; -1 = all GPU layers
25
+ n_gpu_layers: int = 0 # 0 = CPU; -1 = all GPU layers (legacy global default)
26
26
  goal_file: str = "Goal.md" # session goal spec file name
27
+ # --- Desktop takeover safety policy (see modules/tools/desktop_safety.py) -
28
+ desktop_policy: dict = field(default_factory=dict) # empty -> safe defaults
29
+ # --- Per-role device placement -----------------------------------------
30
+ # Maps a model role to "gpu" (offload all layers) or "cpu" (RAM only).
31
+ # Roles: commander, worker, vision, vision_small. Any role not present
32
+ # falls back to the legacy `n_gpu_layers` (GPU if -1, else CPU). This is
33
+ # how a low-VRAM host keeps the Commander on the GPU while Workers run in
34
+ # RAM, and the VL model stays on the GPU for the desktop agent.
35
+ device_map: dict = field(default_factory=dict) # empty -> all roles use n_gpu_layers
27
36
 
28
37
  def save(self) -> None:
29
38
  self._path().parent.mkdir(parents=True, exist_ok=True)
@@ -45,6 +54,8 @@ class OpenCommandConfig:
45
54
  "batch_size": self.batch_size,
46
55
  "n_gpu_layers": self.n_gpu_layers,
47
56
  "goal_file": self.goal_file,
57
+ "desktop_policy": self.desktop_policy,
58
+ "device_map": self.device_map,
48
59
  }, indent=2)
49
60
 
50
61
  @classmethod
@@ -68,11 +79,28 @@ class OpenCommandConfig:
68
79
  batch_size=d.get("batch_size", 4),
69
80
  n_gpu_layers=d.get("n_gpu_layers", 0),
70
81
  goal_file=d.get("goal_file", "Goal.md"),
82
+ desktop_policy=d.get("desktop_policy", {}),
83
+ device_map=d.get("device_map", {}),
71
84
  )
72
85
 
73
86
  def _path(self) -> Path:
74
87
  return self.workspace / ".opencommand" / "config.json"
75
88
 
89
+ def n_gpu_layers_for(self, role: str) -> int:
90
+ """Resolve GPU offload layers for a model role.
91
+
92
+ If ``device_map`` pins the role to "gpu" -> -1 (all layers offloaded);
93
+ if pinned to "cpu" -> 0 (RAM only). When the role is absent from
94
+ ``device_map`` we fall back to the legacy global ``n_gpu_layers`` so
95
+ existing configs keep working.
96
+ """
97
+ dev = self.device_map.get(role)
98
+ if dev == "gpu":
99
+ return -1
100
+ if dev == "cpu":
101
+ return 0
102
+ return self.n_gpu_layers
103
+
76
104
  @classmethod
77
105
  def from_host(cls, workspace: Path) -> "OpenCommandConfig":
78
106
  """Build a config tuned to the current host via platform autodetection."""
@@ -81,4 +109,8 @@ class OpenCommandConfig:
81
109
  cfg = cls(workspace=workspace)
82
110
  for k, v in rec["config"].items():
83
111
  setattr(cfg, k, v)
112
+ # device_map is a nested dict; apply it explicitly so it isn't
113
+ # clobbered by the flat setattr loop above.
114
+ if "device_map" in rec["config"]:
115
+ cfg.device_map = dict(rec["config"]["device_map"])
84
116
  return cfg
@@ -18,18 +18,34 @@ class Engine:
18
18
  dedicated instance per worker index for true parallelism.
19
19
  """
20
20
 
21
- def __init__(self, model_dir: Path | None = None, pool_size: int = 4) -> None:
21
+ def __init__(self, model_dir: Path | None = None, pool_size: int = 4,
22
+ device_map: dict | None = None) -> None:
22
23
  self.model_dir = model_dir or DEFAULT_MODEL_DIR
23
24
  self.pool_size = max(1, pool_size)
25
+ # device_map pins a role to "gpu" (-1 layers) or "cpu" (0 layers);
26
+ # roles absent fall back to the legacy global n_gpu_layers (set per-runner
27
+ # via set_gpu_layers). Kept as a dict so callers can mutate it live.
28
+ self.device_map = dict(device_map or {})
24
29
  self._runners: dict[str, LocalRunner] = {}
25
30
  self._pools: dict[str, list[LocalRunner]] = {}
26
31
 
32
+ def _gpu_layers(self, role: str) -> int:
33
+ """Resolve GPU offload layers for a role from the device map."""
34
+ dev = self.device_map.get(role)
35
+ if dev == "gpu":
36
+ return -1
37
+ if dev == "cpu":
38
+ return 0
39
+ return 0 # default: CPU/RAM unless a runner overrides via set_gpu_layers
40
+
27
41
  def runner(self, role: str) -> LocalRunner:
28
42
  if role not in self._runners:
29
43
  spec = CATALOG.get(role)
30
44
  if spec is None:
31
45
  raise KeyError(f"No built-in model for role '{role}'")
32
- self._runners[role] = LocalRunner(spec, self.model_dir / role)
46
+ r = LocalRunner(spec, self.model_dir / role)
47
+ r.set_gpu_layers(self._gpu_layers(role))
48
+ self._runners[role] = r
33
49
  return self._runners[role]
34
50
 
35
51
  def worker_runner(self, role: str, index: int = 0) -> LocalRunner:
@@ -40,7 +56,9 @@ class Engine:
40
56
  pool = self._pools.setdefault(role, [])
41
57
  i = index % self.pool_size
42
58
  while len(pool) <= i:
43
- pool.append(LocalRunner(spec, self.model_dir / role))
59
+ r = LocalRunner(spec, self.model_dir / role)
60
+ r.set_gpu_layers(self._gpu_layers(role))
61
+ pool.append(r)
44
62
  return pool[i]
45
63
 
46
64
  def ensure_downloaded(self, role: str) -> Path:
@@ -62,5 +80,8 @@ class Engine:
62
80
  state = "instantiated (lazy)"
63
81
  else:
64
82
  state = "loaded"
65
- lines.append(f" - {role}: {spec.repo_id} [{spec.kind}] {state}")
83
+ dev = self.device_map.get(role, "default")
84
+ dev_label = f"device={dev}" if dev != "default" else "device=default"
85
+ lines.append(f" - {role}: {spec.repo_id} [{spec.kind}] "
86
+ f"{state} ({dev_label})")
66
87
  return "\n".join(lines)
@@ -87,6 +87,19 @@ class LocalRunner:
87
87
  self.model_dir = model_dir
88
88
  self._llm = None
89
89
  self._lock = threading.Lock()
90
+ # Per-runner GPU offload override. The Engine sets this from the
91
+ # device_map (role -> "gpu"|"cpu"); -1 = all layers, 0 = CPU/RAM.
92
+ self._gpu_layers = spec.n_gpu_layers
93
+
94
+ def set_gpu_layers(self, n: int) -> None:
95
+ """Override how many layers to offload to the GPU for this runner.
96
+
97
+ Must be called before the model is loaded. -1 offloads all layers
98
+ (GPU mode); 0 keeps everything in RAM (CPU mode).
99
+ """
100
+ if self._llm is not None:
101
+ raise RuntimeError("cannot change GPU layers after the model is loaded")
102
+ self._gpu_layers = n
90
103
 
91
104
  def download(self) -> Path:
92
105
  """Ensure weights exist locally; return the model path (no load)."""
@@ -123,6 +136,21 @@ class LocalRunner:
123
136
  pass
124
137
  return Path(path)
125
138
 
139
+ def unload(self) -> None:
140
+ """Release the mapped weights so the RAM/VRAM is freed.
141
+
142
+ Used by the Commander to drop itself from the GPU while the VL desktop
143
+ agent runs, then re-load on demand. Safe to call when not loaded.
144
+ """
145
+ with self._lock:
146
+ if self._llm is not None:
147
+ log.info("unloading model role=%s", self.spec.role)
148
+ try:
149
+ self._llm.close()
150
+ except Exception: # noqa: BLE001
151
+ pass
152
+ self._llm = None
153
+
126
154
  def _ensure_loaded(self) -> None:
127
155
  # Guard construction with the lock so concurrent first-use from multiple
128
156
  # worker threads can't build two Llama instances into the same slot.
@@ -135,7 +163,7 @@ class LocalRunner:
135
163
  kwargs: dict[str, Any] = {
136
164
  "model_path": str(self._resolve_path()),
137
165
  "n_ctx": self.spec.n_ctx,
138
- "n_gpu_layers": self.spec.n_gpu_layers,
166
+ "n_gpu_layers": self._gpu_layers,
139
167
  "verbose": False,
140
168
  }
141
169
  if self.spec.kind == "vision":
@@ -0,0 +1,41 @@
1
+ ---
2
+ name: desktop-linux
3
+ category: desktop-safety
4
+ updated: 2026-07-12
5
+ tags: [desktop, linux, safety, pyautogui]
6
+ ---
7
+
8
+ # Desktop Agent — Safe Operation on Linux
9
+
10
+ Guidance for the OpenCommand desktop takeover agent when driving a **Linux**
11
+ host (X11/Wayland). The goal is to accomplish UI goals without destabilizing
12
+ the session.
13
+
14
+ ## Hard rules (enforced by policy, do not bypass)
15
+ - Never trigger `Ctrl+Alt+Del`/`Ctrl+Alt+Backspace` (session kill),
16
+ `Ctrl+Alt+F1..F12` (VT switch), or `Super+L` (lock) unless the goal requires it.
17
+ - Never type destructive shell commands: `rm -rf`, `dd if=`, `mkfs`, `shutdown`,
18
+ `reboot`, `:(){:|:&};:` (fork bomb), or anything piped to `| sh` / `| bash`.
19
+ - Do not escalate to root (no `sudo`, no `pkexec`). If a polkit prompt appears,
20
+ STOP and report.
21
+
22
+ ## pyautogui on Linux
23
+ - `pyautogui.FAILSAFE = True` is always on: move the mouse to the top-left
24
+ corner to abort. Respect that escape hatch.
25
+ - Screenshot support needs `scrot` (or `gnome-screenshot`); install it if
26
+ `locateOnScreen` is needed. Wayland may require `gnome-screenshot`/`grim`
27
+ instead of `scrot`.
28
+ - `type()` targets the focused window. Click into the field first and confirm
29
+ focus before typing.
30
+ - Prefer `click(x, y)` via image matching over blind key combos.
31
+
32
+ ## Safe interaction patterns
33
+ 1. Capture screen → locate the target control (image match or window title).
34
+ 2. Click the control; `wait(1)` for the UI to settle.
35
+ 3. For text entry, click the field, then `type()` a small specific string.
36
+ 4. Verify with another capture before `done`.
37
+
38
+ ## Verification before `done`
39
+ - Confirm the intended window/state is visible and no error dialog is open.
40
+ - If an unexpected confirmation/save dialog appeared, report it rather than
41
+ dismissing system dialogs blindly.
@@ -0,0 +1,42 @@
1
+ ---
2
+ name: desktop-macos
3
+ category: desktop-safety
4
+ updated: 2026-07-12
5
+ tags: [desktop, macos, safety, pyautogui]
6
+ ---
7
+
8
+ # Desktop Agent — Safe Operation on macOS
9
+
10
+ Guidance for the OpenCommand desktop takeover agent when driving a **macOS**
11
+ host. The goal is to accomplish UI goals without destabilizing the session.
12
+
13
+ ## Hard rules (enforced by policy, do not bypass)
14
+ - Never trigger `Cmd+Q` (quit all / app), `Cmd+Shift+Q` (log out),
15
+ `Cmd+Ctrl+Power` (force restart), `Ctrl+Power` (dialog), or `Cmd+Option+Esc`
16
+ (force quit) unless the goal explicitly requires it.
17
+ - Never type destructive shell commands: `rm -rf`, `sudo`, `diskutil erase`,
18
+ `shutdown`, `reboot`, or anything piped to `| sh` / `| bash`.
19
+ - Do not approve a Touch ID / admin password prompt on the agent's behalf. If
20
+ one appears, STOP and report.
21
+
22
+ ## pyautogui on macOS
23
+ - `pyautogui.FAILSAFE = True` is always on: move the mouse to the top-left
24
+ corner to abort. Respect that escape hatch.
25
+ - macOS key names differ: use `command` (not `cmd`), `option` (not `alt`),
26
+ `control`, `shift`, `return`, `escape`, `space`, `delete` (backspace),
27
+ `backspace` (forward delete). The policy normalizes these.
28
+ - `type()` targets the focused window. Click into the field first and confirm
29
+ focus before typing. Note `typewrite` is the underlying call; accented
30
+ characters may need explicit key names.
31
+ - Prefer `click(x, y)` via image matching over blind key combos.
32
+
33
+ ## Safe interaction patterns
34
+ 1. Capture screen → locate the target control (image match or window title).
35
+ 2. Click the control; `wait(1)` for the UI to settle.
36
+ 3. For text entry, click the field, then `type()` a small specific string.
37
+ 4. Verify with another capture before `done`.
38
+
39
+ ## Verification before `done`
40
+ - Confirm the intended window/state is visible and no error dialog is open.
41
+ - If an unexpected confirmation/save dialog appeared, report it rather than
42
+ dismissing system dialogs blindly.
@@ -0,0 +1,44 @@
1
+ ---
2
+ name: desktop-windows
3
+ category: desktop-safety
4
+ updated: 2026-07-12
5
+ tags: [desktop, windows, safety, pyautogui]
6
+ ---
7
+
8
+ # Desktop Agent — Safe Operation on Windows
9
+
10
+ Guidance for the OpenCommand desktop takeover agent when driving a **Windows**
11
+ host. The goal is to accomplish UI goals without ever destabilizing the system.
12
+
13
+ ## Hard rules (enforced by policy, do not bypass)
14
+ - Never trigger `Win+L` (lock), `Win+R` (Run), `Win+X` (admin menu), `Win+I`
15
+ (Settings), `Alt+F4` on the desktop/shell, or `Ctrl+Alt+Del`.
16
+ - Never type destructive shell commands: `rmdir /s`, `del /q`, `format`,
17
+ `shutdown`, `deltree`, or anything piped to `iex`/`powershell -enc`.
18
+ - Never open an admin/UAC prompt on the agent's own behalf. If a UAC dialog
19
+ appears, STOP and report — do not click "Yes".
20
+
21
+ ## pyautogui on Windows
22
+ - `pyautogui.FAILSAFE = True` is always on: a human can slam the mouse into the
23
+ top-left corner to abort the agent instantly. Respect that escape hatch.
24
+ - `pyautogui.PAUSE` is set (default 0.5s) so actions are paced and interruptible.
25
+ - Use `click(x, y)` with coordinates from `locateOnScreen`/`locateCenterOnScreen`
26
+ rather than blind `key` combos wherever possible.
27
+ - Prefer `pyautogui.hotkey('ctrl', 'c')` over `key('ctrl+c')` for combos; the
28
+ policy only permits safe combos.
29
+ - `type()` writes to whatever has focus — only type after you have clicked into
30
+ the correct field and confirmed focus.
31
+
32
+ ## Safe interaction patterns
33
+ 1. Capture screen → identify the target control by image match or by reading
34
+ the window.
35
+ 2. Click the control (single left click). Wait for the UI to settle (use
36
+ `wait(1)`).
37
+ 3. If text entry is needed, click the field first, then `type()` a small,
38
+ specific string. Avoid pasting long shell one-liners.
39
+ 4. Verify the result with another screen capture before declaring done.
40
+
41
+ ## Verification before `done`
42
+ - Confirm the intended window/state is visible and no error dialog is open.
43
+ - If a confirmation or save dialog appeared unexpectedly, report it; do not
44
+ dismiss system dialogs blindly.
@@ -120,6 +120,12 @@ async def _live_verify(commander: Commander, goal: str) -> "LiveVerifyResult":
120
120
  Returns a LiveVerifyResult. Only meaningful when the desktop tool is wired
121
121
  and vision is enabled; otherwise reports a no-op pass so the pipeline can
122
122
  continue without a UI target.
123
+
124
+ Device-placement behavior: by default the Commander is unloaded from the
125
+ GPU while the VL desktop agent runs (freeing VRAM for the vision model),
126
+ then re-loaded so the Commander can review the agent's actions + report and
127
+ continue the loop. If the Commander is pinned to the same device as the VL
128
+ model (both GPU) we skip the unload to avoid an unnecessary reload churn.
123
129
  """
124
130
  from ...modules.tools.desktop import DesktopTool
125
131
 
@@ -128,11 +134,28 @@ async def _live_verify(commander: Commander, goal: str) -> "LiveVerifyResult":
128
134
  if desktop is None or not getattr(desktop, "vision", None):
129
135
  return LiveVerifyResult(ok=True, summary="no desktop tool/vision; skipped",
130
136
  detail="")
137
+ # Unload the Commander from the GPU while the VL agent drives the desktop,
138
+ # unless the Commander is itself pinned to CPU (nothing to free).
139
+ commander_runner = commander.provider
140
+ unloaded = False
131
141
  try:
142
+ if getattr(commander_runner, "spec", None) is not None:
143
+ role = getattr(commander_runner.spec, "role", None)
144
+ # Only unload if the Commander sits on the GPU (would compete with VL).
145
+ if commander.engine.device_map.get(role) == "gpu":
146
+ commander_runner.unload()
147
+ unloaded = True
132
148
  out = desktop.run(f"Verify the goal is satisfied on screen: {goal}", steps=8)
133
149
  except Exception as e: # noqa: BLE001
134
150
  return LiveVerifyResult(ok=False, summary="desktop tool error",
135
151
  detail=str(e))
152
+ finally:
153
+ # Re-load the Commander so it can review the agent's report and continue.
154
+ if unloaded:
155
+ try:
156
+ commander_runner._ensure_loaded()
157
+ except Exception: # noqa: BLE001
158
+ pass
136
159
  ok = "ERROR" not in out and "done" in out.lower()
137
160
  return LiveVerifyResult(ok=ok, summary="interactive check completed", detail=out)
138
161
 
@@ -131,6 +131,25 @@ class CronScheduler:
131
131
  job.goal = goal
132
132
  return job
133
133
 
134
+ def edit(self, name: str, schedule: str | None = None,
135
+ command: str | None = None, goal: str | None = None) -> CronJob:
136
+ """Edit an existing job's schedule/command/goal in place.
137
+
138
+ Returns the updated job, or raises KeyError if no such job exists.
139
+ At least one of schedule/command/goal must be provided.
140
+ """
141
+ for job in self._jobs:
142
+ if job.name == name:
143
+ if schedule is not None:
144
+ job.schedule = schedule
145
+ job._parse()
146
+ if command is not None:
147
+ job.command = command
148
+ if goal is not None:
149
+ job.goal = goal
150
+ return job
151
+ raise KeyError(f"No cron job named {name!r}")
152
+
134
153
  def save(self, path: Path) -> None:
135
154
  """Persist serializable jobs to <workspace>/.opencommand/cron.json."""
136
155
  path.parent.mkdir(parents=True, exist_ok=True)
@@ -179,7 +179,18 @@ def recommend_config(d: dict | None = None) -> dict:
179
179
  d = d or detect()
180
180
  cores = max(1, d["cores"])
181
181
  pool = max(1, min(cores - 1, 8)) # leave cores for OS; cap to avoid RAM blowup
182
- n_gpu_layers = -1 if d["gpu"]["backend"] != "none" else 0
182
+ has_gpu = d["gpu"]["backend"] != "none"
183
+ n_gpu_layers = -1 if has_gpu else 0
184
+ # Per-role device placement heuristic. We don't know VRAM size from detect()
185
+ # alone, so we use a conservative split: if a GPU exists, put the small
186
+ # interactive models (commander planner + the VL desktop agent) on the GPU
187
+ # and keep the heavier worker pool in CPU+RAM (which scales with RAM, not
188
+ # VRAM). On a no-GPU host everything is CPU. This is the same split a
189
+ # low-VRAM host (e.g. 8GB VRAM / 32GB RAM) would pick by hand.
190
+ device_map = {}
191
+ if has_gpu:
192
+ device_map = {"commander": "gpu", "worker": "cpu", "vision": "gpu",
193
+ "vision_small": "gpu"}
183
194
  build_flags = []
184
195
  if d["isa"]["avx512"]:
185
196
  build_flags.append("-DGGML_AVX512=ON")
@@ -188,7 +199,8 @@ def recommend_config(d: dict | None = None) -> dict:
188
199
  return {
189
200
  "config": {
190
201
  "max_workers": pool,
191
- "vision": d["gpu"]["backend"] != "none" or cores >= 4,
202
+ "vision": has_gpu or cores >= 4,
203
+ "device_map": device_map,
192
204
  },
193
205
  "engine": {"pool_size": pool},
194
206
  "runner": {"n_gpu_layers": n_gpu_layers},