lcode-cli 0.37.0__py3-none-any.whl → 0.39.0__py3-none-any.whl

lcode/__init__.py

@@ -3,7 +3,7 @@
 From a program: `from lcode import Session` (see lcode.api).
 """
 
-__version__ = "0.37.0"
+__version__ = "0.39.0"
 
 API = ("Session", "Result", "Options", "SetupError")
 

lcode/agent.py

@@ -99,7 +99,9 @@ def stall_note(trouble: str) -> str:
     }[trouble]
     return (
         f"[lcode] {first} Don't deliberate at length now. Either make your next tool call, or, if you're stuck, "
-        "tell the user in a few lines what you did, what fails, and what they could do."
+        "tell the user in a few lines what you did, what fails, and what they could do. If you were about to check "
+        "something by running it, run it now instead of reasoning about it: for a window, take a screenshot (with "
+        "actions for clicks or keys)."
     )
 
 
@@ -302,6 +304,8 @@ class Progress:
 class Settings:
     model: str  # Ollama model name to call
     context: int
+    mode: str = "code"  # what lcode makes: code, image or video (lcode.modes)
+    comfyui_url: str = "http://127.0.0.1:8188"
     num_batch: int | None = None
     keep_alive: str = "30m"
     think: bool = True
@@ -678,6 +682,7 @@ class Agent:
         data = {
             "cwd": str(self.workspace),  # where `lcode -c` there finds it, wherever the shell moved
             "model": self.settings.model,
+            "mode": self.settings.mode,
             "name": self.session_name,
             "title": self.session_title,
             "messages": self.messages,
@@ -724,6 +729,8 @@ class Agent:
         self.session_id = info.id
         self.checkpoints.load(info.id, data.get("checkpoints") or [])
         self.session_name = data.get("name", "")
+        if data.get("mode") in ("code", "image", "video"):
+            self.settings.mode = data["mode"]  # a session belongs to the mode it was made in
         self.session_title = data.get("title") or sessions.title_from(self.messages)
         self.tools.read_mtimes.clear()  # files may have changed since; the model must read them again
         self.tools.todos = list(data.get("todos") or [])
@@ -961,6 +968,12 @@ class Agent:
             return []
         return checks.unrun_scripts(project_root_or_cwd(self.cwd), list(self.tools.before), commands)
 
+    def unseen_windows(self, tools_used: set[str]) -> dict[str, str]:
+        """Window code the model changed in this request and never looked at (only where the checks would run)."""
+        if self.allowed_tools is not None or self.planning() or self.no_changes or self.settings.checks == "off":
+            return {}
+        return checks.unseen_windows(project_root_or_cwd(self.cwd), list(self.tools.before), tools_used)
+
     def project_check(self) -> checks.Check | None:
         """The project's check command, for the main agent outside plan mode and reviews (None otherwise)."""
         if self.allowed_tools is not None or self.planning() or self.no_changes or self.settings.checks == "off":
@@ -1164,6 +1177,8 @@ class Agent:
         stalls = 0  # replies that went round in circles or ended without a word
         commands: list[str] = []  # shell commands run in this request
         scripts_noted = False  # the model was told about scripts it wrote and never ran
+        tools_used: set[str] = set()  # tools called in this request
+        windows_noted = False  # the model was told about windows it changed and never looked at
         for step in range(self.max_steps):
             if self.cancel is not None and self.cancel.is_set():
                 raise KeyboardInterrupt
@@ -1215,10 +1230,11 @@ class Agent:
                 continue
             if not calls and (self.reply_trouble == "circles" or not (reply.get("content") or "").strip()):
                 stalls += 1
-                self.messages.pop()  # nothing to keep: no answer, and circles would only fill the context
+                dropped = self.messages.pop()  # no answer, and circles would only fill the context
+                kept = self.keep_dropped(dropped)
                 if stalls < MAX_STALLS:
                     self.console.print(
-                        Text(f"  ⎿ {STALLED[self.reply_trouble]}: asking for a short answer", style="yellow")
+                        Text(f"  ⎿ {STALLED[self.reply_trouble]}: asking for a short answer{kept}", style="yellow")
                     )
                     self.messages.append({"role": "user", "content": stall_note(self.reply_trouble)})
                     continue
@@ -1234,7 +1250,7 @@ class Agent:
                 self.console.print(
                     Text(
                         "Stopped: the model's replies went round in circles or ended without an answer, twice in "
-                        "this request. Try again with a smaller step, or tell it what to do differently.",
+                        f"this request{kept}. Try again with a smaller step, or tell it what to do differently.",
                         style="yellow",
                     )
                 )
@@ -1253,6 +1269,14 @@ class Agent:
                     self.console.print(Text(f"  ⎿ never ran {', '.join(unrun)}: asking the model to", style="yellow"))
                     self.messages.append({"role": "user", "content": checks.unrun_note(unrun)})
                     continue
+                unseen = {} if windows_noted else self.unseen_windows(tools_used)
+                if unseen:
+                    windows_noted = True
+                    self.console.print(
+                        Text(f"  ⎿ never looked at {', '.join(unseen)}: asking the model to", style="yellow")
+                    )
+                    self.messages.append({"role": "user", "content": checks.unseen_note(unseen)})
+                    continue
                 check = self.project_check() if changed and not verified else None
                 if check is not None and check_rounds < checks.MAX_ROUNDS:
                     check_rounds += 1
@@ -1345,6 +1369,7 @@ class Agent:
                     )
                     if out_of_reach.stop:
                         blocked = out_of_reach
+                tools_used.add(name)
                 if name == "bash":
                     commands.append(str(args.get("command", "")))
                 if name in ("edit_file", "write_file", "apply_patch") and not result.startswith(REFUSED):
@@ -1412,6 +1437,14 @@ class Agent:
         self.turn_status = "success"
         self.print_stats()
 
+    def keep_dropped(self, reply: dict) -> str:
+        """Save a reply lcode dropped (its thinking too), so the user can see what went round in circles."""
+        text = "\n\n".join(part for part in (reply.get("thinking"), reply.get("content")) if part)
+        if not text.strip():
+            return ""
+        path = context_tools.save_output(text, "dropped reply")
+        return f" (the dropped reply is in {path})" if path else ""
+
     def can_escalate(self) -> bool:
         """Whether a stronger model may be asked here: one is set, and this isn't a subagent or plan mode."""
         model = (self.settings.escalate_model or "").strip().lower()

lcode/api.py

@@ -54,6 +54,7 @@ class Options:
     model: str | None = None
     context: int | str | None = None
     permission_mode: str | None = None  # ask, plan, auto-edit or yolo
+    mode: str | None = None  # code, image or video
     allowed_tools: list[str] | None = None  # only these tools (default: all)
     max_steps: int | None = None  # model steps per request
     think: bool | None = None
@@ -179,6 +180,8 @@ def open_agent(cwd: Path, options: Options, console: Console, hw: Hardware | Non
         think=cfg["think"] if options.think is None else options.think,
         show_thinking=options.show_thinking,
         permission_mode=options.permission_mode or cfg["permission_mode"],
+        mode=options.mode or cfg["mode"],
+        comfyui_url=cfg["comfyui_url"],
         web=switch(options.web, cfg["web"], "on"),
         search_backend=cfg["search_backend"],
         searxng_url=cfg["searxng_url"],

lcode/checks.py

@@ -50,10 +50,28 @@ def find(root: Path, setting: str, trusted: bool) -> Check | None:
     return detect(root)
 
 
-def python() -> str:
+VENVS = (".venv", "venv", "env")
+PYTEST_USE = re.compile(r"^\s*(?:import pytest\b|from pytest\b)|^\s*@pytest\.", re.M)
+
+
+def python(root: Path | None = None) -> str:
+    """The project's own Python (its .venv, venv or env), where its dependencies are; else the one on PATH."""
+    if root is not None:
+        for env in VENVS:
+            for exe in ("bin/python", "Scripts/python.exe"):
+                if (root / env / exe).is_file():
+                    return f"{env}/{exe}"  # the checks run in the project's root
     return "python" if shutil.which("python") else "python3"
 
 
+def uses_pytest(root: Path) -> bool:
+    """Whether the tests are written for pytest (they import it), or the requirements list it, without a config."""
+    for name in ("requirements.txt", "requirements-dev.txt", "requirements_dev.txt", "dev-requirements.txt"):
+        if re.search(r"^\s*pytest\b", text(root / name), re.M | re.I):
+            return True
+    return any(PYTEST_USE.search(text(path)[:20000]) for path in test_files(root)[:50])
+
+
 def text(path: Path) -> str:
     try:
         return path.read_text(errors="replace") if path.is_file() else ""
@@ -61,38 +79,49 @@ def text(path: Path) -> str:
         return ""
 
 
-def python_tests(root: Path) -> bool:
-    """Whether the project has Python test files (a few levels deep, outside environments and caches)."""
+def test_files(root: Path, limit: int = 200) -> list[Path]:
+    """The project's Python test files, a few levels deep, outside environments and caches."""
     from lcode.tools import IGNORE_DIRS
 
-    def walk(folder: Path, depth: int) -> bool:
+    found: list[Path] = []
+
+    def walk(folder: Path, depth: int) -> None:
         try:
-            entries = list(folder.iterdir())
+            entries = sorted(folder.iterdir())
         except OSError:
-            return False
-        if any(e.is_file() and re.fullmatch(r"test_.*\.py|.*_test\.py", e.name) for e in entries):
-            return True
-        return depth > 0 and any(
-            walk(e, depth - 1)
-            for e in entries
-            if e.is_dir() and e.name not in IGNORE_DIRS and not e.name.startswith(".")
-        )
+            return
+        found.extend(e for e in entries if e.is_file() and re.fullmatch(r"test_.*\.py|.*_test\.py", e.name))
+        if depth > 0:
+            for e in entries:
+                if len(found) >= limit:
+                    return
+                if e.is_dir() and e.name not in IGNORE_DIRS and e.name not in VENVS and not e.name.startswith("."):
+                    walk(e, depth - 1)
 
-    return walk(root, 3)
+    walk(root, 3)
+    return found[:limit]
+
+
+def python_tests(root: Path) -> bool:
+    """Whether the project has Python test files (a few levels deep, outside environments and caches)."""
+    return bool(test_files(root, limit=1))
 
 
 def detect(root: Path) -> Check | None:
     """A check command from what the project looks like. Python checks need test files: pytest configured in a
     project without tests "fails" with nothing to run."""
     if python_tests(root):
+        py = python(root)
         configs = text(root / "pyproject.toml") + text(root / "setup.cfg") + text(root / "tox.ini")
         if "pytest" in configs or any((root / name).is_file() for name in ("pytest.ini", "conftest.py")):
-            runner = "uv run pytest -q" if (root / "uv.lock").is_file() else f"{python()} -m pytest -q"
+            runner = "uv run pytest -q" if (root / "uv.lock").is_file() else f"{py} -m pytest -q"
             return Check(runner, "pytest in the project's configuration")
+        if uses_pytest(root):
+            return Check(f"{py} -m pytest -q", "tests written for pytest")
         if list(root.glob("test_*.py")) or list(root.glob("*_test.py")):
-            return Check(f"{python()} -m unittest", "test files")
+            return Check(f"{py} -m unittest", "test files")
         if (root / "tests").is_dir():
-            return Check(f"{python()} -m unittest discover -s tests -t .", "tests/")
+            return Check(f"{py} -m unittest discover -s tests -t .", "tests/")
     package = text(root / "package.json")
     if package:
         try:
@@ -229,3 +258,52 @@ def unrun_note(scripts: list[str]) -> str:
         f"deploy, delete data) or needs something you don't have, don't: tell the user plainly that {it} is untested "
         "and why."
     )
+
+
+# ----------------------------------------------------------------------------- windows it never looked at
+
+GUI_IMPORT = re.compile(r"^\s*(?:from|import)\s+(PyQt[56]|PySide[26]|tkinter|kivy|wx|gi\.repository)\b", re.M)
+LOOKED = frozenset({"screenshot", "view_image"})
+
+
+def unseen_windows(root: Path, changed: list[str], tools_used: set[str]) -> dict[str, str]:
+    """Python files that draw a window (they import a GUI toolkit), changed in this request when the model never
+    looked at one: file -> toolkit.
+
+    A small model checks a window change by importing the module and running unrelated tests, then calls it done;
+    the user finds a blank window. Tests and imports don't show what's on the screen; a screenshot does.
+    """
+    if tools_used & LOOKED:
+        return {}
+    found = {}
+    for rel in changed:
+        path = root / rel
+        if path.suffix != ".py" or not path.is_file():
+            continue
+        try:
+            text = path.read_text(errors="replace")[:200_000]
+        except OSError:
+            continue
+        match = GUI_IMPORT.search(text)
+        if match:
+            found[rel] = match.group(1)
+    return found
+
+
+def unseen_note(found: dict[str, str]) -> str:
+    names = ", ".join(found)
+    kits = sorted(set(found.values()))
+    qt = any(kit.startswith(("PyQt", "PySide")) for kit in kits)
+    how = (
+        "take a screenshot with the screenshot tool: it runs the program without a screen and shows what it draws, "
+        "and what the program printed (Qt names layout mistakes there). For something that only shows after a click "
+        'or a key, give actions, e.g. actions=["double_click Camera 1"]'
+        if qt
+        else "take a screenshot with the screenshot tool: it runs the program on a virtual display and shows what it "
+        "draws"
+    )
+    return (
+        f"[lcode] You changed {names}, which draws a window ({', '.join(kits)}), but never looked at it: tests and "
+        f"imports don't show what's on the screen. Before you say it works, {how}. Check the picture against what the "
+        "user asked for. If the program can't run here, tell the user plainly that you haven't seen it working."
+    )

lcode/cli.py

@@ -714,6 +714,7 @@ def cmd_chat(args) -> None:
         model=args.model,
         context=args.context,
         permission_mode=mode,
+        mode=args.mode,
         allowed_tools=tools,
         max_steps=args.max_steps,
         think=False if args.no_think else None,
@@ -733,6 +734,16 @@ def cmd_chat(args) -> None:
             sys.exit(1)
         fail(str(e))
     exit_code = 0
+    if args.prompt and agent.settings.mode != "code":
+        from lcode import modes
+
+        message = modes.COMING[agent.settings.mode]
+        if quiet:
+            print(json.dumps({"type": "result", "status": "error", "error": message}), flush=True)
+        else:
+            out.print(f"[yellow]{escape(message)}[/]")
+        api.close_agent(agent)
+        sys.exit(2)
     if args.escalate and args.prompt:
         agent.escalate_approved = True  # the flag is the user's OK for this run
     try:
@@ -798,6 +809,7 @@ def build_parser() -> argparse.ArgumentParser:
         help="work in a git worktree of its own, on branch NAME (new, or continue one); removed if nothing changed",
     )
     parser.add_argument("--plan", action="store_true", help="start in plan mode: agree on a plan before any change")
+    parser.add_argument("--mode", choices=config.WORK_MODES, help="what lcode makes: code (default), image or video")
     parser.add_argument("--auto-edit", action="store_true", help="apply file edits without asking")
     parser.add_argument("--yolo", action="store_true", help="never ask for permission (edits and commands)")
     parser.add_argument(

lcode/config.py

@@ -23,6 +23,7 @@ else:
 
 DEFAULT_MODEL = "qwen3.6-35b"
 PERMISSION_MODES = ("ask", "plan", "auto-edit", "yolo")  # Shift+Tab cycles in this order
+WORK_MODES = ("code", "image", "video")  # what lcode makes (lcode.modes)
 WEB_MODES = ("on", "ask", "off")
 SEARCH_BACKENDS = ("auto", "ollama", "brave", "tavily", "searxng")
 MCP_TOOL_MODES = ("auto", "direct", "search")
@@ -40,6 +41,8 @@ SETTINGS: dict[str, tuple[object, type, str]] = {
     "backend": ("ollama", str, "model server: ollama | lmstudio | llama.cpp | vllm | mlx | openai (any OpenAI API)"),
     "base_url": (None, str, "OpenAI-compatible server address, e.g. http://localhost:1234/v1 (default: the backend's)"),
     "permission_mode": ("ask", str, "ask | plan | auto-edit | yolo"),
+    "mode": ("code", str, "what lcode makes when it starts: code | image | video"),
+    "comfyui_url": ("http://127.0.0.1:8188", str, "ComfyUI, for image and video mode"),
     "think": (True, bool, "let the model reason before answering (slower, better)"),
     "web": ("on", str, "web search and page fetching for the model: on | ask | off"),
     "search_backend": ("auto", str, "auto | ollama | brave | tavily | searxng (keys come from environment variables)"),
@@ -203,6 +206,8 @@ def coerce(key: str, value: object) -> object:
         raise ConfigError(f"search_backend must be one of {', '.join(SEARCH_BACKENDS)}")
     if key == "permission_mode" and value not in PERMISSION_MODES:
         raise ConfigError(f"permission_mode must be one of {', '.join(PERMISSION_MODES)}")
+    if key == "mode" and value not in WORK_MODES:
+        raise ConfigError(f"mode must be one of {', '.join(WORK_MODES)}")
     if typ is bool and isinstance(value, str):
         if value.lower() not in ("true", "false", "1", "0", "yes", "no", "on", "off"):
             raise ConfigError(f"{key} must be true or false")

lcode/mcp/catalog.toml

@@ -130,34 +130,6 @@ requires = ["npx"]
 setup = "Runs a headless browser; the first use may download it."
 server = { command = "npx", args = ["-y", "@playwright/mcp@latest", "--headless"] }
 
-[comfyui]
-name = "ComfyUI"
-description = "Generate and edit images with models you run locally in ComfyUI: FLUX, SDXL, SD 1.5, Qwen-Image"
-homepage = "https://github.com/Comfy-Org/comfy-mcp"
-requires = ["uvx"]
-setup = """Needs ComfyUI on this machine (http://127.0.0.1:8188) with at least one image model:
-  uv tool install comfy-cli && comfy install
-  comfy launch --background -- --disable-smart-memory
---disable-smart-memory makes ComfyUI give the GPU back after each image. Before generating, lcode
-unloads its own model so ComfyUI has the GPU; it reloads afterwards. Only the local tools are on;
-Comfy Cloud's partner tools are left out, so nothing leaves this machine."""
-
-[comfyui.server]
-command = "uvx"
-args = ["--python", "3.12", "--from", "comfy-mcp", "--with", "comfy-cli>=1.14.0", "comfy-mcp"]
-env = { COMFYUI_URL = "${COMFYUI_URL}" }
-tools = [
-  "server_info", "generate_image", "run_workflow", "run_template", "search_templates", "get_template", "fetch_template",
-  "list_workflow_slots", "set_workflow_slot", "job", "fetch_outputs", "upload_file", "search_models",
-  "download_model", "system_stats", "free_memory", "launch_comfyui",
-]
-free_gpu = ["generate_image", "run_workflow", "run_template"]
-
-[[comfyui.inputs]]
-var = "COMFYUI_URL"
-prompt = "ComfyUI address (Enter for this machine, 127.0.0.1:8188)"
-optional = true
-
 [context7]
 name = "Context7"
 description = "Current documentation and code examples for thousands of libraries"
@@ -282,23 +254,3 @@ var = "VALOHAI_HOST"
 prompt = "Valohai address (Enter for app.valohai.com; for a self-hosted installation, its URL)"
 optional = true
 
-[xai]
-name = "xAI Grok Imagine"
-description = "Generate and edit images, and generate videos and speech, with xAI's Grok Imagine and voice API (paid, in the cloud)"
-homepage = "https://nasser1941.github.io/lcode/mcp/#xai"
-requires = ["uvx"]
-setup = """Uses xAI's API: your prompts, and images you edit or animate, go to xAI, and every call is billed to your xAI
-account (an image $0.02-0.05, video $0.02-0.08 a second, speech per character). Create an API key at https://console.x.ai.
-Files are saved in generated/ in the project. lcode asks before each call; to stop asking for some tools, list them
-under "allow" for this server in mcp.json. xAI has no music generation."""
-
-[xai.server]
-command = "uvx"
-args = ["--isolated", "--python", "3.12", "--from", "lcode-cli", "lcode-mcp-xai"]
-env = { XAI_API_KEY = "${XAI_API_KEY}" }
-timeout = 900  # a video takes minutes
-
-[[xai.inputs]]
-var = "XAI_API_KEY"
-prompt = "xAI API key (create one at https://console.x.ai)"
-secret = true

lcode/mcp/commands.py

@@ -60,6 +60,12 @@ def start_session(cwd: Path, tool_mode: str, console: Console, interactive: bool
     servers, problems = mcp_config.load_all(cwd, include_project=use_project)
     for problem in problems:
         console.print(f"[yellow]MCP: {escape(problem)}[/]")
+    from lcode import modes
+
+    servers, left_out = modes.code_mode_servers(servers)  # images and videos have their own modes
+    note = modes.note_once(left_out)
+    if note and interactive:
+        console.print(f"[dim]{escape(note)}[/]")
     if not servers:
         return None
     manager = McpManager(cwd, servers, tool_mode)

lcode/modes.py

@@ -0,0 +1,102 @@
+"""What lcode makes: code, images or videos.
+
+- **code** is the coding agent: a language model with tools, and nothing else. It doesn't generate images or
+  videos: the MCP servers that do (`comfyui`, `xai`) aren't loaded in code mode.
+- **image** and **video** generate media on the GPU with image and video models (through ComfyUI, #149). They
+  don't need the big coding model: lcode unloads it when you switch to them, and frees ComfyUI's memory when you
+  switch back, so one model has the GPU at a time.
+
+Until the generation engine lands, image and video mode say what's coming instead of sending the prompt anywhere.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import requests
+
+from lcode import config
+
+if TYPE_CHECKING:
+    from lcode.agent import Agent
+    from lcode.mcp.config import ServerConfig
+
+MODES = ("code", "image", "video")
+TITLES = {"code": "local coding agent", "image": "local image generation", "video": "local video generation"}
+MEDIA_SERVERS = frozenset({"comfyui", "xai"})  # the MCP catalog's image and video servers
+MEDIA_COMMANDS = ("comfy-mcp", "lcode-mcp-xai")  # the same servers, under any name
+NOTED_FILE = "media-servers-noted"
+COMING = {
+    "image": (
+        "Image mode doesn't generate yet: lcode's own ComfyUI engine (#149), the image models (#150) and "
+        "generation (#151) come next. /mode code switches back."
+    ),
+    "video": (
+        "Video mode doesn't generate yet: lcode's own ComfyUI engine (#149), the video models (#150) and "
+        "generation (#153) come next. /mode code switches back."
+    ),
+}
+
+
+def is_media_server(server: ServerConfig) -> bool:
+    """An MCP server that generates images or videos: those belong to image and video mode, not code mode."""
+    if server.name.lower() in MEDIA_SERVERS:
+        return True
+    target = " ".join(str(part) for part in [*server.command, server.url])
+    return any(name in target for name in MEDIA_COMMANDS)
+
+
+def code_mode_servers(servers: list[ServerConfig]) -> tuple[list[ServerConfig], list[str]]:
+    """The servers code mode loads, and the names of the media servers it leaves out."""
+    kept = [s for s in servers if not is_media_server(s)]
+    return kept, [s.name for s in servers if is_media_server(s)]
+
+
+def note_once(left_out: list[str]) -> str:
+    """The note about media servers left out of code mode, the first time only."""
+    if not left_out:
+        return ""
+    marker = config.STATE_DIR / NOTED_FILE
+    if marker.exists():
+        return ""
+    try:
+        marker.parent.mkdir(parents=True, exist_ok=True)
+        marker.write_text(json.dumps(sorted(left_out)) + "\n")
+    except OSError:
+        pass
+    names = ", ".join(left_out)
+    return (
+        f"Code mode makes code only, so the MCP servers that make images and videos ({names}) aren't loaded "
+        "here any more. Image and video generation move to their own modes: /mode image, /mode video."
+    )
+
+
+def free_comfyui(url: str) -> bool:
+    """Ask ComfyUI at `url` to give the GPU back (unload its models, free its memory). Whether it answered."""
+    try:
+        r = requests.post(url.rstrip("/") + "/free", json={"unload_models": True, "free_memory": True}, timeout=3)
+    except requests.RequestException:
+        return False
+    return r.status_code == 200
+
+
+def switch(agent: Agent, new: str) -> list[str]:
+    """Switch the agent to another mode, handing the GPU over. The lines to show."""
+    old = agent.settings.mode
+    if new == old:
+        return [f"Already in {new} mode."]
+    lines = []
+    if old == "code" and getattr(agent.ollama, "kind", "ollama") == "ollama":
+        try:
+            agent.ollama.unload(agent.settings.model)  # the image or video model needs the GPU
+            lines.append(f"Unloaded {agent.settings.model} to free the GPU for {new}s.")
+        except Exception:  # not loaded, or Ollama gone: nothing to free
+            pass
+    if new == "code" and free_comfyui(agent.settings.comfyui_url):
+        lines.append("ComfyUI gave the GPU back.")
+    agent.settings.mode = new
+    lines.insert(0, f"Mode: {new} ({TITLES[new]}).")
+    if new != "code":
+        lines.append(COMING[new])
+    return lines

lcode/repeats.py

@@ -6,7 +6,8 @@ one exactly: the same tool, the same arguments and the same result. At the `limi
 failed: one sooner) the result gets a note that tells the model so, and if it still goes on, lcode ends the
 request.
 
-Results that change aren't repeats: a test run again after a fix, a background job read while it runs. And
+Results that change aren't repeats: a test run again after a fix, a background job read while it runs. Timings
+don't count as a change, though: `75 passed in 0.12s` and then `in 0.14s` is the same result. And
 a call that may have changed something (a file edit, a shell command that isn't read-only) starts the count
 again for every other call, so checking a build after each edit never adds up. Its own repeats still count:
 running the same failing command over and over is a loop.
@@ -23,6 +24,9 @@ from dataclasses import dataclass
 from lcode.permissions import is_read_only
 
 STOP_AFTER_NOTE = 2  # repeats after the note before the request ends
+TIMING = re.compile(
+    r"\b\d+(?:\.\d+)?\s?(?:ms|µs|us|s|secs?|seconds?|mins?|minutes?)\b|\b\d{1,2}:\d{2}:\d{2}(?:[.,]\d+)?\b", re.I
+)
 FAILED = ("Error:", "A deny rule", "The user denied", "This needs the user's permission")
 UNCHANGING = frozenset(
     {
@@ -64,7 +68,7 @@ class Repeats:
         """Count this call and its result."""
         if self.limit <= 0:
             return Verdict(1)
-        key = hashlib.sha256(json.dumps([name, args, result], sort_keys=True, default=str).encode()).hexdigest()
+        key = hashlib.sha256(json.dumps([name, args, steady(result)], sort_keys=True, default=str).encode()).hexdigest()
         count = self.seen.get(key, 0) + 1
         if changes_things(name, args, result):
             self.seen.clear()
@@ -95,6 +99,11 @@ class Repeats:
         )
 
 
+def steady(result: str) -> str:
+    """The result without its timings and clock times, which change on every run of the same command."""
+    return TIMING.sub("#", result)
+
+
 def changes_things(name: str, args: dict, result: str) -> bool:
     """Whether the call may have changed files (or anything else a later call could see)."""
     if name in UNCHANGING:

lcode/repl.py

@@ -36,6 +36,7 @@ from lcode import (
     extensions,
     gitflow,
     limits,
+    modes,
     residency,
     sessions,
     web,
@@ -75,7 +76,8 @@ COMMANDS = {
     "/think": "Toggle model reasoning on/off",
     "/verbose": "Toggle showing the model's reasoning text",
     "/plan": "Plan before changing anything: /plan <request>, or /plan to show the approved plan",
-    "/mode": "Permission mode: ask | plan | auto-edit | yolo (Shift+Tab cycles)",
+    "/mode": "What lcode makes: code, image or video, e.g. /mode image",
+    "/permissions": "Permission mode: ask | plan | auto-edit | yolo (Shift+Tab cycles)",
     "/cd": "Change the working directory",
     "/todos": "Show the current todo list",
     "/status": "The task sheet: your requests, the plan and notes, files changed, the last check",
@@ -193,7 +195,7 @@ def build_session(agent: Agent) -> PromptSession:
         color = {"ask": "ansigreen", "plan": "ansicyan", "auto-edit": "ansiyellow", "yolo": "ansired"}[agent.perms.mode]
         return HTML(
             f" <b>{html.escape(s.model)}</b> · ctx {format_tokens(agent.ctx_used)}/{format_tokens(s.context)} "
-            f"({pct:.0f}%) · mode <{color}>{agent.perms.mode}</{color}> (shift+tab) · "
+            f"({pct:.0f}%) · <b>{s.mode}</b> · <{color}>{agent.perms.mode}</{color}> (shift+tab) · "
             f"think {'on' if s.think else 'off'} · {html.escape(agent.cwd.name)}/"
             + (" · <ansiyellow>loading the model into memory…</ansiyellow>" if agent.load_state == "loading" else "")
             + (f" · <b>{html.escape(agent.session_name)}</b>" if agent.session_name else "")
@@ -216,14 +218,15 @@ def banner(agent: Agent) -> None:
     agent.console.print(
         Panel(
             Group(
-                Text.from_markup(f"[bold cyan]lcode[/] [dim]v{__version__}[/] — local coding agent\n"),
+                Text.from_markup(f"[bold cyan]lcode[/] [dim]v{__version__}[/] — {modes.TITLES[s.mode]}\n"),
                 Text.from_markup(
                     f"[dim]model[/]    {escape(s.model)}"
                     + (f" [dim]· {escape(agent.ollama.describe())}[/]" if backends.is_openai(agent.ollama) else "")
                     + "\n"
                     f"[dim]context[/]  {format_tokens(s.context)} tokens\n"
                     f"[dim]cwd[/]      {agent.cwd}\n"
-                    f"[dim]mode[/]     {agent.perms.mode} [dim](Shift+Tab to cycle)[/]\n"
+                    f"[dim]mode[/]     {s.mode} [dim](/mode code, image or video)[/]\n"
+                    f"[dim]perms[/]    {agent.perms.mode} [dim](Shift+Tab to cycle)[/]\n"
                     f"[dim]web[/]      {web_status(agent)}\n"
                     + (f"[dim]mcp[/]      {escape(', '.join(agent.mcp.servers))} [dim](/mcp)[/]\n" if agent.mcp else "")
                     + (
@@ -826,6 +829,30 @@ def restore_checkpoint(agent: Agent, target: Checkpoint, rewind: bool) -> None:
     agent.save()
 
 
+def mode_command(agent: Agent, arg: str) -> None:
+    """/mode: show the mode, or switch to code, image or video (handing the GPU over)."""
+    c = agent.console
+    if not arg:
+        c.print(
+            f"Mode: [bold]{agent.settings.mode}[/] ({modes.TITLES[agent.settings.mode]}). Switch with /mode code, "
+            "/mode image or /mode video. The permission mode is /permissions."
+        )
+        return
+    if arg in PERMISSION_MODES:  # what /mode used to set
+        agent.perms.mode = arg
+        c.print(f"Permission mode: {arg}. [dim](/permissions sets it now; /mode switches code, image and video.)[/]")
+        return
+    if arg not in modes.MODES:
+        c.print(f"[red]Unknown mode {escape(arg)!r}: choose code, image or video.[/]")
+        return
+    lines = modes.switch(agent, arg)
+    c.print(f"[green]{escape(lines[0])}[/]")
+    for line in lines[1:]:
+        c.print(f"[dim]{escape(line)}[/]")
+    if arg == "code":
+        preload(agent)  # the coding model loads in the background, as at the start
+
+
 def model_command(agent: Agent, arg: str, hardware: Hardware) -> None:
     """/models and /model: the numbered list to pick from, or straight to a number or a name (/model 2)."""
     from lcode.cli import model_choices, print_models
@@ -1081,7 +1108,10 @@ def handle_command(agent: Agent, line: str, hardware: Hardware) -> bool:
     elif cmd in ("/context", "/ctx"):
         context_command(agent, arg, hardware)
     elif cmd in ("/models", "/model"):
-        model_command(agent, arg, hardware)
+        if s.mode != "code":
+            c.print(f"[yellow]{escape(modes.COMING[s.mode])}[/]")
+        else:
+            model_command(agent, arg, hardware)
     elif cmd == "/think":
         s.think = (arg == "on") if arg in ("on", "off") else not s.think
         c.print(f"Reasoning {'on' if s.think else 'off'}.")
@@ -1103,10 +1133,12 @@ def handle_command(agent: Agent, line: str, hardware: Hardware) -> bool:
         c.print(Rule(style="dim"))
         run_safely(agent, arg)
     elif cmd == "/mode":
+        mode_command(agent, arg.strip().lower())
+    elif cmd == "/permissions":
         if arg in PERMISSION_MODES:
             agent.perms.mode = arg
         elif arg:
-            c.print(f"[red]Unknown mode; choose from {', '.join(PERMISSION_MODES)}[/]")
+            c.print(f"[red]Unknown permission mode; choose from {', '.join(PERMISSION_MODES)}[/]")
         c.print(f"Permission mode: {agent.perms.mode}")
     elif cmd == "/cd":
         p = Path(os.path.expanduser(arg or "~"))
@@ -1256,6 +1288,9 @@ def repl(agent: Agent, prompt: str | None, hardware: Hardware, cont: bool = Fals
         if info:
             resume_session(agent, info)
     session_start(agent)
+    if prompt and agent.settings.mode != "code":
+        agent.console.print(f"[yellow]{escape(modes.COMING[agent.settings.mode])}[/]")
+        prompt = None
     if prompt:
         run_safely(agent, prompt)
         return
@@ -1266,12 +1301,16 @@ def repl(agent: Agent, prompt: str | None, hardware: Hardware, cont: bool = Fals
     if warning:
         agent.say(warning, warning=True)
     session = build_session(agent)
-    preload(agent, session)
+    if agent.settings.mode == "code":
+        preload(agent, session)
+    else:
+        agent.console.print(f"[yellow]{escape(modes.COMING[agent.settings.mode])}[/]")
     while True:
         try:
             agent.console.print()
             unsent, agent.unsent = agent.unsent, ""
-            line = session.prompt(HTML("<ansicyan><b>❯ </b></ansicyan>"), default=unsent).strip()
+            marker = "❯ " if agent.settings.mode == "code" else f"{agent.settings.mode} ❯ "
+            line = session.prompt(HTML(f"<ansicyan><b>{marker}</b></ansicyan>"), default=unsent).strip()
         except KeyboardInterrupt:
             continue
         except EOFError:
@@ -1284,12 +1323,20 @@ def repl(agent: Agent, prompt: str | None, hardware: Hardware, cont: bool = Fals
             if not handle_command(agent, line, hardware):
                 break
             continue
-        agent.console.print(Rule(style="dim"))
-        run_safely(agent, line)
+        respond(agent, line)
     memory_notes.reflect(agent)
     agent.console.print("[dim]Bye.[/]")
 
 
+def respond(agent: Agent, line: str) -> None:
+    """What a typed request does: the coding agent works on it, or image and video mode take it."""
+    if agent.settings.mode != "code":  # image and video: no coding model involved
+        agent.console.print(f"[yellow]{escape(modes.COMING[agent.settings.mode])}[/]")
+        return
+    agent.console.print(Rule(style="dim"))
+    run_safely(agent, line)
+
+
 def preload(agent: Agent, session: PromptSession | None = None) -> None:
     """Start loading the model in the background so the first answer comes sooner; the status bar says so."""
     if is_cloud(agent.settings.model):

lcode/screenshots.py

@@ -6,7 +6,9 @@ returns what the model can't otherwise see: a picture (described by the vision m
 
 - A Python Qt program (PyQt5/6, PySide2/6) runs on Qt's "offscreen" platform, with a screen of the asked size. A
   small startup hook (a sitecustomize module on PYTHONPATH) waits for the program's event loop, then grabs its
-  windows, measures them and their widgets, and quits it.
+  windows, measures them and their widgets, and quits it. First it can take `actions`, the way a user would: click
+  or double-click a point or a widget by its text, press a key, type, wait. Bugs that only show after a click need
+  that, and Qt's warnings (in what the program printed) often name the cause.
 - Other desktop programs run on a virtual X display (Xvfb, Linux), captured with ImageMagick's `import` or `xwd`.
   The Qt hook still measures a Python Qt program there.
 - A web page, or a local server started with `command`, opens in headless Chrome (or Chromium, Edge), driven
@@ -43,6 +45,14 @@ from pathlib import Path
 DEFAULT_SCREEN = (1920, 1080)
 DEFAULT_WAIT = 3  # seconds for a program or page to start and draw
 MAX_WAIT = 60
+MAX_ACTIONS = 20
+ACTION_PAUSE = 0.4  # seconds after each action, for the program to react and redraw
+MOUSE_ACTIONS = ("click", "double_click", "right_click")
+KEY_NAMES = {"enter": "Return", "return": "Return", "esc": "Escape", "escape": "Escape", "space": "Space",
+             "tab": "Tab", "backspace": "Backspace", "delete": "Delete", "del": "Delete", "home": "Home", "end": "End",
+             "pageup": "PageUp", "pagedown": "PageDown", "up": "Up", "down": "Down", "left": "Left",
+             "right": "Right"}  # fmt: skip
+MODIFIERS = ("ctrl", "shift", "alt", "meta")
 SERVER_TIMEOUT = 60  # seconds for a server started with `command` to answer at `url`
 GRACE = 25  # seconds beyond `wait` before a program that never showed a window is stopped
 QT_BINDINGS = ("PyQt6", "PyQt5", "PySide6", "PySide2")
@@ -169,15 +179,19 @@ def _lcode_shot():
     out = _os.environ.get("LCODE_SHOT_OUT")
     wait_ms = int(float(_os.environ.get("LCODE_SHOT_WAIT", "3")) * 1000)
     quit_after = _os.environ.get("LCODE_SHOT_QUIT", "1") == "1"
+    actions = json.loads(_os.environ.get("LCODE_SHOT_ACTIONS") or "[]")
+    pause_ms = int(float(_os.environ.get("LCODE_SHOT_PAUSE", "0.4")) * 1000)
     bindings = ("PyQt6", "PyQt5", "PySide6", "PySide2")
     state = {"scheduled": False}
 
-    def capture(binding):
+    def capture(binding, steps=None):
         QtCore = _sys.modules[binding + ".QtCore"]
         QtGui = _sys.modules[binding + ".QtGui"]
         QtWidgets = _sys.modules.get(binding + ".QtWidgets")
         app = QtGui.QGuiApplication.instance()
         result = {"binding": binding, "windows": [], "screen": None, "platform": app.platformName()}
+        if steps is not None:
+            result["actions"] = steps
         screen = app.primaryScreen()
         if screen is not None:
             g, a = screen.geometry(), screen.availableGeometry()
@@ -257,6 +271,100 @@ def _lcode_shot():
             app.quit()
             QtCore.QTimer.singleShot(3000, lambda: _os._exit(0))
 
+    def text_of(widget):
+        for getter in ("text", "windowTitle", "title", "placeholderText"):
+            fn = getattr(widget, getter, None)
+            if callable(fn):
+                try:
+                    text = str(fn() or "")
+                except Exception:
+                    text = ""
+                if text:
+                    return text
+        return ""
+
+    def act(binding, then):
+        """Take the steps one by one, pausing after each so the program reacts, then call then(log)."""
+        QtCore = _sys.modules[binding + ".QtCore"]
+        QtWidgets = _sys.modules[binding + ".QtWidgets"]
+        Qt = QtCore.Qt
+        try:
+            QTest = importlib.import_module(binding + ".QtTest").QTest
+        except Exception as e:
+            then([{"step": s["raw"], "error": "Qt's QtTest module isn't available (%s)" % e} for s in actions])
+            return
+        app = QtWidgets.QApplication.instance()
+
+        def enum(group, name):  # scoped enums (PyQt6, PySide6), or flat ones (PyQt5, PySide2)
+            scope = getattr(Qt, group, None)
+            if scope is not None and hasattr(scope, name):
+                return getattr(scope, name)
+            return getattr(Qt, name)
+
+        def window():
+            tops = [w for w in app.topLevelWidgets() if w.isVisible() and w.width() > 1 and w.height() > 1]
+            active = app.activeWindow()
+            return active if active in tops else (tops[0] if tops else None)
+
+        def describe(widget):
+            text = text_of(widget)[:40]
+            return type(widget).__name__ + (' "%s"' % text if text else "")
+
+        def by_text(top, wanted):
+            wanted = wanted.casefold()
+            found = [c for c in top.findChildren(QtWidgets.QWidget)
+                     if c.isVisible() and wanted in text_of(c).casefold()]  # fmt: skip
+            return min(found, key=lambda c: c.width() * c.height()) if found else None
+
+        log = []
+
+        def step(i):
+            if i >= len(actions):
+                then(log)
+                return
+            s, note, pause = actions[i], {"step": actions[i]["raw"]}, pause_ms
+            try:
+                top = window()
+                if s["do"] == "wait":
+                    pause = int(s["seconds"] * 1000)
+                elif top is None:
+                    note["error"] = "no window to act on"
+                elif s["do"] in ("click", "double_click", "right_click"):
+                    if "text" in s:
+                        target = by_text(top, s["text"])
+                        pos = target.rect().center() if target is not None else None
+                    else:
+                        point = QtCore.QPoint(s["x"], s["y"])
+                        target = top.childAt(point) or top
+                        pos = target.mapFrom(top, point)
+                    if target is None:
+                        note["error"] = 'no visible widget shows "%s"' % s["text"]
+                    else:
+                        button = enum("MouseButton", "RightButton" if s["do"] == "right_click" else "LeftButton")
+                        none = enum("KeyboardModifier", "NoModifier")
+                        at = target.mapTo(top, pos)  # before the click moves things around
+                        note["target"] = "%s at (%d, %d)" % (describe(target), at.x(), at.y())
+                        click = QTest.mouseDClick if s["do"] == "double_click" else QTest.mouseClick
+                        click(target, button, none, pos)
+                else:
+                    target = app.focusWidget() or top
+                    if s["do"] == "key":
+                        mods = enum("KeyboardModifier", "NoModifier")
+                        names = {"ctrl": "ControlModifier", "shift": "ShiftModifier", "alt": "AltModifier",
+                                 "meta": "MetaModifier"}  # fmt: skip
+                        for m in s["mods"]:
+                            mods = mods | enum("KeyboardModifier", names[m])
+                        QTest.keyClick(target, enum("Key", "Key_" + s["key"]), mods)
+                    else:
+                        QTest.keyClicks(target, s["text"])
+                    note["target"] = describe(target)
+            except Exception as e:  # one step failing doesn't stop the picture
+                note["error"] = "%s: %s" % (type(e).__name__, e)
+            log.append(note)
+            QtCore.QTimer.singleShot(pause, lambda: step(i + 1))
+
+        step(0)
+
     def patch(binding):
         """Wrap exec() of the application classes loaded so far (QtWidgets may come after QtGui)."""
         QtCore = _sys.modules.get(binding + ".QtCore")
@@ -275,14 +383,23 @@ def _lcode_shot():
                 state["scheduled"] = True
                 open(_os.path.join(out, "started"), "w").close()  # the event loop runs: a capture is coming
 
-                def run():
+                def run(steps=None):
                     try:
-                        capture(binding)
+                        capture(binding, steps)
                     except Exception as e:  # the program goes on; lcode reports what went wrong
                         with open(_os.path.join(out, "qt.json"), "w") as f:
                             json.dump({"error": "%s: %s" % (type(e).__name__, e)}, f)
 
-                QtCore.QTimer.singleShot(wait_ms, run)
+                def start():
+                    if not actions:
+                        run()
+                        return
+                    try:
+                        act(binding, run)
+                    except Exception as e:
+                        run([{"step": "actions", "error": "%s: %s" % (type(e).__name__, e)}])
+
+                QtCore.QTimer.singleShot(wait_ms, start)
 
         class Wrapped:
             """In place of exec(): schedules the capture, then runs the original, static or not."""
@@ -372,6 +489,69 @@ def hook_dir(tmp: Path) -> Path:
 # --------------------------------------------------------------------------------------------- running a program
 
 
+def parse_actions(actions) -> list[dict]:
+    """The steps to take before the picture, from the tool's `actions` (a list, or one string split on ; and lines).
+
+    `click X,Y`, `double_click X,Y`, `right_click X,Y` (a point in the window), or the same with a widget's text
+    (`double_click "Camera 1"`); `key Escape` or `key ctrl+s`; `type some text`; `wait 2` (seconds).
+    """
+    if not actions:
+        return []
+    if isinstance(actions, str):
+        actions = [part for line in actions.splitlines() for part in line.split(";")]
+    steps = []
+    for raw in actions:
+        raw = str(raw).strip()
+        if not raw:
+            continue
+        verb, _, rest = raw.partition(" ")
+        verb, rest = verb.lower().replace("-", "_"), rest.strip()
+        if verb == "doubleclick":
+            verb = "double_click"
+        step: dict = {"do": verb, "raw": raw}
+        if verb in MOUSE_ACTIONS:
+            point = re.fullmatch(r"\(?\s*(-?\d+)\s*[, ]\s*(-?\d+)\s*\)?", rest)
+            if point:
+                step["x"], step["y"] = int(point.group(1)), int(point.group(2))
+            elif rest.strip("\"'"):
+                step["text"] = rest.strip("\"'")
+            else:
+                raise ShotError(f"{raw!r}: say where, as X,Y in the window or the widget's text, e.g. {verb} 200,150")
+        elif verb == "key":
+            parts = [p for p in re.split(r"[+\s]+", rest.lower()) if p]
+            if not parts:
+                raise ShotError(f"{raw!r}: name the key, e.g. key Escape or key ctrl+s")
+            *mods, key = parts
+            if any(m not in MODIFIERS for m in mods):
+                raise ShotError(f"{raw!r}: modifiers are {', '.join(MODIFIERS)}")
+            if key in KEY_NAMES:
+                key = KEY_NAMES[key]
+            elif re.fullmatch(r"f([1-9]|1[0-2])", key) or (len(key) == 1 and key.isalnum()):
+                key = key.upper()
+            else:
+                raise ShotError(f"{raw!r}: unknown key {key!r} (Escape, Return, Tab, Space, arrows, F1-F12, a letter)")
+            step["key"], step["mods"] = key, mods
+        elif verb == "type":
+            if not rest:
+                raise ShotError(f"{raw!r}: give the text to type")
+            step["text"] = rest
+        elif verb == "wait":
+            try:
+                step["seconds"] = max(0.0, min(float(rest or 1), 10.0))
+            except ValueError as e:
+                raise ShotError(f"{raw!r}: wait takes seconds, e.g. wait 2") from e
+        else:
+            raise ShotError(f"{raw!r}: actions are click, double_click, right_click, key, type and wait")
+        steps.append(step)
+    if len(steps) > MAX_ACTIONS:
+        raise ShotError(f"at most {MAX_ACTIONS} actions in one screenshot")
+    return steps
+
+
+def actions_time(steps: list[dict]) -> float:
+    return sum(s.get("seconds", 0) for s in steps) + ACTION_PAUSE * len(steps)
+
+
 def run_program(command: str, cwd: Path, env: dict[str, str], log: Path) -> subprocess.Popen:
     out = log.open("wb")
     try:
@@ -405,6 +585,27 @@ def stop(proc: subprocess.Popen | None) -> None:
             proc.wait(timeout=3)
 
 
+QT_WARNING = re.compile(r"^(?:Q[A-Za-z0-9]+::\w+|qt\.[\w.]+:|QML |Qt[: ])")
+NOT_A_PROBLEM = re.compile(
+    r"propagateSizeHints|QFont::|This plugin does not support|qt\.qpa\.fonts"
+)  # offscreen platform noise
+MAX_WARNINGS = 6
+
+
+def qt_warnings(log: Path) -> list[str]:
+    """Qt's own warnings in everything the program printed (not just the end that's shown), without repeats."""
+    try:
+        text = log.read_bytes().decode(errors="replace")
+    except OSError:
+        return []
+    found: list[str] = []
+    for line in text.splitlines():
+        line = line.strip()
+        if QT_WARNING.match(line) and not NOT_A_PROBLEM.search(line) and line[:300] not in found:
+            found.append(line[:300])
+    return found[:MAX_WARNINGS]
+
+
 def printed(log: Path) -> str:
     try:
         text = log.read_bytes().decode(errors="replace")
@@ -423,7 +624,9 @@ def base_env(hook: Path) -> dict[str, str]:
     return env
 
 
-def qt_offscreen(command: str, cwd: Path, screen: tuple[int, int], wait: int, tmp: Path, cancel=None) -> Shot:
+def qt_offscreen(
+    command: str, cwd: Path, screen: tuple[int, int], wait: int, tmp: Path, cancel=None, actions: list | None = None
+) -> Shot:
     """Run a Python Qt program on Qt's offscreen platform and capture its windows."""
     config = tmp / "screens.json"
     w, h = screen
@@ -453,12 +656,14 @@ def qt_offscreen(command: str, cwd: Path, screen: tuple[int, int], wait: int, tm
         LCODE_SHOT_OUT=str(out),
         LCODE_SHOT_WAIT=str(wait),
         LCODE_SHOT_QUIT="1",
+        LCODE_SHOT_ACTIONS=json.dumps(actions or []),
+        LCODE_SHOT_PAUSE=str(ACTION_PAUSE),
     )
     env.pop("DISPLAY", None)
     log = tmp / "output.txt"
     proc = run_program(command, cwd, env, log)
     try:
-        deadline = time.time() + wait + GRACE
+        deadline = time.time() + wait + actions_time(actions or []) + GRACE
         while time.time() < deadline and proc.poll() is None and not (out / "qt.json").exists():
             if cancel is not None and cancel():
                 raise KeyboardInterrupt
@@ -482,6 +687,9 @@ def qt_offscreen(command: str, cwd: Path, screen: tuple[int, int], wait: int, tm
     if image is None and not windows:
         raise ShotError("the program started Qt but showed no window")
     report = qt_report(data)
+    warnings = qt_warnings(log)
+    if warnings:
+        report[:0] = ["Qt printed warnings (they often name the cause):", *[f"- {line}" for line in warnings]]
     if shot_screen != screen:  # Qt 5's offscreen platform has a fixed screen
         report.append(
             f"(Qt used a {shot_screen[0]}x{shot_screen[1]} screen offscreen, not the {screen[0]}x{screen[1]} asked "
@@ -509,8 +717,17 @@ def qt_report(data: dict) -> list[str]:
     screen = data.get("screen") or [0, 0]
     sw, sh = screen
     windows = data.get("windows") or []
+    if data.get("actions"):
+        lines.append("Steps, before the picture:")
+        for note in data["actions"]:
+            done = (
+                f"couldn't: {note['error']}"
+                if note.get("error")
+                else (f"on {note['target']}" if note.get("target") else "done")
+            )
+            lines.append(f"- {note['step']}: {done}")
     if not windows:
-        return ["No visible Qt window."]
+        return [*lines, "No visible Qt window."]
     for win in windows:
         title = f'"{win["title"]}" ' if win.get("title") else ""
         x, y, w, h = win["x"], win["y"], win["width"], win["height"]
@@ -1059,12 +1276,15 @@ def take(
     full_page: bool = False,
     mode: str = "auto",
     cancel=None,
+    actions: list | None = None,
 ) -> Shot:
-    """Take the screenshot. `cancel` is checked while waiting (return True to stop)."""
+    """Take the screenshot. `cancel` is checked while waiting (return True to stop); `actions` come first."""
     wait = max(0, min(int(wait), MAX_WAIT))
     path = new_path(folder, label_for(command, url))
     with tempfile.TemporaryDirectory(prefix="lcode-shot-") as name:
         tmp = Path(name)
+        if actions and url:
+            raise ShotError("actions work for Python Qt programs (PyQt, PySide), not web pages yet")
         if url:
             url = normalize_url(url, cwd)
             server = None
@@ -1086,7 +1306,14 @@ def take(
             raise ShotError("give a command that starts the program, or a url")
         if mode == "auto":
             mode = "qt" if looks_like_qt(command, cwd) or not xvfb_available() else "xvfb"
-        shot = (qt_offscreen if mode == "qt" else virtual_display)(command, cwd, screen, wait, tmp, cancel)
+        if actions and mode != "qt":
+            raise ShotError(
+                "actions work for Python Qt programs (PyQt, PySide) only: take this one's screenshot without them"
+            )
+        if mode == "qt":
+            shot = qt_offscreen(command, cwd, screen, wait, tmp, cancel, actions)
+        else:
+            shot = virtual_display(command, cwd, screen, wait, tmp, cancel)
         shutil.copyfile(shot.image, path)
         kept = []
         for i, window in enumerate(shot.windows):

lcode/tools.py

@@ -192,7 +192,8 @@ SCHEMAS = [
         "widgets (Qt) or of the page, and the browser's console errors. Use it to check GUI and layout work instead "
         "of guessing. Python Qt programs run offscreen; other desktop programs on a virtual display (Linux); web pages "
         "in a headless browser. For a web app, give the command that starts its server and the url (lcode starts and "
-        "stops it); if a server is already running, or for an .html file, give only the url.",
+        "stops it); if a server is already running, or for an .html file, give only the url. For something that "
+        "only shows after a click or a key (a Python Qt program), give `actions`: they run first, then the picture.",
         {
             "command": {"type": "string", "description": "Command that starts the program, or the web server"},
             "url": {"type": "string", "description": "Page to open: a URL, localhost:PORT/path or an .html file"},
@@ -203,6 +204,14 @@ SCHEMAS = [
             "screen": {"type": "string", "description": "Screen (or browser window) size, WxH (default 1920x1080)"},
             "wait": {"type": "integer", "description": "Seconds to let it start and draw first (default 3)"},
             "full_page": {"type": "boolean", "description": "Web pages: the whole page, not just the window"},
+            "actions": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Python Qt programs: steps to take before the picture, as a user would: "
+                '"click X,Y", "double_click X,Y", "right_click X,Y" (a point in the window, as the sizes show), '
+                'or with the widget\'s text: "double_click Camera 1"; "key Escape", "key ctrl+s"; "type some text"; '
+                '"wait 2". The result says what each step hit.',
+            },
             "mode": {
                 "type": "string",
                 "enum": ["auto", "qt", "xvfb"],
@@ -698,6 +707,7 @@ class Toolbox:
         wait: int = screenshots.DEFAULT_WAIT,
         full_page: bool = False,
         mode: str = "auto",
+        actions: list | str | None = None,
     ) -> str:
         command, url = (command or "").strip(), (url or "").strip()
         if not command and not url:
@@ -708,6 +718,7 @@ class Toolbox:
             size = screenshots.parse_screen(screen)
             if url:
                 url = screenshots.normalize_url(url, self.agent.cwd)
+            steps = screenshots.parse_actions(actions)
         except screenshots.ShotError as e:
             raise ToolError(str(e)) from e
         if command:
@@ -748,6 +759,7 @@ class Toolbox:
                     full_page=full_page,
                     mode=mode,
                     cancel=cancelled,
+                    actions=steps,
                 )
         except screenshots.ShotError as e:
             raise ToolError(f"no screenshot: {e}") from e
@@ -762,6 +774,8 @@ class Toolbox:
             lines.append(f"What the {'server' if url else 'program'} printed:\n{shot.output}")
         if self.agent.vision_model():
             what = f"the page {url}" if url else f"the program `{command}` on a {w}x{h} screen"
+            if steps:
+                what += ", after these steps: " + "; ".join(s["raw"] for s in steps)
             ask = f"This is a screenshot of {what}." + (f" {question.strip()}" if question.strip() else "")
             try:
                 seen = self.agent.look(shot.image, ask)

{lcode_cli-0.37.0.dist-info → lcode_cli-0.39.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lcode-cli
-Version: 0.37.0
+Version: 0.39.0
 Summary: A local-first terminal coding agent powered by open-weight models on your own GPU or Mac, via Ollama.
 Project-URL: Homepage, https://nasser1941.github.io/lcode/
 Project-URL: Documentation, https://nasser1941.github.io/lcode/

{lcode_cli-0.37.0.dist-info → lcode_cli-0.39.0.dist-info}/RECORD

@@ -1,21 +1,21 @@
-lcode/__init__.py,sha256=lKIIi9DSWXNWEjJvCuOOtYqAkmR0gtWHsaKllXOu7vc,473
+lcode/__init__.py,sha256=_Rgyn4icfC6nuHTEHeJBx-sny88eraeJgPJOpzzaqK0,473
 lcode/__main__.py,sha256=s5N5pryX2FAu93GGZ3FxUrIXHWGzxz66lKUoaV_h8Ws,35
 lcode/acp.py,sha256=d97VV3sbqXdWKhPQQNbYRaDTm8WrUk0mGbDLiOyX7jE,29058
 lcode/action.py,sha256=wuT5F9EJCdEligPeytzmEi8FpahWBhXer528tpuTj_k,16711
-lcode/agent.py,sha256=vjWiK79BgUTMAbeC6SPslGaYT86YELwTHIGB21tc4to,84751
-lcode/api.py,sha256=mIfmiN3jDv8jg-HOJtqfr-MQ3cOI8nfFmlYYRZR1r1Y,16164
+lcode/agent.py,sha256=BcuJ79IdVodbOmHhaCbnkH6bOU8M_Prg0awBM8YYfl8,86830
+lcode/api.py,sha256=UV91TGBUVUINM9TVNWbkLmX-DwtoYD06dcqfEhtGz7s,16298
 lcode/backends.py,sha256=343alwyDZin-bEhb5-Es4f8ek1oBQUzEvV6RFWR1VZo,21160
 lcode/bench.py,sha256=HxeUfy9Kv_0Rug1HAlDLyHYfYd9dw2bMgbXsvUy9nd4,43096
 lcode/bench_backend.py,sha256=X2fo-zRX92Xp3KMxSsFlWioc2vRupBTtowj5kLME8mw,14287
 lcode/bench_runtime.py,sha256=EjzNgIvUkdbT8cbSy1TpmlyeIk6e-ipjlDFpM7Dnqx0,6835
 lcode/catalog.py,sha256=pZ1azX9OH2r4JE2c6E5OfwywpgS39-W2oXzoCHJhmtM,4141
 lcode/checkpoints.py,sha256=eqPtmPe44-aweWqc2H21ROogwzIt3NpvcUobsaCHoGY,18354
-lcode/checks.py,sha256=iPaNaGOGo7zHc_mue41ELEHSNq0VIH_xVYL_bHkYLXU,9373
-lcode/cli.py,sha256=mJBM3Loqs9kOE94IRe00d1A1izW5caAjSR7ITUxC27I,45042
+lcode/checks.py,sha256=haYYIPhadKavGYEhh8YWAaWrPMcfzrpT2i6638f4xjQ,12933
+lcode/cli.py,sha256=rRD462mr0JkKE0h83aN2617uNfa5kjIe_q3ZgxbvvEI,45560
 lcode/clipboard.py,sha256=nTSdJyCXPdtjH947RjvB3QvRqnL5HZsS9KB8EkC42cc,5138
 lcode/cloud.py,sha256=q5Dl5GuqWvKzBCIso-eh6yomvEOsnQ0LW8ZgaAGwBxc,2131
 lcode/codesearch.py,sha256=Io8leN1nb_Y-Dvlm9xaGtwUWYXX-ooswuznJV6dGU1s,10766
-lcode/config.py,sha256=hAjuHg7plRxwFgdD4gOGnDH8-FwyT7NRV0YI5grd0qE,12604
+lcode/config.py,sha256=SCYQYgRRKKy-C6ktPprjIAEHQcTvmkEMy-Q9g98Kw5E,12974
 lcode/context.py,sha256=wgK16ZcNNnq4ngHx0EmtyKHDFTImciGEmtC5NP8tKJQ,20435
 lcode/escalate.py,sha256=f-_6_moPaHi2kLuJw2JtkzvTkCl6DoBQgYBOCYLhunM,15120
 lcode/extensions.py,sha256=W0HFDl0BvROzr7i_qtabHdIbacBgP5o5qIJLzoM2oNE,16806
@@ -29,23 +29,24 @@ lcode/lsp.py,sha256=NtSW7xapv7fg2HAQLuYK1nwIqT7nG0LK1hiTpOmla_4,26086
 lcode/matching.py,sha256=2N3_qwKJ5gd8el2m_y_egKkFM9rsF1eUhYJcRCVIp98,14015
 lcode/memory.py,sha256=WZEHTp-lVxAwS-yyLQR2xB3TMZ9sejmX2Ivg7T-tl3w,21949
 lcode/models.toml,sha256=lt2G64zzo_Ryxv2ISRSNZoVD_JqCqYqsM01n8WoXGs8,4098
+lcode/modes.py,sha256=gVYuGDP0IGfRPgBYJinEU8DYIrKjFbgjb0gjK3n9jx8,4202
 lcode/notify.py,sha256=uieDeXe7XwuVHjOk4tqVyZEW_85XWkoWQmFOWEf77oc,1538
 lcode/ollama.py,sha256=xFHU2XW47ANJ34-1aeSMQVj28GhqcWiFi6zASH9FiYY,8053
 lcode/permissions.py,sha256=q7CjDsc_Xrblw7t3_BbxA5LUjxFSswGs1RiJmtMi0po,12288
 lcode/planning.py,sha256=Q2upaoZPnXDMQT2F-ETxX1lk_wDntTzhSKNVYIQSoQA,6540
 lcode/render.py,sha256=gMvf4W3qkLUJ4Vot3az_hP_zRDAItJC9pP-68dHFkfA,1375
-lcode/repeats.py,sha256=vpXR5pzKX9GsTzQONGS30V-rEU3gunaj_hBzJxg43Q4,5542
-lcode/repl.py,sha256=elnJeSQYt0oEtE14UzW2k-fkDHVFr_Lf2L6XWFo0ne0,56891
+lcode/repeats.py,sha256=OTrPdb6BKArDLQEgHaXmJZ-tIFsY_1bk4P0wzzANKLY,5964
+lcode/repl.py,sha256=NpYV0D6YoCsddV2YGmIdseY57mowx9Nv3TwH82R_o4U,59075
 lcode/repomap.py,sha256=1GRJgEUjBl9XHSXsF7tSGXBCEgsY-3CzwMohaTudZ_M,13969
 lcode/residency.py,sha256=B9pwqG6yx5aQD2M5xyQ-bMnYmhD7fv3wCVw_X6VIeAo,7012
 lcode/review.py,sha256=0QBIS-R1B2b3zqx5TcC1qZptpKBbrDhYvMKSNCaDAUo,4686
 lcode/sandbox.py,sha256=u_pkrVRXhCGhh-wFDPwuZ3jGG7UeXxXRAeBNsm37ZvE,8608
-lcode/screenshots.py,sha256=bcWSXS4xTsBk8HhpjBy5jvu705uH9qV8bXANr--Mf2M,46405
+lcode/screenshots.py,sha256=pmrw_tp8eXzNyCZa5uot5rTgsxqsi3AdmkGn-XF9saY,57056
 lcode/secretscan.py,sha256=6Euvfk2gCtK7lZbuQD84zgWuSECi6OnSw8V5rFu1tfU,13575
 lcode/sessions.py,sha256=ojkpP453JyIll8_iFeJgW4BG4Zz2FKMRsLiij6spiTo,4185
 lcode/steer.py,sha256=0vaqLxgRKgC0yGp8xoQfV1f9VEOzH3U4rCIb74eFnZ8,12127
 lcode/subagents.py,sha256=thR2ZzFZ1DeoAf0Ctr_oeKjYajhvd3uRs2kGdw6LlJM,29349
-lcode/tools.py,sha256=glp18sbnxaK5iugDxrCFlDaT0SEbO-GTOL_-O3Afp44,68005
+lcode/tools.py,sha256=lpCIlaabydbmutaXWuOZWS4X-Z_7aIsyNUQdHJzkSC4,68902
 lcode/unreachable.py,sha256=TXrv41ljwX_amxdcf4CY6qGF2YkWetORvPUSvwceJXI,8685
 lcode/update.py,sha256=i92bnW252VneYh0vD6djxmXsDOa5MXP20bjlLZUue_c,25236
 lcode/vision.py,sha256=w33QPmdGwP33wM4mvmAH4amFDZ1xzeXhEsSLJ-STSHY,4243
@@ -53,9 +54,9 @@ lcode/web.py,sha256=vyxKbSRJTl1Lj3hj7Yd_NwTgbv0Xi6dup2YfUXjqp-Y,14579
 lcode/mcp/__init__.py,sha256=XujtAmRoXW9S0uq1XPf_CAeCkHh_UvA8FCmoAKP9j1Y,171
 lcode/mcp/auth.py,sha256=vyH3oolgIBNJLYabr2h5Soui99NTY1QxjQEgtczfF6A,16655
 lcode/mcp/catalog.py,sha256=I_O10N7cBl0ZW6O7x2kwKn89pvEXUXODUh8FsPXnB-E,3790
-lcode/mcp/catalog.toml,sha256=N8MKyyS_FeCo-RbV3RVZ3ZODro2yk55yqUlymXsWdcQ,12641
+lcode/mcp/catalog.toml,sha256=LTby0I98ajZS2aXZpaBq09p1nnt77xs6bngpVMWO-t4,10324
 lcode/mcp/client.py,sha256=3Gen7zRvWehngXn_dAMOAdKqFkFkQZIQW96hQcW4N4k,7119
-lcode/mcp/commands.py,sha256=mnp0G-GW5ZsARi2tkHuT3hl883sBGFUVRAV3AapA5oU,16478
+lcode/mcp/commands.py,sha256=hOND7SsJjz5Hp2CY81PHPsB4R9i8CwhBQeQi-oB8Wg4,16721
 lcode/mcp/config.py,sha256=Ej6WX2b_55a84H5WdAwSk9DzERKoq5ifUBQ9uDgSG_o,8780
 lcode/mcp/http.py,sha256=sG9MajwU4_6sqvZhwD4WfyzRKzaR_Lc8C02jwePz18E,7380
 lcode/mcp/manager.py,sha256=79nNWoT5pxgoQD_Pv4ZH0obHIm7rQ-LyB8OmJnQgc6k,15104
@@ -66,8 +67,8 @@ lcode/servers/__init__.py,sha256=qrIhxEodEB8cjKEQ4Cicc0GsQhE2OH-9qlVHzr3-tq8,77
 lcode/servers/encord.py,sha256=WSauawL1UKYZCUAG1oalcKL_An3DVof-JM6xIgFyQSE,12769
 lcode/servers/valohai.py,sha256=e5xZGIoa4cja4f5i85q1GpL9c7cglUjV8jN2co1CFMY,14430
 lcode/servers/xai.py,sha256=h2emAvHSnkuB9WmJInlynsDhvxJwNVD2AHFX7vqMXNc,13627
-lcode_cli-0.37.0.dist-info/METADATA,sha256=fgaRfrIMEYuJXyDpy8Kj18AL7S2IybZqNmvd09c6IgI,13141
-lcode_cli-0.37.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-lcode_cli-0.37.0.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
-lcode_cli-0.37.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
-lcode_cli-0.37.0.dist-info/RECORD,,
+lcode_cli-0.39.0.dist-info/METADATA,sha256=JCZieRCoNShMIppO-MNzLvxmd240V4EFbdPmAbPWN-I,13141
+lcode_cli-0.39.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+lcode_cli-0.39.0.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
+lcode_cli-0.39.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
+lcode_cli-0.39.0.dist-info/RECORD,,