lcode-cli 0.37.0__py3-none-any.whl → 0.38.0__py3-none-any.whl

lcode/__init__.py

@@ -3,7 +3,7 @@
 From a program: `from lcode import Session` (see lcode.api).
 """
 
-__version__ = "0.37.0"
+__version__ = "0.38.0"
 
 API = ("Session", "Result", "Options", "SetupError")
 

lcode/agent.py

@@ -99,7 +99,9 @@ def stall_note(trouble: str) -> str:
     }[trouble]
     return (
         f"[lcode] {first} Don't deliberate at length now. Either make your next tool call, or, if you're stuck, "
-        "tell the user in a few lines what you did, what fails, and what they could do."
+        "tell the user in a few lines what you did, what fails, and what they could do. If you were about to check "
+        "something by running it, run it now instead of reasoning about it: for a window, take a screenshot (with "
+        "actions for clicks or keys)."
     )
 
 
@@ -961,6 +963,12 @@ class Agent:
             return []
         return checks.unrun_scripts(project_root_or_cwd(self.cwd), list(self.tools.before), commands)
 
+    def unseen_windows(self, tools_used: set[str]) -> dict[str, str]:
+        """Window code the model changed in this request and never looked at (only where the checks would run)."""
+        if self.allowed_tools is not None or self.planning() or self.no_changes or self.settings.checks == "off":
+            return {}
+        return checks.unseen_windows(project_root_or_cwd(self.cwd), list(self.tools.before), tools_used)
+
     def project_check(self) -> checks.Check | None:
         """The project's check command, for the main agent outside plan mode and reviews (None otherwise)."""
         if self.allowed_tools is not None or self.planning() or self.no_changes or self.settings.checks == "off":
@@ -1164,6 +1172,8 @@ class Agent:
         stalls = 0  # replies that went round in circles or ended without a word
         commands: list[str] = []  # shell commands run in this request
         scripts_noted = False  # the model was told about scripts it wrote and never ran
+        tools_used: set[str] = set()  # tools called in this request
+        windows_noted = False  # the model was told about windows it changed and never looked at
         for step in range(self.max_steps):
             if self.cancel is not None and self.cancel.is_set():
                 raise KeyboardInterrupt
@@ -1215,10 +1225,11 @@ class Agent:
                 continue
             if not calls and (self.reply_trouble == "circles" or not (reply.get("content") or "").strip()):
                 stalls += 1
-                self.messages.pop()  # nothing to keep: no answer, and circles would only fill the context
+                dropped = self.messages.pop()  # no answer, and circles would only fill the context
+                kept = self.keep_dropped(dropped)
                 if stalls < MAX_STALLS:
                     self.console.print(
-                        Text(f"  ⎿ {STALLED[self.reply_trouble]}: asking for a short answer", style="yellow")
+                        Text(f"  ⎿ {STALLED[self.reply_trouble]}: asking for a short answer{kept}", style="yellow")
                     )
                     self.messages.append({"role": "user", "content": stall_note(self.reply_trouble)})
                     continue
@@ -1234,7 +1245,7 @@ class Agent:
                 self.console.print(
                     Text(
                         "Stopped: the model's replies went round in circles or ended without an answer, twice in "
-                        "this request. Try again with a smaller step, or tell it what to do differently.",
+                        f"this request{kept}. Try again with a smaller step, or tell it what to do differently.",
                         style="yellow",
                     )
                 )
@@ -1253,6 +1264,14 @@ class Agent:
                     self.console.print(Text(f"  ⎿ never ran {', '.join(unrun)}: asking the model to", style="yellow"))
                     self.messages.append({"role": "user", "content": checks.unrun_note(unrun)})
                     continue
+                unseen = {} if windows_noted else self.unseen_windows(tools_used)
+                if unseen:
+                    windows_noted = True
+                    self.console.print(
+                        Text(f"  ⎿ never looked at {', '.join(unseen)}: asking the model to", style="yellow")
+                    )
+                    self.messages.append({"role": "user", "content": checks.unseen_note(unseen)})
+                    continue
                 check = self.project_check() if changed and not verified else None
                 if check is not None and check_rounds < checks.MAX_ROUNDS:
                     check_rounds += 1
@@ -1345,6 +1364,7 @@ class Agent:
                     )
                     if out_of_reach.stop:
                         blocked = out_of_reach
+                tools_used.add(name)
                 if name == "bash":
                     commands.append(str(args.get("command", "")))
                 if name in ("edit_file", "write_file", "apply_patch") and not result.startswith(REFUSED):
@@ -1412,6 +1432,14 @@ class Agent:
         self.turn_status = "success"
         self.print_stats()
 
+    def keep_dropped(self, reply: dict) -> str:
+        """Save a reply lcode dropped (its thinking too), so the user can see what went round in circles."""
+        text = "\n\n".join(part for part in (reply.get("thinking"), reply.get("content")) if part)
+        if not text.strip():
+            return ""
+        path = context_tools.save_output(text, "dropped reply")
+        return f" (the dropped reply is in {path})" if path else ""
+
     def can_escalate(self) -> bool:
         """Whether a stronger model may be asked here: one is set, and this isn't a subagent or plan mode."""
         model = (self.settings.escalate_model or "").strip().lower()

lcode/checks.py

@@ -50,10 +50,28 @@ def find(root: Path, setting: str, trusted: bool) -> Check | None:
     return detect(root)
 
 
-def python() -> str:
+VENVS = (".venv", "venv", "env")
+PYTEST_USE = re.compile(r"^\s*(?:import pytest\b|from pytest\b)|^\s*@pytest\.", re.M)
+
+
+def python(root: Path | None = None) -> str:
+    """The project's own Python (its .venv, venv or env), where its dependencies are; else the one on PATH."""
+    if root is not None:
+        for env in VENVS:
+            for exe in ("bin/python", "Scripts/python.exe"):
+                if (root / env / exe).is_file():
+                    return f"{env}/{exe}"  # the checks run in the project's root
     return "python" if shutil.which("python") else "python3"
 
 
+def uses_pytest(root: Path) -> bool:
+    """Whether the tests are written for pytest (they import it), or the requirements list it, without a config."""
+    for name in ("requirements.txt", "requirements-dev.txt", "requirements_dev.txt", "dev-requirements.txt"):
+        if re.search(r"^\s*pytest\b", text(root / name), re.M | re.I):
+            return True
+    return any(PYTEST_USE.search(text(path)[:20000]) for path in test_files(root)[:50])
+
+
 def text(path: Path) -> str:
     try:
         return path.read_text(errors="replace") if path.is_file() else ""
@@ -61,38 +79,49 @@ def text(path: Path) -> str:
         return ""
 
 
-def python_tests(root: Path) -> bool:
-    """Whether the project has Python test files (a few levels deep, outside environments and caches)."""
+def test_files(root: Path, limit: int = 200) -> list[Path]:
+    """The project's Python test files, a few levels deep, outside environments and caches."""
     from lcode.tools import IGNORE_DIRS
 
-    def walk(folder: Path, depth: int) -> bool:
+    found: list[Path] = []
+
+    def walk(folder: Path, depth: int) -> None:
         try:
-            entries = list(folder.iterdir())
+            entries = sorted(folder.iterdir())
         except OSError:
-            return False
-        if any(e.is_file() and re.fullmatch(r"test_.*\.py|.*_test\.py", e.name) for e in entries):
-            return True
-        return depth > 0 and any(
-            walk(e, depth - 1)
-            for e in entries
-            if e.is_dir() and e.name not in IGNORE_DIRS and not e.name.startswith(".")
-        )
+            return
+        found.extend(e for e in entries if e.is_file() and re.fullmatch(r"test_.*\.py|.*_test\.py", e.name))
+        if depth > 0:
+            for e in entries:
+                if len(found) >= limit:
+                    return
+                if e.is_dir() and e.name not in IGNORE_DIRS and e.name not in VENVS and not e.name.startswith("."):
+                    walk(e, depth - 1)
 
-    return walk(root, 3)
+    walk(root, 3)
+    return found[:limit]
+
+
+def python_tests(root: Path) -> bool:
+    """Whether the project has Python test files (a few levels deep, outside environments and caches)."""
+    return bool(test_files(root, limit=1))
 
 
 def detect(root: Path) -> Check | None:
     """A check command from what the project looks like. Python checks need test files: pytest configured in a
     project without tests "fails" with nothing to run."""
     if python_tests(root):
+        py = python(root)
         configs = text(root / "pyproject.toml") + text(root / "setup.cfg") + text(root / "tox.ini")
         if "pytest" in configs or any((root / name).is_file() for name in ("pytest.ini", "conftest.py")):
-            runner = "uv run pytest -q" if (root / "uv.lock").is_file() else f"{python()} -m pytest -q"
+            runner = "uv run pytest -q" if (root / "uv.lock").is_file() else f"{py} -m pytest -q"
             return Check(runner, "pytest in the project's configuration")
+        if uses_pytest(root):
+            return Check(f"{py} -m pytest -q", "tests written for pytest")
         if list(root.glob("test_*.py")) or list(root.glob("*_test.py")):
-            return Check(f"{python()} -m unittest", "test files")
+            return Check(f"{py} -m unittest", "test files")
         if (root / "tests").is_dir():
-            return Check(f"{python()} -m unittest discover -s tests -t .", "tests/")
+            return Check(f"{py} -m unittest discover -s tests -t .", "tests/")
     package = text(root / "package.json")
     if package:
         try:
@@ -229,3 +258,52 @@ def unrun_note(scripts: list[str]) -> str:
         f"deploy, delete data) or needs something you don't have, don't: tell the user plainly that {it} is untested "
         "and why."
     )
+
+
+# ----------------------------------------------------------------------------- windows it never looked at
+
+GUI_IMPORT = re.compile(r"^\s*(?:from|import)\s+(PyQt[56]|PySide[26]|tkinter|kivy|wx|gi\.repository)\b", re.M)
+LOOKED = frozenset({"screenshot", "view_image"})
+
+
+def unseen_windows(root: Path, changed: list[str], tools_used: set[str]) -> dict[str, str]:
+    """Python files that draw a window (they import a GUI toolkit), changed in this request when the model never
+    looked at one: file -> toolkit.
+
+    A small model checks a window change by importing the module and running unrelated tests, then calls it done;
+    the user finds a blank window. Tests and imports don't show what's on the screen; a screenshot does.
+    """
+    if tools_used & LOOKED:
+        return {}
+    found = {}
+    for rel in changed:
+        path = root / rel
+        if path.suffix != ".py" or not path.is_file():
+            continue
+        try:
+            text = path.read_text(errors="replace")[:200_000]
+        except OSError:
+            continue
+        match = GUI_IMPORT.search(text)
+        if match:
+            found[rel] = match.group(1)
+    return found
+
+
+def unseen_note(found: dict[str, str]) -> str:
+    names = ", ".join(found)
+    kits = sorted(set(found.values()))
+    qt = any(kit.startswith(("PyQt", "PySide")) for kit in kits)
+    how = (
+        "take a screenshot with the screenshot tool: it runs the program without a screen and shows what it draws, "
+        "and what the program printed (Qt names layout mistakes there). For something that only shows after a click "
+        'or a key, give actions, e.g. actions=["double_click Camera 1"]'
+        if qt
+        else "take a screenshot with the screenshot tool: it runs the program on a virtual display and shows what it "
+        "draws"
+    )
+    return (
+        f"[lcode] You changed {names}, which draws a window ({', '.join(kits)}), but never looked at it: tests and "
+        f"imports don't show what's on the screen. Before you say it works, {how}. Check the picture against what the "
+        "user asked for. If the program can't run here, tell the user plainly that you haven't seen it working."
+    )

lcode/repeats.py

@@ -6,7 +6,8 @@ one exactly: the same tool, the same arguments and the same result. At the `limi
 failed: one sooner) the result gets a note that tells the model so, and if it still goes on, lcode ends the
 request.
 
-Results that change aren't repeats: a test run again after a fix, a background job read while it runs. And
+Results that change aren't repeats: a test run again after a fix, a background job read while it runs. Timings
+don't count as a change, though: `75 passed in 0.12s` and then `in 0.14s` is the same result. And
 a call that may have changed something (a file edit, a shell command that isn't read-only) starts the count
 again for every other call, so checking a build after each edit never adds up. Its own repeats still count:
 running the same failing command over and over is a loop.
@@ -23,6 +24,9 @@ from dataclasses import dataclass
 from lcode.permissions import is_read_only
 
 STOP_AFTER_NOTE = 2  # repeats after the note before the request ends
+TIMING = re.compile(
+    r"\b\d+(?:\.\d+)?\s?(?:ms|µs|us|s|secs?|seconds?|mins?|minutes?)\b|\b\d{1,2}:\d{2}:\d{2}(?:[.,]\d+)?\b", re.I
+)
 FAILED = ("Error:", "A deny rule", "The user denied", "This needs the user's permission")
 UNCHANGING = frozenset(
     {
@@ -64,7 +68,7 @@ class Repeats:
         """Count this call and its result."""
         if self.limit <= 0:
             return Verdict(1)
-        key = hashlib.sha256(json.dumps([name, args, result], sort_keys=True, default=str).encode()).hexdigest()
+        key = hashlib.sha256(json.dumps([name, args, steady(result)], sort_keys=True, default=str).encode()).hexdigest()
         count = self.seen.get(key, 0) + 1
         if changes_things(name, args, result):
             self.seen.clear()
@@ -95,6 +99,11 @@ class Repeats:
         )
 
 
+def steady(result: str) -> str:
+    """The result without its timings and clock times, which change on every run of the same command."""
+    return TIMING.sub("#", result)
+
+
 def changes_things(name: str, args: dict, result: str) -> bool:
     """Whether the call may have changed files (or anything else a later call could see)."""
     if name in UNCHANGING:

lcode/screenshots.py

@@ -6,7 +6,9 @@ returns what the model can't otherwise see: a picture (described by the vision m
 
 - A Python Qt program (PyQt5/6, PySide2/6) runs on Qt's "offscreen" platform, with a screen of the asked size. A
   small startup hook (a sitecustomize module on PYTHONPATH) waits for the program's event loop, then grabs its
-  windows, measures them and their widgets, and quits it.
+  windows, measures them and their widgets, and quits it. First it can take `actions`, the way a user would: click
+  or double-click a point or a widget by its text, press a key, type, wait. Bugs that only show after a click need
+  that, and Qt's warnings (in what the program printed) often name the cause.
 - Other desktop programs run on a virtual X display (Xvfb, Linux), captured with ImageMagick's `import` or `xwd`.
   The Qt hook still measures a Python Qt program there.
 - A web page, or a local server started with `command`, opens in headless Chrome (or Chromium, Edge), driven
@@ -43,6 +45,14 @@ from pathlib import Path
 DEFAULT_SCREEN = (1920, 1080)
 DEFAULT_WAIT = 3  # seconds for a program or page to start and draw
 MAX_WAIT = 60
+MAX_ACTIONS = 20
+ACTION_PAUSE = 0.4  # seconds after each action, for the program to react and redraw
+MOUSE_ACTIONS = ("click", "double_click", "right_click")
+KEY_NAMES = {"enter": "Return", "return": "Return", "esc": "Escape", "escape": "Escape", "space": "Space",
+             "tab": "Tab", "backspace": "Backspace", "delete": "Delete", "del": "Delete", "home": "Home", "end": "End",
+             "pageup": "PageUp", "pagedown": "PageDown", "up": "Up", "down": "Down", "left": "Left",
+             "right": "Right"}  # fmt: skip
+MODIFIERS = ("ctrl", "shift", "alt", "meta")
 SERVER_TIMEOUT = 60  # seconds for a server started with `command` to answer at `url`
 GRACE = 25  # seconds beyond `wait` before a program that never showed a window is stopped
 QT_BINDINGS = ("PyQt6", "PyQt5", "PySide6", "PySide2")
@@ -169,15 +179,19 @@ def _lcode_shot():
     out = _os.environ.get("LCODE_SHOT_OUT")
     wait_ms = int(float(_os.environ.get("LCODE_SHOT_WAIT", "3")) * 1000)
     quit_after = _os.environ.get("LCODE_SHOT_QUIT", "1") == "1"
+    actions = json.loads(_os.environ.get("LCODE_SHOT_ACTIONS") or "[]")
+    pause_ms = int(float(_os.environ.get("LCODE_SHOT_PAUSE", "0.4")) * 1000)
     bindings = ("PyQt6", "PyQt5", "PySide6", "PySide2")
     state = {"scheduled": False}
 
-    def capture(binding):
+    def capture(binding, steps=None):
         QtCore = _sys.modules[binding + ".QtCore"]
         QtGui = _sys.modules[binding + ".QtGui"]
         QtWidgets = _sys.modules.get(binding + ".QtWidgets")
         app = QtGui.QGuiApplication.instance()
         result = {"binding": binding, "windows": [], "screen": None, "platform": app.platformName()}
+        if steps is not None:
+            result["actions"] = steps
         screen = app.primaryScreen()
         if screen is not None:
             g, a = screen.geometry(), screen.availableGeometry()
@@ -257,6 +271,100 @@ def _lcode_shot():
             app.quit()
             QtCore.QTimer.singleShot(3000, lambda: _os._exit(0))
 
+    def text_of(widget):
+        for getter in ("text", "windowTitle", "title", "placeholderText"):
+            fn = getattr(widget, getter, None)
+            if callable(fn):
+                try:
+                    text = str(fn() or "")
+                except Exception:
+                    text = ""
+                if text:
+                    return text
+        return ""
+
+    def act(binding, then):
+        """Take the steps one by one, pausing after each so the program reacts, then call then(log)."""
+        QtCore = _sys.modules[binding + ".QtCore"]
+        QtWidgets = _sys.modules[binding + ".QtWidgets"]
+        Qt = QtCore.Qt
+        try:
+            QTest = importlib.import_module(binding + ".QtTest").QTest
+        except Exception as e:
+            then([{"step": s["raw"], "error": "Qt's QtTest module isn't available (%s)" % e} for s in actions])
+            return
+        app = QtWidgets.QApplication.instance()
+
+        def enum(group, name):  # scoped enums (PyQt6, PySide6), or flat ones (PyQt5, PySide2)
+            scope = getattr(Qt, group, None)
+            if scope is not None and hasattr(scope, name):
+                return getattr(scope, name)
+            return getattr(Qt, name)
+
+        def window():
+            tops = [w for w in app.topLevelWidgets() if w.isVisible() and w.width() > 1 and w.height() > 1]
+            active = app.activeWindow()
+            return active if active in tops else (tops[0] if tops else None)
+
+        def describe(widget):
+            text = text_of(widget)[:40]
+            return type(widget).__name__ + (' "%s"' % text if text else "")
+
+        def by_text(top, wanted):
+            wanted = wanted.casefold()
+            found = [c for c in top.findChildren(QtWidgets.QWidget)
+                     if c.isVisible() and wanted in text_of(c).casefold()]  # fmt: skip
+            return min(found, key=lambda c: c.width() * c.height()) if found else None
+
+        log = []
+
+        def step(i):
+            if i >= len(actions):
+                then(log)
+                return
+            s, note, pause = actions[i], {"step": actions[i]["raw"]}, pause_ms
+            try:
+                top = window()
+                if s["do"] == "wait":
+                    pause = int(s["seconds"] * 1000)
+                elif top is None:
+                    note["error"] = "no window to act on"
+                elif s["do"] in ("click", "double_click", "right_click"):
+                    if "text" in s:
+                        target = by_text(top, s["text"])
+                        pos = target.rect().center() if target is not None else None
+                    else:
+                        point = QtCore.QPoint(s["x"], s["y"])
+                        target = top.childAt(point) or top
+                        pos = target.mapFrom(top, point)
+                    if target is None:
+                        note["error"] = 'no visible widget shows "%s"' % s["text"]
+                    else:
+                        button = enum("MouseButton", "RightButton" if s["do"] == "right_click" else "LeftButton")
+                        none = enum("KeyboardModifier", "NoModifier")
+                        at = target.mapTo(top, pos)  # before the click moves things around
+                        note["target"] = "%s at (%d, %d)" % (describe(target), at.x(), at.y())
+                        click = QTest.mouseDClick if s["do"] == "double_click" else QTest.mouseClick
+                        click(target, button, none, pos)
+                else:
+                    target = app.focusWidget() or top
+                    if s["do"] == "key":
+                        mods = enum("KeyboardModifier", "NoModifier")
+                        names = {"ctrl": "ControlModifier", "shift": "ShiftModifier", "alt": "AltModifier",
+                                 "meta": "MetaModifier"}  # fmt: skip
+                        for m in s["mods"]:
+                            mods = mods | enum("KeyboardModifier", names[m])
+                        QTest.keyClick(target, enum("Key", "Key_" + s["key"]), mods)
+                    else:
+                        QTest.keyClicks(target, s["text"])
+                    note["target"] = describe(target)
+            except Exception as e:  # one step failing doesn't stop the picture
+                note["error"] = "%s: %s" % (type(e).__name__, e)
+            log.append(note)
+            QtCore.QTimer.singleShot(pause, lambda: step(i + 1))
+
+        step(0)
+
     def patch(binding):
         """Wrap exec() of the application classes loaded so far (QtWidgets may come after QtGui)."""
         QtCore = _sys.modules.get(binding + ".QtCore")
@@ -275,14 +383,23 @@ def _lcode_shot():
                 state["scheduled"] = True
                 open(_os.path.join(out, "started"), "w").close()  # the event loop runs: a capture is coming
 
-                def run():
+                def run(steps=None):
                     try:
-                        capture(binding)
+                        capture(binding, steps)
                     except Exception as e:  # the program goes on; lcode reports what went wrong
                         with open(_os.path.join(out, "qt.json"), "w") as f:
                             json.dump({"error": "%s: %s" % (type(e).__name__, e)}, f)
 
-                QtCore.QTimer.singleShot(wait_ms, run)
+                def start():
+                    if not actions:
+                        run()
+                        return
+                    try:
+                        act(binding, run)
+                    except Exception as e:
+                        run([{"step": "actions", "error": "%s: %s" % (type(e).__name__, e)}])
+
+                QtCore.QTimer.singleShot(wait_ms, start)
 
         class Wrapped:
             """In place of exec(): schedules the capture, then runs the original, static or not."""
@@ -372,6 +489,69 @@ def hook_dir(tmp: Path) -> Path:
 # --------------------------------------------------------------------------------------------- running a program
 
 
+def parse_actions(actions) -> list[dict]:
+    """The steps to take before the picture, from the tool's `actions` (a list, or one string split on ; and lines).
+
+    `click X,Y`, `double_click X,Y`, `right_click X,Y` (a point in the window), or the same with a widget's text
+    (`double_click "Camera 1"`); `key Escape` or `key ctrl+s`; `type some text`; `wait 2` (seconds).
+    """
+    if not actions:
+        return []
+    if isinstance(actions, str):
+        actions = [part for line in actions.splitlines() for part in line.split(";")]
+    steps = []
+    for raw in actions:
+        raw = str(raw).strip()
+        if not raw:
+            continue
+        verb, _, rest = raw.partition(" ")
+        verb, rest = verb.lower().replace("-", "_"), rest.strip()
+        if verb == "doubleclick":
+            verb = "double_click"
+        step: dict = {"do": verb, "raw": raw}
+        if verb in MOUSE_ACTIONS:
+            point = re.fullmatch(r"\(?\s*(-?\d+)\s*[, ]\s*(-?\d+)\s*\)?", rest)
+            if point:
+                step["x"], step["y"] = int(point.group(1)), int(point.group(2))
+            elif rest.strip("\"'"):
+                step["text"] = rest.strip("\"'")
+            else:
+                raise ShotError(f"{raw!r}: say where, as X,Y in the window or the widget's text, e.g. {verb} 200,150")
+        elif verb == "key":
+            parts = [p for p in re.split(r"[+\s]+", rest.lower()) if p]
+            if not parts:
+                raise ShotError(f"{raw!r}: name the key, e.g. key Escape or key ctrl+s")
+            *mods, key = parts
+            if any(m not in MODIFIERS for m in mods):
+                raise ShotError(f"{raw!r}: modifiers are {', '.join(MODIFIERS)}")
+            if key in KEY_NAMES:
+                key = KEY_NAMES[key]
+            elif re.fullmatch(r"f([1-9]|1[0-2])", key) or (len(key) == 1 and key.isalnum()):
+                key = key.upper()
+            else:
+                raise ShotError(f"{raw!r}: unknown key {key!r} (Escape, Return, Tab, Space, arrows, F1-F12, a letter)")
+            step["key"], step["mods"] = key, mods
+        elif verb == "type":
+            if not rest:
+                raise ShotError(f"{raw!r}: give the text to type")
+            step["text"] = rest
+        elif verb == "wait":
+            try:
+                step["seconds"] = max(0.0, min(float(rest or 1), 10.0))
+            except ValueError as e:
+                raise ShotError(f"{raw!r}: wait takes seconds, e.g. wait 2") from e
+        else:
+            raise ShotError(f"{raw!r}: actions are click, double_click, right_click, key, type and wait")
+        steps.append(step)
+    if len(steps) > MAX_ACTIONS:
+        raise ShotError(f"at most {MAX_ACTIONS} actions in one screenshot")
+    return steps
+
+
+def actions_time(steps: list[dict]) -> float:
+    return sum(s.get("seconds", 0) for s in steps) + ACTION_PAUSE * len(steps)
+
+
 def run_program(command: str, cwd: Path, env: dict[str, str], log: Path) -> subprocess.Popen:
     out = log.open("wb")
     try:
@@ -405,6 +585,27 @@ def stop(proc: subprocess.Popen | None) -> None:
             proc.wait(timeout=3)
 
 
+QT_WARNING = re.compile(r"^(?:Q[A-Za-z0-9]+::\w+|qt\.[\w.]+:|QML |Qt[: ])")
+NOT_A_PROBLEM = re.compile(
+    r"propagateSizeHints|QFont::|This plugin does not support|qt\.qpa\.fonts"
+)  # offscreen platform noise
+MAX_WARNINGS = 6
+
+
+def qt_warnings(log: Path) -> list[str]:
+    """Qt's own warnings in everything the program printed (not just the end that's shown), without repeats."""
+    try:
+        text = log.read_bytes().decode(errors="replace")
+    except OSError:
+        return []
+    found: list[str] = []
+    for line in text.splitlines():
+        line = line.strip()
+        if QT_WARNING.match(line) and not NOT_A_PROBLEM.search(line) and line[:300] not in found:
+            found.append(line[:300])
+    return found[:MAX_WARNINGS]
+
+
 def printed(log: Path) -> str:
     try:
         text = log.read_bytes().decode(errors="replace")
@@ -423,7 +624,9 @@ def base_env(hook: Path) -> dict[str, str]:
     return env
 
 
-def qt_offscreen(command: str, cwd: Path, screen: tuple[int, int], wait: int, tmp: Path, cancel=None) -> Shot:
+def qt_offscreen(
+    command: str, cwd: Path, screen: tuple[int, int], wait: int, tmp: Path, cancel=None, actions: list | None = None
+) -> Shot:
     """Run a Python Qt program on Qt's offscreen platform and capture its windows."""
     config = tmp / "screens.json"
     w, h = screen
@@ -453,12 +656,14 @@ def qt_offscreen(command: str, cwd: Path, screen: tuple[int, int], wait: int, tm
         LCODE_SHOT_OUT=str(out),
         LCODE_SHOT_WAIT=str(wait),
         LCODE_SHOT_QUIT="1",
+        LCODE_SHOT_ACTIONS=json.dumps(actions or []),
+        LCODE_SHOT_PAUSE=str(ACTION_PAUSE),
     )
     env.pop("DISPLAY", None)
     log = tmp / "output.txt"
     proc = run_program(command, cwd, env, log)
     try:
-        deadline = time.time() + wait + GRACE
+        deadline = time.time() + wait + actions_time(actions or []) + GRACE
         while time.time() < deadline and proc.poll() is None and not (out / "qt.json").exists():
             if cancel is not None and cancel():
                 raise KeyboardInterrupt
@@ -482,6 +687,9 @@ def qt_offscreen(command: str, cwd: Path, screen: tuple[int, int], wait: int, tm
     if image is None and not windows:
         raise ShotError("the program started Qt but showed no window")
     report = qt_report(data)
+    warnings = qt_warnings(log)
+    if warnings:
+        report[:0] = ["Qt printed warnings (they often name the cause):", *[f"- {line}" for line in warnings]]
     if shot_screen != screen:  # Qt 5's offscreen platform has a fixed screen
         report.append(
             f"(Qt used a {shot_screen[0]}x{shot_screen[1]} screen offscreen, not the {screen[0]}x{screen[1]} asked "
@@ -509,8 +717,17 @@ def qt_report(data: dict) -> list[str]:
     screen = data.get("screen") or [0, 0]
     sw, sh = screen
     windows = data.get("windows") or []
+    if data.get("actions"):
+        lines.append("Steps, before the picture:")
+        for note in data["actions"]:
+            done = (
+                f"couldn't: {note['error']}"
+                if note.get("error")
+                else (f"on {note['target']}" if note.get("target") else "done")
+            )
+            lines.append(f"- {note['step']}: {done}")
     if not windows:
-        return ["No visible Qt window."]
+        return [*lines, "No visible Qt window."]
     for win in windows:
         title = f'"{win["title"]}" ' if win.get("title") else ""
         x, y, w, h = win["x"], win["y"], win["width"], win["height"]
@@ -1059,12 +1276,15 @@ def take(
     full_page: bool = False,
     mode: str = "auto",
     cancel=None,
+    actions: list | None = None,
 ) -> Shot:
-    """Take the screenshot. `cancel` is checked while waiting (return True to stop)."""
+    """Take the screenshot. `cancel` is checked while waiting (return True to stop); `actions` come first."""
     wait = max(0, min(int(wait), MAX_WAIT))
     path = new_path(folder, label_for(command, url))
     with tempfile.TemporaryDirectory(prefix="lcode-shot-") as name:
         tmp = Path(name)
+        if actions and url:
+            raise ShotError("actions work for Python Qt programs (PyQt, PySide), not web pages yet")
         if url:
             url = normalize_url(url, cwd)
             server = None
@@ -1086,7 +1306,14 @@ def take(
             raise ShotError("give a command that starts the program, or a url")
         if mode == "auto":
             mode = "qt" if looks_like_qt(command, cwd) or not xvfb_available() else "xvfb"
-        shot = (qt_offscreen if mode == "qt" else virtual_display)(command, cwd, screen, wait, tmp, cancel)
+        if actions and mode != "qt":
+            raise ShotError(
+                "actions work for Python Qt programs (PyQt, PySide) only: take this one's screenshot without them"
+            )
+        if mode == "qt":
+            shot = qt_offscreen(command, cwd, screen, wait, tmp, cancel, actions)
+        else:
+            shot = virtual_display(command, cwd, screen, wait, tmp, cancel)
         shutil.copyfile(shot.image, path)
         kept = []
         for i, window in enumerate(shot.windows):

lcode/tools.py

@@ -192,7 +192,8 @@ SCHEMAS = [
         "widgets (Qt) or of the page, and the browser's console errors. Use it to check GUI and layout work instead "
         "of guessing. Python Qt programs run offscreen; other desktop programs on a virtual display (Linux); web pages "
         "in a headless browser. For a web app, give the command that starts its server and the url (lcode starts and "
-        "stops it); if a server is already running, or for an .html file, give only the url.",
+        "stops it); if a server is already running, or for an .html file, give only the url. For something that "
+        "only shows after a click or a key (a Python Qt program), give `actions`: they run first, then the picture.",
         {
             "command": {"type": "string", "description": "Command that starts the program, or the web server"},
             "url": {"type": "string", "description": "Page to open: a URL, localhost:PORT/path or an .html file"},
@@ -203,6 +204,14 @@ SCHEMAS = [
             "screen": {"type": "string", "description": "Screen (or browser window) size, WxH (default 1920x1080)"},
             "wait": {"type": "integer", "description": "Seconds to let it start and draw first (default 3)"},
             "full_page": {"type": "boolean", "description": "Web pages: the whole page, not just the window"},
+            "actions": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Python Qt programs: steps to take before the picture, as a user would: "
+                '"click X,Y", "double_click X,Y", "right_click X,Y" (a point in the window, as the sizes show), '
+                'or with the widget\'s text: "double_click Camera 1"; "key Escape", "key ctrl+s"; "type some text"; '
+                '"wait 2". The result says what each step hit.',
+            },
             "mode": {
                 "type": "string",
                 "enum": ["auto", "qt", "xvfb"],
@@ -698,6 +707,7 @@ class Toolbox:
         wait: int = screenshots.DEFAULT_WAIT,
         full_page: bool = False,
         mode: str = "auto",
+        actions: list | str | None = None,
     ) -> str:
         command, url = (command or "").strip(), (url or "").strip()
         if not command and not url:
@@ -708,6 +718,7 @@ class Toolbox:
             size = screenshots.parse_screen(screen)
             if url:
                 url = screenshots.normalize_url(url, self.agent.cwd)
+            steps = screenshots.parse_actions(actions)
         except screenshots.ShotError as e:
             raise ToolError(str(e)) from e
         if command:
@@ -748,6 +759,7 @@ class Toolbox:
                     full_page=full_page,
                     mode=mode,
                     cancel=cancelled,
+                    actions=steps,
                 )
         except screenshots.ShotError as e:
             raise ToolError(f"no screenshot: {e}") from e
@@ -762,6 +774,8 @@ class Toolbox:
             lines.append(f"What the {'server' if url else 'program'} printed:\n{shot.output}")
         if self.agent.vision_model():
             what = f"the page {url}" if url else f"the program `{command}` on a {w}x{h} screen"
+            if steps:
+                what += ", after these steps: " + "; ".join(s["raw"] for s in steps)
             ask = f"This is a screenshot of {what}." + (f" {question.strip()}" if question.strip() else "")
             try:
                 seen = self.agent.look(shot.image, ask)

{lcode_cli-0.37.0.dist-info → lcode_cli-0.38.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lcode-cli
-Version: 0.37.0
+Version: 0.38.0
 Summary: A local-first terminal coding agent powered by open-weight models on your own GPU or Mac, via Ollama.
 Project-URL: Homepage, https://nasser1941.github.io/lcode/
 Project-URL: Documentation, https://nasser1941.github.io/lcode/

{lcode_cli-0.37.0.dist-info → lcode_cli-0.38.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-lcode/__init__.py,sha256=lKIIi9DSWXNWEjJvCuOOtYqAkmR0gtWHsaKllXOu7vc,473
+lcode/__init__.py,sha256=TsW2cvGFfg21bkCrYKlSelvpwqk5kcoGy9fkV40Sr0Q,473
 lcode/__main__.py,sha256=s5N5pryX2FAu93GGZ3FxUrIXHWGzxz66lKUoaV_h8Ws,35
 lcode/acp.py,sha256=d97VV3sbqXdWKhPQQNbYRaDTm8WrUk0mGbDLiOyX7jE,29058
 lcode/action.py,sha256=wuT5F9EJCdEligPeytzmEi8FpahWBhXer528tpuTj_k,16711
-lcode/agent.py,sha256=vjWiK79BgUTMAbeC6SPslGaYT86YELwTHIGB21tc4to,84751
+lcode/agent.py,sha256=4vujxmltW3OdDIoQeJXZ74NV6bgyfN0_9s7VO257jZg,86511
 lcode/api.py,sha256=mIfmiN3jDv8jg-HOJtqfr-MQ3cOI8nfFmlYYRZR1r1Y,16164
 lcode/backends.py,sha256=343alwyDZin-bEhb5-Es4f8ek1oBQUzEvV6RFWR1VZo,21160
 lcode/bench.py,sha256=HxeUfy9Kv_0Rug1HAlDLyHYfYd9dw2bMgbXsvUy9nd4,43096
@@ -10,7 +10,7 @@ lcode/bench_backend.py,sha256=X2fo-zRX92Xp3KMxSsFlWioc2vRupBTtowj5kLME8mw,14287
 lcode/bench_runtime.py,sha256=EjzNgIvUkdbT8cbSy1TpmlyeIk6e-ipjlDFpM7Dnqx0,6835
 lcode/catalog.py,sha256=pZ1azX9OH2r4JE2c6E5OfwywpgS39-W2oXzoCHJhmtM,4141
 lcode/checkpoints.py,sha256=eqPtmPe44-aweWqc2H21ROogwzIt3NpvcUobsaCHoGY,18354
-lcode/checks.py,sha256=iPaNaGOGo7zHc_mue41ELEHSNq0VIH_xVYL_bHkYLXU,9373
+lcode/checks.py,sha256=haYYIPhadKavGYEhh8YWAaWrPMcfzrpT2i6638f4xjQ,12933
 lcode/cli.py,sha256=mJBM3Loqs9kOE94IRe00d1A1izW5caAjSR7ITUxC27I,45042
 lcode/clipboard.py,sha256=nTSdJyCXPdtjH947RjvB3QvRqnL5HZsS9KB8EkC42cc,5138
 lcode/cloud.py,sha256=q5Dl5GuqWvKzBCIso-eh6yomvEOsnQ0LW8ZgaAGwBxc,2131
@@ -34,18 +34,18 @@ lcode/ollama.py,sha256=xFHU2XW47ANJ34-1aeSMQVj28GhqcWiFi6zASH9FiYY,8053
 lcode/permissions.py,sha256=q7CjDsc_Xrblw7t3_BbxA5LUjxFSswGs1RiJmtMi0po,12288
 lcode/planning.py,sha256=Q2upaoZPnXDMQT2F-ETxX1lk_wDntTzhSKNVYIQSoQA,6540
 lcode/render.py,sha256=gMvf4W3qkLUJ4Vot3az_hP_zRDAItJC9pP-68dHFkfA,1375
-lcode/repeats.py,sha256=vpXR5pzKX9GsTzQONGS30V-rEU3gunaj_hBzJxg43Q4,5542
+lcode/repeats.py,sha256=OTrPdb6BKArDLQEgHaXmJZ-tIFsY_1bk4P0wzzANKLY,5964
 lcode/repl.py,sha256=elnJeSQYt0oEtE14UzW2k-fkDHVFr_Lf2L6XWFo0ne0,56891
 lcode/repomap.py,sha256=1GRJgEUjBl9XHSXsF7tSGXBCEgsY-3CzwMohaTudZ_M,13969
 lcode/residency.py,sha256=B9pwqG6yx5aQD2M5xyQ-bMnYmhD7fv3wCVw_X6VIeAo,7012
 lcode/review.py,sha256=0QBIS-R1B2b3zqx5TcC1qZptpKBbrDhYvMKSNCaDAUo,4686
 lcode/sandbox.py,sha256=u_pkrVRXhCGhh-wFDPwuZ3jGG7UeXxXRAeBNsm37ZvE,8608
-lcode/screenshots.py,sha256=bcWSXS4xTsBk8HhpjBy5jvu705uH9qV8bXANr--Mf2M,46405
+lcode/screenshots.py,sha256=pmrw_tp8eXzNyCZa5uot5rTgsxqsi3AdmkGn-XF9saY,57056
 lcode/secretscan.py,sha256=6Euvfk2gCtK7lZbuQD84zgWuSECi6OnSw8V5rFu1tfU,13575
 lcode/sessions.py,sha256=ojkpP453JyIll8_iFeJgW4BG4Zz2FKMRsLiij6spiTo,4185
 lcode/steer.py,sha256=0vaqLxgRKgC0yGp8xoQfV1f9VEOzH3U4rCIb74eFnZ8,12127
 lcode/subagents.py,sha256=thR2ZzFZ1DeoAf0Ctr_oeKjYajhvd3uRs2kGdw6LlJM,29349
-lcode/tools.py,sha256=glp18sbnxaK5iugDxrCFlDaT0SEbO-GTOL_-O3Afp44,68005
+lcode/tools.py,sha256=lpCIlaabydbmutaXWuOZWS4X-Z_7aIsyNUQdHJzkSC4,68902
 lcode/unreachable.py,sha256=TXrv41ljwX_amxdcf4CY6qGF2YkWetORvPUSvwceJXI,8685
 lcode/update.py,sha256=i92bnW252VneYh0vD6djxmXsDOa5MXP20bjlLZUue_c,25236
 lcode/vision.py,sha256=w33QPmdGwP33wM4mvmAH4amFDZ1xzeXhEsSLJ-STSHY,4243
@@ -66,8 +66,8 @@ lcode/servers/__init__.py,sha256=qrIhxEodEB8cjKEQ4Cicc0GsQhE2OH-9qlVHzr3-tq8,77
 lcode/servers/encord.py,sha256=WSauawL1UKYZCUAG1oalcKL_An3DVof-JM6xIgFyQSE,12769
 lcode/servers/valohai.py,sha256=e5xZGIoa4cja4f5i85q1GpL9c7cglUjV8jN2co1CFMY,14430
 lcode/servers/xai.py,sha256=h2emAvHSnkuB9WmJInlynsDhvxJwNVD2AHFX7vqMXNc,13627
-lcode_cli-0.37.0.dist-info/METADATA,sha256=fgaRfrIMEYuJXyDpy8Kj18AL7S2IybZqNmvd09c6IgI,13141
-lcode_cli-0.37.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-lcode_cli-0.37.0.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
-lcode_cli-0.37.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
-lcode_cli-0.37.0.dist-info/RECORD,,
+lcode_cli-0.38.0.dist-info/METADATA,sha256=xLaLc5ISpBKTUGO-4sL99E3iSmTBj_pPqo5E1FyTXk8,13141
+lcode_cli-0.38.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+lcode_cli-0.38.0.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
+lcode_cli-0.38.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
+lcode_cli-0.38.0.dist-info/RECORD,,