lcode-cli 0.28.0__py3-none-any.whl → 0.28.1__py3-none-any.whl

lcode/__init__.py

@@ -3,7 +3,7 @@
 From a program: `from lcode import Session` (see lcode.api).
 """
 
-__version__ = "0.28.0"
+__version__ = "0.28.1"
 
 API = ("Session", "Result", "Options", "SetupError")
 

lcode/agent.py

@@ -6,6 +6,7 @@ import base64
 import contextlib
 import datetime as dt
 import json
+import os
 import platform
 import re
 import shutil
@@ -81,6 +82,8 @@ REFUSED = ("Error:", "A deny rule", "The user denied", "This needs the user's pe
 MAX_MALFORMED_CALL_RETRIES = 2  # Ollama rejects tool calls whose arguments aren't valid JSON
 # How Ollama reports a tool call it couldn't parse (Qwen3.6's calls are XML)
 MALFORMED_CALL = ("error parsing tool call", "XML syntax error")
+RESULT_SHARE = 0.25  # of the context window, at most, for one tool result
+MIN_RESULT_CHARS = 8000
 READING_NOTE_TOKENS = 4000  # new input this large gets "Reading … new tokens" in the status line
 AUTO_COMPACT_RATIO = 0.85  # summarize the history when the context is this full
 PRUNED_ENOUGH = 0.6  # after pruning old tool output, summarize only if the context is still this full
@@ -320,7 +323,9 @@ class Agent:
         self.hardware = None  # an lcode.hardware.Hardware, set by the CLI: for what fits on the GPU
         self.made_room_for = ""  # why the model was unloaded (the vision model, an MCP tool), for the reload line
         self.load_state = ""  # "loading" while the model loads in the background at startup (lcode.repl)
+        self.autosave = False  # save the session after every step (sessions people resume; not the bench's)
         self.sent_chars = 0  # characters of the conversation the model had already read
+        self.counted_chars: int | None = None  # characters that ctx_used covers (None: all of them)
         self.turn_status = ""  # how the last request ended: success, max_steps, loop
         self.repeats = Repeats(settings.repeat_limit)  # notices when the model repeats itself
         self.response = None  # the streaming response, so another thread can abort it
@@ -576,6 +581,7 @@ class Agent:
         self.tools.read_mtimes.clear()
         self.reflected = len(self.messages)
         self.ctx_used = len(self.messages[0]["content"]) // 3
+        self.counted_chars = None
 
     def session_file(self) -> Path:
         return sessions.sessions_dir() / f"{self.session_id}.json"
@@ -597,7 +603,10 @@ class Agent:
             "messages": self.messages,
             "checkpoints": self.checkpoints.to_json(),
         }
-        f.write_text(json.dumps(data))
+        # Written next to it and then renamed: a crash halfway leaves the previous version, not a broken file.
+        partial = f.with_name(f".{f.name}.partial")
+        partial.write_text(json.dumps(data))
+        os.replace(partial, f)
 
     def new_session(self) -> None:
         self.reset()
@@ -634,6 +643,7 @@ class Agent:
         self.tools.read_mtimes.clear()  # files may have changed since; the model must read them again
         self.reflected = len(self.messages)  # checked for notes to remember when that session ended
         self.ctx_used = sum(len(json.dumps(m)) for m in self.messages) // 3
+        self.counted_chars = None
         return note
 
     def load_latest(self) -> bool:
@@ -797,7 +807,7 @@ class Agent:
             self.usage["output_tokens"] += final.get("eval_count", 0)
             self.usage["output_ns"] += final.get("eval_duration", 0)
             self.ctx_used = final.get("prompt_eval_count", 0) + final.get("eval_count", 0)
-            self.sent_chars = conversation_chars(self.messages)
+            self.sent_chars = self.counted_chars = conversation_chars(self.messages)
             seconds = final.get("eval_duration", 0) / 1e9
             self.last_speed = final.get("eval_count", 0) / seconds if seconds else 0.0
         return message
@@ -943,6 +953,8 @@ class Agent:
         for step in range(self.max_steps):
             if self.cancel is not None and self.cancel.is_set():
                 raise KeyboardInterrupt
+            if step and self.autosave:
+                self.save()  # a crash or a closed terminal loses at most this step
             self.maybe_compact()
             try:
                 reply = self.assistant_step()
@@ -1027,6 +1039,9 @@ class Agent:
                     self.console.print(Text(f"  {result[:300]}", style="red"))
                 elif name in ("read_file", "grep", "glob", "list_dir"):
                     self.console.print(Text(f"  ⎿ {result.count(chr(10)) + 1} line(s)", style="dim"))
+                budget = self.result_budget()
+                if len(result) > budget and name not in ("agent", "skill"):
+                    result = context_tools.shorten(result, budget, f"{name} result")
                 verdict = self.repeats.check(name, args, result)
                 if verdict.note:
                     result += verdict.note
@@ -1161,8 +1176,21 @@ class Agent:
             limit = min(limit, self.settings.compact_at)
         return limit
 
+    def next_request_tokens(self) -> int:
+        """The size of the next request: what the model last counted, plus what was added since (tool output,
+        attachments). Deciding on the last count alone let one large tool result overflow the window."""
+        if self.counted_chars is None:
+            return self.ctx_used
+        return self.ctx_used + max(0, conversation_chars(self.messages) - self.counted_chars) // 3
+
+    def result_budget(self) -> int:
+        """Characters one tool result may take: a share of the window, so one large result can't fill it."""
+        return max(MIN_RESULT_CHARS, int(self.settings.context * 3 * RESULT_SHARE))
+
     def maybe_compact(self) -> None:
         limit = self.compact_limit()
+        if self.next_request_tokens() > limit:
+            self.ctx_used, self.counted_chars = self.next_request_tokens(), None
         if self.ctx_used <= limit:
             return
         enough = int(limit * PRUNED_ENOUGH / AUTO_COMPACT_RATIO)  # after pruning, a summary isn't needed below this
@@ -1231,6 +1259,7 @@ class Agent:
             )
             self.skills_loaded.clear()
         self.ctx_used = sum(len(m["content"]) for m in self.messages) // 3
+        self.counted_chars = None
         self.summarized_size = self.ctx_used
         self.sent_chars = 0
         self.reflected = len(self.messages)

lcode/api.py

@@ -200,6 +200,7 @@ def open_agent(cwd: Path, options: Options, console: Console, hw: Hardware | Non
     )
     agent = Agent(ollama, settings, cwd, console=console)
     agent.hardware = hw
+    agent.autosave = True
     agent.interactive = options.interactive
     if options.max_steps:
         agent.max_steps = options.max_steps

lcode/context.py

@@ -14,8 +14,10 @@ full output is saved to a file the model can read if it needs more.
 from __future__ import annotations
 
 import json
+import os
 import re
 import time
+from collections import deque
 from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -253,6 +255,72 @@ def shorten(text: str, limit: int, label: str = "output") -> str:
     return f"{head}{note}{tail}{where}"
 
 
+class OutputSpool:
+    """Output collected line by line with bounded memory: once it's long, everything goes to a file, and only the
+    start, the end and the error-looking lines stay in memory. What `shorten` returns comes out the same."""
+
+    def __init__(self, limit: int, label: str = "output") -> None:
+        self.limit, self.label = limit, label
+        self.head: list[str] = []  # everything, until the output gets long; then its start
+        self.tail: deque[str] = deque()
+        self.tail_chars = 0
+        self.errors: list[str] = []
+        self.chars = 0
+        self.file = None
+        self.path: Path | None = None
+
+    def add(self, line: str) -> None:
+        self.chars += len(line)
+        if self.file is None:
+            self.head.append(line)
+            if self.chars > self.limit * 2:
+                self._spool()
+            return
+        try:
+            self.file.write(line)
+        except OSError:
+            pass
+        if ERROR_LINE.search(line) and len(self.errors) < MAX_ERROR_LINES:
+            self.errors.append(line.strip()[:300])
+        self.tail.append(line)
+        self.tail_chars += len(line)
+        while self.tail_chars > self.limit // 4 and len(self.tail) > 1:
+            self.tail_chars -= len(self.tail.popleft())
+
+    def _spool(self) -> None:
+        text = "".join(self.head)
+        self.path = save_output(text, self.label)
+        if self.path is not None:
+            try:
+                self.file = self.path.open("a")
+            except OSError:
+                self.file = None
+        if self.file is None:  # nowhere to keep it: keep only the start and carry on counting
+            self.file = open(os.devnull, "w")  # noqa: SIM115
+        head = text[: self.limit * 3 // 5]
+        self.head = [head]
+
+    def text(self) -> str:
+        if self.file is None:
+            return shorten("".join(self.head), self.limit, self.label)
+        self.file.close()
+        head, tail = "".join(self.head), "".join(self.tail)
+        cut = self.chars - len(head) - len(tail)
+        note = f"\n\n... [{cut} characters cut"
+        if self.errors:
+            note += f"; {len(self.errors)} line(s) from them that look like errors:]\n" + "\n".join(self.errors)
+            note += "\n[...]\n\n"
+        else:
+            note += "] ...\n\n"
+        where = (
+            f"\n[The full {self.label} ({self.chars} characters) is in {self.path}: read parts of it with read_file "
+            "if you need them.]"
+            if self.path
+            else ""
+        )
+        return f"{head}{note}{tail}{where}"
+
+
 # ----------------------------------------------------------------------------- what uses the context
 
 

lcode/permissions.py

@@ -22,51 +22,173 @@ SAFE_GIT = {"status", "log", "diff", "show", "branch", "ls-files", "rev-parse",
 MULTI_WORD = {"git", "npm", "pnpm", "yarn", "pip", "uv", "docker", "cargo", "go", "kubectl", "python", "python3", "brew"}  # fmt: skip  # noqa: E501
 
 
-def bash_key(command: str) -> str:
-    """The key "always allow" remembers: the program (plus subcommand for tools like git), ignoring `cd dir &&`."""
-    command = re.sub(r"^\s*(cd\s+\S+\s*&&\s*)+", "", command)
+# `cd <plain path> &&` in front of a command: harmless, and models write it all the time.
+NOT_ASKED = (
+    "This needs the user's permission, and nobody can be asked in this run, so it was not allowed. Do without it "
+    "if you can; otherwise finish and say what permission you need."
+)
+LEADING_CD = re.compile(r"""^\s*(?:cd\s+(?:[\w./~@+,=:-]+|'[^']*'|"[^"$`\\]*")\s*&&\s*)+""")
+
+
+# Redirections that write nothing anyone keeps: stderr into stdout, or output thrown away.
+HARMLESS_REDIRECT = re.compile(r"(?<![\w>&])(?:[12]?>&[12]|[12]?>\s*/dev/null)(?=\s|$)")
+OPERATORS = {"|", "||", "&&", ";", "&", ">", ">>", "<", "<<", "<<<", ">&", "<&", ">|", "(", ")", ";;"}
+
+
+def split_command(command: str) -> tuple[list[list[str]], list[str]] | None:
+    """The command's segments (as words) and the operators between them, quotes respected. None when it can't be
+    read: command substitution, process substitution or unbalanced quotes."""
+    if any(tok in command for tok in ("`", "$(", "<(", ">(")) or "\n" in command.strip():
+        return None
+    lexer = shlex.shlex(command, posix=True, punctuation_chars=True)
+    lexer.whitespace_split = True
     try:
-        words = shlex.split(command)
+        tokens = list(lexer)
     except ValueError:
-        words = command.split()
+        return None
+    segments: list[list[str]] = [[]]
+    operators: list[str] = []
+    for token in tokens:
+        if token in OPERATORS or (token and set(token) <= set("|&;<>()")):
+            operators.append(token)
+            segments.append([])
+        else:
+            segments[-1].append(token)
+    return [s for s in segments if s], operators
+
+
+def segment_key(words: list[str]) -> str:
+    """The program, plus the subcommand for tools like git (`git commit`, `python -m pytest`)."""
+    while words and re.fullmatch(r"\w+=\S*", words[0]):
+        words = words[1:]  # VAR=value in front
     if not words:
-        return "bash:"
+        return ""
     if words[0] in MULTI_WORD and len(words) > 1:
         if words[1] in ("-m", "-c") and len(words) > 2:  # python -m <module>
-            return f"bash:{' '.join(words[:3])}"
-        return f"bash:{words[0]} {words[1]}"
-    return f"bash:{words[0]}"
+            return " ".join(words[:3])
+        return f"{words[0]} {words[1]}"
+    return words[0]
 
 
-# `cd <plain path> &&` in front of a command: harmless, and models write it all the time.
-NOT_ASKED = (
-    "This needs the user's permission, and nobody can be asked in this run, so it was not allowed. Do without it "
-    "if you can; otherwise finish and say what permission you need."
-)
-LEADING_CD = re.compile(r"""^\s*(?:cd\s+(?:[\w./~@+,=:-]+|'[^']*'|"[^"$`\\]*")\s*&&\s*)+""")
+def bash_key(command: str) -> str:
+    """What "always allow" remembers. For one program, the program (plus subcommand for git and the like); for a
+    chain such as `npm test && rm -rf build`, every program in it, so allowing `npm test` doesn't allow the chain.
+    `cd dir &&` in front doesn't count."""
+    command = HARMLESS_REDIRECT.sub(" ", LEADING_CD.sub("", command)).strip()
+    parsed = split_command(command)
+    if parsed is None:
+        return "bash:" + " ".join(command.split())  # can't be read: only this exact command
+    segments, operators = parsed
+    keys, redirected = [], False
+    for i, words in enumerate(segments):
+        before = operators[i - 1] if i else ""
+        if before and before not in ("|", "&&", "||", ";", "&"):
+            redirected = True  # a file name after > or <, not a program
+            continue
+        keys.append(segment_key(words))
+    key = " + ".join(k for k in keys if k) + (" (redirected)" if redirected else "")
+    return f"bash:{key}"
 
 
-def is_read_only(command: str) -> bool:
-    command = LEADING_CD.sub("", command)
-    if any(tok in command for tok in (";", "&", ">", "`", "$(", "<(")) or "\n" in command:
+def option_values(words: list[str], *options: str) -> list[str]:
+    """The values of options such as `-o FILE`, `-oFILE` or `--output=FILE`."""
+    found = []
+    for i, word in enumerate(words):
+        for option in options:
+            if word == option and i + 1 < len(words):
+                found.append(words[i + 1])
+            elif option.startswith("--") and word.startswith(option + "="):
+                found.append(word.split("=", 1)[1])
+            elif not option.startswith("--") and word.startswith(option) and len(word) > len(option):
+                found.append(word[len(option) :])
+    return found
+
+
+def flags(words: list[str]) -> list[str]:
+    return [w for w in words[1:] if w.startswith("-")]
+
+
+def positional(words: list[str]) -> list[str]:
+    return [w for w in words[1:] if not w.startswith("-")]
+
+
+# Branch and remote listing; anything else creates, deletes, renames or reconfigures.
+BRANCH_LISTING = {
+    "-a", "--all", "-r", "--remotes", "-l", "--list", "-v", "-vv", "--verbose", "--show-current", "--color",
+    "--no-color", "--column", "--no-column", "--abbrev", "--no-abbrev", "-i", "--ignore-case", "--omit-empty",
+}  # fmt: skip
+BRANCH_WITH_VALUE = {"--contains", "--no-contains", "--merged", "--no-merged", "--points-at", "--sort", "--format"}
+NVIDIA_QUERY = ("-q", "--query", "-L", "--list-gpus", "--format", "-i", "--id", "-d", "--display", "-x", "--xml-format",
+                "-h", "--help", "--query-gpu", "--query-compute-apps")  # fmt: skip
+
+
+def safe_git(words: list[str]) -> bool:
+    if len(words) < 2 or words[1] not in SAFE_GIT:
+        return False  # also `git -c …` and `git -C …`, which can run anything
+    sub, args = words[1], words[2:]
+    if sub in ("log", "show", "diff", "shortlog") and option_values(words, "--output"):
+        return False  # writes the output to a file
+    if sub == "branch":
+        listing = any(a in ("-l", "--list") for a in args)
+        i = 0
+        while i < len(args):
+            arg = args[i]
+            name = arg.split("=", 1)[0]
+            if name in BRANCH_WITH_VALUE:
+                i += 1 if "=" in arg else 2
+                continue
+            if arg.startswith("-") and name not in BRANCH_LISTING:
+                return False  # -d, -D, -m, -c, -f, -u, --set-upstream-to, --edit-description, …
+            if not arg.startswith("-") and not listing:
+                return False  # `git branch name` creates a branch
+            i += 1
+    if sub == "remote":
+        return not args or all(a in ("-v", "--verbose") for a in args) or args[0] in ("show", "get-url")
+    return True
+
+
+def safe_segment(words: list[str]) -> bool:
+    program = words[0]
+    if program == "git":
+        return safe_git(words)
+    if program not in SAFE_COMMANDS:
         return False
-    if re.search(r"-exec|-delete|-ok\b|-fprint", command):
+    if program == "sort" and (
+        option_values(words, "-o", "--output") or any(f.startswith("--compress-program") for f in flags(words))
+    ):
         return False
-    for segment in command.split("|"):
-        try:
-            words = shlex.split(segment)
-        except ValueError:
-            return False
-        if not words:
-            return False
-        if words[0] == "git":
-            if len(words) < 2 or words[1] not in SAFE_GIT:
-                return False
-        elif words[0] not in SAFE_COMMANDS:
-            return False
+    if program == "uniq" and len(positional(words)) > 1:
+        return False  # `uniq in out` writes out
+    if program == "tree" and option_values(words, "-o"):
+        return False
+    if program == "rg" and any(f.split("=")[0] in ("--pre", "--pre-glob") for f in flags(words)):
+        return False  # runs a program on each file
+    if program == "find" and any(
+        w in ("-exec", "-execdir", "-ok", "-okdir", "-delete", "-fprint", "-fprint0", "-fprintf", "-fls") for w in words
+    ):
+        return False
+    if program == "date" and (option_values(words, "-s", "--set") or "-s" in words):
+        return False
+    if program == "file" and any(f in ("-C", "--compile") for f in flags(words)):
+        return False
+    if program == "nvidia-smi":
+        return all(f.split("=")[0] in NVIDIA_QUERY for f in flags(words))
     return True
 
 
+def is_read_only(command: str) -> bool:
+    """Whether a command only reads: known read-only programs with arguments that don't write, joined only by
+    pipes. `cd dir &&` in front, `2>&1` and `>/dev/null` are fine."""
+    command = HARMLESS_REDIRECT.sub(" ", LEADING_CD.sub("", command)).strip()
+    parsed = split_command(command)
+    if parsed is None:
+        return False
+    segments, operators = parsed
+    if not segments or any(op != "|" for op in operators):
+        return False
+    return all(safe_segment(words) for words in segments)
+
+
 class Permissions:
     def __init__(self, console: Console, mode: str = "ask"):
         from lcode.hooks import Rules

lcode/repl.py

@@ -754,6 +754,7 @@ def restore_checkpoint(agent: Agent, target: Checkpoint, rewind: bool) -> None:
         agent.messages = agent.messages[:index]
         agent.reflected = min(agent.reflected, index)
         agent.ctx_used = sum(len(json.dumps(m)) for m in agent.messages) // 3
+        agent.counted_chars = None
         c.print(f"The conversation is back to before request {target.n} as well.")
     elif plan.restore or plan.remove:
         what = "undid the file changes from" if not rewind else "rolled the files back to before"

lcode/tools.py

@@ -6,6 +6,7 @@ import difflib
 import fnmatch
 import hashlib
 import inspect
+import itertools
 import json
 import os
 import queue
@@ -16,7 +17,9 @@ import signal
 import subprocess
 import threading
 import time
+from collections import deque
 from collections.abc import Callable
+from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -26,7 +29,7 @@ from rich.syntax import Syntax
 from rich.text import Text
 
 from lcode import secretscan, web
-from lcode.context import shorten
+from lcode.context import OutputSpool, shorten
 from lcode.jobs import JobError, describe, stop_leftovers
 from lcode.permissions import bash_key, is_read_only
 from lcode.planning import BLOCKED, PLAN_MODE_TOOLS
@@ -41,6 +44,7 @@ NO_NETWORK = re.compile(
     r"EAI_AGAIN|Name or service not known|network is unreachable|No route to host"
 )
 MAX_TOOL_OUTPUT = 30_000  # characters returned to the model per tool call
+PENDING_LINES = 5000  # output lines waiting to be read, at most
 LEFTOVER_GRACE = 0.5  # seconds to wait for output after a command ends, before stopping what it left running
 IGNORE_DIRS = {
     ".git", "node_modules", "__pycache__", ".venv", "venv", "env", ".mypy_cache", ".pytest_cache",
@@ -289,6 +293,48 @@ def truncate(text: str, limit: int = MAX_TOOL_OUTPUT) -> str:
     return f"{head}\n\n... [{len(text) - len(head) - len(tail)} characters truncated] ...\n\n{tail}"
 
 
+@dataclass
+class TextFormat:
+    """How a file stores its text, so an edit writes it back the same way."""
+
+    encoding: str = "utf-8"
+    crlf: bool = False  # Windows line endings
+    bom: bool = False  # a UTF-8 byte order mark
+
+
+def decode(raw: bytes) -> tuple[str, TextFormat]:
+    """A file's text with \n line endings, and its format: UTF-8, else Windows-1252 or Latin-1 (which always
+    decode), never with characters replaced. Writing a replaced character back used to corrupt the whole file."""
+    fmt = TextFormat(bom=raw.startswith(b"\xef\xbb\xbf"))
+    text = ""
+    for encoding in ("utf-8-sig" if fmt.bom else "utf-8", "cp1252", "latin-1"):
+        try:
+            text = raw.decode(encoding)
+            fmt.encoding = "utf-8" if encoding == "utf-8-sig" else encoding
+            break
+        except UnicodeDecodeError:
+            continue
+    crlf = text.count("\r\n")
+    if crlf and crlf == text.count("\n"):  # only CRLF: mixed files are left as they are
+        fmt.crlf = True
+        text = text.replace("\r\n", "\n")
+    return text, fmt
+
+
+def encode(text: str, fmt: TextFormat) -> bytes:
+    if fmt.crlf:
+        text = text.replace("\r\n", "\n").replace("\n", "\r\n")
+    try:
+        data = text.encode(fmt.encoding)
+    except UnicodeEncodeError as e:
+        bad = text[e.start : e.end]
+        raise ToolError(
+            f"the file is in {fmt.encoding}, which can't hold {bad!r}: write it without that character, or ask the "
+            "user to convert the file to UTF-8"
+        ) from e
+    return (b"\xef\xbb\xbf" if fmt.bom else b"") + data
+
+
 def is_binary(path: Path) -> bool:
     try:
         with open(path, "rb") as f:
@@ -362,15 +408,28 @@ def parse_text_tool_calls(content: str) -> list[dict]:
             pass
     if calls:
         return calls
-    # Bare JSON objects such as {"name": "grep", "arguments": {...}}, possibly inside ``` fences.
+    # A message that is nothing but JSON calls such as {"name": "grep", "arguments": {...}}, possibly in a ```
+    # fence. JSON inside an answer is an example, not a call: running it once ran a command the model was only
+    # explaining.
+    body = content.strip()
+    lead = re.match(r"([^\n{`]{0,50})\n+(?=```|\{)", body)  # a short lead-in such as "Sure:" is fine
+    if lead:
+        body = body[lead.end() :]
+    body = re.sub(r"^```(?:json)?\s*|\s*```$", "", body)
     decoder = json.JSONDecoder()
-    for m in re.finditer(r"\{", content):
+    position = 0
+    while position < len(body):
+        while position < len(body) and body[position].isspace():
+            position += 1
+        if position == len(body):
+            break
         try:
-            obj, _ = decoder.raw_decode(content, m.start())
+            obj, position = decoder.raw_decode(body, position)
         except json.JSONDecodeError:
-            continue
-        if isinstance(obj, dict) and obj.get("name") in TOOL_NAMES and isinstance(obj.get("arguments", {}), dict):
-            calls.append({"function": {"name": obj["name"], "arguments": obj.get("arguments", {})}})
+            return []  # something else in the message: not a call
+        if not (isinstance(obj, dict) and obj.get("name") in TOOL_NAMES and isinstance(obj.get("arguments", {}), dict)):
+            return []
+        calls.append({"function": {"name": obj["name"], "arguments": obj.get("arguments", {})}})
     return calls
 
 
@@ -385,6 +444,7 @@ class Toolbox:
     def __init__(self, agent: Agent):
         self.agent = agent
         self.read_mtimes: dict[str, float] = {}
+        self.formats: dict[str, TextFormat] = {}  # encoding and line endings of the files read
         self.todos: list[dict] = []
         # An editor's buffers (lcode acp): read unsaved changes and write through the editor.
         self.reader: Callable[[Path], str | None] | None = None
@@ -476,7 +536,7 @@ class Toolbox:
         if problem:
             return f"Error: bad arguments for {name}: {problem}"
         try:
-            return fn(**args)
+            return fn(**self._typed(fn, args))
         except ToolError as e:
             return f"Error: {e}"
         except TypeError as e:
@@ -484,6 +544,23 @@ class Toolbox:
         except Exception as e:  # surface every failure to the model instead of crashing the session
             return f"Error: {type(e).__name__}: {e}"
 
+    @staticmethod
+    def _typed(fn, args: dict) -> dict:
+        """Arguments with text turned into the declared type: some servers and the text fallback send every value
+        as a string, and "false" is true in Python."""
+        params = inspect.signature(fn).parameters
+        typed = dict(args)
+        for name, value in args.items():
+            if not isinstance(value, str) or name not in params:
+                continue
+            kind = str(params[name].annotation).split("|")[0].strip()
+            text = value.strip().lower()
+            if kind == "bool" and text in ("true", "false", "1", "0", "yes", "no"):
+                typed[name] = text in ("true", "1", "yes")
+            elif kind == "int" and re.fullmatch(r"-?\d+", text):
+                typed[name] = int(text)
+        return typed
+
     @staticmethod
     def _argument_problem(fn, args: dict) -> str:
         """Explain unknown or missing arguments in terms the model can act on."""
@@ -639,9 +716,21 @@ class Toolbox:
         self.read_mtimes[str(p)] = p.stat().st_mtime
         if not lines:
             return "(empty file)"
-        out = "\n".join(f"{i:6}\t{line[:2000]}" for i, line in enumerate(chunk, start=offset))
-        end = offset - 1 + len(chunk)
-        if end < len(lines) or offset > 1:
+        budget, shown, size = self.agent.result_budget() - 300, [], 0
+        for i, line in enumerate(chunk, start=offset):
+            numbered = f"{i:6}\t{line[:2000]}"
+            if shown and size + len(numbered) + 1 > budget:
+                break  # the rest wouldn't fit in the context at once
+            shown.append(numbered)
+            size += len(numbered) + 1
+        out = "\n".join(shown)
+        end = offset - 1 + len(shown)
+        if end < offset - 1 + len(chunk):
+            out += (
+                f"\n\n[Showing lines {offset}-{end} of {len(lines)}: more doesn't fit in the context at once. "
+                f"Read on with offset={end + 1}.]"
+            )
+        elif end < len(lines) or offset > 1:
             out += f"\n\n[Showing lines {offset}-{end} of {len(lines)}. Use offset/limit to read more.]"
         return truncate(out, 120_000)
 
@@ -732,14 +821,17 @@ class Toolbox:
             text = self.reader(p)
             if text is not None:
                 return text
-        return p.read_text(errors="replace")
+        text, self.formats[str(p)] = decode(p.read_bytes())
+        return text
 
     def _write(self, p: Path, content: str) -> None:
         p.parent.mkdir(parents=True, exist_ok=True)
+        fmt = self.formats.get(str(p)) or (decode(p.read_bytes())[1] if p.exists() else TextFormat())
+        data = encode(content, fmt)  # before anything is written: it may not be possible
         if self.writer is not None:
             self.writer(p, content)  # the editor updates its buffer (and the file)
-        if not p.exists() or p.read_text(errors="replace") != content:
-            p.write_text(content)  # the file on disk must match, for checkpoints and the tools
+        if not p.exists() or p.read_bytes() != data:
+            p.write_bytes(data)  # the file on disk must match, for checkpoints and the tools
         self.read_mtimes[str(p)] = p.stat().st_mtime
 
     def t_write_file(self, path: str, content: str) -> str:
@@ -749,8 +841,10 @@ class Toolbox:
             if p.is_dir():
                 raise ToolError(f"{path} is a directory")
             self._check_fresh(p)
-            body = self._diff(p, self.read_text(p), content)
+            shown = self.read_text(p)
+            body = self._diff(p, shown, content)
         else:
+            shown = None
             lines = content.splitlines()
             preview = "\n".join(lines[:60]) + (f"\n... ({len(lines) - 60} more lines)" if len(lines) > 60 else "")
             body = Syntax(preview, Syntax.guess_lexer(str(p), content), theme="monokai", line_numbers=True)
@@ -758,13 +852,27 @@ class Toolbox:
         ok, feedback = self.agent.perms.request("edit", "edit", f"{verb} {self.rel(p)}", body, self.project_rel(p))
         if not ok:
             return feedback
-        before = self.read_text(p) if exists else None
+        before = self.read_text(p) if p.exists() else None
+        if before != shown:
+            return self._changed_meanwhile(p)
         self.agent.checkpoint()
         self._write(p, content)
         n = len(content.splitlines())
         self.console.print(f"  [green]✓[/] {'Updated' if exists else 'Created'} {self.rel(p)} ({n} lines)")
         return f"{'Overwrote' if exists else 'Created'} {self.rel(p)} ({n} lines)." + self._check(p, before, content)
 
+    def _changed_meanwhile(self, p: Path) -> str:
+        """The file changed while the user was asked (they edited it): writing now would undo their change."""
+        self.read_mtimes.pop(str(p), None)
+        self.console.print(
+            Text(f"  ⎿ {self.rel(p)} changed while you were asked; the edit wasn't applied", style="yellow")
+        )
+        return (
+            f"Error: {self.rel(p)} changed while the user was being asked (they or another program edited it), so "
+            "lcode didn't apply this change: it would have undone theirs. Read the file again and redo the change "
+            "on its current content."
+        )
+
     def t_edit_file(self, path: str, old_string: str, new_string: str, replace_all: bool = False) -> str:
         p = self.path(path)
         if not p.exists():
@@ -795,6 +903,8 @@ class Toolbox:
         ok, feedback = self.agent.perms.request("edit", "edit", f"Edit {self.rel(p)}", diff, self.project_rel(p))
         if not ok:
             return feedback
+        if not p.exists() or self.read_text(p) != text:
+            return self._changed_meanwhile(p)
         self.agent.checkpoint()
         self._write(p, new_text)
         self.console.print(f"  [green]✓[/] Edited {self.rel(p)}")
@@ -879,13 +989,22 @@ class Toolbox:
             start_new_session=True,
             bufsize=1,
         )
-        lines: queue.Queue[str | None] = queue.Queue()
+        # Bounded, so a command that prints faster than lcode reads waits instead of filling memory.
+        lines: queue.Queue[str | None] = queue.Queue(maxsize=PENDING_LINES)
+        reading = threading.Event()  # cleared when lcode stops reading
+        reading.set()
 
         def pump() -> None:
             assert proc.stdout is not None
-            for line in proc.stdout:
-                lines.put(line)
-            lines.put(None)
+            for line in itertools.chain(proc.stdout, [None]):  # streamed: never the whole output
+                while reading.is_set():
+                    try:
+                        lines.put(line, timeout=0.5)
+                        break
+                    except queue.Full:
+                        continue
+                else:
+                    return
 
         def stop() -> None:
             if sandbox and token:
@@ -893,7 +1012,9 @@ class Toolbox:
             os.killpg(proc.pid, signal.SIGKILL)
 
         threading.Thread(target=pump, daemon=True).start()
-        out: list[str] = []
+        out = OutputSpool(MAX_TOOL_OUTPUT, "output")
+        last = ""  # the latest line, for the end-of-output checks below
+        recent: deque[str] = deque(maxlen=40)
         shown, status, recorded = 0, "", ""
         deadline = time.time() + int(timeout or 180)
         finished = 0.0  # when the command itself ended (its marker arrived)
@@ -902,16 +1023,16 @@ class Toolbox:
             while True:
                 if self.agent.cancel is not None and self.agent.cancel.is_set():
                     raise KeyboardInterrupt
+                if finished and time.time() - finished > LEFTOVER_GRACE:
+                    held_open = True  # also when what it left running keeps printing
+                    break
+                if time.time() > deadline and not finished:  # also while it keeps printing
+                    stop()
+                    status = f"\n[Command timed out after {timeout}s and was killed]"
+                    break
                 try:
                     line = lines.get(timeout=0.2)
                 except queue.Empty:
-                    if finished and time.time() - finished > LEFTOVER_GRACE:
-                        held_open = True
-                        break
-                    if time.time() > deadline:
-                        stop()
-                        status = f"\n[Command timed out after {timeout}s and was killed]"
-                        break
                     continue
                 if line is None:
                     break
@@ -919,14 +1040,19 @@ class Toolbox:
                     recorded = line[len(marker) :].strip()
                     finished = time.time()
                     continue
-                out.append(line)
+                if last:
+                    out.add(last)
+                    recent.append(last)
+                last = line
                 if shown <= 40:
                     msg = "    ... (more output hidden)" if shown == 40 else "    " + line.rstrip("\n")[:300]
                     self.console.print(Text(msg, style="dim"))
                     shown += 1
         except KeyboardInterrupt:
+            reading.clear()
             stop()
             raise
+        reading.clear()
         code = proc.wait()
         if held_open and sandbox and token:
             sandbox.kill(token)  # what it left running in the container
@@ -935,21 +1061,22 @@ class Toolbox:
                 "\n[lcode stopped what this command left running (started with &). To keep a process running, "
                 "start it with background=true: then bash_output reads its output and bash_stop stops it]"
             )
-        if out and out[-1] == "\n":
-            out.pop()  # the blank line printed before the marker
+        if last and last != "\n":  # the blank line printed before the marker isn't output
+            out.add(last)
+            recent.append(last)
         if recorded and Path(recorded).is_dir() and Path(recorded).resolve() != self.agent.cwd:
             if sandbox and not sandbox.contains(Path(recorded).resolve()):
                 status += "\n[That directory is outside the project; the working directory stays the same]"
             else:
                 self.agent.cwd = Path(recorded).resolve()
                 status += f"\n[Working directory is now {self.agent.cwd}]"
-        if sandbox and code != 0 and not sandbox.network and NO_NETWORK.search("".join(out[-40:])):
+        if sandbox and code != 0 and not sandbox.network and NO_NETWORK.search("".join(recent)):
             status += (
                 "\n[The sandbox has no network access. If this needs the network, ask the user to allow it "
                 "with /sandbox network on]"
             )
         self.console.print(Text(f"  exit code {code}", style="green" if code == 0 else "red"))
-        return shorten("".join(out), MAX_TOOL_OUTPUT, "output") + status + f"\n[exit code: {code}]"
+        return out.text() + status + f"\n[exit code: {code}]"
 
     # -- web
     def _web_allowed(self, key: str, title: str, detail: str) -> tuple[bool, str]:

{lcode_cli-0.28.0.dist-info → lcode_cli-0.28.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lcode-cli
-Version: 0.28.0
+Version: 0.28.1
 Summary: A local-first terminal coding agent powered by open-weight models on your own GPU or Mac, via Ollama.
 Project-URL: Homepage, https://nasser1941.github.io/lcode/
 Project-URL: Documentation, https://nasser1941.github.io/lcode/

{lcode_cli-0.28.0.dist-info → lcode_cli-0.28.1.dist-info}/RECORD

@@ -1,9 +1,9 @@
-lcode/__init__.py,sha256=OG44SXHF0F_qxUeC9BH5G84cellTu3wqLwxsfUfYdUU,473
+lcode/__init__.py,sha256=USdcfUSBcDvzY7_LoT19i6HbdafE2NIPUyufHNoj9Vo,473
 lcode/__main__.py,sha256=s5N5pryX2FAu93GGZ3FxUrIXHWGzxz66lKUoaV_h8Ws,35
 lcode/acp.py,sha256=Tpc8lsvffZSYbTRI4t7X4E2chahqZIPcKeYIcL5bV3Y,29035
 lcode/action.py,sha256=wuT5F9EJCdEligPeytzmEi8FpahWBhXer528tpuTj_k,16711
-lcode/agent.py,sha256=HFdmGXK_3oHu57L15z2dmJsQTj_ddjRDq7JnxM1Qq34,63045
-lcode/api.py,sha256=kYq7RghkNyf-A7qsZh6AeMzLoZbc9rTobsFRLtAcvD8,15480
+lcode/agent.py,sha256=ws5i476DOsaEIv5YkYQ_qfmRV5fPitd6QltLDU9QhpE,64804
+lcode/api.py,sha256=Ul_2H_6LzkufaXHaG0Dz1vDIEmOAZ31VHO2qsjVdtE8,15506
 lcode/backends.py,sha256=dtJHrC6ys73bRDg64z0tQz1PavmeAkiODZu1fOO9ExQ,20635
 lcode/bench.py,sha256=7ICipu_2GXW6vE519XeZucc2GSWA44votT8JvqcNv1k,40964
 lcode/bench_backend.py,sha256=X2fo-zRX92Xp3KMxSsFlWioc2vRupBTtowj5kLME8mw,14287
@@ -14,7 +14,7 @@ lcode/cli.py,sha256=M6Mjd5BROSGlZj4J6TrSZMJqI-OVfUEWPoC0TQyOQ28,40831
 lcode/clipboard.py,sha256=nTSdJyCXPdtjH947RjvB3QvRqnL5HZsS9KB8EkC42cc,5138
 lcode/codesearch.py,sha256=Io8leN1nb_Y-Dvlm9xaGtwUWYXX-ooswuznJV6dGU1s,10766
 lcode/config.py,sha256=q1zGzH3p5U23ai6O9Jai3dS-s30CuQ9gc5cWX0_yxWE,11195
-lcode/context.py,sha256=_f0NbXyjA4SyH2RCr49AiWwtAfr_PVDaqyFQ5fZDhkg,14404
+lcode/context.py,sha256=WSMiitscDh2Juy4Hx0x9EPNWrn804IbOWRZly1-DU38,17022
 lcode/extensions.py,sha256=W0HFDl0BvROzr7i_qtabHdIbacBgP5o5qIJLzoM2oNE,16806
 lcode/frontmatter.py,sha256=Bx0n4vLPhp7ZCWx-g7xqzqNy5VoKBxyeeikh5UFB_gA,2441
 lcode/gitflow.py,sha256=6j_nnCBZXkX1qhTI7K4CyZ5D0NAZLpH96weFEo64AkQ,27913
@@ -27,18 +27,18 @@ lcode/memory.py,sha256=WZEHTp-lVxAwS-yyLQR2xB3TMZ9sejmX2Ivg7T-tl3w,21949
 lcode/models.toml,sha256=lt2G64zzo_Ryxv2ISRSNZoVD_JqCqYqsM01n8WoXGs8,4098
 lcode/notify.py,sha256=uieDeXe7XwuVHjOk4tqVyZEW_85XWkoWQmFOWEf77oc,1538
 lcode/ollama.py,sha256=u5tnvL28cfaFS4GhmiEsHhTQe6w3Ng_Tej05UUzLQTQ,7715
-lcode/permissions.py,sha256=i04YLtqYCxwApgawomPp4d35uk9HAJtpKY__gfw8j9s,6347
+lcode/permissions.py,sha256=3OPVVth6uTE5kcAWMZ8X_vUnFRbf6STEHAJimZX0Yu4,12118
 lcode/planning.py,sha256=Q2upaoZPnXDMQT2F-ETxX1lk_wDntTzhSKNVYIQSoQA,6540
 lcode/render.py,sha256=gMvf4W3qkLUJ4Vot3az_hP_zRDAItJC9pP-68dHFkfA,1375
 lcode/repeats.py,sha256=uzyAewX03nJf5dKwYFH4y9ZDnh0Lvy2k0qC2vdntM5Y,4061
-lcode/repl.py,sha256=Et8Tf4COLJ9_FsXtjcOciah_33lLgUS5sl4QAer1m8Y,50652
+lcode/repl.py,sha256=ayrKySMbbRbax_WMjqYBN4fkcDpzuHniefJC3DZ4-2Y,50687
 lcode/repomap.py,sha256=1GRJgEUjBl9XHSXsF7tSGXBCEgsY-3CzwMohaTudZ_M,13969
 lcode/residency.py,sha256=bDB2J0SZULDOz6wYuMCnkjacYk7pWi5f75iY83ycAXQ,6912
 lcode/sandbox.py,sha256=u_pkrVRXhCGhh-wFDPwuZ3jGG7UeXxXRAeBNsm37ZvE,8608
 lcode/secretscan.py,sha256=6Euvfk2gCtK7lZbuQD84zgWuSECi6OnSw8V5rFu1tfU,13575
 lcode/sessions.py,sha256=wOR-J0EHgJxLSPboKPHnvBcp5VTUYTj3tZWgpsa6-K4,4013
 lcode/subagents.py,sha256=Z2RHp7an16dhOGReucVIEOAnXjxM8jx7sHvWd_Fp2V4,29142
-lcode/tools.py,sha256=OFltvnRZlSUMtzNLEUF84RFoyKQkAFpb7_ZBVhVcVp4,48426
+lcode/tools.py,sha256=h7C6r27_cBoCkW8tZ4BLmxdgRByYa2cUKh8AWfmEucI,54193
 lcode/update.py,sha256=i92bnW252VneYh0vD6djxmXsDOa5MXP20bjlLZUue_c,25236
 lcode/vision.py,sha256=w33QPmdGwP33wM4mvmAH4amFDZ1xzeXhEsSLJ-STSHY,4243
 lcode/web.py,sha256=vyxKbSRJTl1Lj3hj7Yd_NwTgbv0Xi6dup2YfUXjqp-Y,14579
@@ -58,8 +58,8 @@ lcode/servers/__init__.py,sha256=qrIhxEodEB8cjKEQ4Cicc0GsQhE2OH-9qlVHzr3-tq8,77
 lcode/servers/encord.py,sha256=WSauawL1UKYZCUAG1oalcKL_An3DVof-JM6xIgFyQSE,12769
 lcode/servers/valohai.py,sha256=e5xZGIoa4cja4f5i85q1GpL9c7cglUjV8jN2co1CFMY,14430
 lcode/servers/xai.py,sha256=h2emAvHSnkuB9WmJInlynsDhvxJwNVD2AHFX7vqMXNc,13627
-lcode_cli-0.28.0.dist-info/METADATA,sha256=b-CcZWIZTz_P0M5H9xP1cwtQpICZpHlaGWdBRlSB52U,12620
-lcode_cli-0.28.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-lcode_cli-0.28.0.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
-lcode_cli-0.28.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
-lcode_cli-0.28.0.dist-info/RECORD,,
+lcode_cli-0.28.1.dist-info/METADATA,sha256=96ojptFvjK6CP51H1expKcg1ZtjKmxSQce4hOk052XI,12620
+lcode_cli-0.28.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+lcode_cli-0.28.1.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
+lcode_cli-0.28.1.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
+lcode_cli-0.28.1.dist-info/RECORD,,