lcode-cli 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

lcode/__init__.py

@@ -3,7 +3,7 @@
 From a program: `from lcode import Session` (see lcode.api).
 """
 
-__version__ = "0.25.0"
+__version__ = "0.27.0"
 
 API = ("Session", "Result", "Options", "SetupError")
 

lcode/action.py

@@ -302,7 +302,14 @@ def answer(hub: GitHub, task: Task, inputs: Inputs, cwd: Path) -> None:
         reply = result.text.strip() or "(no answer)"
         changed = git(cwd, "status", "--porcelain").strip()
         if changed:
-            from lcode import gitflow
+            from lcode import gitflow, secretscan
+
+            found = secretscan.in_files(cwd, ["."], staged=False)
+            if found:
+                raise ActionError(
+                    secretscan.report("commit", found, excerpts=False) + "\n\nSo lcode didn't commit or push its "
+                    "changes. Secrets belong in the repository's secrets or in configuration that git ignores."
+                )
 
             data = gitflow.ask(
                 session.agent,

lcode/agent.py

@@ -233,6 +233,8 @@ class Settings:
     notify: bool = False  # desktop notifications after long requests (lcode.notify)
     notify_after: int = 30  # seconds: a request this long notifies when it's done or waits for an answer
     max_parallel_agents: int = 1
+    secret_check: bool = True  # scan what git commit and git push would add (lcode.secretscan)
+    compact_at: int = 0  # tokens: make room at this size even when the window is larger (0: only at 85%)
     repeat_limit: int = 3  # identical calls with identical results before the model is told (lcode.repeats)
 
 
@@ -269,6 +271,7 @@ class Agent:
         self.agent_runs: list[subagents.Record] = []  # subagents run in this session, for /agents
         self.plan = ""  # the plan the user approved; kept through compaction
         self.pruned = self.compacted = 0  # how often the context was pruned or summarized automatically
+        self.summarized_size = 0  # tokens right after the last summary (0: none in this conversation)
         self._extensions: tuple[Path, extensions.Extensions] | None = None
         self.skills_loaded: set[str] = set()  # skills whose instructions are in the conversation
         # Set on subagents (see lcode.subagents):
@@ -513,6 +516,7 @@ class Agent:
         return WEB_PROMPT.format(tools=tools, date=dt.date.today().isoformat())
 
     def reset(self) -> None:
+        self.summarized_size = 0
         self.messages = [{"role": "system", "content": self.system_prompt()}]
         self._mcp_prompt = ""
         self.tools.read_mtimes.clear()
@@ -1045,13 +1049,27 @@ class Agent:
         return text + ("\n\n" + "\n\n".join(attached) if attached else "")
 
     # -- context management
+    def compact_limit(self) -> int:
+        """The tokens in use at which lcode makes room: 85% of the window, or `compact_at` when that's lower.
+        Small models lose track in long contexts well before a large window is full."""
+        limit = int(AUTO_COMPACT_RATIO * self.settings.context)
+        if self.settings.compact_at > 0:
+            limit = min(limit, self.settings.compact_at)
+        return limit
+
     def maybe_compact(self) -> None:
-        if self.ctx_used <= AUTO_COMPACT_RATIO * self.settings.context:
+        limit = self.compact_limit()
+        if self.ctx_used <= limit:
             return
+        enough = int(limit * PRUNED_ENOUGH / AUTO_COMPACT_RATIO)  # after pruning, a summary isn't needed below this
         if self.settings.prune:
-            saved = context_tools.prune(self.messages)  # tool output from before the last two requests
-            if self.ctx_used - saved // 3 > PRUNED_ENOUGH * self.settings.context:
-                saved += context_tools.prune(self.messages, keep=1)  # not enough: before the last request
+            removed: list[str] = []  # files and pages whose only copy goes
+            saved = context_tools.prune(self.messages, removed=removed)  # tool output from before the last two requests
+            if self.ctx_used - saved // 3 > enough:
+                saved += context_tools.prune(
+                    self.messages, keep=1, removed=removed
+                )  # not enough: before the last request
+            context_tools.note_removed(self.messages, removed)
             if saved:
                 self.pruned += 1
                 self.ctx_used = max(len(self.messages[0]["content"]) // 3, self.ctx_used - saved // 3)
@@ -1059,9 +1077,17 @@ class Agent:
                     f"[dim]Removed old tool output from the conversation (~{format_tokens(saved // 3)} tokens) to "
                     "make room.[/]"
                 )
-                if self.ctx_used <= PRUNED_ENOUGH * self.settings.context:
+                if self.ctx_used <= enough:
                     return
-        self.console.print("[yellow]Context is nearly full — compacting the conversation…[/]")
+        window_full = self.ctx_used > AUTO_COMPACT_RATIO * self.settings.context
+        if not window_full and self.summarized_size and self.ctx_used - self.summarized_size < limit // 4:
+            return  # summarized not long ago: summarizing again so soon would only lose more
+        if limit < AUTO_COMPACT_RATIO * self.settings.context:
+            self.console.print(
+                f"[yellow]The conversation reached {format_tokens(limit)} tokens (compact_at) — summarizing it…[/]"
+            )
+        else:
+            self.console.print("[yellow]Context is nearly full — compacting the conversation…[/]")
         self.compacted += 1
         self.compact()
 
@@ -1082,9 +1108,13 @@ class Agent:
         with self.console.status("[cyan]Compacting…[/]"):
             for chunk in self.chat([*self.messages, {"role": "user", "content": instructions}], None, False):
                 summary += chunk.get("message", {}).get("content", "")
+        requests = context_tools.user_requests(self.messages)
         self.messages = [
             {"role": "system", "content": self.system_prompt()},
-            {"role": "user", "content": f"[Summary of our conversation so far]\n\n{summary}"},
+            {
+                "role": "user",
+                "content": f"{sessions.SUMMARY_PREFIX}\n\n{summary}" + context_tools.requests_block(requests),
+            },
             {"role": "assistant", "content": "Got it — I have the context from the summary and will continue."},
         ]
         if self.plan:  # the approved plan stays word for word
@@ -1096,6 +1126,7 @@ class Agent:
             )
             self.skills_loaded.clear()
         self.ctx_used = sum(len(m["content"]) for m in self.messages) // 3
+        self.summarized_size = self.ctx_used
         self.reflected = len(self.messages)
         self.console.print(Panel(Markdown(summary), title="Compacted summary", border_style="blue"))
 

lcode/api.py

@@ -194,7 +194,9 @@ def open_agent(cwd: Path, options: Options, console: Console, hw: Hardware | Non
         embed_model=cfg["embed_model"],
         notify=cfg["notify"],
         notify_after=cfg["notify_after"],
+        secret_check=cfg["secret_check"],
         repeat_limit=cfg["repeat_limit"],
+        compact_at=cfg["compact_at"],
     )
     agent = Agent(ollama, settings, cwd, console=console)
     agent.interactive = options.interactive

lcode/cli.py

@@ -579,6 +579,7 @@ def cmd_bench(args) -> None:
             web="off",
             checkpoints=False,
             prune=not args.no_prune,
+            compact_at=cfg["compact_at"] if args.compact_at is None else args.compact_at,
             repo_map=args.repo_map,
             embed_model=cfg["embed_model"] if args.code_search else "off",
         )
@@ -678,6 +679,14 @@ def cmd_chat(args) -> None:
         sys.exit(exit_code)
 
 
+def compact_tokens(value: str) -> int:
+    """`--compact-at`: a token count such as 64k, or 0."""
+    try:
+        return config.parse_compact_at(value)
+    except ConfigError as e:
+        raise argparse.ArgumentTypeError(str(e)) from e
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="lcode",
@@ -765,6 +774,12 @@ def build_subparsers() -> dict[str, argparse.ArgumentParser]:
     p.add_argument("--repo-map", action="store_true", help="give the model the repository map (off by default)")
     p.add_argument("--code-search", action="store_true", help="index each task's folder and offer semantic code search")
     p.add_argument("--no-prune", action="store_true", help="don't remove old tool output (to compare, with --session)")
+    p.add_argument(
+        "--compact-at",
+        type=compact_tokens,
+        metavar="TOKENS",
+        help="summarize at this many tokens, e.g. 64k (default: the compact_at setting; 0: only at 85%%)",
+    )
     subs["bench"] = p
     p = argparse.ArgumentParser(
         prog="lcode index",

lcode/config.py

@@ -60,6 +60,8 @@ SETTINGS: dict[str, tuple[object, type, str]] = {
     "notify": (True, bool, "desktop notification when a long request is done or waits for you"),
     "notify_after": (30, int, "seconds a request runs before it notifies (with notify on)"),
     "repeat_limit": (3, int, "identical tool calls with identical results before lcode steps in (0: never)"),
+    "secret_check": (True, bool, "before git commit and git push, look for passwords, keys and tokens they'd add"),
+    "compact_at": (65536, int, "summarize at this many tokens even when the window is larger (0: only at 85%)"),
     "max_parallel_agents": (1, int, "subagents that may run at the same time (more needs OLLAMA_NUM_PARALLEL)"),
 }
 ENV_OVERRIDES = {
@@ -80,6 +82,19 @@ class ConfigError(ValueError):
     pass
 
 
+MIN_COMPACT_AT = 16384  # below this, the system prompt and a summary alone come close to the ceiling
+
+
+def parse_compact_at(value: str | int) -> int:
+    """compact_at: a token count like parse_context's, at least 16K, or 0 (off)."""
+    if str(value).strip().lower() in ("0", "off", "false", "none"):
+        return 0
+    tokens = parse_context(value)
+    if tokens < MIN_COMPACT_AT:
+        raise ConfigError(f"compact_at {tokens} is too small: use at least 16k, or 0 to turn it off")
+    return tokens
+
+
 def parse_context(value: str | int) -> int:
     """Parse '131072', '128k', '128K' or '1m' into a token count (k = 1024)."""
     if isinstance(value, int):
@@ -117,6 +132,8 @@ def coerce(key: str, value: object) -> object:
     _, typ, _ = SETTINGS[key]
     if key == "context":
         return parse_context(value)  # type: ignore[arg-type]
+    if key == "compact_at":
+        return parse_compact_at(value)  # type: ignore[arg-type]
     if key in ("ollama_host", "searxng_url"):
         return normalize_host(str(value))
     if key == "base_url":

lcode/context.py

@@ -29,6 +29,10 @@ if TYPE_CHECKING:
 KEEP_RECENT_REQUESTS = 2  # tool results of the last requests stay as they are
 MIN_PRUNE_CHARS = 400  # shorter results aren't worth replacing
 PRUNED = "[lcode: "  # how stubs start, so they're never pruned twice
+GONE = (  # a removed file read: firm, because models answer from the stub instead of reading it again
+    "earlier; its content was removed to save context and isn't in this conversation any more. Read it again "
+    "before answering anything about it.]"
+)
 KEEP_TOOLS = {"agent", "skill", "memory", "todo_write", "present_plan"}  # short, or still guiding the work
 ERROR_LINE = re.compile(
     r"error|exception|traceback|failed|failure|fatal|panic|assert|denied|not found|cannot|can't|undefined", re.I
@@ -37,6 +41,56 @@ MAX_ERROR_LINES = 40
 OUTPUT_DAYS = 7  # saved full outputs are deleted after this
 
 
+# ----------------------------------------------------------------------------- the user's requests
+
+REQUESTS_HEADER = "[The user's requests in this conversation, word for word, oldest first]"
+MAX_REQUESTS = 8  # the most recent ones are kept through a summary
+MAX_REQUEST_CHARS = 1500
+# What lcode appends to the user's words: attached files, images and folders, and its own notes.
+ATTACHED = re.compile(r"\n\n(?:<file path=|<image path=|<directory path=|\[lcode[:\]] |\[The user wants the )")
+
+
+def request_text(content: str) -> str:
+    """The user's own words in a message, without what lcode attached to it."""
+    cut = ATTACHED.search(content)
+    return (content[: cut.start()] if cut else content).strip()
+
+
+def user_requests(messages: list[dict]) -> list[str]:
+    """What the user asked in the conversation, oldest first, including the requests an earlier summary kept."""
+    requests: list[str] = []
+    for message in messages:
+        content = message.get("content") or ""
+        if message.get("role") != "user":
+            continue
+        if content.startswith(SUMMARY_PREFIX):
+            start = content.find(REQUESTS_HEADER)
+            if start >= 0:
+                rest = content[start + len(REQUESTS_HEADER) :]
+                try:
+                    kept, _ = json.JSONDecoder().raw_decode(rest[rest.index("[") :])
+                    requests += [str(r) for r in kept]
+                except ValueError:
+                    pass
+            continue
+        if content.startswith("[lcode]"):
+            continue  # lcode's own nudges
+        text = request_text(content)
+        if text:
+            requests.append(text)
+    return requests
+
+
+def requests_block(requests: list[str]) -> str:
+    """The user's latest requests, word for word, to keep the goal through a summary."""
+    if not requests:
+        return ""
+    kept = [r if len(r) <= MAX_REQUEST_CHARS else f"{r[: MAX_REQUEST_CHARS - 200]} […] {r[-150:]}" for r in requests]
+    older = len(kept) - MAX_REQUESTS
+    note = f" (and {older} earlier, in the summary)" if older > 0 else ""
+    return f"\n\n{REQUESTS_HEADER}{note}\n{json.dumps(kept[-MAX_REQUESTS:], ensure_ascii=False, indent=1)}"
+
+
 # ----------------------------------------------------------------------------- pruning
 
 
@@ -63,7 +117,7 @@ def stub(name: str, args: dict, content: str) -> str:
         if args.get("offset") or args.get("limit"):
             start = int(args.get("offset") or 1)
             span = f" from line {start}"
-        return f"{PRUNED}read {args.get('path', 'a file')}{span} earlier; removed to save context. Read it again if you need it.]"
+        return f"{PRUNED}read {args.get('path', 'a file')}{span} {GONE}"
     if name == "bash":
         command = " ".join(str(args.get("command", "")).split())
         command = command if len(command) <= 80 else command[:79] + "…"
@@ -92,8 +146,9 @@ def boundary(messages: list[dict], keep: int = KEEP_RECENT_REQUESTS) -> int:
     return requests[-keep] if len(requests) >= keep else 0
 
 
-def prune(messages: list[dict], keep: int = KEEP_RECENT_REQUESTS) -> int:
-    """Replace stale tool results with stubs, in place. Returns the number of characters saved."""
+def prune(messages: list[dict], keep: int = KEEP_RECENT_REQUESTS, removed: list[str] | None = None) -> int:
+    """Replace stale tool results with stubs, in place. Returns the number of characters saved; the files and
+    pages whose only copy went are added to `removed`."""
     calls = calls_for_results(messages)
     edge = boundary(messages, keep)
     # A read is superseded by a later read of the whole file, or of the same lines: not by a later
@@ -118,12 +173,34 @@ def prune(messages: list[dict], keep: int = KEEP_RECENT_REQUESTS) -> int:
             continue
         new = stub(name, args, content)
         if reread:
-            new = new.replace("earlier; removed", "earlier (read again later); removed")
+            new = new.replace(
+                GONE, "earlier (read again later); removed to save context. Read it again if you need it.]"
+            )
+        elif removed is not None and name in ("read_file", "web_fetch"):
+            what = str(args.get("path") or args.get("url") or "")
+            if what and what not in removed:
+                removed.append(what)
         messages[i]["content"] = new
         saved += len(content) - len(new)
     return saved
 
 
+def note_removed(messages: list[dict], removed: list[str]) -> None:
+    """Tell the model, next to the request it's working on, which files it no longer has: a stub deep in the
+    conversation is easy to overlook, and models then answer from memory instead of reading again."""
+    if not removed:
+        return
+    for message in reversed(messages):
+        content = str(message.get("content", ""))
+        if message.get("role") == "user" and not content.startswith(("[lcode]", SUMMARY_PREFIX)):
+            names = ", ".join(removed[:10]) + (f" and {len(removed) - 10} more" if len(removed) > 10 else "")
+            message["content"] = (
+                f"{content}\n\n[lcode] To make room, lcode removed what you read earlier from: {names}. It isn't in "
+                "this conversation any more: read it again before answering anything about it, don't answer from memory."
+            )
+            return
+
+
 # ----------------------------------------------------------------------------- long output
 
 

lcode/gitflow.py

@@ -24,6 +24,7 @@ from typing import TYPE_CHECKING
 
 from rich.markup import escape
 from rich.panel import Panel
+from rich.text import Text
 
 if TYPE_CHECKING:
     from lcode.agent import Agent
@@ -316,6 +317,9 @@ def commit_command(agent: Agent, hint: str = "") -> None:
         chosen = [f for f in answer.get("files") or [] if f in paths]
         files = chosen or paths
     left_out = [p for p in paths if p not in files]
+    from lcode import secretscan
+
+    secrets_found = secretscan.in_files(root, files, bool(staged)) if agent.settings.secret_check else []
     while True:
         c.print(Panel(escape(message), title="Commit message", title_align="left", border_style="cyan"))
         c.print(f"  [bold]Files[/] {escape(', '.join(files))}")
@@ -323,6 +327,9 @@ def commit_command(agent: Agent, hint: str = "") -> None:
             c.print(f"  [dim]Left out (unrelated, the model thinks): {escape(', '.join(left_out))}[/]")
         if risky:
             c.print(f"  [dim]Left out (may hold secrets; stage them yourself if needed): {escape(', '.join(risky))}[/]")
+        if secrets_found:
+            c.print(Text(secretscan.report("commit", secrets_found), style="red"))
+            c.print("  [dim]Commit them only if they aren't secrets; otherwise move them out of the code first.[/]")
         choice = confirm(agent, "  [y] commit · [e] edit the message · [n] cancel: ", message)
         if choice == "e":
             from lcode.planning import edit_text
@@ -552,6 +559,14 @@ def pr_command(agent: Agent, arg: str = "") -> None:
 
 
 def push(agent: Agent, root: Path, remote: str, branch: str) -> bool:
+    from lcode import secretscan
+
+    found = secretscan.outgoing(root, branch) if agent.settings.secret_check else []
+    if found:
+        agent.console.print(Text(secretscan.report("push", found), style="red"))
+        if confirm(agent, "  Push anyway? [y]es / [n]o: ") != "y":
+            agent.console.print("Not pushed.")
+            return False
     try:
         with agent.console.status(f"Pushing {escape(branch)}…", spinner="dots"):
             git(root, "push", "-q", "-u", remote, branch, timeout=300)

lcode/repl.py

@@ -242,7 +242,8 @@ def context_command(agent: Agent, arg: str, hardware: Hardware) -> None:
     c.print(
         f"In use: {format_tokens(agent.ctx_used)} of {format_tokens(s.context)} tokens "
         f"({100 * agent.ctx_used / s.context:.0f}%) in {len(agent.messages)} message"
-        f"{'' if len(agent.messages) == 1 else 's'}; auto-compacts at 85%."
+        f"{'' if len(agent.messages) == 1 else 's'}; makes room at {format_tokens(agent.compact_limit())} "
+        + ("(compact_at)." if agent.compact_limit() < AUTO_COMPACT_RATIO * s.context else "(85%).")
     )
     if agent.mcp and agent.mcp.ready():
         how = "found on demand" if agent.mcp.searching(s.context) else "sent with every request"

lcode/secretscan.py

@@ -0,0 +1,315 @@
+"""Stop secrets before they're committed or pushed.
+
+Before the model runs `git commit` or `git push`, lcode scans what that would add to the repository's
+history: the lines a commit adds (including files a `git add` in the same command stages), or every commit
+a push would send. Passwords and tokens in code, private keys, credentials in URLs (`rtsp://user:pass@…`)
+and `.env` files are reported with their file and line. In `ask` mode the user decides; in the other modes
+the command doesn't run, and the model is told to move the secret out of the code.
+
+A line that holds a value which only looks like a secret can carry `lcode: allow-secret` (or
+`gitleaks:allow`) in a comment, and is skipped.
+"""
+
+from __future__ import annotations
+
+import re
+import shlex
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from lcode.memory import SECRET_PATTERNS
+
+ALLOW = ("lcode: allow-secret", "lcode:allow-secret", "gitleaks:allow")
+MAX_FINDINGS = 10
+GIT_TIMEOUT = 20
+MAX_FILE = 1_000_000  # bytes of an untracked file that are scanned
+
+# Values that only stand in for a secret.
+PLACEHOLDER = re.compile(
+    r"^(?:change[_-]?me\w*|your[_-]?\w*|x+|\*+|\.+|…+|<[^>]*>"
+    r"|(?:dummy|example|placeholder|redacted|secret|password|passwd|pass|pwd|token|test|changeit|none|null"
+    r"|true|false|required|optional|hidden|masked)(?:[_-]?here)?)$",
+    re.I,
+)
+# Words that only appear in made-up values: "change-me-in-production", "test-token", "example-key"…
+FAKE_WORDS = (
+    "example",
+    "sample",
+    "dummy",
+    "fake",
+    "changeme",
+    "change-me",
+    "change_me",
+    "placeholder",
+    "redacted",
+    "test",
+)
+# A password, key or token written in code or configuration as a literal value.
+ASSIGNED = re.compile(
+    r"""(?P<name>\b[\w.-]*(?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key"""
+    r"""|client[_-]?secret)[\w.-]*)['"]?\s*(?::|=|:=|=>)\s*(?P<quote>['"]?)(?P<value>[^\s'"`,;)}\]]{6,})"""
+    r"""(?P=quote)""",
+    re.I,
+)
+# Names that hold something about a secret (where it is, how it looks), not the secret itself.
+ABOUT = re.compile(
+    r"(?:url|uri|endpoint|file|path|dir|env|var|variable|name|field|header|type|label|prompt|hint|length|len|min|"
+    r"max|policy|pattern|regex|format|count|expires?|expiry|ttl|timeout|limit|required|enabled|hash|salt_rounds)$",
+    re.I,
+)
+EXTRA = [
+    ("an xAI API key", re.compile(r"\bxai-[A-Za-z0-9]{20,}")),
+    ("a Hugging Face token", re.compile(r"\bhf_[A-Za-z0-9]{30,}")),
+    ("a Stripe key", re.compile(r"\b(?:sk|rk)_live_[A-Za-z0-9]{20,}")),
+]
+CONFIG_FILES = re.compile(r"(?:\.(?:env|ini|cfg|conf|config|properties|ya?ml|toml)|(?:^|/)\.env[\w.-]*)$", re.I)
+URL_PASSWORD = re.compile(r"\b[a-z][a-z0-9+.\-]*://(?P<user>[^\s/:@'\"]+):(?P<value>[^\s/@'\"]+)@", re.I)
+STRONG = [p for p in SECRET_PATTERNS if p[0] not in ("a password in a URL", "a password or key")] + EXTRA
+ENV_FILE = re.compile(r"(?:^|/)\.env(?:\.[\w-]+)?$")
+ENV_EXAMPLES = re.compile(r"\.(?:example|sample|template|dist|defaults?)$")
+
+
+@dataclass
+class Finding:
+    path: str
+    line: int  # 0: the whole file
+    label: str  # what it looks like, e.g. "a password"
+    excerpt: str  # the line, with the secret masked
+    commit: str = ""  # for a push: the commit that adds it
+
+    def describe(self, excerpt: bool = True) -> str:
+        where = f"{self.path}:{self.line}" if self.line else self.path
+        if self.commit:
+            where += f" (commit {self.commit})"
+        return f"{where}: {self.label}" + (f": {self.excerpt}" if self.excerpt and excerpt else "")
+
+
+def mask(value: str) -> str:
+    return value[:2] + "…" if len(value) > 4 else "…"
+
+
+def placeholder(value: str) -> bool:
+    """A value that stands in for a secret: CHANGE_ME, <token>, ${PASSWORD}, a path, a web address…"""
+    if PLACEHOLDER.match(value) or any(c in value for c in "{}$%<>"):
+        return True
+    if any(word in value.lower() for word in FAKE_WORDS):
+        return True
+    return value.startswith(("/", "./", "~/", "http:", "https:", "os.", "process.env", "env."))
+
+
+def scan_line(text: str, config: bool = False) -> tuple[str, str] | None:
+    """What kind of secret the line holds, and the line with it masked. In configuration files (`config`) a
+    value counts without quotes too, as in `PASSWORD=hunter22` or `password: hunter22`."""
+    if any(marker in text for marker in ALLOW):
+        return None
+    shown = text.strip()[:160]
+    for label, pattern in STRONG:
+        found = pattern.search(text)
+        if found:
+            return label, shown.replace(found.group(0), mask(found.group(0)))
+    found = URL_PASSWORD.search(text)
+    if found and not placeholder(found.group("value")) and not placeholder(found.group("user")):
+        return "a password in a URL", shown.replace(found.group("value"), mask(found.group("value")))
+    for found in ASSIGNED.finditer(text):
+        value = found.group("value")
+        if placeholder(value) or ABOUT.search(found.group("name")):
+            continue
+        if not found.group("quote") and not (config and re.match(r"^\s*(?:export\s+)?[\w.-]+\s*[:=]", text)):
+            continue  # in code, only a quoted literal is a value: `password = args.password` isn't one
+        return "a password or key", shown.replace(value, mask(value))
+    return None
+
+
+def scan_file(path: str, lines: list[tuple[int, str]], commit: str = "") -> list[Finding]:
+    if ENV_FILE.search(path) and not ENV_EXAMPLES.search(path):
+        return [Finding(path, 0, "an environment file (.env), which usually holds secrets", "", commit)]
+    findings = []
+    config = bool(CONFIG_FILES.search(path))
+    for number, text in lines:
+        hit = scan_line(text, config)
+        if hit:
+            findings.append(Finding(path, number, hit[0], hit[1], commit))
+    return findings
+
+
+def scan_diff(diff: str) -> list[Finding]:
+    """The secrets in the lines a unified diff adds (`git diff`, or `git log -p` with commit: lines)."""
+    findings: list[Finding] = []
+    path, number, commit = "", 0, ""
+    added: list[tuple[int, str]] = []
+
+    def flush() -> None:
+        if path and (added or ENV_FILE.search(path)):
+            findings.extend(scan_file(path, added, commit))
+        added.clear()
+
+    for line in diff.splitlines():
+        if line.startswith("commit:"):
+            flush()
+            commit, path = line.split(":", 1)[1].strip(), ""
+        elif line.startswith("+++ "):
+            flush()
+            target = line[4:].strip()
+            path = "" if target == "/dev/null" else target.removeprefix("b/")
+        elif line.startswith("@@"):
+            hunk = re.search(r"\+(\d+)", line)
+            number = int(hunk.group(1)) if hunk else 0
+        elif line.startswith("+") and path:
+            added.append((number, line[1:]))
+            number += 1
+        elif line.startswith(" "):
+            number += 1
+    flush()
+    return findings
+
+
+def git(cwd: Path, *args: str) -> str:
+    try:
+        done = subprocess.run(
+            ["git", *args], cwd=cwd, capture_output=True, text=True, errors="replace", timeout=GIT_TIMEOUT
+        )
+    except (OSError, subprocess.SubprocessError):
+        return ""
+    return done.stdout if done.returncode == 0 else ""
+
+
+GIT_VERB = re.compile(r"\bgit\b(?:\s+-[-\w=.]+(?:\s+[^\s'\"-]\S*)?)*\s+(commit|push)\b")
+
+
+@dataclass
+class GitSteps:
+    adds: list[list[str]]  # the arguments of each `git add`
+    commit: list[str] | None  # the arguments of `git commit`
+    push: list[str] | None  # the arguments of `git push`
+
+
+def git_steps(command: str) -> GitSteps | None:
+    """The git add, commit and push a shell command runs, or None when it commits and pushes nothing."""
+    steps = GitSteps([], None, None)
+    for segment in re.split(r"&&|\|\||;|\n|\|", command):
+        try:
+            words = shlex.split(segment)
+        except ValueError:
+            words = segment.split()
+        while words and re.fullmatch(r"\w+=\S*", words[0]):
+            words = words[1:]  # VAR=value in front
+        if len(words) < 2 or words[0] != "git":
+            continue
+        i = 1
+        while i < len(words) and words[i].startswith("-"):
+            i += 2 if words[i] in ("-C", "-c") else 1  # git -C dir / -c key=value
+        if i >= len(words):
+            continue
+        verb, rest = words[i].strip("'\""), words[i + 1 :]
+        if verb == "add":
+            steps.adds.append(rest)
+        elif verb == "commit":
+            steps.commit = rest
+        elif verb == "push":
+            steps.push = rest
+    # Inside quotes, as in bash -c "git add . && git commit …", sh -c or eval, what gets staged can't be told
+    # from the words: assume everything.
+    nested = [verb for m in re.finditer(r"(['\"])(.*?)\1", command, re.S) for verb in GIT_VERB.findall(m.group(2))]
+    if steps.commit is None and steps.push is None:
+        nested += GIT_VERB.findall(command)
+    if "commit" in nested:
+        steps.commit, steps.adds = ["--all"], [["."]]
+    if "push" in nested and steps.push is None:
+        steps.push = []
+    return steps if steps.commit is not None or steps.push is not None else None
+
+
+def untracked(cwd: Path, paths: list[str]) -> list[Finding]:
+    findings = []
+    for name in git(cwd, "ls-files", "--others", "--exclude-standard", "--", *paths).splitlines():
+        file = cwd / name
+        try:
+            if file.stat().st_size > MAX_FILE:
+                continue
+            text = file.read_text(errors="strict")
+        except (OSError, UnicodeDecodeError):
+            continue  # binary or unreadable
+        findings.extend(scan_file(name, list(enumerate(text.splitlines(), 1))))
+    return findings
+
+
+def to_commit(cwd: Path, steps: GitSteps) -> list[Finding]:
+    """What `git add` and `git commit` in the command would commit: the staged changes, plus what it stages."""
+    args = steps.commit or []
+    everything = any(a in ("-a", "--all") or re.fullmatch(r"-[a-zA-Z]*a[a-zA-Z]*", a) for a in args if a != "--amend")
+    findings = scan_diff(git(cwd, "diff", "--cached", "--no-color", "--no-ext-diff", "-U0"))
+    if everything:
+        findings += scan_diff(git(cwd, "diff", "--no-color", "--no-ext-diff", "-U0"))
+    for add in steps.adds:
+        options = [a for a in add if a.startswith("-")]
+        paths = [a for a in add if not a.startswith("-")] or (["."] if {"-A", "--all"} & set(options) else [])
+        if not paths and "-u" not in options and "--update" not in options:
+            continue
+        paths = paths or ["."]
+        findings += scan_diff(git(cwd, "diff", "--no-color", "--no-ext-diff", "-U0", "--", *paths))
+        if "-u" not in options and "--update" not in options:
+            findings += untracked(cwd, paths)
+    return findings
+
+
+def to_push(cwd: Path, steps: GitSteps) -> list[Finding]:
+    """The secrets in the commits a push would send that no remote has yet."""
+    refs = [a for a in (steps.push or []) if not a.startswith("-")]
+    branches = [r.split(":", 1)[0].lstrip("+") for r in refs[1:]] or ["HEAD"]
+    findings: list[Finding] = []
+    for branch in branches:
+        if not branch or not git(cwd, "rev-parse", "--verify", "--quiet", f"{branch}^{{commit}}").strip():
+            continue
+        log = git(
+            cwd, "log", "-p", "--no-color", "--no-ext-diff", "-U0", "--format=commit:%h", branch, "--not", "--remotes"
+        )
+        findings += scan_diff(log)
+    return findings
+
+
+def check(cwd: Path, command: str) -> tuple[str, list[Finding]]:
+    """('commit' or 'push', the secrets it would add to the history) for a shell command."""
+    steps = git_steps(command)
+    if steps is None or not git(cwd, "rev-parse", "--git-dir").strip():
+        return "", []
+    findings: list[Finding] = []
+    if steps.commit is not None:
+        findings += to_commit(cwd, steps)
+    if steps.push is not None:
+        findings += to_push(cwd, steps)
+    unique = list({(f.path, f.line, f.label): f for f in findings}.values())
+    return ("push" if steps.push is not None and steps.commit is None else "commit"), unique
+
+
+def in_files(cwd: Path, paths: list[str], staged: bool) -> list[Finding]:
+    """For lcode's /commit: the secrets in the staged changes, or in the changes to these files."""
+    if staged:
+        return scan_diff(git(cwd, "diff", "--cached", "--no-color", "--no-ext-diff", "-U0"))
+    if not paths:
+        return []
+    diffs = [
+        git(cwd, "diff", *staged_flag, "--no-color", "--no-ext-diff", "-U0", "--", *paths)
+        for staged_flag in ([], ["--cached"])
+    ]
+    return scan_diff("\n".join(diffs)) + untracked(cwd, paths)
+
+
+def outgoing(cwd: Path, branch: str = "HEAD") -> list[Finding]:
+    """For lcode's /pr: the secrets in the commits of `branch` that no remote has yet."""
+    return to_push(cwd, GitSteps([], None, ["origin", branch]))
+
+
+def report(action: str, findings: list[Finding], excerpts: bool = True) -> str:
+    """The findings, one per line; without `excerpts` for places others read, like a public comment."""
+    lines = [f"- {f.describe(excerpts)}" for f in findings[:MAX_FINDINGS]]
+    if len(findings) > MAX_FINDINGS:
+        lines.append(f"- … and {len(findings) - MAX_FINDINGS} more")
+    return f"This {action} would add what looks like a secret to the repository's history:\n" + "\n".join(lines)
+
+
+ADVICE = (
+    "lcode didn't run the command. Don't commit secrets: read them from an environment variable or from a "
+    "configuration file that git ignores (and commit an example with placeholders instead). If one of these "
+    "isn't a secret, tell the user: they decide."
+)

lcode/tools.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import difflib
 import fnmatch
+import hashlib
 import inspect
 import json
 import os
@@ -19,11 +20,12 @@ from collections.abc import Callable
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from rich.console import Group
 from rich.panel import Panel
 from rich.syntax import Syntax
 from rich.text import Text
 
-from lcode import web
+from lcode import secretscan, web
 from lcode.context import shorten
 from lcode.jobs import JobError, describe, stop_leftovers
 from lcode.permissions import bash_key, is_read_only
@@ -823,7 +825,28 @@ class Toolbox:
         verdict = self.agent.perms.rule("bash", command)
         if verdict is not None and not verdict[0]:  # deny rules apply to read-only commands too
             return verdict[1]
-        if not is_read_only(command) and not (sandbox and self.agent.perms.mode == "auto-edit") and verdict is None:
+        approved = False
+        if self.agent.settings.secret_check and not is_read_only(command):
+            action, found = secretscan.check(self.agent.cwd, command)
+            if found:
+                text = secretscan.report(action, found)
+                if self.agent.perms.mode != "ask":  # nobody looks at each command: it doesn't run
+                    self.console.print(Text(f"  ⎿ {text}", style="red"))
+                    return f"Error: lcode's secret check stopped this command. {text}\n\n{secretscan.ADVICE}"
+                body = Group(Text(text + "\n", style="red"), Syntax(command, "bash", theme="monokai", word_wrap=True))
+                key = "secret:" + hashlib.sha256(text.encode()).hexdigest()[:16]
+                ok, feedback = self.agent.perms.request(
+                    key, "secret", f"This {action} looks like it holds a secret", body, command
+                )
+                if not ok:
+                    return f"{feedback}\n\n{text}\n{secretscan.ADVICE}"
+                approved = True
+        if (
+            not approved
+            and not is_read_only(command)
+            and not (sandbox and self.agent.perms.mode == "auto-edit")
+            and verdict is None
+        ):
             body = Syntax(command, "bash", theme="monokai", word_wrap=True)
             where = "in the sandbox" if sandbox else f"in {self.agent.cwd}"
             ok, feedback = self.agent.perms.request(bash_key(command), "bash", f"Run command ({where})", body, command)

{lcode_cli-0.25.0.dist-info → lcode_cli-0.27.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: lcode-cli
-Version: 0.25.0
+Version: 0.27.0
 Summary: A local-first terminal coding agent powered by open-weight models on your own GPU or Mac, via Ollama.
 Project-URL: Homepage, https://nasser1941.github.io/lcode/
 Project-URL: Documentation, https://nasser1941.github.io/lcode/

{lcode_cli-0.25.0.dist-info → lcode_cli-0.27.0.dist-info}/RECORD

@@ -1,23 +1,23 @@
-lcode/__init__.py,sha256=znAn1x1nESR2keV7zRFdQd7reQE9a4zKFE1_LsWuoas,473
+lcode/__init__.py,sha256=JUcNjfBPUQX6hKCFSklgC04DhC6xrDnysFFhCU7p5Xc,473
 lcode/__main__.py,sha256=s5N5pryX2FAu93GGZ3FxUrIXHWGzxz66lKUoaV_h8Ws,35
 lcode/acp.py,sha256=Tpc8lsvffZSYbTRI4t7X4E2chahqZIPcKeYIcL5bV3Y,29035
-lcode/action.py,sha256=W-QqdGt4VAVLq7mcIdvL9UrEOwDEWvkZ26gNDv8k1kg,16330
-lcode/agent.py,sha256=naua7OJW9BXhv8mpF1QLdclnFy6RHZAjw9LYTt_dBy0,56201
-lcode/api.py,sha256=FQ6aVLB936Cu2XEBmuOoq0u1DfOiExmgap0Y3WxJ2mQ,15376
+lcode/action.py,sha256=wuT5F9EJCdEligPeytzmEi8FpahWBhXer528tpuTj_k,16711
+lcode/agent.py,sha256=tGNjbs8cw2X7IUBb70juI3JH5IpXmmIXOcteTsMhOtA,57936
+lcode/api.py,sha256=aqWnGCy1jNuIktJEHo9ncbHNTXn2QO3BoNiggyo9ZmQ,15456
 lcode/backends.py,sha256=dtJHrC6ys73bRDg64z0tQz1PavmeAkiODZu1fOO9ExQ,20635
 lcode/bench.py,sha256=7ICipu_2GXW6vE519XeZucc2GSWA44votT8JvqcNv1k,40964
 lcode/bench_backend.py,sha256=X2fo-zRX92Xp3KMxSsFlWioc2vRupBTtowj5kLME8mw,14287
 lcode/bench_runtime.py,sha256=EjzNgIvUkdbT8cbSy1TpmlyeIk6e-ipjlDFpM7Dnqx0,6835
 lcode/catalog.py,sha256=pZ1azX9OH2r4JE2c6E5OfwywpgS39-W2oXzoCHJhmtM,4141
 lcode/checkpoints.py,sha256=eqPtmPe44-aweWqc2H21ROogwzIt3NpvcUobsaCHoGY,18354
-lcode/cli.py,sha256=cH5mcYi54Aofcv0PDNgFmFFQX8OQjhwTGdbWMUM_FG8,40289
+lcode/cli.py,sha256=M6Mjd5BROSGlZj4J6TrSZMJqI-OVfUEWPoC0TQyOQ28,40831
 lcode/clipboard.py,sha256=nTSdJyCXPdtjH947RjvB3QvRqnL5HZsS9KB8EkC42cc,5138
 lcode/codesearch.py,sha256=Io8leN1nb_Y-Dvlm9xaGtwUWYXX-ooswuznJV6dGU1s,10766
-lcode/config.py,sha256=fGjK4fM_YRY0qDQmbIVT3FAfL2I2UxLfYEFxnAI-Xu4,10365
-lcode/context.py,sha256=qJymLE3eYeaF6qVOzlUwDURqlky__8cskl_6vXeU3AA,10701
+lcode/config.py,sha256=q1zGzH3p5U23ai6O9Jai3dS-s30CuQ9gc5cWX0_yxWE,11195
+lcode/context.py,sha256=_f0NbXyjA4SyH2RCr49AiWwtAfr_PVDaqyFQ5fZDhkg,14404
 lcode/extensions.py,sha256=W0HFDl0BvROzr7i_qtabHdIbacBgP5o5qIJLzoM2oNE,16806
 lcode/frontmatter.py,sha256=Bx0n4vLPhp7ZCWx-g7xqzqNy5VoKBxyeeikh5UFB_gA,2441
-lcode/gitflow.py,sha256=y78dqqh80is-MiDTnSHnK9vjx0vFovor_SZ8hW1NJaU,27167
+lcode/gitflow.py,sha256=6j_nnCBZXkX1qhTI7K4CyZ5D0NAZLpH96weFEo64AkQ,27913
 lcode/hardware.py,sha256=2VbknZCxTZJwJUD8YWdQ7ryEQo_LcjtoOBOHOU9Y0fw,3085
 lcode/hooks.py,sha256=K0itOiAuH4rxndJ9gne_uCrqby6oWAzFPRrfB3tCq-E,9324
 lcode/jobs.py,sha256=P1P3FqI6EqL4LgdMX9YGZwCW0Bh5BRXFA_FnQxpHNjE,5618
@@ -31,12 +31,13 @@ lcode/permissions.py,sha256=i04YLtqYCxwApgawomPp4d35uk9HAJtpKY__gfw8j9s,6347
 lcode/planning.py,sha256=Q2upaoZPnXDMQT2F-ETxX1lk_wDntTzhSKNVYIQSoQA,6540
 lcode/render.py,sha256=gMvf4W3qkLUJ4Vot3az_hP_zRDAItJC9pP-68dHFkfA,1375
 lcode/repeats.py,sha256=uzyAewX03nJf5dKwYFH4y9ZDnh0Lvy2k0qC2vdntM5Y,4061
-lcode/repl.py,sha256=ZKd4l4wDGKHVaGz8bxjLGc2gRc2W-RpHPQYG9M2A_Zw,49441
+lcode/repl.py,sha256=Y53tTvdoTP6CZOtq96CzbvPcwVlYNc9fppMV0hAlRSY,49573
 lcode/repomap.py,sha256=1GRJgEUjBl9XHSXsF7tSGXBCEgsY-3CzwMohaTudZ_M,13969
 lcode/sandbox.py,sha256=u_pkrVRXhCGhh-wFDPwuZ3jGG7UeXxXRAeBNsm37ZvE,8608
+lcode/secretscan.py,sha256=6Euvfk2gCtK7lZbuQD84zgWuSECi6OnSw8V5rFu1tfU,13575
 lcode/sessions.py,sha256=wOR-J0EHgJxLSPboKPHnvBcp5VTUYTj3tZWgpsa6-K4,4013
 lcode/subagents.py,sha256=Z2RHp7an16dhOGReucVIEOAnXjxM8jx7sHvWd_Fp2V4,29142
-lcode/tools.py,sha256=pOdA4ihQHSs8yWUzfxhiLw3GajF5Dr1edxVX0Akr6CU,47257
+lcode/tools.py,sha256=jqxMcyvvDZs0rV0AROYxpJ4krxbU6LbkOWnxiLEqq1o,48426
 lcode/update.py,sha256=i92bnW252VneYh0vD6djxmXsDOa5MXP20bjlLZUue_c,25236
 lcode/vision.py,sha256=w33QPmdGwP33wM4mvmAH4amFDZ1xzeXhEsSLJ-STSHY,4243
 lcode/web.py,sha256=vyxKbSRJTl1Lj3hj7Yd_NwTgbv0Xi6dup2YfUXjqp-Y,14579
@@ -56,8 +57,8 @@ lcode/servers/__init__.py,sha256=qrIhxEodEB8cjKEQ4Cicc0GsQhE2OH-9qlVHzr3-tq8,77
 lcode/servers/encord.py,sha256=WSauawL1UKYZCUAG1oalcKL_An3DVof-JM6xIgFyQSE,12769
 lcode/servers/valohai.py,sha256=e5xZGIoa4cja4f5i85q1GpL9c7cglUjV8jN2co1CFMY,14430
 lcode/servers/xai.py,sha256=h2emAvHSnkuB9WmJInlynsDhvxJwNVD2AHFX7vqMXNc,13627
-lcode_cli-0.25.0.dist-info/METADATA,sha256=j0rf2PzRDAvWLYN-2RizBuLuel-tmvhfgf_m_pvTHkE,12620
-lcode_cli-0.25.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-lcode_cli-0.25.0.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
-lcode_cli-0.25.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
-lcode_cli-0.25.0.dist-info/RECORD,,
+lcode_cli-0.27.0.dist-info/METADATA,sha256=pASpYtQ_MeN-50aD2sdpTfxMKuX2Tz00LaJa2pT1DmM,12620
+lcode_cli-0.27.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+lcode_cli-0.27.0.dist-info/entry_points.txt,sha256=Hxsn_itbnbgft3U0FwxWdmEt0yoAV2LB-nJDnL9WT20,172
+lcode_cli-0.27.0.dist-info/licenses/LICENSE,sha256=JWjmoL7nNIC1XUa-dZm2hwLUgjUQMFo6k8APb0idNzs,1074
+lcode_cli-0.27.0.dist-info/RECORD,,