coderouter-cli 2.11.0__py3-none-any.whl → 2.11.2__py3-none-any.whl

coderouter/translation/tool_repair.py

@@ -90,12 +90,28 @@ from __future__ import annotations
 
 import contextlib
 import json
+import logging
 import re
 import uuid
+from array import array
 from typing import Any
 
 __all__ = ["deduplicate_tool_calls", "repair_tool_calls_in_text"]
 
+# Plain stdlib logger (same idiom as coderouter/translation/anthropic.py).
+# This module is deliberately import-free of the rest of the package:
+# benchmarks/tool-repair/run_offline.py loads it straight from its source
+# path, with no `coderouter` package on sys.path.
+logger = logging.getLogger(__name__)
+
+# Hard ceiling on the text handed to the bare-JSON brace scanners. The
+# scanners themselves are linear, but every candidate they emit still costs a
+# JSON parse attempt plus a prose-cue check, and an assistant message this
+# large is never a tool call anyway. Above the limit the bare-JSON step is
+# skipped outright; the fenced / XML / R4a / R4c paths are regex-driven and
+# linear, so they keep running.
+_MAX_BARE_SCAN_CHARS = 262_144
+
 
 # ------------------------------------------------------------------
 # R2: shape detection + normalisation (with key aliases)
@@ -528,6 +544,139 @@ def _in_spans(pos: int, spans: list[tuple[int, int]]) -> bool:
     return any(start <= pos < end for start, end in spans)
 
 
+# ------------------------------------------------------------------
+# Brace scanning (bare-JSON candidate discovery)
+#
+# Both scanners below answer the same question for every ``{`` in the text:
+# "does a *fresh* scan started right here reach a balanced close?".  The
+# obvious implementation re-scans to end-of-text once per ``{``, which is
+# O(k*n) for k unclosed braces — a 48 KB assistant message full of stray
+# braces took ~20 s, and because ``to_anthropic_response`` is called from the
+# request path that stalled the whole server.
+#
+# Instead we precompute, in ONE right-to-left sweep, a table that answers
+# "starting at offset j outside any string, where is the first ``}`` that is
+# still at brace depth 0?".  The closing brace of a ``{`` at offset q is then
+# simply ``table[q + 1]``, so the driver below is a plain left-to-right walk
+# and the whole scan is linear in the length of the text.
+#
+# The table is defined by a right-to-left recurrence over the scanner's state
+# machine (outside a string / inside a string / previous char was a
+# backslash).  Only the "outside a string" row needs random access — the ``{``
+# case has to jump over the nested object it just resolved — so the
+# in-string rows are carried as rolling scalars instead of arrays.
+# ------------------------------------------------------------------
+
+
+def _scan_object_spans(text: str, close_table: array[int]) -> list[tuple[int, int, str]]:
+    """Walk ``text`` left to right emitting balanced ``{...}`` spans.
+
+    ``close_table`` comes from :func:`_strict_close_table` or
+    :func:`_lenient_close_table` and supplies, for each offset, the index of
+    the first depth-0 ``}`` at or after it (``-1`` when there is none). The
+    driver is shared; the flavour lives entirely in the table.
+
+    Returns (start, end_exclusive, substring) triples. See the two callers'
+    docstrings for the (deliberate, load-bearing) return-value contract.
+    """
+    out: list[tuple[int, int, str]] = []
+    n = len(text)
+    i = 0
+    while i < n:
+        start = text.find("{", i)
+        if start < 0:
+            break
+        close = close_table[start + 1]
+        if close < 0:
+            # Never closes — drop this `{` and resume at the next one. The
+            # restart deliberately begins from a clean "outside a string"
+            # state (see the contract note in the callers).
+            i = start + 1
+            continue
+        out.append((start, close + 1, text[start : close + 1]))
+        i = close + 1
+    return out
+
+
+def _strict_close_table(text: str) -> array[int]:
+    """Depth-0 close-brace lookup table with *strict* JSON string rules.
+
+    ``table[j]`` is the offset of the first ``}`` that a scanner starting at
+    ``j`` — outside any string literal, at brace depth 0 — would reach while
+    still at depth 0, or ``-1`` if it runs off the end. Only ``"`` opens a
+    string; ``\\`` escapes the next character inside one.
+
+    Built in a single right-to-left sweep, so building plus scanning is
+    O(len(text)) regardless of how many unbalanced braces the text contains.
+    """
+    n = len(text)
+    outside = array("i", [-1]) * (n + 1)
+    in_str = -1  # answer when the scan *enters* offset j+1 inside a "..."
+    escaped = -1  # ... inside a "..." with the previous char a backslash
+    for j in range(n - 1, -1, -1):
+        c = text[j]
+        nxt_outside = outside[j + 1]
+        nxt_in_str = in_str
+        nxt_escaped = escaped
+        # Escaped char: consumed verbatim, then we are back inside the string.
+        escaped = nxt_in_str
+        # Inside a string literal.
+        if c == "\\":
+            in_str = nxt_escaped
+        elif c == '"':
+            in_str = nxt_outside
+        else:
+            in_str = nxt_in_str
+        # Outside any string literal.
+        if c == '"':
+            outside[j] = nxt_in_str
+        elif c == "{":
+            inner = nxt_outside  # close of the nested object opening at j
+            outside[j] = -1 if inner < 0 else outside[inner + 1]
+        elif c == "}":
+            outside[j] = j
+        else:
+            outside[j] = nxt_outside
+    return outside
+
+
+def _lenient_close_table(text: str) -> array[int]:
+    """Depth-0 close-brace lookup table tolerating single-quoted strings.
+
+    Same contract as :func:`_strict_close_table`, but BOTH ``'`` and ``"``
+    open a string literal (and only the matching quote closes it), which is
+    what the lenient pass needs to survive Python-repr / single-quoted
+    malformed objects. Five scanner states instead of three; again only the
+    "outside a string" row needs to be materialised as an array.
+    """
+    n = len(text)
+    outside = array("i", [-1]) * (n + 1)
+    in_sq = in_dq = esc_sq = esc_dq = -1
+    for j in range(n - 1, -1, -1):
+        c = text[j]
+        nxt_outside = outside[j + 1]
+        nxt_in_sq, nxt_in_dq = in_sq, in_dq
+        nxt_esc_sq, nxt_esc_dq = esc_sq, esc_dq
+        esc_sq, esc_dq = nxt_in_sq, nxt_in_dq
+        if c == "\\":
+            in_sq, in_dq = nxt_esc_sq, nxt_esc_dq
+        else:
+            in_sq = nxt_outside if c == "'" else nxt_in_sq
+            in_dq = nxt_outside if c == '"' else nxt_in_dq
+        if c == "'":
+            outside[j] = nxt_in_sq
+        elif c == '"':
+            outside[j] = nxt_in_dq
+        elif c == "{":
+            inner = nxt_outside
+            outside[j] = -1 if inner < 0 else outside[inner + 1]
+        elif c == "}":
+            outside[j] = j
+        else:
+            outside[j] = nxt_outside
+    return outside
+
+
 def _find_balanced_json_objects(text: str) -> list[tuple[int, int, str]]:
     """Find top-level `{...}` JSON substrings by a brace-counter scan.
 
@@ -539,45 +688,27 @@ def _find_balanced_json_objects(text: str) -> list[tuple[int, int, str]]:
     brace balancing. Single-quoted / unquoted malformed objects are found by
     :func:`_find_candidate_object_spans` instead, which brace-balances without
     assuming JSON string syntax.
+
+    Return-value contract (load-bearing — downstream recall depends on all
+    three; do NOT "simplify" this into a single global stack pass):
+
+    1. **Top-level objects only.** Once a ``{`` closes, the scan resumes after
+       its ``}``, so objects nested inside a *closed* object are not returned.
+       ``'{"a": {"b": 1}}'`` yields the outer object alone.
+    2. **Nested objects surface when the outer never closes.** An unbalanced
+       ``{`` is dropped and the scan restarts at the next ``{``, so
+       ``'{"a": {"b":1}'`` yields ``[(6, 13, '{"b":1}')]`` — the inner
+       object is still repairable even though the outer one is broken.
+    3. **Every restart begins with a fresh string state.** The scan between
+       candidates does not track quotes, so a stray/unbalanced quote before a
+       ``{`` cannot swallow it. This is what keeps prose apostrophes
+       (``"I'll call the tool now."``) and quoted braces (``'{ "z{}" '`` ->
+       ``[(4, 6, '{}')]``) from hiding a real call. A single global pass that
+       carried string state across candidates loses both.
     """
-    out: list[tuple[int, int, str]] = []
-    n = len(text)
-    i = 0
-    while i < n:
-        if text[i] != "{":
-            i += 1
-            continue
-        # Scan forward to find a balanced close.
-        depth = 0
-        j = i
-        in_str = False
-        escape = False
-        while j < n:
-            c = text[j]
-            if escape:
-                escape = False
-            elif in_str:
-                if c == "\\":
-                    escape = True
-                elif c == '"':
-                    in_str = False
-            else:
-                if c == '"':
-                    in_str = True
-                elif c == "{":
-                    depth += 1
-                elif c == "}":
-                    depth -= 1
-                    if depth == 0:
-                        out.append((i, j + 1, text[i : j + 1]))
-                        i = j + 1
-                        break
-            j += 1
-        else:
-            # Ran off the end without closing — skip this `{` and move on.
-            i += 1
-            continue
-    return out
+    if "{" not in text:
+        return []
+    return _scan_object_spans(text, _strict_close_table(text))
 
 
 def _find_candidate_object_spans(text: str) -> list[tuple[int, int, str]]:
@@ -587,43 +718,17 @@ def _find_candidate_object_spans(text: str) -> list[tuple[int, int, str]]:
     respecting BOTH ``'`` and ``"`` string delimiters, so it can locate
     Python-repr / single-quoted malformed objects for the lenient parser.
     Returns (start, end_exclusive, substring).
+
+    The same three-point return-value contract documented on
+    :func:`_find_balanced_json_objects` applies here: top-level objects only,
+    nested objects surface when the outer never closes, and every restart
+    begins from a fresh string state. Point 3 matters most on this flavour —
+    ``'`` opens a string here, so carrying state across candidates would let
+    an ordinary prose apostrophe swallow every following object.
     """
-    out: list[tuple[int, int, str]] = []
-    n = len(text)
-    i = 0
-    while i < n:
-        if text[i] != "{":
-            i += 1
-            continue
-        depth = 0
-        j = i
-        quote: str | None = None
-        escape = False
-        while j < n:
-            c = text[j]
-            if escape:
-                escape = False
-            elif quote is not None:
-                if c == "\\":
-                    escape = True
-                elif c == quote:
-                    quote = None
-            else:
-                if c == '"' or c == "'":
-                    quote = c
-                elif c == "{":
-                    depth += 1
-                elif c == "}":
-                    depth -= 1
-                    if depth == 0:
-                        out.append((i, j + 1, text[i : j + 1]))
-                        i = j + 1
-                        break
-            j += 1
-        else:
-            i += 1
-            continue
-    return out
+    if "{" not in text:
+        return []
+    return _scan_object_spans(text, _lenient_close_table(text))
 
 
 # ------------------------------------------------------------------
@@ -1405,12 +1510,24 @@ def repair_tool_calls_in_text(
     #    Skip anything inside a protected code fence or introduced by a prose
     #    example cue. Walk back-to-front so slicing removals don't shift the
     #    indices of earlier matches.
+    #    Oversized inputs skip this step entirely (see _MAX_BARE_SCAN_CHARS):
+    #    the per-candidate JSON parsing / cue checking is the expensive part
+    #    and a message that large is not a tool call. Steps 1/2/2b/2c above
+    #    are regex-driven and linear, so they are NOT skipped.
     protected = _protected_code_spans(cleaned)
     spans_to_remove: list[tuple[int, int]] = []
     repaired_from_bare: list[dict[str, Any]] = []
 
+    strict_spans: list[tuple[int, int, str]] = []
+    lenient_spans: list[tuple[int, int, str]] = []
+    if len(cleaned) > _MAX_BARE_SCAN_CHARS:
+        logger.warning("tool-repair-input-too-large", extra={"chars": len(cleaned)})
+    else:
+        strict_spans = _find_balanced_json_objects(cleaned)
+        lenient_spans = _find_candidate_object_spans(cleaned)
+
     # First pass: strict-JSON balanced objects.
-    for start, end, substr in _find_balanced_json_objects(cleaned):
+    for start, end, substr in strict_spans:
         if _in_spans(start, protected):
             continue
         if _preceded_by_prose_cue(cleaned, start):
@@ -1436,7 +1553,7 @@ def repair_tool_calls_in_text(
 
     # Second pass: lenient objects (single-quoted / unquoted / trailing comma /
     # double-brace) that strict parsing missed. Skip spans already claimed.
-    for start, end, substr in _find_candidate_object_spans(cleaned):
+    for start, end, substr in lenient_spans:
         if _in_spans(start, spans_to_remove):
             continue
         if _in_spans(start, protected):

coderouter/vscode_init.py

@@ -16,11 +16,21 @@ The design contract:
   without ``--force`` and print a unified diff so the operator decides.
 * ``--dry-run`` is byte-identical to the write path minus the actual
   ``os.replace`` at the end.
-* All writes are atomic (tmp file + ``os.replace``) so a partial write
-  cannot corrupt an existing ``settings.json``. This matters more here
-  than in most CodeRouter modules because the file we're editing is
-  workspace-critical: a broken JSON blob would silently break every
-  VSCode terminal env until a human debugged it.
+* All writes are atomic (``tempfile.mkstemp`` + ``fsync`` +
+  ``os.replace``) so a partial write cannot corrupt an existing
+  ``settings.json``. This matters more here than in most CodeRouter
+  modules because the file we're editing is workspace-critical: a
+  broken JSON blob would silently break every VSCode terminal env
+  until a human debugged it.
+* Nothing an operator wrote is destroyed without a copy: before any
+  existing file is rewritten, its previous bytes (and mode) land in
+  ``<name>.bak``. ``--force`` on ``.envrc`` is a whole-file replace,
+  not a merge, so this is the only thing standing between the user and
+  a lost ``source_env_if_exists .envrc.local`` line.
+* A freshly generated ``.envrc`` is created 0600 — it carries
+  ``ANTHROPIC_AUTH_TOKEN`` and sits next to the operator's real
+  secrets. An existing file's mode is preserved across rewrites
+  instead of being reset to the umask default by ``os.replace``.
 
 Files touched (opt-in per flag):
 
@@ -40,8 +50,12 @@ directly without argparse in the way.
 
 from __future__ import annotations
 
+import contextlib
 import json
 import os
+import shutil
+import stat
+import tempfile
 from dataclasses import dataclass
 from difflib import unified_diff
 from pathlib import Path
@@ -62,6 +76,15 @@ DEFAULT_PORT = 8088
 # then flag the unused directive.)
 DEFAULT_TOKEN = "dummy"
 
+# v2.11 (H-11): a generated .envrc carries ANTHROPIC_AUTH_TOKEN, and
+# users routinely add their real key next to it (or point
+# ``source_env_if_exists .envrc.local`` at one). ``env_security``
+# already WARNs on any ``.env`` whose mode has 0o077 bits set and tells
+# the operator to ``chmod 0600``; a file we generate ourselves should
+# not be born failing that check. Applies to newly created files only —
+# an existing file keeps whatever mode its owner chose.
+_ENVRC_MODE = 0o600
+
 # VSCode reads terminal env from a separate key per host OS. Writing all
 # three means the same ``.vscode/`` folder works whether the developer
 # opens it on macOS, Linux, or WSL/Windows without a second scaffolder
@@ -261,7 +284,9 @@ def _apply_settings_json(
             diff=diff,
             reason=(
                 "existing settings.json has different value(s) for "
-                f"{', '.join(sorted(conflicts))}. Re-run with --force to overwrite."
+                f"{', '.join(sorted(conflicts))}. Re-run with --force to "
+                "overwrite those keys (unrelated keys are preserved; the "
+                f"previous file is saved to {_backup_path(path).name})."
             ),
         )
 
@@ -270,7 +295,24 @@ def _apply_settings_json(
         return FileOutcome(path=path, action="unchanged")
 
     diff = _text_diff(old_text, new_text, path)
+    reason = ""
     if not dry_run:
+        # H-11: same deal as .envrc. The merge preserves unrelated keys
+        # by design, but a bug in _merge_terminal_env would rewrite a
+        # hand-tuned settings.json with no way back — so keep the old
+        # bytes. Unlike .envrc this is not gated on --force: an additive
+        # merge rewrites the file without --force too, and that is
+        # exactly the path a merge bug would take.
+        if old_text:
+            backup = _make_backup(path)
+            reason = (
+                f"previous contents backed up to {backup}"
+                if backup is not None
+                else (
+                    f"could not write {_backup_path(path).name} — "
+                    "previous contents were NOT preserved"
+                )
+            )
         try:
             path.parent.mkdir(parents=True, exist_ok=True)
             _atomic_write(path, new_text)
@@ -281,11 +323,14 @@ def _apply_settings_json(
                 reason=f"write failed: {exc}",
                 diff=diff,
             )
+    elif old_text:
+        reason = f"would back up previous contents to {_backup_path(path)}"
 
     return FileOutcome(
         path=path,
         action="updated" if old_text else "created",
         diff=diff,
+        reason=reason,
     )
 
 
@@ -374,15 +419,32 @@ def _apply_envrc(
             diff=diff,
             reason=(
                 ".envrc already exists and its contents differ. "
-                "Re-run with --force to overwrite. "
+                "Re-run with --force to replace the WHOLE file with the "
+                "generated contents (this is not a merge — custom lines "
+                "such as `source_env_if_exists .envrc.local` are dropped); "
+                f"the previous contents are saved to {_backup_path(path).name}. "
                 "(Tip: keep secrets in a separate .envrc.local and "
                 "``source_env_if_exists`` from .envrc.)"
             ),
         )
 
+    reason = ""
     if not dry_run:
+        # H-11: --force replaces the whole file, so stash the old bytes
+        # (mode included) before they are gone. Inside the dry_run guard
+        # on purpose — --dry-run must not touch the filesystem at all.
+        if old_text:
+            backup = _make_backup(path)
+            reason = (
+                f"previous contents backed up to {backup}"
+                if backup is not None
+                else (
+                    f"could not write {_backup_path(path).name} — "
+                    "previous contents were NOT preserved"
+                )
+            )
         try:
-            _atomic_write(path, new_text)
+            _atomic_write(path, new_text, mode=_ENVRC_MODE)
         except OSError as exc:
             return FileOutcome(
                 path=path,
@@ -390,11 +452,14 @@ def _apply_envrc(
                 reason=f"write failed: {exc}",
                 diff=diff,
             )
+    elif old_text:
+        reason = f"would back up previous contents to {_backup_path(path)}"
 
     return FileOutcome(
         path=path,
         action="updated" if old_text else "created",
         diff=diff,
+        reason=reason,
     )
 
 
@@ -538,21 +603,112 @@ def _dump_json(obj: object) -> str:
     return json.dumps(obj, indent=2, ensure_ascii=False) + "\n"
 
 
-def _atomic_write(path: Path, text: str) -> None:
-    """Write ``text`` to ``path`` atomically (tmp + os.replace).
+def _default_file_mode() -> int:
+    """Return the mode a plain ``open(..., "w")`` would produce.
+
+    ``tempfile.mkstemp`` deliberately creates its file 0600. That is the
+    right default for a secret-bearing ``.envrc``, but it would silently
+    tighten ``settings.json`` (previously umask-derived, typically
+    0644). Reading the umask back keeps the pre-existing behaviour for
+    files that have no explicit mode requirement.
+    """
+    current = os.umask(0)
+    os.umask(current)
+    return 0o666 & ~current
+
+
+def _atomic_write(path: Path, text: str, *, mode: int | None = None) -> None:
+    """Write ``text`` to ``path`` atomically (tmp + ``os.replace``).
 
     A partial write cannot corrupt the target file: the tmp file lives
     on the same filesystem (same parent directory) so ``os.replace`` is
     an atomic inode swap on POSIX. On Windows ``os.replace`` is also
     atomic since Python 3.3.
 
-    The tmp file uses the parent + ``<name>.tmp`` pattern rather than
-    ``Path.with_suffix`` because ``.envrc`` has no stem/suffix in the
-    ``Path`` sense and ``with_suffix`` would silently misbehave.
+    v2.11 hardening (H-11). The tmp file used to be the predictable
+    ``<name>.tmp`` next to the target, written with ``Path.write_text``:
+
+    * a pre-planted symlink at that exact path was *followed*, so the
+      write landed on the link's target (CWE-377);
+    * two concurrent ``vscode-init`` runs on the same workspace
+      clobbered each other's tmp file;
+    * ``os.replace`` swaps inodes, so the target's mode was reset to the
+      umask default — a 0600 ``.envrc`` holding ``ANTHROPIC_AUTH_TOKEN``
+      silently became 0644;
+    * without ``fsync`` a crash right after the rename could leave a
+      zero-length file.
+
+    So: ``tempfile.mkstemp`` (random name, ``O_EXCL``, no symlink
+    following), ``fsync`` before the rename, mode restored from the
+    file being replaced (or from ``mode`` for a fresh file), and the tmp
+    file unlinked in ``finally`` so a failed write leaves no debris.
+
+    Parameters
+    ----------
+    mode:
+        POSIX mode for a *newly created* file. Ignored when ``path``
+        already exists — an existing file's own mode wins, so we never
+        loosen (or tighten) what the operator chose. ``None`` means
+        "whatever the umask would have given us".
+    """
+    existing_mode: int | None = None
+    try:
+        existing_mode = stat.S_IMODE(path.stat().st_mode)
+    except OSError:
+        existing_mode = None
+
+    target_mode = existing_mode if existing_mode is not None else mode
+    if target_mode is None:
+        target_mode = _default_file_mode()
+
+    fd, tmp_name = tempfile.mkstemp(
+        dir=str(path.parent),
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+    )
+    tmp = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(text)
+            handle.flush()
+            os.fsync(handle.fileno())
+        # Windows has no POSIX mode bits worth honouring; os.chmod there
+        # only toggles the read-only flag, which would be a surprising
+        # side effect of "preserve the mode".
+        if os.name != "nt":
+            os.chmod(tmp, target_mode)
+        os.replace(tmp, path)
+    finally:
+        # Best effort: a leftover tmp file is noise, not corruption, and
+        # cleaning it up must never mask the original exception.
+        with contextlib.suppress(OSError):
+            tmp.unlink(missing_ok=True)
+
+
+def _backup_path(path: Path) -> Path:
+    """Return the ``.bak`` sibling used to preserve ``path``'s old bytes."""
+    return path.with_name(path.name + ".bak")
+
+
+def _make_backup(path: Path) -> Path | None:
+    """Copy ``path`` to ``<name>.bak``, preserving mode and mtime.
+
+    Returns the backup path, or ``None`` when the copy failed (the
+    caller then proceeds without a backup rather than refusing to write
+    — but the failure is not silent, see the ``reason`` plumbing).
+
+    The destination is unlinked first: ``shutil.copy2`` opens the
+    destination for writing and would happily follow a symlink planted
+    at ``<name>.bak``. ``Path.unlink`` removes the link itself, so the
+    subsequent copy always creates a fresh regular file.
     """
-    tmp = path.parent / (path.name + ".tmp")
-    tmp.write_text(text, encoding="utf-8")
-    os.replace(tmp, path)
+    backup = _backup_path(path)
+    try:
+        backup.unlink(missing_ok=True)
+        shutil.copy2(path, backup)
+    except OSError:
+        return None
+    return backup
 
 
 def _text_diff(old: str, new: str, path: Path) -> str:

{coderouter_cli-2.11.0.dist-info → coderouter_cli-2.11.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coderouter-cli
-Version: 2.11.0
+Version: 2.11.2
 Summary: Local-first, free-first, fallback-built-in LLM router. Claude Code / OpenAI compatible.
 Project-URL: Homepage, https://github.com/zephel01/CodeRouter
 Project-URL: Repository, https://github.com/zephel01/CodeRouter