localcode 0.3.8__tar.gz → 0.3.10__tar.gz

{localcode-0.3.8/src/localcode.egg-info → localcode-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localcode
-Version: 0.3.8
+Version: 0.3.10
 Summary: High-performance AI coding on consumer hardware.
 Author: LocalCode contributors
 License-Expression: Apache-2.0
@@ -35,6 +35,8 @@ Requires-Dist: prompt_toolkit>=3.0.52
 Requires-Dist: rich>=13.7.1
 Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: textual>=8.2.4
+Requires-Dist: tree-sitter>=0.23.0
+Requires-Dist: tree-sitter-language-pack>=1.0.0
 Requires-Dist: tomli>=2.0.0; python_version < "3.11"
 Provides-Extra: voice
 Requires-Dist: pywhispercpp>=1.2.0; extra == "voice"

{localcode-0.3.8 → localcode-0.3.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "localcode"
-version = "0.3.8"
+version = "0.3.10"
 description = "High-performance AI coding on consumer hardware."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -40,6 +40,8 @@ dependencies = [
   "rich>=13.7.1",
   "scikit-learn>=1.3.0",
   "textual>=8.2.4",
+  "tree-sitter>=0.23.0",              # in-process, offline syntax checking (Tier-1 verify)
+  "tree-sitter-language-pack>=1.0.0", # bundled grammars, 100+ languages, no external tools
   "tomli>=2.0.0;python_version<'3.11'",
 ]
 

{localcode-0.3.8 → localcode-0.3.10}/src/localcode/__init__.py

@@ -1,3 +1,3 @@
 __all__ = ["__version__"]
 
-__version__ = "0.3.8"
+__version__ = "0.3.10"

{localcode-0.3.8 → localcode-0.3.10}/src/localcode/agent/loop.py

@@ -838,6 +838,25 @@ def run_agent_loop(
         content = _stream_result.content
         tool_calls = _stream_result.tool_calls
 
+        # Recover tool calls the model emitted but the server failed to parse.
+        # Quantized Qwen quants sometimes emit `<tool_call><function=…>` (XML)
+        # or `<tool_call>{…}` (JSON) shapes --jinja doesn't recognize, so they
+        # arrive as plain CONTENT with no structured tool_calls: the call never
+        # runs (wasted turn) and leaked <think> reasoning makes the model narrate
+        # "the user wants…". Parse them out of content, execute them, and strip
+        # the raw XML + leaked thinking from what's stored/shown.
+        if content and ("<tool_call>" in content or "</think>" in content):
+            try:
+                from ..tools.tool_call_repair import repair_tool_calls
+                _cleaned, _recovered = repair_tool_calls(content)
+                if _recovered and not tool_calls:
+                    tool_calls = _recovered
+                    _stream_result.tool_calls = _recovered
+                if _cleaned != content:
+                    content = _cleaned
+            except Exception:
+                pass
+
         # Clear the indicator before rendering output
         out._stop_indicator()
         sys.stdout.write("\r\033[K")  # clear indicator line

localcode-0.3.10/src/localcode/tools/syntax_check.py

@@ -0,0 +1,122 @@
+"""One-shot syntax validation for files the agent writes — in-process, offline.
+
+Tier-1 verification. The big agents use a persistent language server (LSP);
+opencode's own docs say that's often worse than running diagnostics directly
+(LSPs "get out of sync, use significant memory, slow down agent workflows") —
+decisive for a 16 GB Mac already running a large local model.
+
+Crucially this must work with NO tools the user has to install and NO network:
+it's bundled. We parse with tree-sitter (grammars compiled into the wheel — one
+~4.5 MB native extension covering 100+ languages) and flag any syntax error
+in-process. So when a small quantized model drifts into a structural typo
+(`useState(0]`, an unterminated string, a missing bracket), the write tool
+appends the exact error and line, and the model fixes THAT line instead of
+re-reading the broken file and reproducing the error (the self-conditioning
+loop). Python/JSON keep their native checkers for sharper messages.
+
+Semantic/type errors (wrong names, missing imports) are caught separately by
+project_check at completion (tsc/ruff/go/cargo). Disable with
+LOCALCODE_SYNTAX_CHECK=0.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+
+_TIMEOUT = 5.0
+
+# File extension → tree-sitter language name (all bundled in the language pack).
+_TS_LANG = {
+    ".js": "javascript", ".mjs": "javascript", ".cjs": "javascript",
+    ".jsx": "tsx", ".ts": "typescript", ".mts": "typescript", ".cts": "typescript",
+    ".tsx": "tsx", ".go": "go", ".rs": "rust", ".rb": "ruby", ".php": "php",
+    ".sh": "bash", ".bash": "bash", ".zsh": "bash", ".lua": "lua",
+    ".c": "c", ".h": "c", ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp",
+    ".hpp": "cpp", ".hh": "cpp", ".java": "java", ".kt": "kotlin",
+    ".swift": "swift", ".cs": "csharp", ".scala": "scala",
+    ".css": "css", ".scss": "scss", ".html": "html", ".xml": "xml",
+    ".yaml": "yaml", ".yml": "yaml", ".toml": "toml", ".sql": "sql",
+}
+
+
+def check_syntax(path: str, content: str) -> str | None:
+    """Return a one-line syntax-error string, or None if OK / not checkable."""
+    if os.environ.get("LOCALCODE_SYNTAX_CHECK") == "0":
+        return None
+    ext = os.path.splitext(path)[1].lower()
+    try:
+        # Native checkers give the sharpest messages; use them where free.
+        if ext == ".json":
+            return _check_json(content)
+        if ext in (".py", ".pyi"):
+            return _check_python(content, path)
+        lang = _TS_LANG.get(ext)
+        if lang:
+            return _check_treesitter(content, lang)
+    except Exception:
+        return None
+    return None
+
+
+def _check_json(content: str) -> str | None:
+    try:
+        json.loads(content)
+        return None
+    except json.JSONDecodeError as e:
+        return f"JSON syntax error at line {e.lineno} col {e.colno}: {e.msg}"
+
+
+def _check_python(content: str, path: str) -> str | None:
+    try:
+        compile(content, path, "exec")
+        return None
+    except SyntaxError as e:
+        loc = f"line {e.lineno}" + (f" col {e.offset}" if e.offset else "")
+        return f"Python syntax error at {loc}: {e.msg}"
+
+
+def _check_treesitter(content: str, lang: str) -> str | None:
+    """Parse with a bundled tree-sitter grammar; report the first syntax error.
+
+    No external tools, no network — the grammars are compiled into the wheel.
+    Returns None if tree-sitter isn't available (graceful) or the file parses.
+    """
+    try:
+        from tree_sitter_language_pack import get_language
+        from tree_sitter import Parser
+    except Exception:
+        return None
+    try:
+        parser = Parser(get_language(lang))
+        tree = parser.parse(content.encode("utf-8"))
+    except Exception:
+        return None
+    root = tree.root_node
+    if not root.has_error:
+        return None
+    node = _first_error_node(root)
+    if node is None:
+        return f"syntax error detected in this {lang} file (unbalanced brackets/quotes or a stray token)"
+    line = node.start_point[0] + 1
+    col = node.start_point[1] + 1
+    snippet = ""
+    try:
+        src_line = content.splitlines()[node.start_point[0]].strip()
+        snippet = f" — near: {src_line[:80]}"
+    except Exception:
+        snippet = ""
+    kind = "missing token" if node.is_missing else "unexpected token"
+    return f"syntax error (line {line} col {col}): {kind}{snippet}"
+
+
+def _first_error_node(root):
+    """DFS for the first ERROR or MISSING node (the actual syntax fault)."""
+    stack = [root]
+    while stack:
+        n = stack.pop()
+        if n.is_error or n.is_missing:
+            return n
+        # push children in order so we return the earliest error
+        stack.extend(reversed(n.children))
+    return None

localcode-0.3.10/src/localcode/tools/tool_call_repair.py

@@ -0,0 +1,157 @@
+"""Recover tool calls a quantized model emitted but the server failed to parse.
+
+Small local models sometimes emit tool calls in a slightly off shape that
+llama-server's --jinja parser doesn't recognize, so they come back as plain
+assistant CONTENT with zero structured tool_calls: the call never executes (a
+wasted round), the raw markup pollutes context, and leaked reasoning makes the
+model narrate about "the user". This is the fallback the deleted tool_parsing.py
+used to provide — restored here and made comprehensive across ALL model
+families, because every family's format leaks the same way with no parser.
+
+Handles (verified Qwen from real logs; others from each family's chat template):
+  Qwen  XML : <tool_call><function=NAME><parameter=KEY>\nVALUE\n</parameter>…</function></tool_call>
+  Qwen  JSON: <tool_call>{"name": "NAME", "arguments": {…}}</tool_call>
+  Gemma     : <|tool_call>call:NAME{…}<tool_call|>
+  Cohere    : <|START_ACTION|>[{"tool_name": "NAME", "parameters": {…}}]<|END_ACTION|>
+
+Reasoning strippers cover every family: <think>…</think> (Qwen/DeepSeek),
+<unused25>/<|channel>thought (Gemma), <|START_THINKING|>…<|END_THINKING|> (Cohere).
+"""
+from __future__ import annotations
+
+import json
+import re
+
+# ── tool-call shapes ──
+_QWEN_BLOCK = re.compile(r"<tool_call>\s*(.*?)\s*</tool_call>", re.DOTALL)
+_FUNC_NAME = re.compile(r"<function\s*=\s*([A-Za-z0-9_]+)\s*>", re.DOTALL)
+_PARAM = re.compile(r"<parameter\s*=\s*([A-Za-z0-9_]+)\s*>(.*?)</parameter>", re.DOTALL)
+_GEMMA_CALL = re.compile(r"<\|tool_call\>\s*call\s*:\s*(\w+)\s*(\{.*?\})\s*<\s*tool_call\s*\|>", re.DOTALL)
+_COHERE_ACTION = re.compile(r"<\|START_ACTION\|>(.*?)<\|END_ACTION\|>", re.DOTALL)
+
+# ── reasoning (every family) ──
+_THINK_BLOCKS = [
+    re.compile(r"<think>.*?</think>", re.DOTALL),
+    re.compile(r"<unused25>.*?<unused25>", re.DOTALL),
+    re.compile(r"<\|START_THINKING\|>.*?<\|END_THINKING\|>", re.DOTALL),
+    re.compile(r"<\|channel>thought.*?<channel\|>", re.DOTALL),
+]
+# Streaming can drop the OPEN marker, leaving reasoning + a stray CLOSE: drop up
+# to and including the first close marker.
+_THINK_LEAD = re.compile(r"^.*?(?:</think>|<\|END_THINKING\|>|<unused25>|<channel\|>)", re.DOTALL)
+_STRAY = [re.compile(p) for p in (r"</think>", r"<think>", r"<unused25>",
+                                  r"<\|channel>thought\n?", r"<channel\|>\n?",
+                                  r"<\|START_THINKING\|>", r"<\|END_THINKING\|>")]
+
+_MARKERS = ("<tool_call>", "<|tool_call>", "<|START_ACTION|>")
+
+
+def has_leaked_markup(content: str) -> bool:
+    if not content:
+        return False
+    return any(m in content for m in _MARKERS) or "</think>" in content \
+        or "<|END_THINKING|>" in content or "<unused25>" in content
+
+
+def repair_tool_calls(content: str) -> tuple[str, list[dict]]:
+    """Return (cleaned_content, tool_calls). tool_calls is [] if none leaked.
+
+    Each call: {"id", "type": "function", "function": {"name", "arguments"}}
+    with `arguments` as a JSON string (the shape the executor expects).
+    """
+    if not content:
+        return content or "", []
+    calls: list[dict] = []
+
+    for block in _QWEN_BLOCK.findall(content):
+        p = _parse_qwen_block(block)
+        if p:
+            calls.append(p)
+    for name, payload in _GEMMA_CALL.findall(content):
+        args = _loads_or_raw(payload)
+        calls.append(_mk(name, args))
+    for payload in _COHERE_ACTION.findall(content):
+        calls.extend(_parse_cohere(payload))
+
+    cleaned = content
+    cleaned = _QWEN_BLOCK.sub("", cleaned)
+    cleaned = _GEMMA_CALL.sub("", cleaned)
+    cleaned = _COHERE_ACTION.sub("", cleaned)
+    cleaned = _strip_thinking(cleaned).strip()
+
+    # Re-number ids after collecting from all shapes.
+    for i, c in enumerate(calls):
+        c["id"] = f"repair_{i}"
+    return cleaned, calls
+
+
+def _mk(name: str, args) -> dict:
+    if not isinstance(args, dict):
+        args = {"_raw": args} if args else {}
+    return {"id": "repair", "type": "function",
+            "function": {"name": str(name).strip(), "arguments": json.dumps(args, ensure_ascii=False)}}
+
+
+def _loads_or_raw(s: str):
+    try:
+        return json.loads(s)
+    except Exception:
+        return {"_raw": s.strip()}
+
+
+def _parse_qwen_block(block: str) -> dict | None:
+    block = block.strip()
+    if block.startswith("{"):
+        try:
+            obj = json.loads(block)
+        except Exception:
+            return None
+        name = obj.get("name") or obj.get("function", {}).get("name")
+        args = obj.get("arguments") or obj.get("parameters") or {}
+        if isinstance(args, str):
+            args = _loads_or_raw(args)
+        return _mk(name, args) if name else None
+    m = _FUNC_NAME.search(block)
+    if not m:
+        return None
+    args: dict = {}
+    for key, val in _PARAM.findall(block):
+        v = val
+        if v.startswith("\n"):
+            v = v[1:]
+        if v.endswith("\n"):
+            v = v[:-1]
+        args[key] = v
+    return _mk(m.group(1), args)
+
+
+def _parse_cohere(payload: str) -> list[dict]:
+    try:
+        arr = json.loads(payload.strip())
+    except Exception:
+        return []
+    if isinstance(arr, dict):
+        arr = [arr]
+    out = []
+    for item in arr if isinstance(arr, list) else []:
+        if not isinstance(item, dict):
+            continue
+        name = item.get("tool_name") or item.get("name")
+        args = item.get("parameters") or item.get("arguments") or {}
+        if name:
+            out.append(_mk(name, args))
+    return out
+
+
+def _strip_thinking(content: str) -> str:
+    if not content:
+        return content or ""
+    out = content
+    for pat in _THINK_BLOCKS:
+        out = pat.sub("", out)
+    # Stray close marker left by streaming → drop reasoning up to it.
+    if any(t in out for t in ("</think>", "<|END_THINKING|>", "<unused25>", "<channel|>")):
+        out = _THINK_LEAD.sub("", out, count=1)
+    for pat in _STRAY:
+        out = pat.sub("", out)
+    return out

{localcode-0.3.8 → localcode-0.3.10/src/localcode.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localcode
-Version: 0.3.8
+Version: 0.3.10
 Summary: High-performance AI coding on consumer hardware.
 Author: LocalCode contributors
 License-Expression: Apache-2.0
@@ -35,6 +35,8 @@ Requires-Dist: prompt_toolkit>=3.0.52
 Requires-Dist: rich>=13.7.1
 Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: textual>=8.2.4
+Requires-Dist: tree-sitter>=0.23.0
+Requires-Dist: tree-sitter-language-pack>=1.0.0
 Requires-Dist: tomli>=2.0.0; python_version < "3.11"
 Provides-Extra: voice
 Requires-Dist: pywhispercpp>=1.2.0; extra == "voice"

{localcode-0.3.8 → localcode-0.3.10}/src/localcode.egg-info/SOURCES.txt

@@ -886,6 +886,7 @@ src/localcode/tools/read_file.py
 src/localcode/tools/skill_tool.py
 src/localcode/tools/syntax_check.py
 src/localcode/tools/todo_write.py
+src/localcode/tools/tool_call_repair.py
 src/localcode/tools/web_fetch.py
 src/localcode/tools/web_search.py
 src/localcode/tools/write_file.py

{localcode-0.3.8 → localcode-0.3.10}/src/localcode.egg-info/requires.txt

@@ -9,6 +9,8 @@ prompt_toolkit>=3.0.52
 rich>=13.7.1
 scikit-learn>=1.3.0
 textual>=8.2.4
+tree-sitter>=0.23.0
+tree-sitter-language-pack>=1.0.0
 
 [:python_version < "3.11"]
 tomli>=2.0.0

{localcode-0.3.8 → localcode-0.3.10}/tests/test_architecture.py

@@ -296,6 +296,11 @@ def test_gemma_specific_literals_live_only_in_model_families():
     ALLOWED = {
         (SRC / "model_families.py").resolve(),
         (SRC / "runtime_diffusion.py").resolve(),
+        # The cross-family tool-call repair parser legitimately needs every
+        # family's markup (it recovers tool calls the server failed to parse,
+        # for ALL families) — the one place besides the adapters that must know
+        # the literals directly.
+        (SRC / "tools" / "tool_call_repair.py").resolve(),
     }
     violations: list[str] = []
     for path in _iter_py(SRC):

localcode-0.3.8/src/localcode/tools/syntax_check.py

@@ -1,159 +0,0 @@
-"""One-shot syntax validation for files the agent writes.
-
-Poor-man's LSP. The big agents (Claude Code, opencode) run a persistent
-language server and feed its diagnostics back after each edit; codex uses
-verified structured patches. A persistent LSP is too heavy for localcode's
-target — a 16 GB Mac already running a large local model. So instead we run a
-FAST, ONE-SHOT syntax check after each write and, if it fails, append the exact
-error to the tool result.
-
-Why this matters for a LOCAL model specifically: a small quantized model drifts
-into structural typos (`useState(0]`, an unterminated string, a missing bracket,
-`getCardsForCard`). With no verification the model has to NOTICE its own typo —
-which weak models are bad at — so it re-reads the broken file, the broken code
-enters its context, and it reproduces the error (self-conditioning loop). A
-deterministic check gives it GROUND TRUTH ("line 37: unexpected token") so it
-fixes the exact line instead of guessing.
-
-Best-effort and non-blocking: the write ALWAYS succeeds; this only appends a
-warning. Any checker failure/absence returns None (no false alarms). Disable
-with LOCALCODE_SYNTAX_CHECK=0.
-"""
-from __future__ import annotations
-
-import json
-import os
-import subprocess
-
-_TIMEOUT = 5.0
-
-
-def check_syntax(path: str, content: str) -> str | None:
-    """Return a one-line syntax-error string, or None if OK / not checkable.
-
-    In-process for JSON/Python (instant, zero deps); a fast one-shot subprocess
-    for JS (node --check) and TS/JSX (esbuild, when the project has it).
-    """
-    if os.environ.get("LOCALCODE_SYNTAX_CHECK") == "0":
-        return None
-    ext = os.path.splitext(path)[1].lower()
-    try:
-        if ext == ".json":
-            return _check_json(content)
-        if ext in (".py", ".pyi"):
-            return _check_python(content, path)
-        if ext in (".js", ".mjs", ".cjs"):
-            return _check_node(path)
-        if ext in (".ts", ".tsx", ".jsx", ".mts", ".cts"):
-            return _check_esbuild(path)
-        # Per-file syntax linters for other languages (best-effort — only if the
-        # interpreter/tool is on PATH). Whole-project TYPE checks (go/rust/tsc)
-        # run separately at completion in project_check.
-        if ext == ".rb":
-            return _check_cli(["ruby", "-wc", path], "ruby")
-        if ext == ".php":
-            return _check_cli(["php", "-l", path], "php")
-        if ext in (".sh", ".bash"):
-            return _check_cli(["bash", "-n", path], "bash")
-        if ext == ".lua":
-            return _check_cli(["luac", "-p", path], "luac")
-        if ext == ".rs":
-            # rustc syntax-only parse (no full build): --emit=metadata is slow;
-            # a lightweight parse-only pass isn't stable, so skip unless a fast
-            # tool exists. cargo check at completion covers Rust type errors.
-            return None
-    except Exception:
-        return None
-    return None
-
-
-def _check_cli(argv: list, tool: str) -> str | None:
-    import shutil
-    if not shutil.which(tool):
-        return None
-    r = subprocess.run(argv, capture_output=True, text=True, timeout=_TIMEOUT)
-    if r.returncode != 0:
-        return _first_error_line((r.stderr or "") + "\n" + (r.stdout or ""))
-    return None
-
-
-def _check_json(content: str) -> str | None:
-    try:
-        json.loads(content)
-        return None
-    except json.JSONDecodeError as e:
-        return f"JSON syntax error at line {e.lineno} col {e.colno}: {e.msg}"
-
-
-def _check_python(content: str, path: str) -> str | None:
-    try:
-        compile(content, path, "exec")
-        return None
-    except SyntaxError as e:
-        loc = f"line {e.lineno}" + (f" col {e.offset}" if e.offset else "")
-        return f"Python syntax error at {loc}: {e.msg}"
-
-
-def _check_node(path: str) -> str | None:
-    import shutil
-    if not shutil.which("node"):
-        return None
-    r = subprocess.run(
-        ["node", "--check", path],
-        capture_output=True, text=True, timeout=_TIMEOUT,
-    )
-    if r.returncode != 0:
-        return _first_error_line(r.stderr)
-    return None
-
-
-def _find_bin(start: str, name: str) -> str | None:
-    """Walk up from the file's dir to find node_modules/.bin/<name>."""
-    d = os.path.dirname(os.path.abspath(start))
-    for _ in range(25):
-        cand = os.path.join(d, "node_modules", ".bin", name)
-        if os.path.exists(cand):
-            return cand
-        parent = os.path.dirname(d)
-        if parent == d:
-            break
-        d = parent
-    return None
-
-
-def _check_esbuild(path: str) -> str | None:
-    # esbuild parses TS/TSX/JSX and reports syntax errors instantly. Only used
-    # when the project has it (Vite/most JS projects do post-install); if not
-    # present we skip rather than risk a false alarm from a wrong parser.
-    esbuild = _find_bin(path, "esbuild")
-    if not esbuild:
-        return None
-    loader = "tsx" if path.endswith((".tsx", ".jsx")) else "ts"
-    r = subprocess.run(
-        [esbuild, path, f"--loader={loader}", "--log-level=error"],
-        capture_output=True, text=True, timeout=_TIMEOUT,
-    )
-    if r.returncode != 0 and r.stderr.strip():
-        return _first_error_line(r.stderr)
-    return None
-
-
-def _first_error_line(stderr: str) -> str | None:
-    """Pull the human-meaningful error out of a node/esbuild stderr dump.
-
-    Prefer the actual `SyntaxError: …` / `error: …` message, skipping node's
-    internal loader/stack frames (`node:internal/…`, `    at …`).
-    """
-    lines = [l.rstrip() for l in stderr.splitlines()]
-    # First choice: the real error message line.
-    for s in lines:
-        t = s.strip()
-        if t.startswith(("SyntaxError", "TypeError", "ReferenceError")) or " error:" in t.lower() or t.lower().startswith("error"):
-            return t[:200]
-    # Fallback: the "file:line" location node prints first (skip internals/stack).
-    for s in lines:
-        t = s.strip()
-        if not t or t.startswith(("node:internal", "at ", "^")):
-            continue
-        return t[:200]
-    return None