localtools-sql-formatter 1.0.0__tar.gz → 1.0.1__tar.gz

{localtools_sql_formatter-1.0.0 → localtools_sql_formatter-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: localtools-sql-formatter
-Version: 1.0.0
+Version: 1.0.1
 Summary: Lint, fix and transpile SQL in any major dialect with sqlfluff and sqlglot (localtools, offline)
 Project-URL: Source, https://github.com/chayprabs/localtools
 License-Expression: MIT
@@ -34,7 +34,7 @@ The script installs a self-contained build (bundled Python) into `~/.localtools/
 - **Fix**: the fixed SQL with a diff, Copy and Download, plus what still needs a human.
 - **Transpile**: rewrite between dialects (e.g. T-SQL `TOP`/`ISNULL` to Postgres `LIMIT`/`COALESCE`), with notes on unsupported constructs.
 
-Options: rule selection/exclusion (`LT01,CP01`, `core`, `layout`), templater (Jinja, or placeholders `:name`, `?`, `$1`, `%(name)s`, `@name`), and an uploaded `.sqlfluff` / `setup.cfg` / `tox.ini` / `pyproject.toml`.
+Options: rule selection/exclusion (`LT01,CP01`, `core`, `layout`, names, legacy aliases and wildcards), templater (Raw, Jinja, or placeholders `:name`, `?`, `$1`, `$name`, `%(name)s`, `%s`), and an uploaded `.sqlfluff` / `setup.cfg` / `tox.ini` / `pyproject.toml`. Config/default uses SQLFluff's Jinja default unless the uploaded config chooses another templater. Unknown rule selectors and placeholder styles are rejected.
 
 ```sh
 localtools-sql-formatter lint models/*.sql --dialect postgres --format json   # or human | sarif; exit 1 on violations
@@ -67,6 +67,7 @@ API: `POST /api/lint`, `/api/fix`, `/api/transpile` (multipart: `sql` and/or `fi
 - Config files are parsed, never executed: `library_path`, macro/loader paths (any `*_path` / `*_dir` key), the python and
   dbt templaters and `processes` are ignored and listed in the result. sqlfluff's 20 KB "skip large file" default is off.
 - Files that do not parse are not fixed (like the sqlfluff CLI); the parse error shows line and column.
+- In-place fixes use an atomic replacement and retain the file mode, UTF-8 BOM and CRLF convention. Old crashed request folders are swept after one hour at startup; active owners and linked folders are preserved.
 
 ## Cut
 

{localtools_sql_formatter-1.0.0 → localtools_sql_formatter-1.0.1}/README.md

@@ -21,7 +21,7 @@ The script installs a self-contained build (bundled Python) into `~/.localtools/
 - **Fix**: the fixed SQL with a diff, Copy and Download, plus what still needs a human.
 - **Transpile**: rewrite between dialects (e.g. T-SQL `TOP`/`ISNULL` to Postgres `LIMIT`/`COALESCE`), with notes on unsupported constructs.
 
-Options: rule selection/exclusion (`LT01,CP01`, `core`, `layout`), templater (Jinja, or placeholders `:name`, `?`, `$1`, `%(name)s`, `@name`), and an uploaded `.sqlfluff` / `setup.cfg` / `tox.ini` / `pyproject.toml`.
+Options: rule selection/exclusion (`LT01,CP01`, `core`, `layout`, names, legacy aliases and wildcards), templater (Raw, Jinja, or placeholders `:name`, `?`, `$1`, `$name`, `%(name)s`, `%s`), and an uploaded `.sqlfluff` / `setup.cfg` / `tox.ini` / `pyproject.toml`. Config/default uses SQLFluff's Jinja default unless the uploaded config chooses another templater. Unknown rule selectors and placeholder styles are rejected.
 
 ```sh
 localtools-sql-formatter lint models/*.sql --dialect postgres --format json   # or human | sarif; exit 1 on violations
@@ -54,6 +54,7 @@ API: `POST /api/lint`, `/api/fix`, `/api/transpile` (multipart: `sql` and/or `fi
 - Config files are parsed, never executed: `library_path`, macro/loader paths (any `*_path` / `*_dir` key), the python and
   dbt templaters and `processes` are ignored and listed in the result. sqlfluff's 20 KB "skip large file" default is off.
 - Files that do not parse are not fixed (like the sqlfluff CLI); the parse error shows line and column.
+- In-place fixes use an atomic replacement and retain the file mode, UTF-8 BOM and CRLF convention. Old crashed request folders are swept after one hour at startup; active owners and linked folders are preserved.
 
 ## Cut
 

{localtools_sql_formatter-1.0.0 → localtools_sql_formatter-1.0.1}/localtools_sql_formatter/__main__.py

@@ -1,6 +1,9 @@
 import argparse
 import json
 import os
+import re
+import shutil
+import stat
 import select
 import socket
 import subprocess
@@ -49,6 +52,50 @@ JOBS = int(os.environ.get("LOCALTOOLS_SQL_JOBS") or max(1, min(4, (os.cpu_count(
 SLOTS = threading.BoundedSemaphore(JOBS)
 
 
+def process_alive(pid):
+    if os.name == "nt":
+        import ctypes
+        from ctypes import wintypes
+        kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+        kernel.OpenProcess.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD)
+        kernel.OpenProcess.restype = wintypes.HANDLE
+        kernel.GetExitCodeProcess.argtypes = (wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD))
+        kernel.CloseHandle.argtypes = (wintypes.HANDLE,)
+        handle = kernel.OpenProcess(0x1000, False, pid)
+        if not handle:
+            return ctypes.get_last_error() != 87  # access denied is not proof of death
+        try:
+            code = wintypes.DWORD()
+            return not kernel.GetExitCodeProcess(handle, ctypes.byref(code)) or code.value == 259
+        finally:
+            kernel.CloseHandle(handle)
+    try:
+        os.kill(pid, 0)
+        return True
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+
+
+def sweep_jobs():
+    """Recover crashed jobs only: exact tool names, >1h, never traverse links/junctions."""
+    for path in Path(tempfile.gettempdir()).iterdir():
+        match = re.fullmatch(r"lt-sqlf-(?:(\d+)-)?[a-z0-9_]{8}", path.name)
+        if not match:
+            continue
+        try:
+            info = path.lstat()
+            if not stat.S_ISDIR(info.st_mode) or path.is_symlink() or (hasattr(path, "is_junction") and path.is_junction()):
+                continue
+            if time.time() - info.st_mtime <= 3600 or (match[1] and process_alive(int(match[1]))):
+                continue
+            # rmtree does not follow directory symlinks (or Windows junctions on Python >=3.8).
+            shutil.rmtree(path)
+        except OSError:
+            pass  # a busy/locked or concurrently removed folder is safe to leave for next startup
+
+
 def text(data, name):
     try:
         return engine.newlines(data.decode("utf-8-sig"))
@@ -93,11 +140,13 @@ def client_gone(req):
 
 def run_worker(req, job):
     """One worker process per request: killed at the deadline, when the client disconnects, and (stdin EOF)
-    when this server exits. At most JOBS run at once; more get 503."""
-    if not SLOTS.acquire(blocking=False):
+    when this server exits. At most JOBS run at once; more get 503.
+    An aborted fetch frees its slot on the next poll, after the follow-up click, so wait for that
+    cancellation instead of answering 503."""
+    if not (SLOTS.acquire(blocking=False) or SLOTS.acquire(timeout=1)):
         raise HTTPError(503, f"busy: {JOBS} SQL job(s) already running; try again when they finish")
     try:
-        with tempfile.TemporaryDirectory(prefix="lt-sqlf-") as d:
+        with tempfile.TemporaryDirectory(prefix=f"lt-sqlf-{os.getpid()}-") as d:
             jp = Path(d, "job.json")
             jp.write_text(json.dumps(job), encoding="utf-8")
             cmd = [sys.executable, *(["-I"] if sys.flags.isolated else []), "-m", __package__, "_worker", str(jp)]
@@ -112,7 +161,7 @@ def run_worker(req, job):
                     if client_gone(req):
                         raise HTTPError(499, "client disconnected")
                     try:
-                        p.wait(0.2)
+                        p.wait(0.05)
                     except subprocess.TimeoutExpired:
                         pass
             finally:
@@ -143,7 +192,12 @@ def job(op, files, f):
 def api(op):
     def handler(req):
         files, f = read_form(req)
-        res = run_worker(req, job(op, files, f))
+        try:
+            res = run_worker(req, job(op, files, f))
+        except HTTPError as e:
+            if e.code == 499:
+                return  # the cancelled browser is gone; sending its error would raise a socket traceback
+            raise
         if op == "lint" and f.get("format") == "sarif":
             body = json.dumps(engine.sarif(res["files"], sqlfluff_version()), indent=2).encode()
             req.send_response(200)
@@ -173,6 +227,27 @@ def api_meta(req):
 
 
 # ---------- CLI ----------
+def atomic_rewrite(name, sql):
+    target = Path(name).resolve()  # preserve a user's symlink and replace its actual file
+    original = target.read_bytes()
+    encoding = "utf-8-sig" if original.startswith(b"\xef\xbb\xbf") else "utf-8"
+    if b"\r\n" in original and b"\n" not in original.replace(b"\r\n", b""):
+        sql = sql.replace("\n", "\r\n")
+    tmp = None
+    try:
+        with tempfile.NamedTemporaryFile(prefix=f".{target.name}.", suffix=".tmp", dir=target.parent,
+                                         mode="w", encoding=encoding, newline="", delete=False) as f:
+            tmp = Path(f.name)
+            f.write(sql)
+            f.flush()
+            os.fsync(f.fileno())
+        os.chmod(tmp, stat.S_IMODE(target.stat().st_mode))
+        os.replace(tmp, target)
+    finally:
+        if tmp is not None:
+            tmp.unlink(missing_ok=True)
+
+
 def read_inputs(paths):
     out = []
     for p in paths:
@@ -251,7 +326,7 @@ def cli_fix(args):
     for f in res:
         if a.in_place:
             if f["changed"]:
-                Path(f["name"]).write_text(f["fixed"], encoding="utf-8", newline="")
+                atomic_rewrite(f["name"], f["fixed"])
             print(f"{f['name']}: {'fixed' if f['changed'] else 'unchanged'}", file=sys.stderr)
         elif a.diff:
             sys.stdout.write(f["diff"])
@@ -304,6 +379,8 @@ tool = Tool(slug="sql-formatter", port=47117, usage=USAGE, ui=UI,
 
 
 def main():
+    if len(sys.argv) == 1 or sys.argv[1] == "serve" or sys.argv[1] in ("--no-open", "--port"):
+        sweep_jobs()
     sys.exit(run(tool))
 
 

{localtools_sql_formatter-1.0.0 → localtools_sql_formatter-1.0.1}/localtools_sql_formatter/engine.py

@@ -1,5 +1,6 @@
 """sqlfluff lint/fix and sqlglot transpile, safe config loading, SARIF output and the worker process entry."""
 import difflib
+import fnmatch
 import json
 import logging
 import os
@@ -12,8 +13,9 @@ MAX_BYTES = 512 << 10  # per request (server); sqlfluff lints roughly 1-2 KB/s o
 MAX_STATEMENTS = 2000
 DOCS = "https://docs.sqlfluff.com/en/stable/reference/rules/{group}.html#{code}"
 TEMPLATERS = ("raw", "jinja", "placeholder")
-PARAM_STYLES = ("colon", "colon_nospaces", "numeric_colon", "question_mark", "numeric_dollar", "dollar",
-                "percent", "pyformat", "at", "ampersand", "apache_ignite", "flyway_var", "dollar_brace")
+from sqlfluff.core.templaters.placeholder import KNOWN_STYLES
+
+PARAM_STYLES = tuple(KNOWN_STYLES)
 
 
 class SQLError(ValueError):
@@ -84,7 +86,7 @@ def load_user_config(text, name):
 
 
 def make_config(dialect="", rules="", exclude_rules="", templater="", config="", config_name=".sqlfluff"):
-    """templater: raw | jinja | placeholder:<param_style>; empty dialect/templater = from the config file, else ansi/raw."""
+    """Empty dialect/templater uses the uploaded config, then SQLFluff's ansi/Jinja defaults."""
     from sqlfluff.core import FluffConfig
     from sqlfluff.core.errors import SQLFluffUserError
     cfg, dropped = load_user_config(config, config_name) if config.strip() else ({}, [])
@@ -108,7 +110,20 @@ def make_config(dialect="", rules="", exclude_rules="", templater="", config="",
         if style:
             cfg.setdefault("templater", {}).setdefault("placeholder", {})["param_style"] = style
     try:
-        return FluffConfig(configs=cfg, overrides=ov, ignore_local_config=True), dropped
+        result = FluffConfig(configs=cfg, overrides=ov, ignore_local_config=True)
+        from sqlfluff.core.rules import get_ruleset
+        references = get_ruleset().rule_reference_map()
+        for key in ("rule_allowlist", "rule_denylist"):
+            for selector in result.get(key) or []:
+                if not fnmatch.filter(references, selector):
+                    raise SQLError(f'unknown rule selector "{selector}" (use a SQLFluff code, name, alias or group)')
+        if result.get("templater") == "placeholder":
+            from sqlfluff.core.templaters.placeholder import PlaceholderTemplater
+            try:
+                PlaceholderTemplater().get_context(fname=None, config=result)
+            except (ValueError, KeyError) as e:
+                raise SQLError(f"invalid placeholder options: {e}") from None
+        return result, dropped
     except SQLFluffUserError as e:
         raise SQLError(str(e)) from None
 
@@ -128,8 +143,6 @@ def violation(v):
 def lint_string(linter, sql, name, fix=False):
     try:
         return linter.lint_string(sql, fname=name, fix=fix)
-    except Exception:
-        raise
     except BaseException as e:  # sqlfluffrs raises pyo3 PanicException (a BaseException) when its parser gives up
         if type(e).__name__ != "PanicException":
             raise
@@ -171,10 +184,15 @@ def fix(files, cfg, passes=5):
         out.append({"name": name, "fixed": cur, "changed": cur != sql, "unparsable": unparsable,
                     "fixed_count": max(0, len(first or []) - len(left)),
                     "remaining": [violation(v) for v in left],
-                    "diff": "".join(difflib.unified_diff(sql.splitlines(True), cur.splitlines(True), f"a/{name}", f"b/{name}"))})
+                    "diff": unified_diff(sql, cur, name)})
     return out
 
 
+def unified_diff(before, after, name):
+    return "".join(line if line.endswith("\n") else line + "\n\\n"
+                   for line in difflib.unified_diff(before.splitlines(True), after.splitlines(True), f"a/{name}", f"b/{name}"))
+
+
 class _Collect(logging.Handler):
     def __init__(self):
         super().__init__(logging.WARNING)

{localtools_sql_formatter-1.0.0 → localtools_sql_formatter-1.0.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "localtools-sql-formatter"
-version = "1.0.0"
+version = "1.0.1"
 description = "Lint, fix and transpile SQL in any major dialect with sqlfluff and sqlglot (localtools, offline)"
 readme = "README.md"
 license = "MIT"

{localtools_sql_formatter-1.0.0 → localtools_sql_formatter-1.0.1}/ui/index.html

@@ -7,6 +7,7 @@
 <link rel="stylesheet" href="kit/localtools.css">
 <style>
   #sql { min-height: 240px; }
+  #sql, .lt-pre { font-variant-ligatures: none; } /* show SQL operators (>=, <>) as typed */
   .lt-files button { border: 0; background: none; color: var(--muted); font: 700 13px var(--mono); cursor: pointer; padding: 0 var(--s1); }
   .lt-files button:hover { color: var(--accent); }
   .lt-files li span:first-child { margin-right: auto; }
@@ -67,7 +68,8 @@
         <div class="lt-field">
           <label for="templater">Templater</label>
           <select class="lt-select" id="templater">
-            <option value="">none (or config)</option>
+            <option value="">config / default (Jinja)</option>
+            <option value="raw">raw (no templating)</option>
             <option value="jinja">jinja {{ }}</option>
             <option value="placeholder:colon">:name</option>
             <option value="placeholder:question_mark">?</option>
@@ -75,7 +77,6 @@
             <option value="placeholder:dollar">$name</option>
             <option value="placeholder:pyformat">%(name)s</option>
             <option value="placeholder:percent">%s</option>
-            <option value="placeholder:at">@name</option>
           </select>
         </div>
         <div class="lt-field">
@@ -133,6 +134,17 @@ const el = (tag, props = {}, ...kids) => { const e = Object.assign(document.crea
 const MAX_ROWS = 2000;
 const picked = [];
 let cfgFile = null;
+let revision = 0, pending = null;
+function invalidate() {
+  revision++;
+  pending?.abort();
+  pending = null;
+  $("out").replaceChildren(el("div", { className: "lt-empty", textContent: "No output. Run with the current input and options." }));
+  $("status").textContent = "";
+  $("err").hidden = true;
+}
+for (const id of ["sql", "dialect", "templater", "rules", "exclude", "read", "write", "mode"])
+  $(id).addEventListener("input", invalidate);
 
 function fileList(ul, list, onRemove) {
   ul.replaceChildren(...list.map((f, i) => {
@@ -142,10 +154,10 @@ function fileList(ul, list, onRemove) {
     return el("li", {}, el("span", { textContent: f.name }), el("span", { textContent: fmtBytes(f.size) }), x);
   }));
 }
-const renderPicked = () => fileList($("files"), picked, (i) => { picked.splice(i, 1); renderPicked(); });
-const renderCfg = () => fileList($("cfg-files"), cfgFile ? [cfgFile] : [], () => { cfgFile = null; renderCfg(); });
-dropZone($("drop"), null, (fs) => { picked.push(...fs); $("file").value = ""; renderPicked(); });
-dropZone($("cfg-drop"), null, (fs) => { cfgFile = fs[0] || cfgFile; $("cfg").value = ""; renderCfg(); });
+const renderPicked = () => fileList($("files"), picked, (i) => { picked.splice(i, 1); invalidate(); renderPicked(); });
+const renderCfg = () => fileList($("cfg-files"), cfgFile ? [cfgFile] : [], () => { cfgFile = null; invalidate(); renderCfg(); });
+dropZone($("drop"), null, (fs) => { picked.push(...fs); $("file").value = ""; invalidate(); renderPicked(); });
+dropZone($("cfg-drop"), null, (fs) => { cfgFile = fs[0] || cfgFile; $("cfg").value = ""; invalidate(); renderCfg(); });
 
 const mode = () => document.querySelector('input[name="mode"]:checked').value;
 $("mode").onchange = () => {
@@ -240,11 +252,11 @@ const render = {
         ...body, ...(r ? table(f.remaining) : []));
     });
   },
-  transpile(res) {
+  transpile(res, submitted) {
     return res.files.map((f) => {
       const blob = new Blob([f.sql], { type: "application/sql" });
       return el("article", { className: "file" },
-        bar(f.name, [0, `${$("read").value || "generic"} > ${$("write").value || "generic"}`],
+        bar(f.name, [0, `${submitted.read || "generic"} > ${submitted.write || "generic"}`],
           btn("Copy", (b) => copy(f.sql, b)), btn("Download", () => download(blob, sqlName(f.name)))),
         ...f.warnings.map((w) => el("div", { className: "note", textContent: "Unsupported: " + w })),
         el("pre", { className: "lt-pre", textContent: f.sql }));
@@ -253,6 +265,8 @@ const render = {
 };
 
 async function go(m = mode()) {
+  invalidate();
+  const submittedRevision = revision;
   const t0 = performance.now();
   const { files, sql } = sources();
   const fields = { sql };
@@ -265,13 +279,18 @@ async function go(m = mode()) {
   $("out").replaceChildren(el("div", { className: "lt-empty", textContent: m === "transpile" ? "Transpiling..." : "sqlfluff is working. Big files take a while." }));
   $("status").textContent = "";
   let res;
+  const controller = pending = new AbortController();
   try {
-    const r = await fetch("/api/" + m, { method: "POST", body });
+    const r = await fetch("/api/" + m, { method: "POST", body, signal: controller.signal });
     res = await r.json().catch(() => ({ error: `${r.status} ${r.statusText}` }));
+    if (submittedRevision !== revision) return;
     if (!r.ok) throw new Error(res.error);
   } catch (e) {
+    if (submittedRevision !== revision) return;
     $("out").replaceChildren(el("div", { className: "lt-empty", textContent: "No output." }));
     throw e.message === "Failed to fetch" ? new Error("Cannot reach the local server. Is the tool still running?") : e;
+  } finally {
+    if (pending === controller) pending = null;
   }
   const head = [];
   if (res.ignored_config?.length) head.push(el("div", { className: "note", textContent: "Ignored config settings: " + res.ignored_config.join(", ") }));
@@ -285,7 +304,7 @@ async function go(m = mode()) {
   } else {
     $("status").textContent = `${res.files.length} file${res.files.length > 1 ? "s" : ""} · ${fmtMs(performance.now() - t0)}`;
   }
-  $("out").replaceChildren(...head, ...render[m](res));
+  $("out").replaceChildren(...head, ...render[m](res, fields));
 }
 
 $("go").onclick = () => run($("go"), $("err"), () => go());