structured-text-formatter 0.1.0__py3-none-any.whl

st_formatter/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

st_formatter/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

st_formatter/align.py

@@ -0,0 +1,115 @@
+"""Column-alignment passes: consecutive `:=`, consecutive `=>`, and
+consecutive trailing `(* ... *)` comments.
+
+Runs after indent.py, so leading indentation is already final. Each pass
+re-tokenizes the current text (cheap at this file scale) and works purely
+by splicing whitespace between two token boundaries on a line -- it never
+touches the operator token, the comment token, or anything else.
+"""
+from __future__ import annotations
+
+from .regions import Regions, detect
+from .tokenizer import Token, TokenType, tokenize
+
+_NON_SIG = (TokenType.WHITESPACE, TokenType.NEWLINE, TokenType.EOF)
+
+LineInfo = tuple[int, list[Token], int, bool]  # (line_no, sig_tokens, leading_width, protected)
+
+
+def _group_lines(tokens: list[Token]) -> dict[int, list[Token]]:
+    by_line: dict[int, list[Token]] = {}
+    for t in tokens:
+        by_line.setdefault(t.line, []).append(t)
+    return by_line
+
+
+def _build_lines_info(text: str) -> tuple[list[str], list[LineInfo], Regions]:
+    regions = detect(text)
+    tokens = tokenize(text)
+    by_line = _group_lines(tokens)
+    lines = text.splitlines(keepends=True)
+
+    info: list[LineInfo] = []
+    for line_no in range(1, len(lines) + 1):
+        line_toks = by_line.get(line_no, [])
+        sig = [t for t in line_toks if t.type not in _NON_SIG]
+        width = len(line_toks[0].text) if line_toks and line_toks[0].type == TokenType.WHITESPACE else 0
+        protected = regions.is_protected(line_no - 1)
+        info.append((line_no, sig, width, protected))
+    return lines, info, regions
+
+
+def _get_assign_op(sig: list[Token]) -> Token | None:
+    if not sig or (len(sig) == 1 and sig[0].type == TokenType.COMMENT):
+        return None
+    for t in sig:
+        if t.type in (TokenType.ASSIGN, TokenType.ARROW):
+            return t if t.type == TokenType.ASSIGN else None
+    return None
+
+
+def _get_arrow_op(sig: list[Token]) -> Token | None:
+    if not sig or (len(sig) == 1 and sig[0].type == TokenType.COMMENT):
+        return None
+    for t in sig:
+        if t.type in (TokenType.ASSIGN, TokenType.ARROW):
+            return t if t.type == TokenType.ARROW else None
+    return None
+
+
+def _get_trailing_comment_op(sig: list[Token]) -> Token | None:
+    if len(sig) < 2 or sig[-1].type != TokenType.COMMENT:
+        return None
+    return sig[-1]
+
+
+def _run_align(lines: list[str], lines_info: list[LineInfo], get_op) -> None:
+    i = 0
+    n = len(lines_info)
+    while i < n:
+        line_no, sig, width, protected = lines_info[i]
+        op = None if protected else get_op(sig)
+        if op is None:
+            i += 1
+            continue
+
+        run = [i]
+        j = i + 1
+        while j < n:
+            _, sig2, width2, prot2 = lines_info[j]
+            if prot2 or width2 != width or get_op(sig2) is None:
+                break
+            run.append(j)
+            j += 1
+
+        target = 0
+        for k in run:
+            sig_k = lines_info[k][1]
+            op_k = get_op(sig_k)
+            idx = sig_k.index(op_k)
+            before_end = (sig_k[idx - 1].col + len(sig_k[idx - 1].text)) if idx > 0 else 0
+            target = max(target, before_end + 1)
+
+        for k in run:
+            line_no_k, sig_k, _, _ = lines_info[k]
+            op_k = get_op(sig_k)
+            idx = sig_k.index(op_k)
+            before_end = (sig_k[idx - 1].col + len(sig_k[idx - 1].text)) if idx > 0 else 0
+            pad = max(1, target - before_end)
+            raw = lines[line_no_k - 1]
+            lines[line_no_k - 1] = raw[:before_end] + (" " * pad) + raw[op_k.col:]
+
+        i = j if j > i else i + 1
+
+
+def _run_pass(text: str, get_op) -> str:
+    lines, info, _ = _build_lines_info(text)
+    _run_align(lines, info, get_op)
+    return "".join(lines)
+
+
+def apply(text: str) -> str:
+    text = _run_pass(text, _get_assign_op)
+    text = _run_pass(text, _get_arrow_op)
+    text = _run_pass(text, _get_trailing_comment_op)
+    return text

st_formatter/blocks.py

@@ -0,0 +1,135 @@
+"""Block-nesting analysis, shared by indent.py (reindentation) and
+validator.py (Check B: nesting-tree equality).
+
+Confirmed against real usage in the repo: PROGRAM/ACTION/TYPE/FUNCTION
+bodies sit flush with their header (they are POU-level wrappers spanning
+almost the whole file), while VAR*/STRUCT/IF/CASE/FOR/WHILE/REPEAT bodies
+are genuinely indented one level relative to their header.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .tokenizer import Token, TokenType
+
+# Opener keyword -> its closer keyword. VAR_INPUT/VAR_OUTPUT/VAR_IN_OUT/
+# VAR_GLOBAL all close on END_VAR, matching real usage.
+OPENER_TO_CLOSER = {
+    "PROGRAM": "END_PROGRAM",
+    "FUNCTION": "END_FUNCTION",
+    "FUNCTION_BLOCK": "END_FUNCTION_BLOCK",
+    "ACTION": "END_ACTION",
+    "TYPE": "END_TYPE",
+    "STRUCT": "END_STRUCT",
+    "VAR": "END_VAR",
+    "VAR_INPUT": "END_VAR",
+    "VAR_OUTPUT": "END_VAR",
+    "VAR_IN_OUT": "END_VAR",
+    "VAR_GLOBAL": "END_VAR",
+    "IF": "END_IF",
+    "CASE": "END_CASE",
+    "FOR": "END_FOR",
+    "WHILE": "END_WHILE",
+    "REPEAT": "END_REPEAT",
+}
+OPENER_KEYWORDS = set(OPENER_TO_CLOSER)
+CLOSER_KEYWORDS = set(OPENER_TO_CLOSER.values())
+
+# Openers whose own body is NOT indented relative to their header line.
+FLAT_OPENERS = {"PROGRAM", "FUNCTION", "FUNCTION_BLOCK", "ACTION", "TYPE"}
+
+MID_KEYWORDS = {"THEN", "ELSE", "ELSIF", "UNTIL"}
+
+_NON_SIG = (TokenType.WHITESPACE, TokenType.NEWLINE, TokenType.EOF)
+
+
+@dataclass
+class Frame:
+    kind: str
+    closer: str
+    header_level: int
+    body_level: int
+    content_level: int
+
+
+def significant(tokens: list[Token]) -> list[Token]:
+    return [t for t in tokens if t.type not in _NON_SIG]
+
+
+def is_case_label_line(sig: list[Token], in_case: bool) -> bool:
+    """A CASE label line: a comma/`..`-separated list of idents/numbers
+    followed by a bare `:` (not `:=`), as the first thing on the line."""
+    if not in_case or not sig:
+        return False
+    saw_value = False
+    for t in sig:
+        if t.type in (TokenType.IDENT, TokenType.NUMBER):
+            saw_value = True
+        elif t.type in (TokenType.DOTDOT, TokenType.COMMA):
+            continue
+        elif t.type == TokenType.COLON:
+            return saw_value
+        else:
+            return False
+    return False
+
+
+class NestingWalker:
+    """Walks a file's content lines in order, maintaining the block-frame
+    stack, and returns the indentation level for each line visited."""
+
+    def __init__(self) -> None:
+        self.stack: list[Frame] = []
+
+    def current_level(self) -> int:
+        return self.stack[-1].content_level if self.stack else 0
+
+    def visit_line(self, sig: list[Token]) -> int:
+        if not sig:
+            return self.current_level()
+
+        tok0 = sig[0]
+        kw0 = tok0.text.upper() if tok0.type == TokenType.KEYWORD else None
+
+        if kw0 in OPENER_KEYWORDS:
+            level = self.current_level()
+            body_level = level if kw0 in FLAT_OPENERS else level + 1
+            self.stack.append(Frame(
+                kind=kw0, closer=OPENER_TO_CLOSER[kw0],
+                header_level=level, body_level=body_level,
+                content_level=body_level,
+            ))
+            return level
+
+        if kw0 in CLOSER_KEYWORDS:
+            if self.stack and self.stack[-1].closer == kw0:
+                return self.stack.pop().header_level
+            # Malformed/mismatched nesting: best-effort, never crash.
+            return self.current_level()
+
+        in_case = bool(self.stack) and self.stack[-1].kind == "CASE"
+        if kw0 in MID_KEYWORDS or is_case_label_line(sig, in_case):
+            if self.stack:
+                level = self.stack[-1].body_level
+                self.stack[-1].content_level = self.stack[-1].body_level + 1
+            else:
+                level = 0
+            return level
+
+        return self.current_level()
+
+
+def nesting_events(tokens: list[Token]) -> list[str]:
+    """Flat OPEN/CLOSE event sequence from a token stream's KEYWORD
+    tokens alone -- independent of line/indentation, used by the
+    validator to assert nesting is unchanged by formatting."""
+    events: list[str] = []
+    for t in tokens:
+        if t.type != TokenType.KEYWORD:
+            continue
+        kw = t.text.upper()
+        if kw in OPENER_KEYWORDS:
+            events.append(f"OPEN:{kw}")
+        elif kw in CLOSER_KEYWORDS:
+            events.append(f"CLOSE:{kw}")
+    return events

st_formatter/cli.py

@@ -0,0 +1,181 @@
+"""CLI entry point for the ST formatter.
+
+    st-formatter <paths...> [options]
+    python -m st_formatter <paths...> [options]
+
+There is no implicit default path: callers must name the directory/files
+to format explicitly (e.g. `code`), so the tool stays safe to point at a
+small test-fixtures directory too. The tool never invokes git itself.
+"""
+from __future__ import annotations
+
+import argparse
+import difflib
+import fnmatch
+import sys
+from pathlib import Path
+
+from . import __version__
+from .config import resolve_config
+from .formatter import format_text
+
+EXIT_CLEAN = 0
+EXIT_WOULD_CHANGE = 1
+EXIT_VALIDATION_FAILED = 2
+EXIT_FATAL = 3
+
+
+def _is_excluded(path: Path, exclude_patterns: tuple) -> bool:
+    posix = path.as_posix()
+    return any(fnmatch.fnmatch(posix, pat) for pat in exclude_patterns)
+
+
+def _iter_source_files(paths: list, extensions: tuple, exclude_patterns: tuple, verbose: bool = False) -> list:
+    files = []
+    for raw in paths:
+        p = Path(raw)
+        if p.is_dir():
+            for f in sorted(p.rglob("*")):
+                if not f.is_file() or f.suffix.lower() not in extensions:
+                    continue
+                if _is_excluded(f, exclude_patterns):
+                    continue
+                files.append(f)
+        elif p.is_file():
+            if verbose and p.suffix.lower() not in extensions:
+                print(f"note: {p} has an unrecognized extension, formatting it anyway (explicit file argument)")
+            files.append(p)
+        else:
+            raise FileNotFoundError(f"no such file or directory: {raw}")
+    return files
+
+
+def _read(path: Path) -> str:
+    with path.open("r", encoding="latin-1", newline="") as f:
+        return f.read()
+
+
+def _write(path: Path, text: str) -> None:
+    with path.open("w", encoding="latin-1", newline="") as f:
+        f.write(text)
+
+
+def main(argv: list = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="clang-format-style formatter for IEC 61131-3 Structured "
+                     "Text (.st) and Bodas/CoDeSys 2.3 .EXP exports."
+    )
+    parser.add_argument("paths", nargs="+", help="Files and/or directories to format (.st and .EXP by default)")
+    parser.add_argument("--write", action="store_true", help="Apply changes in place (default: dry run)")
+    parser.add_argument("--check", action="store_true", help="Explicit dry-run alias (default behavior)")
+    parser.add_argument("--diff", action="store_true", help="Print a unified diff for each file that would change")
+    parser.add_argument("--config", metavar="PATH", help="Use this config file instead of auto-discovery")
+    parser.add_argument("--extensions", metavar="EXT,...", help="Comma-separated extensions to treat as ST source, "
+                                                                  "e.g. .st,.exp (default: .st,.exp)")
+    parser.add_argument("--indent-size", type=int, default=None)
+    parser.add_argument("--tab-width", type=int, default=None)
+    parser.add_argument("--no-align", action="store_true", default=None,
+                         help="Skip the := / => / comment alignment pass")
+    parser.add_argument("--no-indent", action="store_true", default=None, help="Skip the reindentation pass")
+    parser.add_argument("--report", metavar="PATH", help="Write the full summary/diff/failure log to this file")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    parser.add_argument("--version", action="version", version=f"st-formatter {__version__}")
+    args = parser.parse_args(argv)
+
+    try:
+        config = resolve_config(
+            paths=args.paths,
+            explicit_config=args.config,
+            cli_overrides={
+                "indent_size": args.indent_size,
+                "tab_width": args.tab_width,
+                "do_indent": None if args.no_indent is None else not args.no_indent,
+                "do_align": None if args.no_align is None else not args.no_align,
+                "extensions": tuple(e.strip() for e in args.extensions.split(",")) if args.extensions else None,
+            },
+        )
+    except (FileNotFoundError, OSError, ValueError) as exc:
+        print(f"error: invalid config: {exc}", file=sys.stderr)
+        return EXIT_FATAL
+
+    try:
+        files = _iter_source_files(args.paths, config.extensions, config.exclude, verbose=args.verbose)
+    except FileNotFoundError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_FATAL
+
+    unchanged = reformatted = failed = 0
+    report_lines: list = []
+
+    for path in files:
+        try:
+            original = _read(path)
+        except OSError as exc:
+            print(f"error: cannot read {path}: {exc}", file=sys.stderr)
+            failed += 1
+            continue
+
+        result = format_text(
+            original,
+            indent_size=config.indent_size,
+            tab_width=config.tab_width,
+            do_indent=config.do_indent,
+            do_align=config.do_align,
+        )
+
+        if not result.ok:
+            failed += 1
+            msg = f"FAILED VALIDATION: {path}"
+            print(msg, file=sys.stderr)
+            report_lines.append(msg)
+            for f in result.failures:
+                print(f"    {f}", file=sys.stderr)
+                report_lines.append(f"    {f}")
+            continue
+
+        if not result.changed:
+            unchanged += 1
+            if args.verbose:
+                print(f"unchanged: {path}")
+            continue
+
+        reformatted += 1
+        if args.verbose:
+            print(f"reformatted: {path}" if args.write else f"would reformat: {path}")
+
+        if args.diff:
+            diff = "".join(difflib.unified_diff(
+                original.splitlines(keepends=True),
+                result.formatted_text.splitlines(keepends=True),
+                fromfile=str(path), tofile=str(path),
+            ))
+            print(diff)
+            report_lines.append(diff)
+
+        if args.write:
+            try:
+                _write(path, result.formatted_text)
+            except OSError as exc:
+                print(f"error: cannot write {path}: {exc}", file=sys.stderr)
+                failed += 1
+
+    verb = "reformatted" if args.write else "would be reformatted"
+    summary = (
+        f"{len(files)} files: {unchanged} unchanged, {reformatted} {verb}, "
+        f"{failed} FAILED VALIDATION (skipped, left untouched)"
+    )
+    print(summary)
+    report_lines.append(summary)
+
+    if args.report:
+        Path(args.report).write_text("\n".join(report_lines) + "\n", encoding="utf-8")
+
+    if failed:
+        return EXIT_VALIDATION_FAILED
+    if reformatted and not args.write:
+        return EXIT_WOULD_CHANGE
+    return EXIT_CLEAN
+
+
+if __name__ == "__main__":
+    sys.exit(main())

st_formatter/config.py

@@ -0,0 +1,108 @@
+"""Project configuration: discovery, precedence, and merging.
+
+Mirrors the black/ruff convention: look for a dedicated `.stformat.toml`
+first, then a `[tool.st_formatter]` table in `pyproject.toml`, walking
+upward from the target path(s). CLI flags always win over a config file,
+and a config file always wins over the built-in defaults.
+"""
+from __future__ import annotations
+
+import os
+import sys
+import tomllib
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+
+_DEDICATED_FILENAME = ".stformat.toml"
+_PYPROJECT_FILENAME = "pyproject.toml"
+_KNOWN_KEYS = {"indent_size", "tab_width", "indent", "align", "extensions", "exclude"}
+
+
+@dataclass
+class Config:
+    indent_size: int = 2
+    tab_width: int = 4
+    do_indent: bool = True
+    do_align: bool = True
+    extensions: tuple = (".st", ".exp")
+    exclude: tuple = field(default_factory=tuple)
+
+
+def _normalize_extensions(values) -> tuple:
+    return tuple(v if v.startswith(".") else f".{v}" for v in (e.lower() for e in values))
+
+
+def _table_to_overrides(table: dict) -> dict:
+    unknown = set(table) - _KNOWN_KEYS
+    if unknown:
+        print(f"warning: ignoring unknown st_formatter config key(s): {', '.join(sorted(unknown))}", file=sys.stderr)
+
+    overrides = {}
+    if "indent_size" in table:
+        overrides["indent_size"] = int(table["indent_size"])
+    if "tab_width" in table:
+        overrides["tab_width"] = int(table["tab_width"])
+    if "indent" in table:
+        overrides["do_indent"] = bool(table["indent"])
+    if "align" in table:
+        overrides["do_align"] = bool(table["align"])
+    if "extensions" in table:
+        overrides["extensions"] = _normalize_extensions(table["extensions"])
+    if "exclude" in table:
+        overrides["exclude"] = tuple(table["exclude"])
+    return overrides
+
+
+def _load_toml_table(path: Path) -> dict:
+    with path.open("rb") as f:
+        data = tomllib.load(f)
+    if path.name == _PYPROJECT_FILENAME:
+        return data.get("tool", {}).get("st_formatter", {})
+    return data
+
+
+def _find_config_file(start_dir: Path) -> Path | None:
+    current = start_dir.resolve()
+    while True:
+        dedicated = current / _DEDICATED_FILENAME
+        if dedicated.is_file():
+            return dedicated
+        pyproject = current / _PYPROJECT_FILENAME
+        if pyproject.is_file():
+            try:
+                if _load_toml_table(pyproject):
+                    return pyproject
+            except tomllib.TOMLDecodeError:
+                raise
+        if current.parent == current:
+            return None
+        current = current.parent
+
+
+def _common_start_dir(paths: list) -> Path:
+    resolved = [Path(p).resolve() for p in paths]
+    dirs = [p if p.is_dir() else p.parent for p in resolved]
+    return Path(os.path.commonpath(dirs))
+
+
+def resolve_config(paths: list, explicit_config: str = None, cli_overrides: dict = None) -> Config:
+    overrides = {}
+
+    if explicit_config:
+        config_path = Path(explicit_config)
+        if not config_path.is_file():
+            raise FileNotFoundError(f"no such config file: {explicit_config}")
+        overrides.update(_table_to_overrides(_load_toml_table(config_path)))
+    elif paths:
+        found = _find_config_file(_common_start_dir(paths))
+        if found is not None:
+            overrides.update(_table_to_overrides(_load_toml_table(found)))
+
+    if cli_overrides:
+        for key, value in cli_overrides.items():
+            if value is not None:
+                overrides[key] = value
+
+    known_fields = {f.name for f in fields(Config)}
+    overrides = {k: v for k, v in overrides.items() if k in known_fields}
+    return Config(**overrides)

st_formatter/formatter.py

@@ -0,0 +1,40 @@
+"""Orchestrates formatting one file's text: classify -> indent -> align ->
+validate. Never touches disk itself -- callers (cli.py) decide that."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from . import align, indent, validator
+from .regions import FileClass, detect
+
+
+@dataclass
+class FormatResult:
+    changed: bool
+    ok: bool
+    formatted_text: str
+    failures: list = field(default_factory=list)
+
+
+def format_text(
+    original_text: str,
+    indent_size: int = 2,
+    tab_width: int = 4,
+    do_indent: bool = True,
+    do_align: bool = True,
+) -> FormatResult:
+    regions = detect(original_text)
+    if regions.file_class == FileClass.LIBRARY_MANIFEST:
+        return FormatResult(changed=False, ok=True, formatted_text=original_text)
+
+    text = original_text
+    if do_indent:
+        text = indent.apply(text, regions, indent_size=indent_size, tab_width=tab_width)
+    if do_align:
+        text = align.apply(text)
+
+    result = validator.check(original_text, text)
+    if not result.ok:
+        return FormatResult(changed=False, ok=False, formatted_text=original_text, failures=result.failures)
+
+    return FormatResult(changed=(text != original_text), ok=True, formatted_text=text)

st_formatter/indent.py

@@ -0,0 +1,89 @@
+"""Reindentation pass: recomputes each content line's leading indentation
+from block-keyword nesting (blocks.py).
+
+This is a line-based, non-reflowing transform: it never merges/splits
+lines or reorders tokens. It only ever rewrites WHITESPACE token text (or
+inserts a new leading-whitespace token where a line previously had none) --
+every other token (keyword/identifier/number/string/comment/operator/
+newline) is always re-emitted verbatim. That invariant is what makes the
+validator's Check A (structural token-stream equality) hold by
+construction.
+"""
+from __future__ import annotations
+
+from .blocks import NestingWalker, significant
+from .regions import Regions
+from .tokenizer import Token, TokenType, tokenize
+
+_NON_SIG = (TokenType.WHITESPACE, TokenType.NEWLINE, TokenType.EOF)
+
+
+def _group_lines(tokens: list[Token]) -> dict[int, list[Token]]:
+    by_line: dict[int, list[Token]] = {}
+    for t in tokens:
+        by_line.setdefault(t.line, []).append(t)
+    return by_line
+
+
+def _paren_depth_at_line_start(tokens: list[Token]) -> dict[int, int]:
+    depth_at_start: dict[int, int] = {}
+    depth = 0
+    seen_line = 0
+    for t in tokens:
+        if t.line != seen_line:
+            depth_at_start[t.line] = depth
+            seen_line = t.line
+        if t.type == TokenType.LPAREN:
+            depth += 1
+        elif t.type == TokenType.RPAREN:
+            depth = max(0, depth - 1)
+    return depth_at_start
+
+
+def _reindent_tokens(tokens: list[Token], regions: Regions, indent_size: int, tab_width: int) -> list[Token]:
+    by_line = _group_lines(tokens)
+    if not by_line:
+        return tokens
+    depth_at_start = _paren_depth_at_line_start(tokens)
+    walker = NestingWalker()
+
+    out: list[Token] = []
+    for line_no in range(1, max(by_line) + 1):
+        line_toks = by_line.get(line_no)
+        if not line_toks:
+            continue
+
+        sig = significant(line_toks)
+        protected = regions.is_protected(line_no - 1)
+        continuation = depth_at_start.get(line_no, 0) > 0
+
+        if not sig or protected:
+            out.extend(line_toks)
+            continue
+
+        level = None if continuation else walker.visit_line(sig)
+
+        has_leading_ws = line_toks[0].type == TokenType.WHITESPACE
+        if has_leading_ws:
+            lead = line_toks[0]
+            new_text = (" " * (level * indent_size)) if level is not None else lead.text.expandtabs(tab_width)
+            out.append(Token(TokenType.WHITESPACE, new_text, lead.line, 0, lead.line))
+            rest = line_toks[1:]
+        else:
+            if level is not None and level > 0:
+                out.append(Token(TokenType.WHITESPACE, " " * (level * indent_size), line_no, 0, line_no))
+            rest = line_toks
+
+        for t in rest:
+            if t.type == TokenType.WHITESPACE and "\t" in t.text:
+                out.append(Token(TokenType.WHITESPACE, t.text.replace("\t", " "), t.line, t.col, t.end_line))
+            else:
+                out.append(t)
+
+    return out
+
+
+def apply(text: str, regions: Regions, indent_size: int = 2, tab_width: int = 4) -> str:
+    tokens = tokenize(text)
+    new_tokens = _reindent_tokens(tokens, regions, indent_size, tab_width)
+    return "".join(t.text for t in new_tokens)