fmstyle 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fmstyle/__init__.py +68 -0
- fmstyle/cli.py +163 -0
- fmstyle/config.py +105 -0
- fmstyle/lexer.py +272 -0
- fmstyle/parser.py +343 -0
- fmstyle/presets.py +72 -0
- fmstyle/printer.py +304 -0
- fmstyle/rules.py +70 -0
- fmstyle/skill/SKILL.md +101 -0
- fmstyle/web/index.html +1338 -0
- fmstyle-0.2.0.dist-info/METADATA +283 -0
- fmstyle-0.2.0.dist-info/RECORD +15 -0
- fmstyle-0.2.0.dist-info/WHEEL +5 -0
- fmstyle-0.2.0.dist-info/entry_points.txt +2 -0
- fmstyle-0.2.0.dist-info/top_level.txt +1 -0
fmstyle/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""fmstyle - deterministic formatter + style engine for FileMaker calculations.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
|
|
5
|
+
format_calc(source, style) -> formatted text (token-preservation verified)
|
|
6
|
+
lint_calc(source, style) -> [(rule_id, message), ...]
|
|
7
|
+
Style -> configuration (an org's mechanical rules)
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .config import Style
|
|
13
|
+
from .lexer import LexError, tokenize
|
|
14
|
+
from .parser import ParseError, Parser
|
|
15
|
+
from .printer import Printer
|
|
16
|
+
from .rules import lint as _lint
|
|
17
|
+
|
|
18
|
+
__version__ = "0.2.0"
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"format_calc",
|
|
22
|
+
"lint_calc",
|
|
23
|
+
"Style",
|
|
24
|
+
"LexError",
|
|
25
|
+
"ParseError",
|
|
26
|
+
"FormatSafetyError",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class FormatSafetyError(RuntimeError):
|
|
31
|
+
"""Formatting would have changed the token stream - refused."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _signature(source: str):
|
|
35
|
+
code = []
|
|
36
|
+
comments = []
|
|
37
|
+
for tok in tokenize(source):
|
|
38
|
+
comments.extend(c.text for c in tok.pre_comments)
|
|
39
|
+
if tok.kind != "EOF":
|
|
40
|
+
text = tok.text.lower() if tok.kind == "OP" else tok.text
|
|
41
|
+
code.append((tok.kind, text))
|
|
42
|
+
return code, comments
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def format_calc(source: str, style: Style | None = None) -> str:
|
|
46
|
+
"""Format a FileMaker calculation. Guaranteed semantics-preserving: the
|
|
47
|
+
output must re-tokenize to the exact same code tokens and comments as the
|
|
48
|
+
input, otherwise FormatSafetyError is raised and nothing is changed."""
|
|
49
|
+
style = style or Style()
|
|
50
|
+
tokens = tokenize(source)
|
|
51
|
+
if len(tokens) == 1: # comments only (e.g. a commented-out calc) - keep as is
|
|
52
|
+
return source
|
|
53
|
+
node = Parser(tokens).parse()
|
|
54
|
+
out = Printer(style).format(node)
|
|
55
|
+
if _signature(source) != _signature(out):
|
|
56
|
+
raise FormatSafetyError(
|
|
57
|
+
"formatting would alter the token stream; refusing (please report this input)"
|
|
58
|
+
)
|
|
59
|
+
return out
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def lint_calc(source: str, style: Style | None = None):
|
|
63
|
+
"""Run guideline lint rules over a calculation."""
|
|
64
|
+
style = style or Style()
|
|
65
|
+
tokens = tokenize(source)
|
|
66
|
+
if len(tokens) == 1: # comments only - nothing to lint
|
|
67
|
+
return []
|
|
68
|
+
return _lint(Parser(tokens).parse(), style)
|
fmstyle/cli.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""fmstyle CLI - format / lint FileMaker calculations.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 ok, 1 check/lint findings, 2 input could not be parsed.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
|
|
14
|
+
from . import FormatSafetyError, LexError, ParseError, __version__, format_calc, lint_calc
|
|
15
|
+
from .config import Style
|
|
16
|
+
from .presets import PRESETS, preset_dict, preset_names
|
|
17
|
+
|
|
18
|
+
SKILL_RAW_URL = (
|
|
19
|
+
"https://raw.githubusercontent.com/oogi-io/fm-code-formatter/main/fmstyle/skill/SKILL.md"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def cmd_install_skill(args) -> int:
|
|
24
|
+
"""Install the fmstyle Claude Code skill, or check its freshness.
|
|
25
|
+
|
|
26
|
+
--check compare the installed skill against the one shipped with this
|
|
27
|
+
fmstyle version (offline). 0 = up to date, 1 = differs, 2 = absent.
|
|
28
|
+
--remote compare against GitHub main instead (network).
|
|
29
|
+
Neither flag: (re)install the packaged skill.
|
|
30
|
+
"""
|
|
31
|
+
import hashlib
|
|
32
|
+
import shutil
|
|
33
|
+
|
|
34
|
+
src = Path(__file__).resolve().parent / "skill" / "SKILL.md"
|
|
35
|
+
dest_dir = Path("~/.claude/skills/fmstyle").expanduser()
|
|
36
|
+
dest = dest_dir / "SKILL.md"
|
|
37
|
+
|
|
38
|
+
def digest(b: bytes) -> str:
|
|
39
|
+
return hashlib.sha256(b).hexdigest()[:12]
|
|
40
|
+
|
|
41
|
+
if args.check or args.remote:
|
|
42
|
+
if not dest.exists():
|
|
43
|
+
print(f"fmstyle skill is not installed ({dest} missing). Run: fmstyle install-skill")
|
|
44
|
+
return 2
|
|
45
|
+
installed = dest.read_bytes()
|
|
46
|
+
if args.remote:
|
|
47
|
+
import urllib.request
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
ref = urllib.request.urlopen(SKILL_RAW_URL, timeout=10).read()
|
|
51
|
+
except Exception as exc: # noqa: BLE001 - offline is fine, just report
|
|
52
|
+
print(f"Could not fetch {SKILL_RAW_URL}: {exc}")
|
|
53
|
+
return 2
|
|
54
|
+
ref_label = "GitHub main"
|
|
55
|
+
hint = "pipx upgrade fmstyle (or git pull), then: fmstyle install-skill"
|
|
56
|
+
else:
|
|
57
|
+
ref = src.read_bytes()
|
|
58
|
+
ref_label = f"the skill shipped with fmstyle {__version__}"
|
|
59
|
+
hint = "run: fmstyle install-skill"
|
|
60
|
+
if digest(installed) == digest(ref):
|
|
61
|
+
print(f"fmstyle skill is up to date with {ref_label}.")
|
|
62
|
+
return 0
|
|
63
|
+
print(
|
|
64
|
+
f"fmstyle skill DIFFERS from {ref_label} "
|
|
65
|
+
f"(installed {digest(installed)}, reference {digest(ref)}). To update: {hint}"
|
|
66
|
+
)
|
|
67
|
+
return 1
|
|
68
|
+
|
|
69
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
70
|
+
shutil.copy(src, dest)
|
|
71
|
+
print(f"Installed the 'fmstyle' skill (fmstyle {__version__}) -> {dest_dir}")
|
|
72
|
+
print("Claude Code can now format FileMaker calculations from any directory -")
|
|
73
|
+
print("just write or paste a calc, or ask to format one.")
|
|
74
|
+
print("Freshness check anytime: fmstyle install-skill --check (or --remote for GitHub main)")
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _load_style(config: str | None, preset: str | None) -> Style:
|
|
79
|
+
base = dict(preset_dict(preset)) if preset else {}
|
|
80
|
+
path = Path(config) if config else Path("fmstyle.json")
|
|
81
|
+
if config or (not preset and path.exists()):
|
|
82
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
83
|
+
functions = {**base.get("functions", {}), **data.get("functions", {})}
|
|
84
|
+
base.update(data)
|
|
85
|
+
if functions:
|
|
86
|
+
base["functions"] = functions
|
|
87
|
+
return Style.from_dict(base) if base else Style()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _read_sources(paths: list[str]) -> list[tuple[str, str]]:
|
|
91
|
+
if not paths or paths == ["-"]:
|
|
92
|
+
return [("<stdin>", sys.stdin.read())]
|
|
93
|
+
return [(p, Path(p).read_text(encoding="utf-8")) for p in paths]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def main(argv: list[str] | None = None) -> int:
|
|
97
|
+
ap = argparse.ArgumentParser(
|
|
98
|
+
prog="fmstyle",
|
|
99
|
+
description="Deterministic formatter and style checker for FileMaker calculations.",
|
|
100
|
+
)
|
|
101
|
+
ap.add_argument("--version", action="version", version=f"fmstyle {__version__}")
|
|
102
|
+
ap.add_argument("--config", help="path to fmstyle.json (default: ./fmstyle.json if present)")
|
|
103
|
+
ap.add_argument("--preset", help="named style pack (see `fmstyle presets`); --config keys override it")
|
|
104
|
+
sub = ap.add_subparsers(dest="command", required=True)
|
|
105
|
+
|
|
106
|
+
sub.add_parser("presets", help="list available style presets")
|
|
107
|
+
|
|
108
|
+
fmt = sub.add_parser("format", help="format calculation(s) (stdin by default)")
|
|
109
|
+
fmt.add_argument("paths", nargs="*", help="files to format, '-' or empty for stdin")
|
|
110
|
+
fmt.add_argument("-w", "--write", action="store_true", help="rewrite files in place")
|
|
111
|
+
fmt.add_argument("--check", action="store_true", help="exit 1 if any file would change")
|
|
112
|
+
|
|
113
|
+
ln = sub.add_parser("lint", help="run guideline rules over calculation(s)")
|
|
114
|
+
ln.add_argument("paths", nargs="*", help="files to lint, '-' or empty for stdin")
|
|
115
|
+
|
|
116
|
+
ins = sub.add_parser("install-skill", help="install the Claude Code skill globally (~/.claude/skills)")
|
|
117
|
+
ins.add_argument("--check", action="store_true", help="compare installed skill vs this version's")
|
|
118
|
+
ins.add_argument("--remote", action="store_true", help="compare installed skill vs GitHub main")
|
|
119
|
+
|
|
120
|
+
args = ap.parse_args(argv)
|
|
121
|
+
|
|
122
|
+
if args.command == "presets":
|
|
123
|
+
for name in preset_names():
|
|
124
|
+
print(f"{name:10s} {PRESETS[name]['description']}")
|
|
125
|
+
return 0
|
|
126
|
+
|
|
127
|
+
if args.command == "install-skill":
|
|
128
|
+
return cmd_install_skill(args)
|
|
129
|
+
|
|
130
|
+
try:
|
|
131
|
+
style = _load_style(args.config, args.preset)
|
|
132
|
+
except KeyError as exc:
|
|
133
|
+
print(f"error: {exc.args[0]}", file=sys.stderr)
|
|
134
|
+
return 2
|
|
135
|
+
rc = 0
|
|
136
|
+
|
|
137
|
+
for name, text in _read_sources(args.paths):
|
|
138
|
+
try:
|
|
139
|
+
if args.command == "format":
|
|
140
|
+
out = format_calc(text, style)
|
|
141
|
+
if args.check:
|
|
142
|
+
if out != text:
|
|
143
|
+
print(f"would reformat {name}")
|
|
144
|
+
rc = max(rc, 1)
|
|
145
|
+
elif args.write and name != "<stdin>":
|
|
146
|
+
if out != text:
|
|
147
|
+
Path(name).write_text(out, encoding="utf-8")
|
|
148
|
+
print(f"reformatted {name}")
|
|
149
|
+
else:
|
|
150
|
+
sys.stdout.write(out)
|
|
151
|
+
else:
|
|
152
|
+
for rule, message in lint_calc(text, style):
|
|
153
|
+
print(f"{name}: {rule}: {message}")
|
|
154
|
+
rc = max(rc, 1)
|
|
155
|
+
except (LexError, ParseError, FormatSafetyError) as exc:
|
|
156
|
+
print(f"{name}: error: {exc}", file=sys.stderr)
|
|
157
|
+
rc = 2
|
|
158
|
+
|
|
159
|
+
return rc
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
if __name__ == "__main__":
|
|
163
|
+
sys.exit(main())
|
fmstyle/config.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Style configuration - the machine-readable half of an organisation's
|
|
2
|
+
FileMaker guidelines. Defaults implement the Thomas DS style guide
|
|
3
|
+
(filemaker_development_style_guide.md, sections 3-4).
|
|
4
|
+
|
|
5
|
+
Per-function rules: every FileMaker (or custom) function name can get its own
|
|
6
|
+
entry under "functions", controlling how that call is laid out:
|
|
7
|
+
|
|
8
|
+
"functions": {
|
|
9
|
+
"let": { "layout": "let", "multiline": "always" },
|
|
10
|
+
"while": { "layout": "while", "multiline": "always" },
|
|
11
|
+
"case": { "layout": "pairs" },
|
|
12
|
+
"if": { "multiline": "always" }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
layout: "args" - one argument per line (default for any function)
|
|
16
|
+
"pairs" - condition ; result pairs per line (Case-style)
|
|
17
|
+
"let" - the mandatory Let block shape
|
|
18
|
+
"while" - the mandatory While block shape
|
|
19
|
+
"auto" - same as "args"
|
|
20
|
+
multiline: "auto" - explode only when the call exceeds the line width
|
|
21
|
+
"always" - always explode, even when it would fit
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
from dataclasses import dataclass, field, fields
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
_SIMPLE_KEYS = (
|
|
31
|
+
"width",
|
|
32
|
+
"let_blank_lines",
|
|
33
|
+
"lowercase_keywords",
|
|
34
|
+
"space_before_semicolon",
|
|
35
|
+
"result_name",
|
|
36
|
+
"local_variable_pattern",
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
VALID_LAYOUTS = {"auto", "args", "pairs", "let", "while", "leading"}
|
|
40
|
+
VALID_MULTILINE = {"auto", "always"}
|
|
41
|
+
|
|
42
|
+
DEFAULT_FUNCTIONS = {
|
|
43
|
+
"let": {"layout": "let", "multiline": "always"},
|
|
44
|
+
"while": {"layout": "while", "multiline": "always"},
|
|
45
|
+
"case": {"layout": "pairs"},
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _default_functions() -> dict:
|
|
50
|
+
return {name: dict(rule) for name, rule in DEFAULT_FUNCTIONS.items()}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class Style:
|
|
55
|
+
indent: str = " "
|
|
56
|
+
width: int = 96
|
|
57
|
+
let_blank_lines: bool = True
|
|
58
|
+
lowercase_keywords: bool = True
|
|
59
|
+
space_before_semicolon: bool = True
|
|
60
|
+
result_name: str = "result"
|
|
61
|
+
local_variable_pattern: str = r"^[_a-z][A-Za-z0-9]*$"
|
|
62
|
+
functions: dict = field(default_factory=_default_functions)
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def from_dict(cls, data: dict) -> "Style":
|
|
66
|
+
known = {f.name for f in fields(cls)} | {"force_multiline", "indent"}
|
|
67
|
+
unknown = set(data) - known
|
|
68
|
+
if unknown:
|
|
69
|
+
raise ValueError(f"unknown style option(s): {', '.join(sorted(unknown))}")
|
|
70
|
+
|
|
71
|
+
kwargs = {k: data[k] for k in _SIMPLE_KEYS if k in data}
|
|
72
|
+
if "indent" in data:
|
|
73
|
+
v = data["indent"]
|
|
74
|
+
if v == "tab":
|
|
75
|
+
kwargs["indent"] = "\t"
|
|
76
|
+
elif isinstance(v, int):
|
|
77
|
+
kwargs["indent"] = " " * v
|
|
78
|
+
else:
|
|
79
|
+
kwargs["indent"] = v
|
|
80
|
+
|
|
81
|
+
functions = _default_functions()
|
|
82
|
+
if "force_multiline" in data: # legacy shorthand for multiline: always
|
|
83
|
+
for rule in functions.values():
|
|
84
|
+
rule.pop("multiline", None)
|
|
85
|
+
for name in data["force_multiline"]:
|
|
86
|
+
functions.setdefault(name.lower(), {})["multiline"] = "always"
|
|
87
|
+
if "functions" in data:
|
|
88
|
+
for name, rule in data["functions"].items():
|
|
89
|
+
if not isinstance(rule, dict):
|
|
90
|
+
raise ValueError(f"functions.{name} must be an object")
|
|
91
|
+
bad = set(rule) - {"layout", "multiline"}
|
|
92
|
+
if bad:
|
|
93
|
+
raise ValueError(f"functions.{name}: unknown key(s) {', '.join(sorted(bad))}")
|
|
94
|
+
if "layout" in rule and rule["layout"] not in VALID_LAYOUTS:
|
|
95
|
+
raise ValueError(f"functions.{name}.layout must be one of {sorted(VALID_LAYOUTS)}")
|
|
96
|
+
if "multiline" in rule and rule["multiline"] not in VALID_MULTILINE:
|
|
97
|
+
raise ValueError(f"functions.{name}.multiline must be one of {sorted(VALID_MULTILINE)}")
|
|
98
|
+
functions.setdefault(name.lower(), {}).update(rule)
|
|
99
|
+
kwargs["functions"] = functions
|
|
100
|
+
|
|
101
|
+
return cls(**kwargs)
|
|
102
|
+
|
|
103
|
+
@classmethod
|
|
104
|
+
def load(cls, path: str | Path) -> "Style":
|
|
105
|
+
return cls.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
|
fmstyle/lexer.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""Tokenizer for FileMaker calculation expressions.
|
|
2
|
+
|
|
3
|
+
Design rules:
|
|
4
|
+
|
|
5
|
+
- Verbatim tokens: a token's text is the exact source slice (field names keep
|
|
6
|
+
their original interior spacing), so the printer can never alter a name.
|
|
7
|
+
- Comments ride on the next code token (`pre_comments`); the formatter's safety
|
|
8
|
+
check compares code tokens and comment texts of input vs output, so nothing
|
|
9
|
+
can be dropped silently.
|
|
10
|
+
- Anything we do not understand raises LexError - the formatter then leaves the
|
|
11
|
+
source untouched instead of guessing.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
|
|
18
|
+
KEYWORD_OPS = {"and", "or", "xor", "not"}
|
|
19
|
+
TWO_CHAR_OPS = ("<>", "<=", ">=")
|
|
20
|
+
SINGLE_CHAR_OPS = set("&+-*/^=<>≠≤≥") # incl. != / <= / >= unicode forms
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class LexError(ValueError):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Comment:
|
|
29
|
+
text: str
|
|
30
|
+
own_line: bool # True when only whitespace precedes it on its line
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Token:
|
|
35
|
+
kind: str # NAME NUMBER STRING OP LPAREN RPAREN LBRACKET RBRACKET SEMI EOF
|
|
36
|
+
text: str
|
|
37
|
+
line: int
|
|
38
|
+
pre_comments: list = field(default_factory=list)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
UNICODE_OPS = "≠≤≥¶"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _is_name_char(ch: str) -> bool:
|
|
45
|
+
# ~ # . appear in real-world FileMaker names: $$~DISABLETRIGGERS,
|
|
46
|
+
# #ScriptResultJSON, Triggers.Disable, $sub.errorCode, Field~extra.
|
|
47
|
+
# Any non-ASCII char that is not an operator is a name char too
|
|
48
|
+
# (field names like "Rep 5¢ Commission" exist in the wild).
|
|
49
|
+
if ch.isalnum() or ch in "_~#.":
|
|
50
|
+
return True
|
|
51
|
+
return ord(ch) > 127 and ch not in UNICODE_OPS
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _is_name_start(ch: str) -> bool:
|
|
55
|
+
if ch.isalpha() or ch in "_~#":
|
|
56
|
+
return True
|
|
57
|
+
return ord(ch) > 127 and ch not in UNICODE_OPS
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _extend_name_run(s: str, end: int) -> int:
|
|
61
|
+
"""Merge following space-separated words into one name (FileMaker field
|
|
62
|
+
names may contain spaces). Stops at keywords, operators, newlines, and
|
|
63
|
+
words that start a function call."""
|
|
64
|
+
n = len(s)
|
|
65
|
+
while True:
|
|
66
|
+
k = end
|
|
67
|
+
while k < n and s[k] in " \t":
|
|
68
|
+
k += 1
|
|
69
|
+
if k == end or k >= n or not _is_name_char(s[k]):
|
|
70
|
+
break
|
|
71
|
+
j = k
|
|
72
|
+
while j < n and _is_name_char(s[j]):
|
|
73
|
+
j += 1
|
|
74
|
+
if s[k:j].lower() in KEYWORD_OPS:
|
|
75
|
+
break
|
|
76
|
+
m = j
|
|
77
|
+
while m < n and s[m] in " \t":
|
|
78
|
+
m += 1
|
|
79
|
+
if m < n and s[m] == "(": # that word is a function name
|
|
80
|
+
break
|
|
81
|
+
end = j
|
|
82
|
+
return end
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _qualify(s: str, end: int, line: int) -> int:
|
|
86
|
+
"""Extend a name across a `::` field qualifier if present."""
|
|
87
|
+
n = len(s)
|
|
88
|
+
m = end
|
|
89
|
+
while m < n and s[m] in " \t":
|
|
90
|
+
m += 1
|
|
91
|
+
if not s.startswith("::", m):
|
|
92
|
+
return end
|
|
93
|
+
m += 2
|
|
94
|
+
while m < n and s[m] in " \t":
|
|
95
|
+
m += 1
|
|
96
|
+
if s.startswith("${", m):
|
|
97
|
+
k = s.find("}", m)
|
|
98
|
+
if k == -1:
|
|
99
|
+
raise LexError(f"unterminated ${{...}} name (line {line})")
|
|
100
|
+
return k + 1
|
|
101
|
+
if m < n and _is_name_char(s[m]):
|
|
102
|
+
j = m
|
|
103
|
+
while j < n and _is_name_char(s[j]):
|
|
104
|
+
j += 1
|
|
105
|
+
return _extend_name_run(s, j)
|
|
106
|
+
raise LexError(f"expected a field name after '::' (line {line})")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def tokenize(source: str) -> list[Token]:
|
|
110
|
+
n = len(source)
|
|
111
|
+
i = 0
|
|
112
|
+
line = 1
|
|
113
|
+
fresh = True # only whitespace so far on the current line
|
|
114
|
+
pending: list[Comment] = []
|
|
115
|
+
tokens: list[Token] = []
|
|
116
|
+
|
|
117
|
+
def add(kind: str, text: str) -> None:
|
|
118
|
+
nonlocal pending, fresh
|
|
119
|
+
tokens.append(Token(kind, text, line, pending))
|
|
120
|
+
pending = []
|
|
121
|
+
fresh = False
|
|
122
|
+
|
|
123
|
+
while i < n:
|
|
124
|
+
ch = source[i]
|
|
125
|
+
|
|
126
|
+
if ch == "\n":
|
|
127
|
+
line += 1
|
|
128
|
+
i += 1
|
|
129
|
+
fresh = True
|
|
130
|
+
continue
|
|
131
|
+
if ch in " \t\r":
|
|
132
|
+
i += 1
|
|
133
|
+
continue
|
|
134
|
+
|
|
135
|
+
if source.startswith("//", i):
|
|
136
|
+
j = source.find("\n", i)
|
|
137
|
+
j = n if j == -1 else j
|
|
138
|
+
pending.append(Comment(source[i:j].rstrip(), fresh))
|
|
139
|
+
fresh = False
|
|
140
|
+
i = j
|
|
141
|
+
continue
|
|
142
|
+
|
|
143
|
+
if source.startswith("/*", i):
|
|
144
|
+
depth = 1
|
|
145
|
+
j = i + 2
|
|
146
|
+
while j < n and depth:
|
|
147
|
+
if source.startswith("/*", j):
|
|
148
|
+
depth += 1
|
|
149
|
+
j += 2
|
|
150
|
+
elif source.startswith("*/", j):
|
|
151
|
+
depth -= 1
|
|
152
|
+
j += 2
|
|
153
|
+
else:
|
|
154
|
+
if source[j] == "\n":
|
|
155
|
+
line += 1
|
|
156
|
+
j += 1
|
|
157
|
+
if depth:
|
|
158
|
+
raise LexError(f"unterminated block comment (line {line})")
|
|
159
|
+
pending.append(Comment(source[i:j], fresh))
|
|
160
|
+
fresh = False
|
|
161
|
+
i = j
|
|
162
|
+
continue
|
|
163
|
+
|
|
164
|
+
if ch == '"':
|
|
165
|
+
j = i + 1
|
|
166
|
+
while j < n:
|
|
167
|
+
if source[j] == "\\":
|
|
168
|
+
j += 2
|
|
169
|
+
continue
|
|
170
|
+
if source[j] == '"':
|
|
171
|
+
break
|
|
172
|
+
if source[j] == "\n":
|
|
173
|
+
line += 1
|
|
174
|
+
j += 1
|
|
175
|
+
if j >= n:
|
|
176
|
+
raise LexError(f"unterminated string (line {line})")
|
|
177
|
+
add("STRING", source[i : j + 1])
|
|
178
|
+
i = j + 1
|
|
179
|
+
continue
|
|
180
|
+
|
|
181
|
+
if ch.isdigit() or (ch == "." and i + 1 < n and source[i + 1].isdigit()):
|
|
182
|
+
j = i
|
|
183
|
+
seen_dot = False
|
|
184
|
+
while j < n and (source[j].isdigit() or (source[j] == "." and not seen_dot)):
|
|
185
|
+
if source[j] == ".":
|
|
186
|
+
seen_dot = True
|
|
187
|
+
j += 1
|
|
188
|
+
add("NUMBER", source[i:j])
|
|
189
|
+
i = j
|
|
190
|
+
continue
|
|
191
|
+
|
|
192
|
+
if ch == "$":
|
|
193
|
+
j = i
|
|
194
|
+
while j < n and source[j] == "$":
|
|
195
|
+
j += 1
|
|
196
|
+
if j - i > 2:
|
|
197
|
+
raise LexError(f"too many '$' (line {line})")
|
|
198
|
+
if j < n and source[j] == "{":
|
|
199
|
+
k = source.find("}", j)
|
|
200
|
+
if k == -1:
|
|
201
|
+
raise LexError(f"unterminated ${{...}} name (line {line})")
|
|
202
|
+
end = _qualify(source, k + 1, line)
|
|
203
|
+
add("NAME", source[i:end])
|
|
204
|
+
i = end
|
|
205
|
+
continue
|
|
206
|
+
if j < n and _is_name_char(source[j]):
|
|
207
|
+
k = j
|
|
208
|
+
while k < n and _is_name_char(source[k]):
|
|
209
|
+
k += 1
|
|
210
|
+
add("NAME", source[i:k])
|
|
211
|
+
i = k
|
|
212
|
+
continue
|
|
213
|
+
raise LexError(f"bare '$' (line {line})")
|
|
214
|
+
|
|
215
|
+
if ch == "¶": # pilcrow literal
|
|
216
|
+
add("NAME", ch)
|
|
217
|
+
i += 1
|
|
218
|
+
continue
|
|
219
|
+
|
|
220
|
+
if _is_name_start(ch):
|
|
221
|
+
j = i
|
|
222
|
+
while j < n and _is_name_char(source[j]):
|
|
223
|
+
j += 1
|
|
224
|
+
if source[i:j].lower() in KEYWORD_OPS:
|
|
225
|
+
add("OP", source[i:j])
|
|
226
|
+
i = j
|
|
227
|
+
continue
|
|
228
|
+
end = _extend_name_run(source, j)
|
|
229
|
+
end = _qualify(source, end, line)
|
|
230
|
+
add("NAME", source[i:end])
|
|
231
|
+
i = end
|
|
232
|
+
continue
|
|
233
|
+
|
|
234
|
+
matched = False
|
|
235
|
+
for two in TWO_CHAR_OPS:
|
|
236
|
+
if source.startswith(two, i):
|
|
237
|
+
add("OP", two)
|
|
238
|
+
i += 2
|
|
239
|
+
matched = True
|
|
240
|
+
break
|
|
241
|
+
if matched:
|
|
242
|
+
continue
|
|
243
|
+
|
|
244
|
+
if ch in SINGLE_CHAR_OPS:
|
|
245
|
+
add("OP", ch)
|
|
246
|
+
i += 1
|
|
247
|
+
continue
|
|
248
|
+
if ch == "(":
|
|
249
|
+
add("LPAREN", ch)
|
|
250
|
+
i += 1
|
|
251
|
+
continue
|
|
252
|
+
if ch == ")":
|
|
253
|
+
add("RPAREN", ch)
|
|
254
|
+
i += 1
|
|
255
|
+
continue
|
|
256
|
+
if ch == "[":
|
|
257
|
+
add("LBRACKET", ch)
|
|
258
|
+
i += 1
|
|
259
|
+
continue
|
|
260
|
+
if ch == "]":
|
|
261
|
+
add("RBRACKET", ch)
|
|
262
|
+
i += 1
|
|
263
|
+
continue
|
|
264
|
+
if ch == ";":
|
|
265
|
+
add("SEMI", ch)
|
|
266
|
+
i += 1
|
|
267
|
+
continue
|
|
268
|
+
|
|
269
|
+
raise LexError(f"unexpected character {ch!r} (line {line})")
|
|
270
|
+
|
|
271
|
+
tokens.append(Token("EOF", "", line, pending))
|
|
272
|
+
return tokens
|