nkparser 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
nkparser/lexer.py ADDED
@@ -0,0 +1,299 @@
1
+ """Character-level lexer for Foundry Nuke .nk script files.
2
+
3
+ The lexer converts raw source text into a flat list of Tokens. It handles:
4
+ - Bare words (identifiers, numbers, unquoted values)
5
+ - Quoted strings (double-quoted, with backslash escapes)
6
+ - Brace blocks {...} — fully matched, including multiline and nested braces
7
+ - Bracket expressions [...] — used in set/push operations
8
+ - Shebang lines (#! ...) and comments (# ...)
9
+ - Newlines (emitted as NEWLINE tokens to assist the parser)
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from .errors import LexError
15
+ from .tokens import Token, TokenKind
16
+
17
+
18
+ class Lexer:
19
+ """Tokenise a Nuke .nk source string.
20
+
21
+ Usage::
22
+
23
+ lexer = Lexer(source_text)
24
+ tokens = lexer.tokenize()
25
+ if lexer.errors:
26
+ for err in lexer.errors:
27
+ print(err)
28
+ """
29
+
30
+ def __init__(self, source: str) -> None:
31
+ self._src = source
32
+ self._pos = 0
33
+ self._line = 1
34
+ self._col = 1
35
+ self.errors: list[LexError] = []
36
+
37
+ # ------------------------------------------------------------------
38
+ # Public entry point
39
+ # ------------------------------------------------------------------
40
+
41
+ def tokenize(self) -> list[Token]:
42
+ """Return all tokens for the source text, including a trailing EOF."""
43
+ tokens: list[Token] = []
44
+ while self._pos < len(self._src):
45
+ ch = self._src[self._pos]
46
+
47
+ # Shebang (#! at very start of file)
48
+ if ch == "#" and self._pos == 0 and self._src[self._pos: self._pos + 2] == "#!":
49
+ tokens.append(self._read_shebang())
50
+ continue
51
+
52
+ # Comment
53
+ if ch == "#":
54
+ tokens.append(self._read_comment())
55
+ continue
56
+
57
+ # Newline
58
+ if ch == "\n":
59
+ tok = Token(
60
+ kind=TokenKind.NEWLINE,
61
+ value="\n",
62
+ line=self._line,
63
+ col=self._col,
64
+ end_line=self._line,
65
+ end_col=self._col + 1,
66
+ )
67
+ tokens.append(tok)
68
+ self._advance()
69
+ continue
70
+
71
+ # Carriage return (treat \r\n as single newline)
72
+ if ch == "\r":
73
+ self._advance()
74
+ continue
75
+
76
+ # Whitespace (space, tab — skip silently)
77
+ if ch in (" ", "\t"):
78
+ self._advance()
79
+ continue
80
+
81
+ # Quoted string
82
+ if ch == '"':
83
+ tokens.append(self._read_string())
84
+ continue
85
+
86
+ # Brace block
87
+ if ch == "{":
88
+ tokens.append(self._read_brace_block())
89
+ continue
90
+
91
+ # Bracket expression [stack 0] or [value ...]
92
+ if ch == "[":
93
+ tokens.append(self._read_bracket_expr())
94
+ continue
95
+
96
+ # Bare word (identifier / number / unquoted value including $var refs)
97
+ if self._is_word_char(ch):
98
+ tokens.append(self._read_word())
99
+ continue
100
+
101
+ # Unrecognised character — record error and skip
102
+ err = LexError(
103
+ f"unexpected character {ch!r}",
104
+ self._line,
105
+ self._col,
106
+ self._current_line_text(),
107
+ )
108
+ self.errors.append(err)
109
+ self._advance()
110
+
111
+ tokens.append(
112
+ Token(TokenKind.EOF, "", self._line,
113
+ self._col, self._line, self._col)
114
+ )
115
+ return tokens
116
+
117
+ # ------------------------------------------------------------------
118
+ # Private helpers
119
+ # ------------------------------------------------------------------
120
+
121
+ def _advance(self) -> str:
122
+ """Consume one character and update position tracking."""
123
+ ch = self._src[self._pos]
124
+ self._pos += 1
125
+ if ch == "\n":
126
+ self._line += 1
127
+ self._col = 1
128
+ else:
129
+ self._col += 1
130
+ return ch
131
+
132
+ def _peek(self, offset: int = 0) -> str:
133
+ idx = self._pos + offset
134
+ if idx < len(self._src):
135
+ return self._src[idx]
136
+ return ""
137
+
138
+ @staticmethod
139
+ def _is_word_char(ch: str) -> bool:
140
+ # Allow identifiers, numbers, signs, slashes, dots, colons, $, %, @, ~, ^, *, =, <, >, |, &, +
141
+ return ch not in (" ", "\t", "\n", "\r", "{", "}", "[", "]", '"', "#")
142
+
143
+ def _current_line_text(self) -> str:
144
+ start = self._src.rfind("\n", 0, self._pos)
145
+ start = 0 if start == -1 else start + 1
146
+ end = self._src.find("\n", self._pos)
147
+ end = len(self._src) if end == -1 else end
148
+ return self._src[start:end]
149
+
150
+ # ------------------------------------------------------------------
151
+ # Token readers
152
+ # ------------------------------------------------------------------
153
+
154
+ def _read_shebang(self) -> Token:
155
+ start_line, start_col = self._line, self._col
156
+ buf: list[str] = []
157
+ while self._pos < len(self._src) and self._src[self._pos] != "\n":
158
+ buf.append(self._advance())
159
+ return Token(
160
+ kind=TokenKind.SHEBANG,
161
+ value="".join(buf),
162
+ line=start_line,
163
+ col=start_col,
164
+ end_line=self._line,
165
+ end_col=self._col,
166
+ )
167
+
168
+ def _read_comment(self) -> Token:
169
+ start_line, start_col = self._line, self._col
170
+ buf: list[str] = []
171
+ while self._pos < len(self._src) and self._src[self._pos] != "\n":
172
+ buf.append(self._advance())
173
+ return Token(
174
+ kind=TokenKind.COMMENT,
175
+ value="".join(buf),
176
+ line=start_line,
177
+ col=start_col,
178
+ end_line=self._line,
179
+ end_col=self._col,
180
+ )
181
+
182
+ def _read_string(self) -> Token:
183
+ """Read a double-quoted string, handling backslash escapes."""
184
+ start_line, start_col = self._line, self._col
185
+ buf: list[str] = [self._advance()] # consume opening "
186
+ while self._pos < len(self._src):
187
+ ch = self._src[self._pos]
188
+ if ch == "\\":
189
+ buf.append(self._advance()) # backslash
190
+ if self._pos < len(self._src):
191
+ buf.append(self._advance()) # escaped char
192
+ continue
193
+ buf.append(self._advance())
194
+ if ch == '"':
195
+ break
196
+ else:
197
+ self.errors.append(
198
+ LexError("unterminated string literal", start_line, start_col)
199
+ )
200
+ return Token(
201
+ kind=TokenKind.STRING,
202
+ value="".join(buf),
203
+ line=start_line,
204
+ col=start_col,
205
+ end_line=self._line,
206
+ end_col=self._col,
207
+ )
208
+
209
+ def _read_brace_block(self) -> Token:
210
+ """Read a balanced {...} block, handling arbitrary nesting and strings inside."""
211
+ start_line, start_col = self._line, self._col
212
+ buf: list[str] = []
213
+ depth = 0
214
+ while self._pos < len(self._src):
215
+ ch = self._src[self._pos]
216
+ if ch == "{":
217
+ depth += 1
218
+ buf.append(self._advance())
219
+ elif ch == "}":
220
+ depth -= 1
221
+ buf.append(self._advance())
222
+ if depth == 0:
223
+ break
224
+ elif ch == '"':
225
+ # Consume string inside brace block verbatim to avoid
226
+ # misinterpreting braces inside quoted strings.
227
+ str_start = self._pos
228
+ buf.append(self._advance()) # opening "
229
+ while self._pos < len(self._src):
230
+ c2 = self._src[self._pos]
231
+ if c2 == "\\":
232
+ buf.append(self._advance())
233
+ if self._pos < len(self._src):
234
+ buf.append(self._advance())
235
+ continue
236
+ buf.append(self._advance())
237
+ if c2 == '"':
238
+ break
239
+ _ = str_start # used only for clarity
240
+ else:
241
+ buf.append(self._advance())
242
+ else:
243
+ self.errors.append(
244
+ LexError("unterminated brace block", start_line, start_col)
245
+ )
246
+ return Token(
247
+ kind=TokenKind.BRACE_BLOCK,
248
+ value="".join(buf),
249
+ line=start_line,
250
+ col=start_col,
251
+ end_line=self._line,
252
+ end_col=self._col,
253
+ )
254
+
255
+ def _read_bracket_expr(self) -> Token:
256
+ """Read a [...] expression (used in 'set X [stack 0]')."""
257
+ start_line, start_col = self._line, self._col
258
+ buf: list[str] = []
259
+ depth = 0
260
+ while self._pos < len(self._src):
261
+ ch = self._src[self._pos]
262
+ if ch == "[":
263
+ depth += 1
264
+ buf.append(self._advance())
265
+ elif ch == "]":
266
+ depth -= 1
267
+ buf.append(self._advance())
268
+ if depth == 0:
269
+ break
270
+ else:
271
+ buf.append(self._advance())
272
+ else:
273
+ self.errors.append(
274
+ LexError("unterminated bracket expression",
275
+ start_line, start_col)
276
+ )
277
+ return Token(
278
+ kind=TokenKind.BRACKET_EXPR,
279
+ value="".join(buf),
280
+ line=start_line,
281
+ col=start_col,
282
+ end_line=self._line,
283
+ end_col=self._col,
284
+ )
285
+
286
+ def _read_word(self) -> Token:
287
+ """Read a bare word token (identifier, number, unquoted value, $variable)."""
288
+ start_line, start_col = self._line, self._col
289
+ buf: list[str] = []
290
+ while self._pos < len(self._src) and self._is_word_char(self._src[self._pos]):
291
+ buf.append(self._advance())
292
+ return Token(
293
+ kind=TokenKind.WORD,
294
+ value="".join(buf),
295
+ line=start_line,
296
+ col=start_col,
297
+ end_line=self._line,
298
+ end_col=self._col,
299
+ )