bashwalker 0.1.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
bashwalker/core.py ADDED
@@ -0,0 +1,2380 @@
1
+ """bashwalker's walker: a bash command line in, proven facts per simple command out.
2
+
3
+ It parses a command with the pinned tree-sitter-bash and walks the tree with one handler
4
+ per named node kind of the pinned grammar (``HANDLERS``). Each simple command becomes an
5
+ ``Invocation``: its words, assignments, redirects and where its stdin comes from. What a
6
+ command is *allowed* to do is the caller's policy; this module only says what it *is*.
7
+
8
+ One principle runs through this module: **nothing is proven until it is proven**. Every
9
+ fact it records (the tree is right, a node was walked, a word is literal, stdin comes from
10
+ X) starts as not proven, and becomes proven only through the short lists of forms below.
11
+ Anything else is reported as the worst case.
12
+
13
+ The rules are numbered as in nl-shell's ``policy-gate/v1`` spec, where this walker comes
14
+ from. Rule 4 there (which redirects are safe) is policy and stays with the caller; this
15
+ module records the redirect facts that rule needs.
16
+
17
+ The facts are for one line run on its own in a new, non-interactive shell with no startup
18
+ files and a clean environment (architecture section 3.2); in an interactive shell, or one
19
+ that has already run other lines, they do not hold.
20
+
21
+ - Rule 1 (``untrusted_features``): the tree is trusted only with no ``ERROR`` or
22
+ ``MISSING`` node and none of the features tree-sitter-bash reads differently from bash.
23
+ - Rule 2 (``walk``): every named node is visited exactly once, and every ``command``,
24
+ ``declaration_command`` and ``unset_command`` gives exactly one invocation. Both are
25
+ checked against the tree on every walk; a miss is a ``ParseRefused``.
26
+ - Rule 3 (``Stdin``, ``lex_word``): stdin sources and the literal flag are proven only
27
+ through the forms listed there; anything else is ``unknown`` / not literal.
28
+ - Rule 5 (``Limits``): input over ``max_input_bytes`` is refused before it is parsed, and
29
+ the walk has a total-work budget (``max_work`` nodes, ``max_depth`` levels).
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import re
35
+ from collections.abc import Callable, Iterator
36
+ from dataclasses import dataclass, field, replace
37
+
38
+ import tree_sitter
39
+ import tree_sitter_bash
40
+
41
+ # Rule 5 defaults. tree-sitter-bash's parse time grows with the square of the input, which
42
+ # is why the size cap is checked before parsing.
43
+ MAX_INPUT_BYTES = 8 * 1024
44
+ MAX_WORK = 4096
45
+ MAX_DEPTH = 100
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class Limits:
50
+ """Rule 5: the bounds one ``walk`` runs under."""
51
+
52
+ max_input_bytes: int = MAX_INPUT_BYTES
53
+ max_work: int = MAX_WORK
54
+ max_depth: int = MAX_DEPTH
55
+
56
+
57
+ DEFAULT_LIMITS = Limits()
58
+
59
+ LANGUAGE = tree_sitter.Language(tree_sitter_bash.language())
60
+
61
+ ERROR_KIND = "ERROR"
62
+ SIMPLE_KINDS = ("command", "declaration_command", "unset_command")
63
+ REDIRECT_KINDS = ("file_redirect", "heredoc_redirect", "herestring_redirect")
64
+ # Anonymous tokens that are redirect operators.
65
+ REDIRECT_OPERATORS = frozenset(
66
+ (">", ">>", ">|", "&>", "&>>", "<>", ">&", "<&", ">&-", "<&-", "<", "<<", "<<-", "<<<")
67
+ )
68
+ INPUT_OPERATORS = ("<", "<&", "<&-") # default descriptor 0
69
+ # Operators that take no descriptor: bash passes a number written right before them to the
70
+ # command (`printf %s 0&>out` writes `0`).
71
+ NO_DESCRIPTOR_OPERATORS = ("&>", "&>>")
72
+ _PLAIN_DELIMITER = re.compile(r"[A-Za-z0-9_]+")
73
+ _QUOTED_DELIMITER = re.compile(r"'[^']*'|\"[^\"\\$`]*\"")
74
+ LOW_FD = ("0", "1", "2")
75
+
76
+
77
+ class ParseRefused(Exception):
78
+ """The walker could not prove its facts about the command (Rule 2 or Rule 5). The
79
+ message is fixed text: it never quotes the input. Callers treat the command as the
80
+ worst case."""
81
+
82
+
83
+ def named_kinds() -> tuple[str, ...]:
84
+ """Every named node kind of the pinned grammar, plus ``ERROR``."""
85
+ kinds = {
86
+ LANGUAGE.node_kind_for_id(i)
87
+ for i in range(LANGUAGE.node_kind_count)
88
+ if LANGUAGE.node_kind_is_named(i) and LANGUAGE.node_kind_is_visible(i)
89
+ }
90
+ kinds.add(ERROR_KIND)
91
+ return tuple(sorted(kinds))
92
+
93
+
94
+ def parse_source(source: bytes) -> tree_sitter.Tree:
95
+ """Parse ``source`` with the pinned grammar (the one entry point to the parser)."""
96
+ return tree_sitter.Parser(LANGUAGE).parse(source)
97
+
98
+
99
+ # --- facts ---------------------------------------------------------------------------
100
+
101
+
102
+ @dataclass(frozen=True)
103
+ class Stdin:
104
+ """Where a simple command's standard input comes from (Rule 3).
105
+
106
+ ``kind`` is one of:
107
+
108
+ - ``terminal``: a command outside any function body and outside ``>(…)``, with no
109
+ redirect touching fd 0, first in its pipeline or standing alone;
110
+ - ``pipe``: a later stage of a pipeline, or the command inside ``>(…)``;
111
+ - ``literal``: the simple command's own here-string whose word is literal (``value``
112
+ is the word with the newline bash appends), or its own here-doc whose delimiter is
113
+ quoted (or unquoted with no ``$``, `` ` `` or ``\\`` in its body; ``value`` is the
114
+ body). A literal feed is never passed into a compound command or a substitution
115
+ (their commands share it, and one may consume part of it before another reads), and
116
+ a substitution in a later redirect makes it ``unknown``;
117
+ - ``unknown``: everything else, including every ``<path``: what a path names (a
118
+ file, or through a symlink or ``/dev/fd`` a descriptor such as the pipe) is a
119
+ run-time fact. The redirect and its target stay in ``Invocation.redirects`` for a
120
+ caller that resolves the path when the command runs.
121
+
122
+ Consumers MUST treat ``unknown`` as the worst case for each of their rules: as a pipe
123
+ where a pipe is the danger, and as the terminal where the terminal is.
124
+ """
125
+
126
+ kind: str
127
+ value: str | None = None
128
+
129
+
130
+ TERMINAL = Stdin("terminal")
131
+ PIPE = Stdin("pipe")
132
+ UNKNOWN = Stdin("unknown")
133
+
134
+
135
+ @dataclass(frozen=True)
136
+ class Word:
137
+ """One shell word: ``text`` after quote removal, ``literal`` when bash would expand
138
+ nothing in it (Rule 3), ``source`` as written."""
139
+
140
+ text: str
141
+ literal: bool
142
+ source: str
143
+
144
+
145
+ @dataclass(frozen=True)
146
+ class Assignment:
147
+ """One ``variable_assignment`` node. ``role`` is ``prefix`` (``VAR=x cmd``),
148
+ ``declaration`` (inside ``export``/``declare``/…) or ``standalone``."""
149
+
150
+ name: str
151
+ value: Word | None
152
+ role: str
153
+
154
+
155
+ @dataclass(frozen=True)
156
+ class Redirect:
157
+ """One redirect, as written. Whether it is allowed is the caller's policy.
158
+
159
+ ``kind`` is ``file_redirect``, ``heredoc_redirect``, ``herestring_redirect``, or
160
+ ``loose``: a redirect operator the tree did not attach (inside an ``ERROR`` node, or
161
+ ``<``/``>`` inside ``[ ]``, which bash performs as a redirection; ``in_test`` marks
162
+ the latter). A loose redirect has no proven descriptor or target.
163
+ """
164
+
165
+ kind: str
166
+ descriptor: str | None
167
+ operator: str
168
+ target: Word | None
169
+ readable: bool # False when the parser could not read the target (MISSING, several)
170
+ output: bool # writes a descriptor other than stdin (file redirects only)
171
+ feed: Stdin | None # the stdin a here-doc/here-string gives
172
+ display: str # operator and target as written (to be escaped before display)
173
+ heredoc_operator: bool # its operator bytes are an unquoted `<<` in the text
174
+ start: int
175
+ relative: bool = False # a file target placed by the cwd or `~` (not a dup of 0-2)
176
+ in_test: bool = False # a loose `<`/`>` inside `[ ]`
177
+
178
+
179
+ @dataclass(frozen=True)
180
+ class Invocation:
181
+ """One simple command handed to the command layer."""
182
+
183
+ kind: str # command | declaration | unset
184
+ words: tuple[Word, ...]
185
+ assignments: tuple[Assignment, ...]
186
+ redirects: tuple[Redirect, ...]
187
+ stdin: Stdin
188
+ in_function: bool
189
+ # False when an assignment or redirect comes before the first word: a leading `time`
190
+ # is then the external program, not bash's reserved word (`X=1 time export Y=z`).
191
+ time_reserved: bool = True
192
+
193
+ @property
194
+ def command_word(self) -> Word | None:
195
+ return self.words[0] if self.words else None
196
+
197
+
198
+ @dataclass
199
+ class Walk:
200
+ """Everything the walk proved about one command line."""
201
+
202
+ untrusted: tuple[str, ...]
203
+ invocations: list[Invocation] = field(default_factory=list)
204
+ assignments: list[Assignment] = field(default_factory=list)
205
+ functions: list[str] = field(default_factory=list)
206
+ redirects: list[Redirect] = field(default_factory=list)
207
+ process_substitutions: int = 0
208
+ words: list[Word] = field(default_factory=list)
209
+
210
+
211
+ # --- Rule 3: the literal flag --------------------------------------------------------
212
+
213
+
214
+ def lex_word(source: str) -> tuple[str, frozenset[str]]:
215
+ """Quote-remove ``source`` as bash would and say what bash would expand in it.
216
+
217
+ Returns the text and the set of causes that make it non-literal (empty means
218
+ literal). ``tilde`` alone, at the very start and followed by ``/`` or nothing, is
219
+ reported as ``leading-tilde``, so a caller may allow it in a redirect target.
220
+ """
221
+ out: list[str] = []
222
+ causes: set[str] = set()
223
+ i, n = 0, len(source)
224
+ while i < n:
225
+ c = source[i]
226
+ if c == "\\":
227
+ if i + 1 < n:
228
+ if source[i + 1] == "\n":
229
+ causes.add("line continuation")
230
+ else:
231
+ out.append(source[i + 1])
232
+ i += 2
233
+ else:
234
+ out.append(c)
235
+ i += 1
236
+ continue
237
+ if c == "'":
238
+ end = source.find("'", i + 1)
239
+ if end < 0:
240
+ causes.add("unclosed quote")
241
+ out.append(source[i + 1 :])
242
+ break
243
+ out.append(source[i + 1 : end])
244
+ i = end + 1
245
+ continue
246
+ if c == '"':
247
+ i += 1
248
+ closed = False
249
+ while i < n:
250
+ d = source[i]
251
+ if d == '"':
252
+ closed = True
253
+ i += 1
254
+ break
255
+ if d == "\\" and i + 1 < n and source[i + 1] in '$`"\\\n':
256
+ if source[i + 1] == "\n":
257
+ causes.add("line continuation")
258
+ else:
259
+ out.append(source[i + 1])
260
+ i += 2
261
+ continue
262
+ if d in "$`":
263
+ causes.add("expansion")
264
+ out.append(d)
265
+ i += 1
266
+ if not closed:
267
+ causes.add("unclosed quote")
268
+ continue
269
+ if c == "$" and source[i + 1 : i + 2] == '"':
270
+ # `$"…"`: a translated string, which in the C locale is the double-quoted text.
271
+ # The translation is not known, so the word is not literal.
272
+ causes.add("locale translation")
273
+ i += 1
274
+ continue
275
+ if c == "$" and source[i + 1 : i + 2] == "'":
276
+ # `$'…'`: the text is what bash decodes (`$'PS\064'` is `PS4`). bash expands
277
+ # it, so the word is not literal, but every check reads the decoded text.
278
+ j = i + 2
279
+ while j < n and source[j] != "'":
280
+ j += 2 if source[j] == "\\" else 1
281
+ if j >= n:
282
+ causes.add("unclosed quote")
283
+ out.append(source[i + 2 :])
284
+ break
285
+ causes.add("expansion")
286
+ out.append(_ansi_c(source[i + 2 : j]))
287
+ i = j + 1
288
+ continue
289
+ if c in "$`":
290
+ causes.add("expansion")
291
+ elif c in "*?[":
292
+ causes.add("glob")
293
+ elif c in "{}":
294
+ causes.add("brace")
295
+ elif c in "()":
296
+ causes.add("pattern or substitution")
297
+ elif c in " \t\n;&|<>":
298
+ causes.add("metacharacter")
299
+ elif c == "~":
300
+ if i == 0:
301
+ follows = source[1:2]
302
+ causes.add("leading-tilde" if follows in ("", "/") else "tilde")
303
+ elif source[i - 1] in "=:":
304
+ causes.add("tilde")
305
+ out.append(c)
306
+ i += 1
307
+ return "".join(out), frozenset(causes)
308
+
309
+
310
+ def make_word(source: str) -> Word:
311
+ text, causes = lex_word(source)
312
+ return Word(text=text, literal=not causes, source=source)
313
+
314
+
315
+ def _is_relative_target(operator: str, target: Word | None) -> bool:
316
+ """A file target that the cwd or ``~`` places (not a dup or close of 0-2)."""
317
+ if target is None:
318
+ return False
319
+ if operator in (">&", "<&") and re.fullmatch(r"[012]-?|-", target.text):
320
+ return False
321
+ return target.source.startswith("~") or not target.text.startswith("/")
322
+
323
+
324
+ def _is_output(descriptor: str | None, operator: str, kind: str) -> bool:
325
+ if kind != "file_redirect":
326
+ return False
327
+ if operator in INPUT_OPERATORS and descriptor in (None, "0"):
328
+ return False
329
+ return True
330
+
331
+
332
+ # --- Rule 1: trusting the tree -------------------------------------------------------
333
+
334
+
335
+ def _iter_nodes(root: tree_sitter.Node) -> Iterator[tree_sitter.Node]:
336
+ stack = [root]
337
+ while stack:
338
+ node = stack.pop()
339
+ yield node
340
+ stack.extend(reversed(node.children))
341
+
342
+
343
+ def _masks(root: tree_sitter.Node, size: int) -> tuple[bytearray, bytearray]:
344
+ """Two byte masks over the source: ``single`` (inside single or ANSI-C quotes) and
345
+ ``quoted`` (also inside double-quoted text, here-doc text and comments)."""
346
+ single = bytearray(size)
347
+ quoted = bytearray(size)
348
+ bodies, spans = [], []
349
+ for node in _iter_nodes(root):
350
+ if node.type == "heredoc_body":
351
+ bodies.append(node)
352
+ elif node.type in ("raw_string", "ansi_c_string"):
353
+ spans.append((node, True))
354
+ elif node.type in ("string_content", "heredoc_content", "comment"):
355
+ spans.append((node, False))
356
+ for body in bodies:
357
+ width = body.end_byte - body.start_byte
358
+ quoted[body.start_byte : body.end_byte] = b"\x01" * width
359
+ for child in body.named_children:
360
+ if child.type != "heredoc_content":
361
+ width = child.end_byte - child.start_byte
362
+ quoted[child.start_byte : child.end_byte] = bytes(width)
363
+ for node, is_single in spans:
364
+ width = node.end_byte - node.start_byte
365
+ quoted[node.start_byte : node.end_byte] = b"\x01" * width
366
+ if is_single:
367
+ single[node.start_byte : node.end_byte] = b"\x01" * width
368
+ return single, quoted
369
+
370
+
371
+ def _escaped(source: bytes, i: int) -> bool:
372
+ count = 0
373
+ while i - count - 1 >= 0 and source[i - count - 1] == 0x5C:
374
+ count += 1
375
+ return count % 2 == 1
376
+
377
+
378
+ def _heredoc_runs(source: bytes) -> list[tuple[int, int]]:
379
+ """Every run of exactly two ``<`` in the text: what bash reads as a here-doc
380
+ operator wherever ``<<`` is an operator at all."""
381
+ runs, i, n = [], 0, len(source)
382
+ while i < n:
383
+ if source[i] == 0x3C:
384
+ j = i
385
+ while j < n and source[j] == 0x3C:
386
+ j += 1
387
+ if j - i == 2:
388
+ runs.append((i, j))
389
+ i = j
390
+ else:
391
+ i += 1
392
+ return runs
393
+
394
+
395
+ def _bracket_redirects(node: tree_sitter.Node) -> list[tree_sitter.Node]:
396
+ """The redirect-operator tokens inside a single-bracket test (``[ x > y ]``).
397
+ tree-sitter-bash reads them as test operators; bash runs ``[`` as a command, so for
398
+ bash they are redirections. ``[[ … ]]`` compares strings and is not affected."""
399
+ if node.type != "test_command" or not node.children or node.children[0].type != "[":
400
+ return []
401
+ return [n for n in _iter_nodes(node) if not n.is_named and n.type in REDIRECT_OPERATORS]
402
+
403
+
404
+ # Quoted or escaped text that bash may later read as a variable reference (`printf -v`,
405
+ # `read`, `test -v`, `[[ -v ]]`, `let`, `local`, `declare -n`, `${!v}`): a subscript in
406
+ # it is evaluated, and a substitution there runs although the tree never parsed it
407
+ # (`printf -v 'a[$(id)0]' x`). Any such text is flagged, wherever it is; harmless text
408
+ # such as `echo 'a[$(x)]'` is flagged too, which fails closed.
409
+ _TEXT_KINDS = ("word", "raw_string", "string", "ansi_c_string", "concatenation")
410
+ _ANSI_C_ESCAPE = re.compile(
411
+ r"\\(?:x([0-9A-Fa-f]{1,2})|u([0-9A-Fa-f]{1,4})|U([0-9A-Fa-f]{1,8})|([0-7]{1,3})|c(.)|(.))",
412
+ re.S,
413
+ )
414
+ _ANSI_C_SIMPLE = {"a": "\a", "b": "\b", "e": "\x1b", "E": "\x1b", "f": "\f", "n": "\n",
415
+ "r": "\r", "t": "\t", "v": "\v"}
416
+
417
+
418
+ _SUBSTITUTION_TEXT = re.compile(r"\$\(|`|\$\{")
419
+
420
+
421
+ def _holds_substitution_text(value: Word | None) -> bool:
422
+ """A value that holds a ``$`` or a backtick as text, not as an expansion: pieces
423
+ joined later can spell a substitution (``x='a[$'; x+='(id)0]'``), and a ``$(`` split
424
+ this way leaves the ``$`` in one of them. Harmless text such as ``x='$5'`` is flagged
425
+ too, which fails closed."""
426
+ return value is not None and _text_specials(value.source)
427
+
428
+
429
+ # What may follow `$` to start an expansion; any other `$` is plain text.
430
+ _EXPANSION_START = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_{([@*#?$!-'\"")
431
+
432
+
433
+ def _text_specials(source: str) -> bool:
434
+ """Whether ``source`` yields a ``$`` or a backtick as plain text once bash removes
435
+ the quoting: inside single quotes, escaped with a backslash, decoded from ``$'…'``,
436
+ or a ``$`` that starts no expansion (``"a[$"``)."""
437
+ i, n = 0, len(source)
438
+ in_double = False
439
+ while i < n:
440
+ c = source[i]
441
+ if c == "\\":
442
+ if i + 1 < n and source[i + 1] in "$`":
443
+ return True
444
+ i += 2
445
+ continue
446
+ if not in_double and c == "'":
447
+ end = source.find("'", i + 1)
448
+ body = source[i + 1 :] if end < 0 else source[i + 1 : end]
449
+ if "$" in body or "`" in body:
450
+ return True
451
+ i = n if end < 0 else end + 1
452
+ continue
453
+ if not in_double and c == "$" and source[i + 1 : i + 2] == "'":
454
+ j = i + 2
455
+ while j < n and source[j] != "'":
456
+ j += 2 if source[j] == "\\" else 1
457
+ decoded = _ansi_c(source[i + 2 : j])
458
+ if "$" in decoded or "`" in decoded:
459
+ return True
460
+ i = j + 1
461
+ continue
462
+ if c == '"':
463
+ in_double = not in_double
464
+ i += 1
465
+ continue
466
+ if c == "$":
467
+ after = source[i + 1 : i + 2]
468
+ if not after or after not in _EXPANSION_START or (in_double and after in "'\""):
469
+ return True
470
+ i += 1
471
+ return False
472
+
473
+
474
+ def _is_arithmetic(node: tree_sitter.Node) -> bool:
475
+ if node.type in ("arithmetic_expansion", "c_style_for_statement"):
476
+ return True
477
+ return node.type == "compound_statement" and node.text.startswith(b"((")
478
+
479
+
480
+ def _ansi_c(body: str) -> str:
481
+ """The text of a ``$'…'`` body, as bash decodes its escapes."""
482
+
483
+ def one(m: re.Match) -> str:
484
+ hex2, hex4, hex8, octal, control, other = m.groups()
485
+ if octal is not None:
486
+ return chr(int(octal, 8) & 0xFF) # bash keeps the low byte (`\444` is `$`)
487
+ for digits in (hex2, hex4, hex8):
488
+ if digits is not None:
489
+ code = int(digits, 16)
490
+ return chr(code) if code < 0x110000 else ""
491
+ if control is not None:
492
+ return chr(ord(control) & 0x1F)
493
+ return _ANSI_C_SIMPLE.get(other, other)
494
+
495
+ # bash ends the string at its first NUL (`$'a\0b'` is `a`; also `\c@`).
496
+ return _ANSI_C_ESCAPE.sub(one, body).split("\0", 1)[0]
497
+
498
+
499
+ def _decoded(node: tree_sitter.Node) -> str:
500
+ """A text node as bash reads it after quote removal, ``$'…'`` escapes decoded. The
501
+ patterns checked on it are ASCII, and a node may split an invalid UTF-8 sequence, so
502
+ it decodes leniently."""
503
+ if node.type == "ansi_c_string":
504
+ return _ansi_c(node.text.decode("utf-8", "replace")[2:-1])
505
+ if node.type == "concatenation":
506
+ return "".join(_decoded(child) for child in node.children)
507
+ return lex_word(node.text.decode("utf-8", "replace"))[0]
508
+ # From any `[` on, not only to the next `]`: subscripts nest (`a[b[0]+$(id)]`).
509
+ _HIDDEN_SUBSCRIPT = re.compile(r"\[.*?(?:\$\(|`|\$\{)", re.S)
510
+
511
+
512
+ def _joined_across_lines(node: tree_sitter.Node) -> bool:
513
+ """Whether a simple command's parts are separated by an unescaped line break, which
514
+ ends a command in bash (`HOME=/dev <<<$x` then a new line `if …` parses as one
515
+ command named `if`)."""
516
+ parts = node.children
517
+ for before, after in zip(parts, parts[1:]):
518
+ gap = node.text[before.end_byte - node.start_byte : after.start_byte - node.start_byte]
519
+ if b"\n" in gap.replace(b"\\\n", b""):
520
+ return True
521
+ return False
522
+
523
+
524
+ def _quoted_heredoc(body: tree_sitter.Node) -> bool:
525
+ """Whether a here-doc body's delimiter is quoted, so bash expands nothing in it."""
526
+ redirect = body.parent
527
+ if redirect is None:
528
+ return False
529
+ for child in redirect.children:
530
+ if child.type == "heredoc_start":
531
+ return child.text[:1] in (b"'", b'"')
532
+ return False
533
+
534
+
535
+ def untrusted_features(tree: tree_sitter.Tree, source: bytes) -> tuple[str, ...]:
536
+ """Rule 1. The features that make the tree untrusted (empty means trusted)."""
537
+ root = tree.root_node
538
+ found: list[str] = []
539
+
540
+ def note(feature: str) -> None:
541
+ if feature not in found:
542
+ found.append(feature)
543
+
544
+ heredocs = []
545
+ for node in _iter_nodes(root):
546
+ if node.is_error or node.type == ERROR_KIND:
547
+ note("syntax error")
548
+ if node.is_missing:
549
+ note("missing token")
550
+ if node.type == "heredoc_start":
551
+ text = node.text.decode("utf-8", "surrogatepass")
552
+ if not (_PLAIN_DELIMITER.fullmatch(text) or _QUOTED_DELIMITER.fullmatch(text)):
553
+ note("here-doc delimiter is not a plain or fully quoted word")
554
+ if node.type == "heredoc_redirect":
555
+ heredocs.append(node)
556
+ # tree-sitter-bash parses neither of these backtick forms; bash runs both
557
+ # (nl-shell#213).
558
+ if node.type == "command_substitution" and node.text.startswith(b"`"):
559
+ if b"\\`" in node.text:
560
+ note("backtick substitution nested in a backtick substitution")
561
+ if re.search(rb"(?<!\\)`", node.text[1:-1]):
562
+ # An unescaped backtick cannot be inside one: the grammar merged two
563
+ # (`` `id` `rm x` `` parses as one substitution running "id` `rm").
564
+ note("backtick substitutions the parser merged into one")
565
+ if node.type == "heredoc_body" and b"`" in node.text and not _quoted_heredoc(node):
566
+ note("backtick in an unquoted here-doc body")
567
+ if not node.is_named and node.type == "<<-":
568
+ note("here-doc with <<-")
569
+ if node.type == "word" and re.search(rb"(?<!\\)[ \t\n]", node.text):
570
+ # An unquoted word cannot hold unescaped whitespace: the grammar read two
571
+ # words as one (`inside [ ]` gives the single word "[ ]").
572
+ note("word the parser read across whitespace")
573
+ if node.type in SIMPLE_KINDS and _joined_across_lines(node):
574
+ note("command the parser continued across a line break")
575
+ if _bracket_redirects(node):
576
+ note("'<' or '>' inside [ ], which bash reads as a redirection")
577
+ if node.type in _TEXT_KINDS and _HIDDEN_SUBSCRIPT.search(_decoded(node)):
578
+ note("quoted text holding a subscript with a substitution bash may evaluate")
579
+ if _is_arithmetic(node):
580
+ # Inside `$(( ))`, `(( ))` and `for (( ))` bash expands quoted text again:
581
+ # `$(( '$(id)0' ))` and `$'\x24(id)'` there run `id`.
582
+ for inner in _iter_nodes(node):
583
+ if inner.type in _TEXT_KINDS and _SUBSTITUTION_TEXT.search(_decoded(inner)):
584
+ note("quoted text in an arithmetic context holding a substitution")
585
+ break
586
+ if node.type == "for_statement": # also `select`, which the grammar parses so
587
+ target = node.child_by_field_name("variable")
588
+ name = target.text.decode("utf-8", "replace") if target is not None else ""
589
+ if name in CODE_VARIABLES:
590
+ # `for PS4 in '$(id)'; do set -x; :; done` sets PS4 to each value.
591
+ note("a variable bash runs as code (PS4, PROMPT_COMMAND, …) is set")
592
+ if name in MODE_VARIABLES:
593
+ note("a shell option that changes how bash reads the line is set")
594
+ if name in FD_VARIABLES:
595
+ note("BASH_XTRACEFD, which can close the shell's stdin, is set")
596
+ if node.type == "expansion":
597
+ # `${PS4:='$(id)'}` and `${POSIXLY_CORRECT:=1}` assign the variable.
598
+ assigned = _DEFAULT_ASSIGNMENT.match(node.text.decode("utf-8", "replace"))
599
+ if assigned and assigned.group(1) in CODE_VARIABLES:
600
+ note("a variable bash runs as code (PS4, PROMPT_COMMAND, …) is set")
601
+ if assigned and assigned.group(1) in MODE_VARIABLES:
602
+ note("a shell option that changes how bash reads the line is set")
603
+ if assigned and assigned.group(1) in FD_VARIABLES:
604
+ note("BASH_XTRACEFD, which can close the shell's stdin, is set")
605
+ if node.type == "expansion" and node.text.endswith(b"@P}"):
606
+ # `${v@P}` expands v's value as a prompt, which runs `$(…)` in it; the value
607
+ # is only known at run time.
608
+ note("${…@P} prompt expansion, which runs the value's substitutions")
609
+ if node.type == "subscript":
610
+ # The whole index, whose quoted pieces are separate nodes (`a['$(id)0']=1`).
611
+ index = node.child_by_field_name("index")
612
+ if index is not None and _HIDDEN_SUBSCRIPT.search("[" + _decoded(index)):
613
+ note("quoted text holding a subscript with a substitution bash may evaluate")
614
+ single, quoted = _masks(root, len(source))
615
+ lines: dict[int, int] = {}
616
+ for start, _ in _heredoc_runs(source):
617
+ if quoted[start] or _escaped(source, start):
618
+ continue
619
+ if source[start + 2 : start + 3] == b"-":
620
+ note("here-doc with <<-")
621
+ line = source.count(b"\n", 0, start)
622
+ lines[line] = lines.get(line, 0) + 1
623
+ if any(count > 1 for count in lines.values()):
624
+ note("more than one here-doc on one line")
625
+ if sum(lines.values()) != len(heredocs):
626
+ note("here-doc the parser did not place")
627
+ for i in range(len(source) - 1):
628
+ pair = source[i : i + 2]
629
+ if pair == b"\\\n" and not single[i]:
630
+ note("backslash-newline")
631
+ elif pair[1:] == b"(" and pair[:1] in (b"?", b"*", b"+", b"@", b"!"):
632
+ if not quoted[i] and not _escaped(source, i):
633
+ note("extglob pattern")
634
+ elif pair == b"${" and not single[i]:
635
+ if source[i + 2 : i + 3] in (b" ", b"\t", b"\n", b"|"):
636
+ note("bash 5.3 ${ } substitution")
637
+ return tuple(found)
638
+
639
+
640
+ # --- Rule 2: the walk ----------------------------------------------------------------
641
+
642
+
643
+ @dataclass(frozen=True)
644
+ class Ctx:
645
+ stdin: Stdin
646
+ in_function: bool = False
647
+ role: str | None = None # how a variable_assignment child is used
648
+ # In a simple command's redirects: the stdin its words are expanded with. bash
649
+ # expands words before it performs any redirect.
650
+ command_stdin: Stdin | None = None
651
+
652
+
653
+ Handler = Callable[["Walker", tree_sitter.Node, Ctx, int, tuple], object]
654
+
655
+
656
+ class Walker:
657
+ def __init__(self, source: bytes, layer: Walk, limits: Limits = DEFAULT_LIMITS) -> None:
658
+ self.source = source
659
+ self.layer = layer
660
+ self.limits = limits
661
+ self.visits: dict[tuple[int, int, int], int] = {}
662
+ self.work = 0
663
+ self.heredoc_runs = _heredoc_runs(source)
664
+ # Words written after a file redirect's target (`cat <a b`): tree-sitter-bash puts
665
+ # them under the redirect, bash passes them to the command. Keyed by the
666
+ # redirect's id until the simple command that owns the redirect takes them.
667
+ self.operands: dict[int, list[tuple[int, int]]] = {}
668
+ # Per invocation, in the same order: its command node's byte span and whether a
669
+ # loop or function body around it may run it again after later commands.
670
+ self.spans: list[tuple[int, int, bool]] = []
671
+ # Command and process substitutions walked so far (each may read stdin).
672
+ self.substitutions = 0
673
+
674
+ def note_untrusted(self, feature: str) -> None:
675
+ """Rule 1, for a feature only the walk can see."""
676
+ if feature not in self.layer.untrusted:
677
+ self.layer.untrusted += (feature,)
678
+
679
+ def text(self, node: tree_sitter.Node) -> str:
680
+ return self.source[node.start_byte : node.end_byte].decode("utf-8", "surrogatepass")
681
+
682
+ def span(self, start: int, end: int) -> str:
683
+ return self.source[start:end].decode("utf-8", "surrogatepass")
684
+
685
+ def visit(self, node: tree_sitter.Node, ctx: Ctx, depth: int, bound: tuple = ()):
686
+ self.work += 1
687
+ if self.work > self.limits.max_work:
688
+ raise ParseRefused("work budget exceeded")
689
+ if depth > self.limits.max_depth:
690
+ raise ParseRefused("nesting too deep")
691
+ kind = ERROR_KIND if node.is_error else node.type
692
+ handler = HANDLERS.get(kind)
693
+ if handler is None:
694
+ raise ParseRefused("unhandled node kind")
695
+ key = (node.id, node.start_byte, node.end_byte)
696
+ self.visits[key] = self.visits.get(key, 0) + 1
697
+ return handler(self, node, ctx, depth, bound)
698
+
699
+ # Descriptor written right before a redirect operator that the tree did not attach
700
+ # (`cat 0>&-` parses as the argument `0` and the redirect `>&-`).
701
+ def adjacent_token(self, start: int) -> tuple[int, str] | None:
702
+ i = start
703
+ while i > 0 and self.source[i - 1] not in b" \t\n;&|()<>":
704
+ i -= 1
705
+ if i == start:
706
+ return None
707
+ return i, self.span(i, start)
708
+
709
+ def in_heredoc_run(self, start: int) -> bool:
710
+ """True when ``start`` is inside a ``<<`` run with no descriptor in
711
+ front of it, or a literal 0-2 (``3<<E`` and ``{x}<<E`` are never exempt)."""
712
+ for a, b in self.heredoc_runs:
713
+ if a <= start < b:
714
+ token = self.adjacent_token(a)
715
+ return token is None or token[1] in LOW_FD
716
+ return False
717
+
718
+
719
+ def _children(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
720
+ if bound:
721
+ ctx = _apply_compound(walker, bound, ctx, depth)
722
+ for child in node.named_children:
723
+ walker.visit(child, _async(child, ctx), depth + 1)
724
+
725
+
726
+ def _async(child: tree_sitter.Node, ctx: Ctx) -> Ctx:
727
+ """A command, pipeline or list run with `&` reads `/dev/null` without job control and
728
+ is stopped if it reads the terminal with it: its stdin is not proven (its own
729
+ redirects still set one)."""
730
+ after = child.next_sibling
731
+ if after is not None and not after.is_named and after.type == "&":
732
+ return replace(ctx, stdin=UNKNOWN)
733
+ return ctx
734
+
735
+
736
+ def _consumed(stdin: Stdin, substituted: bool) -> Stdin:
737
+ """A literal feed that a substitution in a later redirect may have read part of
738
+ (``bash <<<'#cmd' >"$(read -N 1 s)"`` gives ``bash`` ``cmd``) is no longer proven."""
739
+ if substituted and stdin.kind == "literal":
740
+ return UNKNOWN
741
+ return stdin
742
+
743
+
744
+ def _apply_compound(walker: Walker, bound: tuple, ctx: Ctx, depth: int) -> Ctx:
745
+ """Redirects on a compound command: record them, and let them set the stdin the
746
+ commands inside inherit.
747
+
748
+ A literal feed is never passed on: the commands inside share it, so one can consume
749
+ part of it before the next reads (``{ read -N 1 s; bash; } <<<'#cmd'`` gives ``bash``
750
+ ``cmd``), and a loop reads on where the last pass stopped. They get ``unknown``."""
751
+ stdin = ctx.stdin
752
+ for node in bound:
753
+ # Each redirect's substitutions read the stdin the earlier ones set.
754
+ before = walker.substitutions
755
+ visited = walker.visit(node, replace(ctx, stdin=stdin), depth + 1)
756
+ stdin = _consumed(stdin, walker.substitutions != before)
757
+ for record in visited:
758
+ stdin = _stdin_after(record, stdin)
759
+ if stdin.kind == "literal":
760
+ stdin = UNKNOWN
761
+ return replace(ctx, stdin=stdin)
762
+
763
+
764
+ def _stdin_after(record: Redirect, stdin: Stdin) -> Stdin:
765
+ if record.kind in ("heredoc_redirect", "herestring_redirect"):
766
+ if record.descriptor in (None, "0"):
767
+ return record.feed or UNKNOWN
768
+ if record.descriptor in LOW_FD:
769
+ return stdin
770
+ return UNKNOWN
771
+ if record.descriptor is not None and not record.descriptor.isdigit():
772
+ return UNKNOWN
773
+ if record.descriptor is None:
774
+ fd = "0" if record.operator in INPUT_OPERATORS or record.operator == "<>" else "1"
775
+ else:
776
+ fd = record.descriptor.lstrip("0") or "0"
777
+ if record.operator in (">&", "<&") and record.target is not None:
778
+ target = record.target
779
+ if not target.literal or re.fullmatch(r"0+-", target.text):
780
+ # `3<&0-` moves fd 0 away and closes it; a target the walker cannot read
781
+ # may do the same.
782
+ return UNKNOWN
783
+ if fd != "0":
784
+ return stdin
785
+ # `<path`: the path may name any descriptor at run time (a symlink to /dev/stdin
786
+ # reads the pipe), so it proves no stdin.
787
+ return UNKNOWN
788
+
789
+
790
+ def _error(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
791
+ for child in node.children:
792
+ loose = (not child.is_named and child.type in REDIRECT_OPERATORS) or (
793
+ child.child_count == 0 and child.is_error
794
+ )
795
+ if loose:
796
+ text = walker.text(child)
797
+ if child.is_named and not any(c in text for c in "<>"):
798
+ continue
799
+ walker.layer.redirects.append(
800
+ Redirect(
801
+ kind="loose",
802
+ descriptor=None,
803
+ operator=child.type,
804
+ target=None,
805
+ readable=False,
806
+ output=False,
807
+ feed=None,
808
+ display=text,
809
+ heredoc_operator=walker.in_heredoc_run(child.start_byte),
810
+ start=child.start_byte,
811
+ )
812
+ )
813
+ _children(walker, node, ctx, depth, bound)
814
+
815
+
816
+ def _test_command(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
817
+ """``[ … ]``: each ``<``/``>`` inside is a redirection bash would perform, with a
818
+ target the tree does not give, so it is recorded as loose (Rule 1 also marks it)."""
819
+ for token in _bracket_redirects(node):
820
+ after = token.next_sibling
821
+ end = after.end_byte if after is not None else token.end_byte
822
+ walker.layer.redirects.append(
823
+ Redirect(
824
+ kind="loose",
825
+ descriptor=None,
826
+ operator=token.type,
827
+ target=None,
828
+ readable=False,
829
+ output=False,
830
+ feed=None,
831
+ display=walker.span(token.start_byte, end),
832
+ heredoc_operator=False,
833
+ start=token.start_byte,
834
+ in_test=True,
835
+ )
836
+ )
837
+ _children(walker, node, ctx, depth, bound)
838
+
839
+
840
+ def _pipeline(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
841
+ stages, later = [], False
842
+ for child in node.children:
843
+ if not child.is_named:
844
+ if child.type in ("|", "|&"):
845
+ later = True
846
+ continue
847
+ stages.append((child, later))
848
+ # A trailing redirect on a pipeline belongs to its last stage, as in bash.
849
+ last = max((i for i, (c, _) in enumerate(stages) if c.type != "comment"), default=-1)
850
+ for i, (child, is_later) in enumerate(stages):
851
+ stage_ctx = replace(ctx, stdin=PIPE) if is_later else ctx
852
+ walker.visit(child, stage_ctx, depth + 1, bound if i == last else ())
853
+ if bound and last < 0:
854
+ _apply_compound(walker, bound, ctx, depth)
855
+
856
+
857
+ def _list(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
858
+ """`a && b`, `a || b`: tree-sitter attaches a trailing redirect to the whole list
859
+ (`x || bash 0<f`), but bash attaches it to the last command only, as in a pipeline."""
860
+ if not bound:
861
+ _children(walker, node, ctx, depth)
862
+ return
863
+ parts = [c for c in node.named_children if c.type != "comment"]
864
+ for child in node.named_children:
865
+ walker.visit(child, _async(child, ctx), depth + 1, bound if child is parts[-1] else ())
866
+
867
+
868
+ def _negated(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
869
+ inner = node.named_children
870
+ if bound and len(inner) == 1:
871
+ walker.visit(inner[0], ctx, depth + 1, bound)
872
+ return
873
+ _children(walker, node, ctx, depth, bound)
874
+
875
+
876
+ def _redirected(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
877
+ body = node.child_by_field_name("body")
878
+ mine = tuple(c for c in node.named_children if body is None or c.id != body.id)
879
+ redirects = tuple(c for c in mine if c.type in REDIRECT_KINDS)
880
+ others = tuple(c for c in mine if c.type not in REDIRECT_KINDS)
881
+ for other in others:
882
+ walker.visit(other, ctx, depth + 1)
883
+ if body is None:
884
+ _apply_compound(walker, redirects + bound, ctx, depth)
885
+ return
886
+ walker.visit(body, ctx, depth + 1, redirects + bound)
887
+
888
+
889
+ def _simple(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
890
+ kind = {"command": "command", "declaration_command": "declaration"}.get(
891
+ node.type, "unset"
892
+ )
893
+ word_nodes: list = [] # name and argument children, named or not
894
+ prefix: list = []
895
+ redirect_nodes: list = list(bound)
896
+ others: list = []
897
+ unread: list = [] # walked, but not words (a zero-width MISSING command name)
898
+ keyword = None
899
+ seen_name = False
900
+ has_name = False
901
+ for i, child in enumerate(node.children):
902
+ fname = node.field_name_for_child(i)
903
+ if not child.is_named:
904
+ if fname == "argument":
905
+ word_nodes.append(child)
906
+ elif keyword is None and kind != "command" and not seen_name:
907
+ keyword = child
908
+ seen_name = True
909
+ continue
910
+ if child.type in REDIRECT_KINDS:
911
+ redirect_nodes.append(child)
912
+ elif fname == "name" and child.end_byte == child.start_byte:
913
+ # `PATH=x >/dev/null` parses with a MISSING command name: bash runs no
914
+ # command there, so this is no name.
915
+ unread.append(child)
916
+ seen_name = True
917
+ elif fname == "name":
918
+ word_nodes.append(child)
919
+ seen_name = True
920
+ has_name = True
921
+ elif child.type == "variable_assignment" and kind == "command" and not seen_name:
922
+ prefix.append(child)
923
+ elif fname == "argument" or kind != "command":
924
+ word_nodes.append(child)
925
+ else:
926
+ others.append(child)
927
+ redirect_nodes.sort(key=lambda n: n.start_byte)
928
+
929
+ # Descriptors the tree left as a separate word (`cat 0>&-`). Only a token that is a
930
+ # descriptor by the same test `_descriptor` uses is taken out of the words; any other
931
+ # word written against a redirect (`rm -rf />/dev/null`) stays an argument.
932
+ fd_tokens = set()
933
+ for r in redirect_nodes:
934
+ if r.child_by_field_name("descriptor") is not None:
935
+ continue
936
+ if _operator(r) not in NO_DESCRIPTOR_OPERATORS:
937
+ token = walker.adjacent_token(r.start_byte)
938
+ if token is not None and _is_descriptor_token(token[1]):
939
+ fd_tokens.add((token[0], r.start_byte))
940
+ fd_words = [w for w in word_nodes if (w.start_byte, w.end_byte) in fd_tokens]
941
+ word_nodes = [w for w in word_nodes if (w.start_byte, w.end_byte) not in fd_tokens]
942
+
943
+ # With no command word (`PATH=x >/dev/null`) the assignment is not a one-command
944
+ # prefix: bash keeps it in the shell, so it is standalone.
945
+ prefix_role = "prefix" if has_name or kind != "command" else "standalone"
946
+ assignments = []
947
+ for child in prefix:
948
+ assignments.append(walker.visit(child, replace(ctx, role=prefix_role), depth + 1))
949
+ role = "declaration" if kind == "declaration" else None
950
+ for child in word_nodes:
951
+ if child.is_named:
952
+ result = walker.visit(child, replace(ctx, role=role), depth + 1)
953
+ if isinstance(result, Assignment):
954
+ assignments.append(result)
955
+ if kind == "declaration":
956
+ # An operand that is not one `variable_assignment` node is still read by bash
957
+ # as `name=value` after quote removal (`export 'SUDO_PROMPT=x'`). Pieces with no
958
+ # space between them are one bash word (`export S'UDO_PROMPT'=x`).
959
+ operands: list[list] = []
960
+ for child in sorted(word_nodes, key=lambda n: n.start_byte):
961
+ if operands and operands[-1][-1].end_byte == child.start_byte:
962
+ operands[-1].append(child)
963
+ else:
964
+ operands.append([child])
965
+ for pieces in operands:
966
+ if len(pieces) == 1 and pieces[0].type == "variable_assignment":
967
+ continue
968
+ _declaration_operand(
969
+ walker, walker.span(pieces[0].start_byte, pieces[-1].end_byte), assignments
970
+ )
971
+ for child in others + unread + [w for w in fd_words if w.is_named]:
972
+ walker.visit(child, ctx, depth + 1)
973
+ # bash performs redirects left to right and expands each just before, so a
974
+ # substitution in one reads the stdin the earlier ones set (`: </x >"$(bash)"` runs
975
+ # /x).
976
+ records: list[Redirect] = []
977
+ redirect_stdin = ctx.stdin
978
+ for child in redirect_nodes:
979
+ before = walker.substitutions
980
+ redirect_ctx = replace(ctx, stdin=redirect_stdin, command_stdin=ctx.stdin)
981
+ visited = walker.visit(child, redirect_ctx, depth + 1)
982
+ redirect_stdin = _consumed(redirect_stdin, walker.substitutions != before)
983
+ for record in visited:
984
+ records.append(record)
985
+ redirect_stdin = _stdin_after(record, redirect_stdin)
986
+ # Words the grammar put after a redirect target are the command's (`cat <a b`).
987
+ recovered: list[tuple[int, int]] = []
988
+ for record in records:
989
+ recovered.extend(walker.operands.pop(id(record), ()))
990
+ if kind == "declaration":
991
+ # They are declaration operands like any other (`declare >/dev/null 'x=1'`).
992
+ for start, end in recovered:
993
+ _declaration_operand(walker, walker.span(start, end), assignments)
994
+
995
+ # Words: tree-sitter splits some bash words in two (`$"x"`); join pieces with no
996
+ # space between them, as bash does.
997
+ words: list[Word] = []
998
+ if keyword is not None:
999
+ words.append(make_word(walker.text(keyword)))
1000
+ groups: list[list[int]] = []
1001
+ for child in sorted(word_nodes + others, key=lambda n: n.start_byte):
1002
+ if groups and groups[-1][1] == child.start_byte:
1003
+ groups[-1][1] = child.end_byte
1004
+ else:
1005
+ groups.append([child.start_byte, child.end_byte])
1006
+ groups.extend([start, end] for start, end in recovered)
1007
+ groups.sort()
1008
+ for start, end in groups:
1009
+ words.append(make_word(walker.span(start, end)))
1010
+ if kind == "command" and not has_name:
1011
+ words.insert(0, Word(text="", literal=False, source=""))
1012
+ walker.layer.words.extend(words)
1013
+ # `command export 'PATH=x'`, `"export" 'PATH=x'` and `builtin unset …` are the
1014
+ # builtins themselves, which the grammar parses as an ordinary command: look through
1015
+ # the prefixes and the quoting.
1016
+ head = _builtin_head(words, _time_reserved(node)) if kind == "command" else None
1017
+ if kind == "command" and _assigns_after_time(words, _time_reserved(node)):
1018
+ walker.note_untrusted("assignment after time, which the walker does not record")
1019
+ if kind == "command" and _skip_prefixes(words, _time_reserved(node)) == UNREADABLE:
1020
+ walker.note_untrusted("command or builtin prefix with an option the walker cannot read")
1021
+ declarer = 0 if kind == "declaration" else head
1022
+ display_only = declarer is not None and _displays_only(words, declarer)
1023
+ if head is not None and words[head].text in DECLARATION_BUILTINS and not display_only:
1024
+ for word in words[head + 1 :]:
1025
+ _declaration_operand(walker, word.source, assignments)
1026
+ if display_only and kind == "declaration":
1027
+ # `declare -p NAME=x` only displays; it assigns nothing.
1028
+ dropped = [a for a in assignments if a.role == "declaration"]
1029
+ assignments[:] = [a for a in assignments if a.role != "declaration"]
1030
+ walker.layer.assignments = [
1031
+ a for a in walker.layer.assignments if not any(a is d for d in dropped)
1032
+ ]
1033
+ if declarer is not None and words[declarer].text in NAMEREF_BUILTINS:
1034
+ for word in words[declarer + 1 :]:
1035
+ option = word.text.startswith("-")
1036
+ if option and (not word.literal or "n" in word.text[1:]):
1037
+ # A nameref can alias any variable, also one bash runs as code
1038
+ # (`declare -n r=PS4; r='$(id)'; set -x; :`).
1039
+ walker.note_untrusted("nameref, which may alias a variable bash runs as code")
1040
+ hash_at = 0 if words and words[0].literal and words[0].text == "hash" else head
1041
+ if hash_at is not None and words[hash_at].text == "hash":
1042
+ for word in words[hash_at + 1 :]:
1043
+ if not word.literal or (word.text.startswith("-") and "p" in word.text):
1044
+ # `hash -p /x ls` makes a later `ls` run /x: command words stop meaning
1045
+ # what they say.
1046
+ walker.note_untrusted("hash -p, which changes what a command name runs")
1047
+ break
1048
+ alias_at = 0 if words and words[0].literal and words[0].text == "alias" else head
1049
+ if alias_at is not None and words[alias_at].text == "alias":
1050
+ if any(not w.literal or "=" in w.text for w in words[alias_at + 1 :]):
1051
+ # An alias changes how bash reads the lines after it, which this tree does
1052
+ # not show (`alias x='exec <payload'` then `x`).
1053
+ walker.note_untrusted("alias definition, which changes how later lines are read")
1054
+ if kind == "unset" or (head is not None and words[head].text == "unset"):
1055
+ # bash evaluates an array subscript in an unset operand after quote removal, so a
1056
+ # quoted operand runs what the tree never parsed (`unset 'a[$(id)0]'`).
1057
+ for word in words[(head or 0) + 1 :]:
1058
+ match = _SUBSCRIPTED.fullmatch(word.text)
1059
+ if match and not _NUMBER_SUBSCRIPT.fullmatch(match.group(1)):
1060
+ walker.note_untrusted("unset operand with a subscript bash evaluates")
1061
+
1062
+ stdin = redirect_stdin
1063
+ walker.layer.invocations.append(
1064
+ Invocation(
1065
+ kind=kind,
1066
+ words=tuple(words),
1067
+ assignments=tuple(a for a in assignments if a is not None),
1068
+ redirects=tuple(records),
1069
+ stdin=stdin,
1070
+ in_function=ctx.in_function,
1071
+ time_reserved=_time_reserved(node),
1072
+ )
1073
+ )
1074
+ walker.spans.append((node.start_byte, node.end_byte, _may_rerun(node)))
1075
+
1076
+
1077
+ _DECLARED = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)(\[[^\]]*\])?\+?=(.*)", re.S)
1078
+ _NUMBER_SUBSCRIPT = re.compile(r"\[[0-9]+\]")
1079
+ _SUBSCRIPTED = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\[.*\])", re.S)
1080
+
1081
+
1082
+ def _declared(source: str) -> tuple[Assignment | None, bool]:
1083
+ """A declaration operand that is not a ``variable_assignment`` node, as bash reads it
1084
+ after quote removal: ``export 'SUDO_PROMPT=x'`` assigns ``SUDO_PROMPT``. An operand
1085
+ whose name the walker cannot read (``export "$x"``) gives the name ``""``; a literal
1086
+ option or plain name (``-x``, ``A``) gives ``None``.
1087
+
1088
+ The flag is True when the operand has a subscript other than a plain number. bash
1089
+ evaluates an indexed array's subscript after quote removal, so a quoted operand runs
1090
+ what the tree never parsed (``declare -a 'x[$(id)]=1'`` runs ``id``)."""
1091
+ word = make_word(source)
1092
+ match = _DECLARED.match(word.text)
1093
+ if match is not None:
1094
+ value = Word(text=match.group(3), literal=word.literal, source=source)
1095
+ subscript = match.group(2)
1096
+ unsafe = subscript is not None and not _NUMBER_SUBSCRIPT.fullmatch(subscript)
1097
+ return Assignment(name=match.group(1), value=value, role="declaration"), unsafe
1098
+ if word.literal:
1099
+ return None, False
1100
+ return Assignment(name="", value=None, role="declaration"), False
1101
+
1102
+
1103
+ def _displays_only(words: list[Word], declarer: int) -> bool:
1104
+ """``declare``/``typeset``/``local`` with ``-p``, ``-f`` or ``-F`` display and assign
1105
+ nothing. An option the walker cannot read may assign, so it does not count."""
1106
+ if words[declarer].text not in NAMEREF_BUILTINS:
1107
+ return False
1108
+ for word in words[declarer + 1 :]:
1109
+ if not word.literal:
1110
+ return False
1111
+ if word.text == "--" or not word.text.startswith(("-", "+")):
1112
+ return False # options end here (`declare PATH=/x -p` assigns)
1113
+ # `-p`, `-f`, `-F` and `+p` display; `+f`/`+F` only turn an attribute off and
1114
+ # still assign.
1115
+ letters = "pfF" if word.text[0] == "-" else "p"
1116
+ if any(c in word.text[1:] for c in letters):
1117
+ return True
1118
+ return False
1119
+
1120
+
1121
+ def _declaration_operand(walker: Walker, source: str, assignments: list) -> None:
1122
+ operand, unsafe = _declared(source)
1123
+ if unsafe:
1124
+ walker.note_untrusted("declaration operand with a subscript bash evaluates")
1125
+ if operand is not None:
1126
+ assignments.append(operand)
1127
+ walker.layer.assignments.append(operand)
1128
+
1129
+
1130
+ def _assignment(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()):
1131
+ name_node = node.child_by_field_name("name")
1132
+ value_node = node.child_by_field_name("value")
1133
+ if name_node is not None and name_node.type == "subscript":
1134
+ inner = name_node.child_by_field_name("name")
1135
+ name_node = inner if inner is not None else name_node
1136
+ name = walker.text(name_node) if name_node is not None else walker.text(node)
1137
+ value = make_word(walker.text(value_node)) if value_node is not None else None
1138
+ record = Assignment(name=name, value=value, role=ctx.role or "standalone")
1139
+ walker.layer.assignments.append(record)
1140
+ if value is not None:
1141
+ walker.layer.words.append(value)
1142
+ _children(walker, node, replace(ctx, role=None), depth, bound)
1143
+ return record
1144
+
1145
+
1146
+ def _function(walker: Walker, node, ctx: Ctx, depth: int, bound: tuple = ()) -> None:
1147
+ name = node.child_by_field_name("name")
1148
+ body = node.child_by_field_name("body")
1149
+ start = name.start_byte if name is not None else node.start_byte
1150
+ end = body.start_byte if body is not None else node.end_byte
1151
+ shown = re.sub(r"\s*\(\s*\)\s*$", "", walker.span(start, end).strip())
1152
+ walker.layer.functions.append(shown)
1153
+ inner = replace(ctx, stdin=UNKNOWN, in_function=True, role=None)
1154
+ redirects = [c for c in node.named_children if c.type in REDIRECT_KINDS]
1155
+ for child in node.named_children:
1156
+ if child.type in REDIRECT_KINDS:
1157
+ continue
1158
+ walker.visit(child, inner, depth + 1)
1159
+ # bash performs a function's redirects each time the function is called, so what
1160
+ # they run is inside the function too (`f() { :; } <"$(cat)"; x | f` gives `cat` the
1161
+ # pipe).
1162
+ for child in list(bound) + redirects:
1163
+ walker.visit(child, inner, depth + 1)
1164
+
1165
+
1166
+ def _command_substitution(walker: Walker, node, ctx, depth, bound=()) -> None:
1167
+ walker.substitutions += 1
1168
+ _children(walker, node, replace(ctx, role=None, stdin=_unshared(ctx.stdin)), depth, bound)
1169
+
1170
+
1171
+ def _unshared(stdin: Stdin) -> Stdin:
1172
+ """The stdin for commands that share a feed: a literal one is not proven for any of
1173
+ them, since one may consume part of it before another reads."""
1174
+ return UNKNOWN if stdin.kind == "literal" else stdin
1175
+
1176
+
1177
+ def _process_substitution(walker: Walker, node, ctx, depth, bound=()) -> None:
1178
+ walker.layer.process_substitutions += 1
1179
+ opener = node.children[0].type if node.child_count else ""
1180
+ if opener != ">(":
1181
+ walker.substitutions += 1 # `>(…)` reads a pipe, not the shell's stdin
1182
+ inner = replace(ctx, role=None, stdin=PIPE if opener == ">(" else _unshared(ctx.stdin))
1183
+ _children(walker, node, inner, depth, bound)
1184
+
1185
+
1186
+ _NUMBER_DESCRIPTOR = re.compile(r"[0-9]+")
1187
+ # bash's `{varname}` redirect: an identifier, optionally with one non-empty subscript.
1188
+ _NAMED_DESCRIPTOR = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*(?:\[[^\]]+\])?\}")
1189
+
1190
+
1191
+ def _is_descriptor_token(text: str) -> bool:
1192
+ """Whether a token written right before a redirect operator is its descriptor, as
1193
+ bash reads it: ASCII digits (``0>&-``) or ``{name}`` / ``{name[subscript]}``
1194
+ (``{fd}>x``, ``{a[0]}>&-``). Anything else is an ordinary word, as in bash:
1195
+ ``echo secret>out`` keeps ``secret``, ``rm ١>x`` keeps ``١`` (not an ASCII digit) and
1196
+ ``rm {a,b}>x`` keeps ``{a,b}`` (brace expansion), all as arguments."""
1197
+ return bool(_NUMBER_DESCRIPTOR.fullmatch(text) or _NAMED_DESCRIPTOR.fullmatch(text))
1198
+
1199
+
1200
+ def _descriptor(walker: Walker, node) -> str | None:
1201
+ child = node.child_by_field_name("descriptor")
1202
+ if child is not None:
1203
+ return walker.text(child)
1204
+ if _operator(node) in NO_DESCRIPTOR_OPERATORS:
1205
+ return None
1206
+ token = walker.adjacent_token(node.start_byte)
1207
+ if token is None or not _is_descriptor_token(token[1]):
1208
+ return None
1209
+ return token[1]
1210
+
1211
+
1212
+ def _operator(node) -> str:
1213
+ for child in node.children:
1214
+ if not child.is_named and child.type in REDIRECT_OPERATORS:
1215
+ return child.type
1216
+ return ""
1217
+
1218
+
1219
+ def _file_redirect(walker: Walker, node, ctx, depth, bound=()) -> list[Redirect]:
1220
+ descriptor = _descriptor(walker, node)
1221
+ operator = _operator(node)
1222
+ targets = node.children_by_field_name("destination")
1223
+ missing = any(t.is_missing or t.has_error for t in targets)
1224
+ # Destinations with no space between them are one bash word; the first such word is
1225
+ # the target and every later one is a word of the command (`cat <a b` reads `a` and
1226
+ # passes `b`). A close (`>&-`) has none.
1227
+ spans: list[list[int]] = []
1228
+ for t in targets:
1229
+ if spans and spans[-1][1] == t.start_byte:
1230
+ spans[-1][1] = t.end_byte
1231
+ else:
1232
+ spans.append([t.start_byte, t.end_byte])
1233
+ target = make_word(walker.span(*spans[0])) if spans else None
1234
+ readable = not missing
1235
+ # Words after the target are the command's, expanded before any redirect, so their
1236
+ # substitutions read the command's stdin (`: </x >/dev/null "$(bash)"` reads the
1237
+ # terminal or pipe, not /x).
1238
+ operand_start = spans[1][0] if len(spans) > 1 else None
1239
+ inner = replace(ctx, role=None, command_stdin=None)
1240
+ word_ctx = replace(inner, stdin=ctx.command_stdin or UNKNOWN)
1241
+ for child in node.named_children:
1242
+ late = operand_start is not None and child.start_byte >= operand_start
1243
+ walker.visit(child, word_ctx if late else inner, depth + 1)
1244
+ op_node = next((c for c in node.children if c.type == operator), None)
1245
+ op_start = op_node.start_byte if op_node is not None else node.start_byte
1246
+ display_end = spans[0][1] if spans else node.end_byte
1247
+ record = Redirect(
1248
+ kind="file_redirect",
1249
+ descriptor=descriptor,
1250
+ operator=operator,
1251
+ target=target,
1252
+ readable=readable,
1253
+ output=_is_output(descriptor, operator, "file_redirect"),
1254
+ feed=None,
1255
+ display=(descriptor or "") + walker.span(op_start, display_end),
1256
+ heredoc_operator=walker.in_heredoc_run(op_start),
1257
+ start=node.start_byte,
1258
+ relative=readable and _is_relative_target(operator, target),
1259
+ )
1260
+ if len(spans) > 1:
1261
+ walker.operands[id(record)] = [(a, b) for a, b in spans[1:]]
1262
+ walker.layer.redirects.append(record)
1263
+ return [record]
1264
+
1265
+
1266
+ def _heredoc_redirect(walker: Walker, node, ctx, depth, bound=()) -> list[Redirect]:
1267
+ descriptor = _descriptor(walker, node)
1268
+ operator = _operator(node)
1269
+ delimiter = body = None
1270
+ nested: list[Redirect] = []
1271
+ nested_substitutions = False
1272
+ piped = False
1273
+ for i, child in enumerate(node.children):
1274
+ fname = node.field_name_for_child(i)
1275
+ if not child.is_named:
1276
+ if child.type in ("|", "|&"):
1277
+ piped = True
1278
+ continue
1279
+ if child.type == "heredoc_start":
1280
+ delimiter = child
1281
+ walker.visit(child, ctx, depth + 1)
1282
+ elif child.type == "heredoc_body":
1283
+ body = child
1284
+ walker.visit(child, replace(ctx, role=None), depth + 1)
1285
+ elif child.type in REDIRECT_KINDS or fname == "redirect":
1286
+ # A redirect written after the here-doc, which tree-sitter hangs under it:
1287
+ # bash performs it after the here-doc, so its substitutions may read the
1288
+ # feed (`bash <<'E' >"$(read -N 1 s)"`). Their stdin is not proven, and the
1289
+ # feed is not proven once they ran.
1290
+ before = walker.substitutions
1291
+ result = walker.visit(child, replace(ctx, stdin=UNKNOWN), depth + 1)
1292
+ nested_substitutions |= walker.substitutions != before
1293
+ nested.extend(result if isinstance(result, list) else [])
1294
+ elif child.type in ("heredoc_end", "file_descriptor"):
1295
+ walker.visit(child, ctx, depth + 1)
1296
+ else:
1297
+ # The rest of the command line, which tree-sitter hangs under the here-doc.
1298
+ if piped or child.type == "pipeline":
1299
+ rest = PIPE
1300
+ elif ctx.stdin.kind == "pipe":
1301
+ rest = UNKNOWN
1302
+ else:
1303
+ rest = ctx.stdin
1304
+ walker.visit(child, replace(ctx, stdin=rest, role=None), depth + 1)
1305
+ feed = UNKNOWN
1306
+ if delimiter is not None and body is not None and operator == "<<":
1307
+ delim = walker.text(delimiter)
1308
+ text = walker.text(body)
1309
+ if delim[:1] in ("'", '"') and _QUOTED_DELIMITER.fullmatch(delim):
1310
+ feed = Stdin("literal", text)
1311
+ elif _PLAIN_DELIMITER.fullmatch(delim) and not any(c in text for c in "$`\\"):
1312
+ feed = Stdin("literal", text)
1313
+ if nested_substitutions:
1314
+ feed = UNKNOWN
1315
+ record = Redirect(
1316
+ kind="heredoc_redirect",
1317
+ descriptor=descriptor,
1318
+ operator=operator,
1319
+ target=None,
1320
+ readable=True,
1321
+ output=False,
1322
+ feed=feed,
1323
+ display=(descriptor or "")
1324
+ + operator
1325
+ + (walker.text(delimiter) if delimiter else ""),
1326
+ heredoc_operator=True,
1327
+ start=node.start_byte,
1328
+ )
1329
+ walker.layer.redirects.append(record)
1330
+ return [record, *nested]
1331
+
1332
+
1333
+ def _herestring_redirect(walker: Walker, node, ctx, depth, bound=()) -> list[Redirect]:
1334
+ _children(walker, node, replace(ctx, role=None), depth)
1335
+ descriptor = _descriptor(walker, node)
1336
+ pieces = [c for c in node.named_children if c.type != "file_descriptor"]
1337
+ word = None
1338
+ if pieces:
1339
+ word = make_word(walker.span(pieces[0].start_byte, pieces[-1].end_byte))
1340
+ # bash appends a newline to a here-string's word.
1341
+ feed = Stdin("literal", word.text + "\n") if word is not None and word.literal else UNKNOWN
1342
+ if word is not None:
1343
+ walker.layer.words.append(word)
1344
+ record = Redirect(
1345
+ kind="herestring_redirect",
1346
+ descriptor=descriptor,
1347
+ operator="<<<",
1348
+ target=word,
1349
+ readable=word is not None,
1350
+ output=False,
1351
+ feed=feed,
1352
+ display=walker.text(node),
1353
+ heredoc_operator=False,
1354
+ start=node.start_byte,
1355
+ )
1356
+ walker.layer.redirects.append(record)
1357
+ return [record]
1358
+
1359
+
1360
+ HANDLERS: dict[str, Handler] = {kind: _children for kind in named_kinds()}
1361
+ HANDLERS.update(
1362
+ {
1363
+ ERROR_KIND: _error,
1364
+ "pipeline": _pipeline,
1365
+ "list": _list,
1366
+ "negated_command": _negated,
1367
+ "test_command": _test_command,
1368
+ "redirected_statement": _redirected,
1369
+ "command": _simple,
1370
+ "declaration_command": _simple,
1371
+ "unset_command": _simple,
1372
+ "variable_assignment": _assignment,
1373
+ "function_definition": _function,
1374
+ "command_substitution": _command_substitution,
1375
+ "process_substitution": _process_substitution,
1376
+ "file_redirect": _file_redirect,
1377
+ "heredoc_redirect": _heredoc_redirect,
1378
+ "herestring_redirect": _herestring_redirect,
1379
+ }
1380
+ )
1381
+
1382
+
1383
+ def _check_walk(tree: tree_sitter.Tree, walker: Walker) -> None:
1384
+ """Rule 2, on every classification: every named node visited exactly once, and one
1385
+ invocation per simple command node."""
1386
+ named = 0
1387
+ simple = 0
1388
+ for node in _iter_nodes(tree.root_node):
1389
+ if not node.is_named:
1390
+ continue
1391
+ named += 1
1392
+ if node.type in SIMPLE_KINDS:
1393
+ simple += 1
1394
+ if walker.visits.get((node.id, node.start_byte, node.end_byte)) != 1:
1395
+ raise ParseRefused("a node was not walked exactly once")
1396
+ if named != len(walker.visits):
1397
+ raise ParseRefused("the walk and the tree disagree")
1398
+ if simple != len(walker.layer.invocations):
1399
+ raise ParseRefused("a simple command gave no invocation")
1400
+
1401
+
1402
+ def walk(source: str, limits: Limits = DEFAULT_LIMITS) -> Walk:
1403
+ """Parse and walk ``source``. Raises ``ParseRefused`` when a fact cannot be proven
1404
+ (Rule 2) or a limit is exceeded (Rule 5); the size cap is checked before parsing."""
1405
+ refused = ParseRefused(f"input over {limits.max_input_bytes // 1024} KiB")
1406
+ # Every character is at least one byte: refuse an oversized string before encoding
1407
+ # it, so a huge input costs no allocation.
1408
+ if len(source) > limits.max_input_bytes:
1409
+ raise refused
1410
+ data = source.encode("utf-8", "surrogatepass")
1411
+ if len(data) > limits.max_input_bytes:
1412
+ raise refused
1413
+ tree = parse_source(data)
1414
+ layer = Walk(untrusted=untrusted_features(tree, data))
1415
+ walker = Walker(data, layer, limits)
1416
+ walker.visit(tree.root_node, Ctx(stdin=TERMINAL), 0)
1417
+ _check_walk(tree, walker)
1418
+ # A line continuation outside single quotes (`OPT\<newline>IND=$X`): bash joins the
1419
+ # pieces into a name the arithmetic checks would match, so they cannot rule it out.
1420
+ if "backslash-newline" in layer.untrusted or _unproven_arithmetic(tree.root_node):
1421
+ walker.note_untrusted("arithmetic on a value the walker cannot prove is a number")
1422
+ if any(_holds_substitution_text(a.value) for a in layer.assignments):
1423
+ # A quoted value that spells a substitution: bash may evaluate it later, also
1424
+ # once pieces are joined (`x='a['; x+='$(id)]'; : $((x))`).
1425
+ walker.note_untrusted("assigned text holding a substitution bash may evaluate")
1426
+ if any(a.name in CODE_VARIABLES for a in layer.assignments) or any(
1427
+ _NAMES_CODE_VARIABLE.match(w.text) for w in layer.words
1428
+ ):
1429
+ # bash runs these variables' values as code later (`PS4='$(id)'; set -x; :`),
1430
+ # however they are set (`printf -v PS4`, `read PS4`, `declare`); the walker sees
1431
+ # no command for it.
1432
+ walker.note_untrusted("a variable bash runs as code (PS4, PROMPT_COMMAND, …) is set")
1433
+ if (
1434
+ any(a.name in MODE_VARIABLES for a in layer.assignments)
1435
+ or any(_NAMES_MODE_VARIABLE.match(w.text) for w in layer.words)
1436
+ # `{POSIXLY_CORRECT}>/dev/null` assigns the descriptor number to the name.
1437
+ or any(
1438
+ r.descriptor is not None
1439
+ and _NAMES_MODE_VARIABLE.match(r.descriptor.strip("{}"))
1440
+ for r in layer.redirects
1441
+ )
1442
+ or any(_changes_line_reading(i) for i in layer.invocations)
1443
+ ):
1444
+ # History expansion, POSIX mode, `keyword` and `interactive_comments` change how
1445
+ # bash reads and runs the commands after the one that sets them (`set -H` on one
1446
+ # line, `!!` on the next); the facts assume them at their defaults (§3.2).
1447
+ walker.note_untrusted("a shell option that changes how bash reads the line is set")
1448
+ elif _unseen_code_runs_first(layer.invocations, walker.spans):
1449
+ # Code the walker does not see (`eval 'set -o posix'`, a trap, a sourced file, a
1450
+ # command word it cannot read) may set one of those options before the others run.
1451
+ walker.note_untrusted("shell code the walker does not see may change how bash reads the line")
1452
+ if (
1453
+ any(a.name in FD_VARIABLES for a in layer.assignments)
1454
+ or any(_NAMES_FD_VARIABLE.match(w.text) for w in layer.words)
1455
+ or any(
1456
+ r.descriptor is not None and _NAMES_FD_VARIABLE.match(r.descriptor.strip("{}"))
1457
+ for r in layer.redirects
1458
+ )
1459
+ ):
1460
+ # Any write or `unset` (`unset BASH_XTRACEFD`, `read BASH_XTRACEFD`,
1461
+ # `{BASH_XTRACEFD}>f`) may close a descriptor, stdin included, that the facts
1462
+ # still report as open.
1463
+ walker.note_untrusted("BASH_XTRACEFD, which can close the shell's stdin, is set")
1464
+ if any(_rebinds_names(i) for i in layer.invocations):
1465
+ # `enable -n command` makes a later `command x` run a `command` program from PATH,
1466
+ # like `hash -p`.
1467
+ walker.note_untrusted("enable with a builtin name, which changes what a command name runs")
1468
+ if walker.operands:
1469
+ # Words after a redirect target with no simple command to take them
1470
+ # (`{ cat; } <a b`, a syntax error in bash): the tree is not bash's.
1471
+ walker.note_untrusted("words after a redirect target with no command to take them")
1472
+ if layer.untrusted:
1473
+ # No stdin fact from a tree the walker does not trust.
1474
+ layer.invocations = [replace(i, stdin=UNKNOWN) for i in layer.invocations]
1475
+ elif any(may_rewire_stdin(i) for i in layer.invocations):
1476
+ # A command that can change the shell's own stdin (`exec </x; sudo tee /y`) makes
1477
+ # every command's stdin unproven: any of them may run after it, in source order or
1478
+ # on a later loop pass, another rewiring command included (`exec </x; exec bash`
1479
+ # runs /x).
1480
+ layer.invocations = [replace(i, stdin=UNKNOWN) for i in layer.invocations]
1481
+ return layer
1482
+
1483
+
1484
+ DECLARATION_BUILTINS = ("declare", "typeset", "local", "export", "readonly")
1485
+ NAMEREF_BUILTINS = ("declare", "typeset", "local") # `-n` makes a nameref (export -n does not)
1486
+ # Variables whose value bash later runs or expands as code: the prompts (an interactive
1487
+ # shell expands PS0-PS2 and runs PROMPT_COMMAND; `set -x` expands PS4), the startup
1488
+ # files a child shell sources, the mail message and the `fc` editor.
1489
+ CODE_VARIABLES = (
1490
+ "PS0", "PS1", "PS2", "PS3", "PS4", "PROMPT_COMMAND", "BASH_ENV", "ENV",
1491
+ "MAILPATH", # an interactive shell command-expands its `?message` on new mail
1492
+ "FCEDIT", # `fc` runs it as the editor
1493
+ "BASH_ALIASES", # defines aliases, like `alias`
1494
+ "BASH_CMDS", # maps a command name to the file bash runs for it, like `hash -p`
1495
+ )
1496
+ # Variables whose value changes how bash reads or runs the line: POSIX mode, the
1497
+ # compatibility level, the `set -o` / `shopt` options themselves, and the history
1498
+ # expansion characters.
1499
+ MODE_VARIABLES = ("POSIXLY_CORRECT", "BASH_COMPAT", "SHELLOPTS", "BASHOPTS", "histchars")
1500
+ _NAMES_MODE_VARIABLE = re.compile(
1501
+ r"(?:-[A-Za-z]*)?(?:" + "|".join(MODE_VARIABLES) + r")(?:\[.*\])?(?:\+?=.*)?$", re.S
1502
+ )
1503
+ # Variables whose write or `unset` changes the shell's own descriptors: unsetting
1504
+ # BASH_XTRACEFD closes the descriptor it names, also 0 (`BASH_XTRACEFD=0; unset
1505
+ # BASH_XTRACEFD; read x` reads from a closed stdin).
1506
+ FD_VARIABLES = ("BASH_XTRACEFD",)
1507
+ _NAMES_FD_VARIABLE = re.compile(
1508
+ r"(?:-[A-Za-z]*)?(?:" + "|".join(FD_VARIABLES) + r")(?:\[.*\])?(?:\+?=.*)?$", re.S
1509
+ )
1510
+ _NAMES_CODE_VARIABLE = re.compile(
1511
+ # Also attached to any options (`printf -vPS4`, `read -aPS4`).
1512
+ r"(?:-[A-Za-z]*)?(?:" + "|".join(CODE_VARIABLES) + r")(?:\[.*\])?(?:\+?=.*)?$", re.S
1513
+ )
1514
+
1515
+
1516
+ def _builtin_head(words: list[Word], time_reserved: bool = True) -> int | None:
1517
+ """The index of the literal word bash runs as the command, after any ``command`` /
1518
+ ``builtin`` prefixes and their options; None when that word is not literal."""
1519
+ i = _skip_prefixes(words, time_reserved)
1520
+ if i is None or i == UNREADABLE or i >= len(words) or not words[i].literal:
1521
+ return None
1522
+ if not words[i].source:
1523
+ return None
1524
+ return i
1525
+
1526
+
1527
+ UNREADABLE = -1
1528
+
1529
+
1530
+ _ASSIGNMENT_WORD = re.compile(r"[A-Za-z_]\w*(?:\[[^\]]*\])?\+?=")
1531
+
1532
+
1533
+ def _assigns_after_time(words: list[Word], time_reserved: bool) -> bool:
1534
+ """Whether an assignment follows the reserved `time` (`time PATH=/tmp git`): the
1535
+ grammar parses it as a word, so it is in no ``Assignment``."""
1536
+ i = 0
1537
+ while time_reserved and i < len(words) and words[i].source == "time":
1538
+ i += 1
1539
+ if i < len(words) and words[i].source == "-p":
1540
+ i += 1
1541
+ if i < len(words) and words[i].source == "--":
1542
+ i += 1
1543
+ if i < len(words) and _ASSIGNMENT_WORD.match(words[i].source):
1544
+ return True
1545
+ return False
1546
+
1547
+
1548
+ def _time_reserved(node: tree_sitter.Node) -> bool:
1549
+ """Whether a leading `time` in this command node is bash's reserved word: only when
1550
+ nothing (an assignment, a redirect) comes before it."""
1551
+ for child in node.children:
1552
+ if child.type == "variable_assignment" or child.type in REDIRECT_KINDS:
1553
+ return False
1554
+ if child.type in ("command_name", "word"):
1555
+ return True
1556
+ return True
1557
+
1558
+
1559
+ def _skip_prefixes(words: list[Word], time_reserved: bool = True) -> int | None:
1560
+ """The index after any ``command`` / ``builtin`` prefixes and their options; None when
1561
+ a prefix only looks a command up (``command -v``/``-V`` runs nothing); ``UNREADABLE``
1562
+ when a prefix has an option the walker cannot read (``command "-$o" exec <x`` may run
1563
+ ``exec``)."""
1564
+ i = 0
1565
+ # `time` is a reserved word only as the first word, unquoted; after `command` or
1566
+ # `builtin` it is the external program, which runs what follows in a child process.
1567
+ while (
1568
+ time_reserved
1569
+ and i < len(words)
1570
+ and words[i].source == "time"
1571
+ and all(w.source in ("time", "-p", "--") for w in words[:i])
1572
+ ):
1573
+ i += 1 # `time time exec …`: each leading `time` is the reserved word
1574
+ # Its option is a word as written, once: `time "-p" git` and `time -p -p git` run `-p`.
1575
+ if i < len(words) and words[i].source == "-p":
1576
+ i += 1
1577
+ if i < len(words) and words[i].source == "--":
1578
+ i += 1 # `time -- eval …`
1579
+ # Assignments may follow, before the command (`time X=1 export …`); the grammar
1580
+ # parses them as words.
1581
+ while i < len(words) and _ASSIGNMENT_WORD.match(words[i].source):
1582
+ i += 1
1583
+ while i < len(words) and words[i].literal and words[i].text in BUILTIN_PREFIXES:
1584
+ prefix = words[i].text
1585
+ i += 1
1586
+ while i < len(words) and words[i].text.startswith("-"):
1587
+ option = words[i]
1588
+ if not option.literal:
1589
+ return UNREADABLE
1590
+ if option.text == "--":
1591
+ i += 1
1592
+ break
1593
+ if prefix == "command" and ("v" in option.text or "V" in option.text):
1594
+ return None
1595
+ i += 1
1596
+ return i
1597
+
1598
+
1599
+ def command_head(invocation: Invocation) -> Word | None:
1600
+ """The command word, with ``command`` and ``builtin`` in front looked through; None
1601
+ when there is no command word."""
1602
+ words = list(invocation.words)
1603
+ i = _skip_prefixes(words, invocation.time_reserved)
1604
+ if i == UNREADABLE:
1605
+ # An option the walker cannot read: what runs is not known, so the head is that
1606
+ # non-literal word, which callers treat as the worst case.
1607
+ return next(w for w in words if not w.literal)
1608
+ if i is None or i >= len(words) or not words[i].source:
1609
+ return None
1610
+ return words[i]
1611
+
1612
+
1613
+ # Builtins that run shell code the walker does not see.
1614
+ SHELL_RUNNERS = ("eval", "source", ".", "fc") # `fc` re-runs history lines
1615
+ # Builtins that install code to run later, which the walker does not see (`trap '…' DEBUG`
1616
+ # runs before every later command).
1617
+ CODE_LATER = ("trap", "alias") # an alias runs its text where it is used
1618
+ BUILTIN_PREFIXES = ("command", "builtin")
1619
+ # Builtins that take shell code in an option, by the option letters that do.
1620
+ # `enable -f` loads a shared object into the shell: native code the walker does not see.
1621
+ CODE_OPTIONS = {
1622
+ "mapfile": "C", "readarray": "C", "complete": "CF", "compgen": "CF", "bind": "x", "enable": "f",
1623
+ "jobs": "x", # `jobs -x cmd` runs cmd in the current shell
1624
+ }
1625
+
1626
+
1627
+ # --- Options that change how bash reads the line -------------------------------------
1628
+ #
1629
+ # The facts assume history expansion off, POSIX mode off, `keyword` off and
1630
+ # `interactive_comments` on (architecture §3.2). A line that changes one of them makes
1631
+ # bash read or run its later commands differently (`set -H` then `!!` on the next line;
1632
+ # `set -o posix` then `time -p cmd`; `set -k` then `echo a=b`). Only the options below are
1633
+ # trusted, by direction; any other, or one the walker cannot read, makes the tree untrusted.
1634
+ _SET_LETTERS_SAFE = {"-": frozenset("abefhmnptuvxBCEPT"), "+": frozenset("abefhmnptuvxBCEPTHk")}
1635
+ _SET_NAMES = frozenset(
1636
+ ("allexport", "braceexpand", "emacs", "errexit", "errtrace", "functrace", "hashall",
1637
+ "ignoreeof", "monitor", "noclobber", "noexec", "noglob", "nolog", "notify", "nounset",
1638
+ "onecmd", "physical", "pipefail", "privileged", "verbose", "vi", "xtrace")
1639
+ )
1640
+ _SET_NAMES_SAFE = {
1641
+ "-": _SET_NAMES | {"interactive-comments"},
1642
+ "+": _SET_NAMES | {"histexpand", "history", "keyword", "posix"},
1643
+ }
1644
+ _SHOPT_NAMES = frozenset(
1645
+ ("autocd", "cdable_vars", "cdspell", "checkhash", "checkjobs", "checkwinsize", "cmdhist",
1646
+ "direxpand", "dirspell", "dotglob", "execfail", "failglob", "force_fignore",
1647
+ "globasciiranges", "globskipdots", "globstar", "gnu_errfmt", "histappend", "histreedit",
1648
+ "histverify", "hostcomplete", "huponexit", "inherit_errexit", "lithist",
1649
+ "localvar_inherit", "localvar_unset", "mailwarn", "no_empty_cmd_completion",
1650
+ "nocaseglob", "nocasematch", "noexpand_translation", "nullglob", "progcomp",
1651
+ "progcomp_alias", "shift_verbose", "sourcepath", "varredir_close", "xpg_echo")
1652
+ )
1653
+ _SHOPT_NAMES_SAFE = {
1654
+ "s": _SHOPT_NAMES | {"interactive_comments"},
1655
+ "u": _SHOPT_NAMES | {"extglob", "expand_aliases", "extdebug", "extquote", "lastpipe",
1656
+ "promptvars", "patsub_replacement", "assoc_expand_once"},
1657
+ }
1658
+
1659
+
1660
+ def _changes_line_reading(invocation: Invocation) -> bool:
1661
+ """Whether ``invocation`` is ``set`` or ``shopt`` (``command`` / ``builtin`` in front
1662
+ looked through) setting an option outside the trusted lists above."""
1663
+ head = command_head(invocation)
1664
+ # The decoded name counts even when it is not literal: `$'set' -H` runs `set`. Any
1665
+ # other unreadable command word is unseen code (`_runs_unseen_code`).
1666
+ if head is None or head.text not in ("set", "shopt"):
1667
+ return False
1668
+ words = list(invocation.words)
1669
+ args = words[words.index(head) + 1 :]
1670
+ if head.text == "set":
1671
+ return _set_changes_reading(args)
1672
+ return _shopt_changes_reading(args)
1673
+
1674
+
1675
+ _ENABLE_LISTING = frozenset("apsn") # with no name, these only list builtins
1676
+
1677
+
1678
+ def _rebinds_names(invocation: Invocation) -> bool:
1679
+ """Whether ``invocation`` is ``enable`` (``command`` / ``builtin`` in front looked
1680
+ through, its decoded name counted like ``set``'s) doing anything but listing: any word
1681
+ other than a literal cluster of ``-a`` ``-p`` ``-s`` ``-n``, including ``--`` and every
1682
+ name, may enable, disable, load or delete a builtin."""
1683
+ head = command_head(invocation)
1684
+ if head is None or head.text != "enable":
1685
+ return False
1686
+ words = list(invocation.words)
1687
+ for word in words[words.index(head) + 1 :]:
1688
+ text = word.text
1689
+ if not word.literal or len(text) < 2 or text[0] != "-" or not set(text[1:]) <= _ENABLE_LISTING:
1690
+ return True
1691
+ return False
1692
+
1693
+
1694
+ def _set_changes_reading(args: list[Word]) -> bool:
1695
+ i = 0
1696
+ while i < len(args):
1697
+ word = args[i]
1698
+ if not word.literal:
1699
+ return True
1700
+ text = word.text
1701
+ if text == "+":
1702
+ i += 1
1703
+ continue # bash skips a lone `+` and goes on reading options
1704
+ if text in ("--", "-") or len(text) < 2 or text[0] not in "-+":
1705
+ return False # the rest are positional parameters
1706
+ sign = text[0]
1707
+ i += 1
1708
+ for letter in text[1:]:
1709
+ if letter == "o":
1710
+ if i >= len(args):
1711
+ continue # a bare `-o` prints the options; the other letters still apply
1712
+ name = args[i]
1713
+ i += 1
1714
+ if not name.literal or name.text not in _SET_NAMES_SAFE[sign]:
1715
+ return True
1716
+ elif letter not in _SET_LETTERS_SAFE[sign]:
1717
+ return True
1718
+ return False
1719
+
1720
+
1721
+ def _shopt_changes_reading(args: list[Word]) -> bool:
1722
+ flags: set[str] = set()
1723
+ i = 0
1724
+ while i < len(args):
1725
+ word = args[i]
1726
+ if not word.literal:
1727
+ return True
1728
+ if word.text == "--":
1729
+ i += 1
1730
+ break
1731
+ if len(word.text) < 2 or word.text[0] != "-":
1732
+ break
1733
+ if any(c not in "psuoq" for c in word.text[1:]):
1734
+ return True
1735
+ flags.update(word.text[1:])
1736
+ i += 1
1737
+ if "s" not in flags and "u" not in flags:
1738
+ return False # it prints or queries
1739
+ if {"s", "u"} <= flags:
1740
+ return True
1741
+ direction = "s" if "s" in flags else "u"
1742
+ for name in args[i:]:
1743
+ if not name.literal:
1744
+ return True
1745
+ if "o" in flags:
1746
+ if name.text not in _SET_NAMES_SAFE["-" if direction == "s" else "+"]:
1747
+ return True
1748
+ elif name.text not in _SHOPT_NAMES_SAFE[direction]:
1749
+ return True
1750
+ return False
1751
+
1752
+
1753
+ _RERUNS = ("while_statement", "for_statement", "c_style_for_statement", "function_definition")
1754
+
1755
+
1756
+ def _may_rerun(node) -> bool:
1757
+ """Whether a loop or function body holds ``node``, so it may run again after commands
1758
+ written later."""
1759
+ parent = node.parent
1760
+ while parent is not None:
1761
+ if parent.type in _RERUNS:
1762
+ return True
1763
+ parent = parent.parent
1764
+ return False
1765
+
1766
+
1767
+ def _unseen_code_runs_first(
1768
+ invocations: list[Invocation], spans: list[tuple[int, int, bool]]
1769
+ ) -> bool:
1770
+ """Whether code the walker does not see may run before another command: an
1771
+ invocation that runs such code and that a loop or function may rerun, or that another
1772
+ invocation starts after."""
1773
+ for invocation, (start, end, rerun) in zip(invocations, spans, strict=True):
1774
+ if not _runs_unseen_code(invocation):
1775
+ continue
1776
+ if rerun or any(other_start >= end for other_start, _, _ in spans):
1777
+ return True
1778
+ return False
1779
+
1780
+
1781
+ def _runs_unseen_code(invocation: Invocation) -> bool:
1782
+ """Whether ``invocation`` runs shell code the walker does not see: ``eval`` /
1783
+ ``source`` / ``.`` / ``fc``, ``trap`` / ``alias``, a builtin with a code option, or a
1784
+ command word the walker cannot read."""
1785
+ head = command_head(invocation)
1786
+ if head is None:
1787
+ return False
1788
+ if not head.literal or head.text in SHELL_RUNNERS or head.text in CODE_LATER:
1789
+ return True
1790
+ if head.text in CODE_OPTIONS:
1791
+ letters = CODE_OPTIONS[head.text]
1792
+ words = list(invocation.words)
1793
+ for word in words[words.index(head) + 1 :]:
1794
+ if not word.literal:
1795
+ return True
1796
+ if word.text.startswith("-") and any(c in word.text[1:] for c in letters):
1797
+ return True
1798
+ return False
1799
+
1800
+
1801
+ def may_rewire_stdin(invocation: Invocation) -> bool:
1802
+ """Whether ``invocation`` can change the stdin of the shell itself: ``exec`` with
1803
+ any redirect, ``eval`` / ``source`` / ``.`` / ``trap`` or a builtin with a code option
1804
+ (``mapfile -C``, ``complete -C``/``-F``, ``bind -x``: code the walker does not see),
1805
+ or a command word the walker cannot read. ``command`` and ``builtin`` in front are
1806
+ looked through."""
1807
+ head = command_head(invocation)
1808
+ if head is None:
1809
+ return False # no command word: a redirect alone applies to nothing
1810
+ # `mapfile -C 'code'` runs code now, `complete -C`/`bind -x` on a later key or
1811
+ # completion: shell code the walker does not see, as with `eval` and `trap`.
1812
+ if _runs_unseen_code(invocation):
1813
+ return True
1814
+ return head.text == "exec" and bool(invocation.redirects)
1815
+
1816
+
1817
+ # --- Rule 1: run-time values in arithmetic -------------------------------------------
1818
+ #
1819
+ # bash evaluates some values as arithmetic, and an array subscript in such a value runs
1820
+ # its command substitutions: with X='a[$(id)]' in the environment, `echo $((X))` runs
1821
+ # `id`. Checked against bash 5.3, these evaluate a value as arithmetic: `$(( ))`, `$[ ]`,
1822
+ # `(( ))`, `for (( ))`, `let`, the numeric tests of `[[ ]]`, array subscripts (also in
1823
+ # names given to builtins and `-v` tests), `${…:off:len}`, and assignments to an integer
1824
+ # variable: one declared `-i`, or one of bash's own (`RANDOM`, `OPTIND`, …).
1825
+ #
1826
+ # The rule is ultra-strict (decided on #12 after eight review rounds kept finding ways
1827
+ # around narrower ones): arithmetic is trusted only when it is literal numbers. Any name,
1828
+ # `$` or backtick in an arithmetic text or an array index, any `-i` declaration, any write
1829
+ # to one of bash's integer variables, any declaration or destination whose name the
1830
+ # walker cannot read, and any indirect `${!name}` make the tree untrusted.
1831
+
1832
+ # bash's own integer variables: an assignment to one evaluates the value as arithmetic.
1833
+ _BASH_INTEGERS = frozenset(
1834
+ ("RANDOM", "SRANDOM", "LINENO", "SECONDS", "EPOCHSECONDS", "BASHPID", "PPID", "UID", "EUID",
1835
+ "BASH_SUBSHELL", "OPTIND", "MAILCHECK", "HISTCMD")
1836
+ )
1837
+ _INT_CONSTANT = re.compile(r"(?<![\w$])(?:0[xX][0-9A-Fa-f]+|[0-9]+#[0-9A-Za-z@_]+|[0-9]+)(?![\w#])")
1838
+ _PURE_ARITHMETIC = re.compile(r"[\s0-9+\-*/%<>=!&|^~?:,()]*")
1839
+ _ARITHMETIC_TESTS = ("-eq", "-ne", "-lt", "-le", "-gt", "-ge")
1840
+ _DECLARERS = ("declare", "typeset", "local", "export", "readonly")
1841
+ _INTEGER_DECLARERS = ("declare", "typeset", "local")
1842
+ # A declaration operand whose name is literal even if its value is not (`PATH="$PATH:/x"`).
1843
+ _STATIC_NAME = re.compile(r"[A-Za-z_]\w*(?:\[[^\]]*\])?\+?=")
1844
+ # Builtins that write variables they are given.
1845
+ _VARIABLE_WRITERS = ("read", "printf", "mapfile", "readarray", "getopts", "wait")
1846
+ # `${!x*}`, `${!x@}`, `${!a[@]}`, `${!a[*]}` list names or keys and evaluate nothing.
1847
+ _DEFAULT_ASSIGNMENT = re.compile(r"\$\{([A-Za-z_]\w*)(?:\[[^\]]*\])?:?=")
1848
+ _INDIRECT_LISTING = re.compile(r"\$\{![A-Za-z_]\w*(?:[*@]|\[[*@]\])\}$")
1849
+
1850
+
1851
+ def _node_text(node: tree_sitter.Node) -> str:
1852
+ return node.text.decode("utf-8", "replace")
1853
+
1854
+
1855
+ def _pure(text: str) -> bool:
1856
+ """Whether an arithmetic text is literal numbers and operators only."""
1857
+ return bool(_PURE_ARITHMETIC.fullmatch(_INT_CONSTANT.sub(" ", text)))
1858
+
1859
+
1860
+ # What bash may expand in a word before it reads the word as arithmetic: `~` ($HOME, which
1861
+ # may carry a subscript), and a process substitution (`<(…)` becomes `/dev/fd/63`).
1862
+ _WORD_EXPANSIONS = ("~", "<(", ">(")
1863
+
1864
+
1865
+ def _index_pure(index: str) -> bool:
1866
+ if any(e in index for e in _WORD_EXPANSIONS):
1867
+ return False
1868
+ return index.strip() in ("@", "*") or _pure(index)
1869
+
1870
+
1871
+ def _word_pure(source: str) -> bool:
1872
+ """Whether a word as written (`let 1+2`, a `[[ ]]` numeric operand) is literal numbers
1873
+ once bash has expanded it: no expansion, glob, brace, tilde or process substitution."""
1874
+ text, causes = lex_word(source)
1875
+ if causes or "{" in source or any(e in source for e in _WORD_EXPANSIONS):
1876
+ return False
1877
+ return _pure(text)
1878
+
1879
+
1880
+ def _name_words(node: tree_sitter.Node) -> list[tuple[int, str, bool, str]]:
1881
+ """A command's words with pieces that touch joined (`a[X]` parses as `a` and `[X]`),
1882
+ each with the index (in ``_word_children``) of its first piece, and literal when bash
1883
+ expands nothing in it but its brackets: an element name like `a[0]` is a name, not a
1884
+ glob. The source as written comes last."""
1885
+ groups: list[tuple[int, str]] = []
1886
+ end = None
1887
+ for k, child in enumerate(_word_children(node)):
1888
+ text = _node_text(child)
1889
+ if groups and end == child.start_byte:
1890
+ first, previous = groups.pop()
1891
+ groups.append((first, previous + text))
1892
+ else:
1893
+ groups.append((k, text))
1894
+ end = child.end_byte
1895
+ result = []
1896
+ for first, source in groups:
1897
+ text, causes = lex_word(source)
1898
+ # A glob is a literal name only as one element `name[…]`, which can only expand to
1899
+ # a name without brackets; `a*` may expand to a file named `a[$(id)]`.
1900
+ literal = not causes or (causes == {"glob"} and bool(_ELEMENT_FORM.fullmatch(text)))
1901
+ result.append((first, text, literal, source))
1902
+ return result
1903
+
1904
+
1905
+ _ELEMENT_FORM = re.compile(r"[A-Za-z_]\w*\[[^\[\]*?]*\]")
1906
+
1907
+
1908
+ def _operands(node: tree_sitter.Node, head: int) -> list[tuple[str, bool, str]]:
1909
+ """The words after the command word (``_word_children`` index ``head``), with element
1910
+ names joined and judged as names (``_name_words``), each with its source."""
1911
+ return [(text, literal, source) for first, text, literal, source in _name_words(node) if first > head]
1912
+
1913
+
1914
+ def _unproven_compound_keys(array: tree_sitter.Node) -> bool:
1915
+ """Whether a compound assignment `([k]=v …)` has a key that is not literal numbers.
1916
+ Bash reads an element that starts with an unquoted `[` and holds a `=` as `[key]=v`,
1917
+ the key up to the matching `]`; the key is read from the source as written, so any
1918
+ quote, expansion or nested bracket in it counts as not proven (`[${X:-1]x}]=v`)."""
1919
+ elements: list[str] = []
1920
+ end = None
1921
+ for piece in array.named_children: # `[b[0]+$1]=1` parses as several touching pieces
1922
+ if elements and piece.start_byte == end:
1923
+ elements[-1] += _node_text(piece)
1924
+ else:
1925
+ elements.append(_node_text(piece))
1926
+ end = piece.end_byte
1927
+ for text in elements:
1928
+ if text.startswith("["):
1929
+ # `[0]=v`; anything else that starts with `[` may be a key bash reads up to
1930
+ # its matching `]`, across nested brackets, quotes or spaces (`[ X ]=1`).
1931
+ match = re.match(r"\[([^\[\]]*)\]\+?=", text)
1932
+ if match is None or not _index_pure(match.group(1)):
1933
+ return True
1934
+ return False
1935
+
1936
+
1937
+ def _element_indexes(text: str) -> list[str]:
1938
+ """The index of every array element a word names, nested ones whole (`a[b[0]+x]`)."""
1939
+ out, depth, start = [], 0, None
1940
+ for i, c in enumerate(text):
1941
+ if c == "[":
1942
+ if depth == 0 and i > 0 and (text[i - 1].isalnum() or text[i - 1] == "_"):
1943
+ start = i + 1
1944
+ depth += 1
1945
+ elif c == "]" and depth:
1946
+ depth -= 1
1947
+ if depth == 0 and start is not None:
1948
+ out.append(text[start:i])
1949
+ start = None
1950
+ return out
1951
+
1952
+
1953
+ def _substring_offset(text: str) -> str | None:
1954
+ """The `off:len` of a `${…:off:len}` expansion, or None: the first `:` outside
1955
+ brackets after the parameter, not followed by `-`, `=`, `?` or `+`."""
1956
+ if not text.startswith("${") or not text.endswith("}"):
1957
+ return None
1958
+ depth = 0
1959
+ for i in range(2, len(text) - 1):
1960
+ c = text[i]
1961
+ if c == "[":
1962
+ depth += 1
1963
+ elif c == "]":
1964
+ depth -= 1
1965
+ elif depth == 0 and c in "#%/^,@":
1966
+ if i > 2:
1967
+ return None # another operator came first (`${x#…}`, `${x/…}`)
1968
+ elif depth == 0 and c == ":":
1969
+ rest = text[i + 1 : -1]
1970
+ return None if rest[:1] in ("-", "=", "?", "+") else rest
1971
+ return None
1972
+
1973
+
1974
+ def _in_double_bracket(node: tree_sitter.Node) -> bool:
1975
+ """Inside `[[ ]]`, whose numeric tests evaluate arithmetic (`[ ]` and `test` do not)."""
1976
+ current = node.parent
1977
+ while current is not None and current.type != "test_command":
1978
+ current = current.parent
1979
+ return current is not None and bool(current.children) and current.children[0].type == "[["
1980
+
1981
+
1982
+ def _lexed(node: tree_sitter.Node) -> list[tuple[str, bool]]:
1983
+ """A command's words (not its redirects or prefix assignments) after quote removal,
1984
+ each with whether it is literal."""
1985
+ out = []
1986
+ for child in _word_children(node):
1987
+ text, causes = lex_word(_node_text(child))
1988
+ out.append((text, not causes))
1989
+ return out
1990
+
1991
+
1992
+ def _word_children(node: tree_sitter.Node) -> list[tree_sitter.Node]:
1993
+ """The children of a command or declaration that are its words: not redirects, not a
1994
+ redirect's `{fd}`, not prefix assignments (`x=1 cmd`)."""
1995
+ return [
1996
+ child
1997
+ for child in node.named_children
1998
+ if not child.type.endswith("_redirect")
1999
+ and child.type != "ERROR" # a mis-parsed piece is no word (the tree is untrusted)
2000
+ and not _redirect_descriptor(child, node)
2001
+ and not (child.type == "variable_assignment" and node.type == "command")
2002
+ ]
2003
+
2004
+
2005
+ def _redirect_descriptor(child: tree_sitter.Node, node: tree_sitter.Node) -> bool:
2006
+ """Whether ``child`` is the `{fd}` or number of a redirect written right after it,
2007
+ which the grammar leaves as a word (`unset B {fd}>x`, `declare -a {fd}>x`)."""
2008
+ # The redirect may hang on a statement further out (`x || unset B {fd}>x`): take the
2009
+ # next node at any level; it must start exactly where the token ends.
2010
+ after, current = child.next_sibling, node
2011
+ while after is None and current is not None:
2012
+ after, current = current.next_sibling, current.parent
2013
+ return (
2014
+ after is not None
2015
+ and after.type.endswith("_redirect")
2016
+ and after.start_byte == child.end_byte
2017
+ and _is_descriptor_token(_node_text(child))
2018
+ )
2019
+
2020
+
2021
+ def _lexed_head(node: tree_sitter.Node) -> int | None:
2022
+ """``_skip_prefixes`` for a command node's words (as ``_lexed`` lists them): the index
2023
+ of the word that runs, None when nothing runs (no word, or `command -v`/`-V`),
2024
+ ``UNREADABLE`` when a prefix option cannot be read."""
2025
+ words = [make_word(_node_text(c)) for c in _word_children(node)]
2026
+ i = _skip_prefixes(words, _time_reserved(node))
2027
+ if i is None or i == UNREADABLE:
2028
+ return i
2029
+ return i if i < len(words) else None
2030
+
2031
+
2032
+
2033
+ def _display_only(words: list[tuple[str, bool]]) -> bool:
2034
+ """`declare -p`, `-f`, `-F` or `+p` before any operand: displays, writes nothing. A
2035
+ word the walker cannot read ends the options (`declare -"$x" -p …` may be `-- -p`)."""
2036
+ for word, literal in words:
2037
+ if not literal or word == "--" or not word.startswith(("-", "+")):
2038
+ return False
2039
+ if any(c in word[1:] for c in ("pfF" if word[0] == "-" else "p")):
2040
+ return True
2041
+ return False
2042
+
2043
+
2044
+
2045
+ _DESTINATION_OPTIONS = {"read": "a", "printf": "v", "wait": "p", "mapfile": "", "readarray": "", "getopts": ""}
2046
+ _DESTINATION_OPERANDS = {"read": "all", "printf": None, "wait": None, "mapfile": 0, "readarray": 0, "getopts": 1}
2047
+ # A `%n` conversion in a printf format writes the count printed so far to the variable
2048
+ # its argument names (`printf %n "$v"`); flags and length modifiers may come between.
2049
+ _PRINTF_COUNT = re.compile(r"%[^a-zA-Z%]*[hlLqjzt]*n")
2050
+ _DATA_OPTIONS = {"read": "dinNptu", "printf": "", "wait": "", "mapfile": "dnOsuCc", "readarray": "dnOsuCc", "getopts": ""}
2051
+
2052
+
2053
+ def _destinations(head: str, words: list[tuple[str, bool]]) -> tuple[list[str], bool]:
2054
+ """The variables (or elements, `a[x]`) a `read`/`printf`/`wait`/`mapfile`/`getopts`
2055
+ call writes, and whether a destination, or a word that could be an option selecting
2056
+ one (`printf "$opt" x`), cannot be read."""
2057
+ names: list[str] = []
2058
+ if head == "getopts": # no options of its own: optstring, name, [args]
2059
+ if words and not words[0][1]:
2060
+ return names, True # `getopts "$dash" …`: it may expand to `--`
2061
+ if words and words[0] == ("--", True):
2062
+ words = words[1:] # `getopts -- ab OPTIND`
2063
+ if len(words) < 2:
2064
+ return names, False
2065
+ return ([words[1][0]], False) if words[1][1] else (names, True)
2066
+ operands: list[tuple[str, bool]] = []
2067
+ by_option = _DESTINATION_OPTIONS[head]
2068
+ takes = by_option + _DATA_OPTIONS[head]
2069
+ i = 0
2070
+ while i < len(words):
2071
+ text, literal = words[i]
2072
+ if not operands and not literal:
2073
+ return names, True # it may be an option (`-v`) or a name
2074
+ if operands or text == "-" or not text.startswith("-"):
2075
+ operands.append((text, literal))
2076
+ i += 1
2077
+ continue
2078
+ if text == "--":
2079
+ operands.extend(words[i + 1 :])
2080
+ break
2081
+ for k, letter in enumerate(text[1:], start=1):
2082
+ if letter not in takes:
2083
+ continue
2084
+ attached = text[k + 1 :]
2085
+ if attached:
2086
+ argument = (attached, True)
2087
+ else:
2088
+ i += 1
2089
+ argument = words[i] if i < len(words) else ("", True)
2090
+ if letter in by_option:
2091
+ if not argument[1]:
2092
+ return names, True
2093
+ names.append(argument[0])
2094
+ break
2095
+ i += 1
2096
+ where = _DESTINATION_OPERANDS[head]
2097
+ picked = operands if where == "all" else (operands[where : where + 1] if where is not None else [])
2098
+ if head == "printf" and len(operands) > 1:
2099
+ fmt, fmt_literal = operands[0]
2100
+ if not fmt_literal:
2101
+ return names, True # `printf "$f" "$v"`: the format may hold `%n`
2102
+ if _PRINTF_COUNT.search(fmt.replace("%%", "")):
2103
+ picked = picked + operands[1:] # `%n` names a variable in any argument
2104
+ for text, literal in picked:
2105
+ if not literal:
2106
+ return names, True
2107
+ names.append(text)
2108
+ return names, False
2109
+
2110
+
2111
+
2112
+ def _unproven_arithmetic(root: tree_sitter.Node) -> bool:
2113
+ for node in _iter_nodes(root):
2114
+ if _unproven_node(node):
2115
+ return True
2116
+ return False
2117
+
2118
+
2119
+ def _unproven_node(node: tree_sitter.Node) -> bool:
2120
+ kind = node.type
2121
+ if kind == "arithmetic_expansion":
2122
+ return not _pure(" ".join(_node_text(c) for c in node.named_children))
2123
+ if kind == "compound_statement" and node.text.startswith(b"(("):
2124
+ return not _pure(_node_text(node)[2:-2])
2125
+ if kind == "c_style_for_statement":
2126
+ return any(
2127
+ not _pure(_node_text(part))
2128
+ for field_name in ("initializer", "condition", "update")
2129
+ for part in node.children_by_field_name(field_name)
2130
+ )
2131
+ if kind == "test_command" and not _node_text(node).startswith("[["):
2132
+ # `[ … ]` is the `test` builtin, whose words the grammar parses as an expression:
2133
+ # judge them as `test`'s (`[ $OP ]` with OP='-v a[X]').
2134
+ inner = _node_text(node)[1:].removesuffix("]")
2135
+ command = parse_source(("test " + inner).encode()).root_node
2136
+ return any(_unproven_node(n) for n in _iter_nodes(command))
2137
+ if kind == "test_command" and _node_text(node).startswith("[[") and any(
2138
+ e in _node_text(node) for e in _WORD_EXPANSIONS
2139
+ ) and any(
2140
+ d.type == "test_operator" and _node_text(d) in _ARITHMETIC_TESTS for d in _iter_nodes(node)
2141
+ ):
2142
+ return True # `[[ ~ -eq 0 ]]`: the grammar reads `~` as an operator, bash as $HOME
2143
+ if kind == "binary_expression" and _in_double_bracket(node) and any(
2144
+ c.type == "test_operator" and _node_text(c) in _ARITHMETIC_TESTS for c in node.children
2145
+ ):
2146
+ sides = [node.child_by_field_name(s) for s in ("left", "right")]
2147
+ return any(side is not None and not _word_pure(_node_text(side)) for side in sides)
2148
+ if kind == "unary_expression" and any(
2149
+ c.type == "test_operator" and _node_text(c) == "-v" for c in node.children
2150
+ ):
2151
+ for operand in node.named_children:
2152
+ if operand.type == "test_operator":
2153
+ continue
2154
+ text, causes = lex_word(_node_text(operand))
2155
+ if causes - {"glob"} or not all(_index_pure(i) for i in _element_indexes(text)):
2156
+ return True # `[[ -v $name ]]`, `[ -v 'a[x]' ]`
2157
+ return False
2158
+ if kind == "array":
2159
+ # `a=([X]=1)`: the keys of a compound assignment are indexes, which the grammar
2160
+ # leaves as plain words.
2161
+ return _unproven_compound_keys(node)
2162
+ if kind == "subscript":
2163
+ index = node.child_by_field_name("index")
2164
+ if index is None:
2165
+ return False
2166
+ text, causes = lex_word(_node_text(index))
2167
+ return bool(causes - {"glob"}) or not _index_pure(text) # `${a[$"1"]}`: translated
2168
+ if kind == "expansion":
2169
+ text = _node_text(node)
2170
+ assigned = _DEFAULT_ASSIGNMENT.match(text)
2171
+ if assigned and assigned.group(1) in _BASH_INTEGERS:
2172
+ return True # `${MAILCHECK:=$1}` assigns to an integer variable
2173
+ if text.startswith("${!") and not _INDIRECT_LISTING.match(text):
2174
+ return True # `${!name}`: name's value may carry a subscript bash evaluates
2175
+ offset = _substring_offset(text)
2176
+ return offset is not None and not _pure(offset)
2177
+ if kind == "variable_assignment":
2178
+ name_node = node.child_by_field_name("name")
2179
+ if name_node is not None and name_node.type == "subscript":
2180
+ name_node = name_node.child_by_field_name("name") or name_node
2181
+ return name_node is not None and _node_text(name_node) in _BASH_INTEGERS
2182
+ if kind == "for_statement":
2183
+ variable = node.child_by_field_name("variable")
2184
+ return variable is not None and _node_text(variable) in _BASH_INTEGERS
2185
+ if kind == "declaration_command":
2186
+ keyword = node.children[0].text.decode() if node.children else ""
2187
+ sources = [_node_text(c) for c in _word_children(node)]
2188
+ words = [(t, lit, src) for (t, lit), src in zip(_lexed(node), sources)]
2189
+ return _unproven_declaration(keyword, words + _words_after_redirect(node))
2190
+ if kind == "unset_command":
2191
+ return any(
2192
+ not literal or not all(_index_pure(i) for i in _element_indexes(text))
2193
+ for text, literal in [(t, lit) for _, t, lit, _ in _name_words(node)]
2194
+ + [(t, lit) for t, lit, _ in _words_after_redirect(node)]
2195
+ )
2196
+ if kind.endswith("_redirect") and _unproven_descriptor(node):
2197
+ return True # `: {a[X]}>x`: bash evaluates the element's index
2198
+ if kind == "command_substitution" and _node_text(node).startswith("$(("):
2199
+ # `$((X))` in an unquoted here-doc body parses as a substitution of a subshell
2200
+ # (tree-sitter-bash#325); bash reads it as arithmetic.
2201
+ text = _node_text(node)
2202
+ return not text.endswith("))") or not _pure(text[3:-2])
2203
+ if kind == "subshell" and _node_text(node).startswith("(("):
2204
+ # `time ((X))`, `! ((X))`: bash's arithmetic command, which the grammar parses as
2205
+ # nested subshells after a reserved word.
2206
+ text = _node_text(node)
2207
+ return not text.endswith("))") or not _pure(text[2:-2])
2208
+ if kind == "command":
2209
+ lexed, i = _lexed(node), _lexed_head(node)
2210
+ if i == UNREADABLE:
2211
+ return True # `command -"$opt" let X`: the command after it may be any
2212
+ sources = [_node_text(c) for c in _word_children(node)]
2213
+ if "coproc" in (sources[:1] + sources[i : i + 1] if i is not None else sources[:1]):
2214
+ return True # `coproc let X`, `time coproc let X`: the walker does not look into coproc
2215
+ if "[[" in sources:
2216
+ # `time [[ X -eq 0 ]]`, `time [[ -v a[X] ]]`: the grammar parses `[[` as a word
2217
+ # here and may split the conditional at `&&`, so it cannot be read.
2218
+ return True
2219
+ # Words before the command that are assignments (`time RANDOM=$X`): bash makes
2220
+ # them, so they are judged as declarations are.
2221
+ assigned = [
2222
+ (text, literal, source)
2223
+ for (text, literal), source in zip(lexed[: len(lexed) if i is None else i], sources)
2224
+ if _ASSIGNMENT_WORD.match(source)
2225
+ ]
2226
+ if assigned and _unproven_declaration("export", assigned):
2227
+ return True
2228
+ extra = _words_after_redirect(node)
2229
+ if i is None:
2230
+ return bool(extra) # `command >/dev/null let X`: the command is under the redirect
2231
+ head = lexed[i][0]
2232
+ # A non-literal command word (`"$cmd" X`) may be any command, `eval` and `let`
2233
+ # included: like `eval`, it is opaque to the walker, and callers treat it so
2234
+ # (`may_rewire_stdin` is true for it).
2235
+ operands = _operands(node, i) + extra
2236
+ if head in _DECLARERS:
2237
+ return _unproven_declaration(head, operands)
2238
+ rest = [(text, literal) for text, literal, _ in operands]
2239
+ if head == "let":
2240
+ return any(not _word_pure(source) for _, _, source in operands)
2241
+ if head == "unset":
2242
+ return any(
2243
+ not literal or not all(_index_pure(i) for i in _element_indexes(text))
2244
+ for text, literal in rest
2245
+ )
2246
+ if head in ("test", "["):
2247
+ if any(_may_split(source) and not literal for _, literal, source in operands):
2248
+ return True # `test $OP`, `test "$@"`, `test {-v,a[X]}`, `test *`
2249
+ for k, (word, literal) in enumerate(rest[:-1]):
2250
+ if word == "-v" or not literal: # `test "$op" 'a[X]'`: $op may be -v
2251
+ target, literal = rest[k + 1]
2252
+ if not literal or not all(_index_pure(i) for i in _element_indexes(target)):
2253
+ return True # `test -v "$name"`, `test -v 'a[x]'`
2254
+ return False
2255
+ if head in _VARIABLE_WRITERS:
2256
+ names, dynamic = _destinations(head, rest)
2257
+ if dynamic:
2258
+ return True # `read "$1"`: may name an element or an integer variable
2259
+ return any(
2260
+ name.split("[", 1)[0] in _BASH_INTEGERS
2261
+ or not all(_index_pure(i) for i in _element_indexes(name))
2262
+ for name in names
2263
+ )
2264
+ return False
2265
+
2266
+
2267
+ def _unproven_descriptor(redirect: tree_sitter.Node) -> bool:
2268
+ """Whether the word written right before a redirect operator ends like a named
2269
+ descriptor with an index (`{a[X]}>x`, `{a[b[0]]}>x`) that is not literal numbers.
2270
+ Read from the source, since the grammar splits or misplaces such words."""
2271
+ root = redirect
2272
+ while root.parent is not None:
2273
+ root = root.parent
2274
+ before = root.text[: redirect.start_byte].decode(errors="replace")
2275
+ token = re.split(r"[\s;&|()<>]", before)[-1]
2276
+ if not token.endswith("]}"):
2277
+ return False
2278
+ match = re.fullmatch(r"\{[A-Za-z_]\w*\[(.*)\]\}", token, re.S)
2279
+ return match is None or not _index_pure(match.group(1))
2280
+
2281
+
2282
+ _MANY_WORDS = re.compile(r"\$[@*]|\$\{[@*!]|\$\{[^}]*\[[@*]\]") # `"${a[@]:0}"` too
2283
+
2284
+
2285
+ def _words_after_redirect(node: tree_sitter.Node) -> list[tuple[str, bool, str]]:
2286
+ """The words the grammar hung under the redirects of the statement ``node`` is the
2287
+ body of (`let >/dev/null X`, `let <<E X`): bash passes every word after a redirect's
2288
+ target to the command. Each is (text, literal, source), in source order."""
2289
+ # A redirect after `a && let >x X` or `a | let >x X` hangs on the whole list or
2290
+ # pipeline; its words still belong to the last command.
2291
+ body, parent = node, node.parent
2292
+ while parent is not None and parent.type in ("list", "pipeline", "negated_command"):
2293
+ if parent.named_children[-1] != body:
2294
+ return []
2295
+ body, parent = parent, parent.parent
2296
+ if parent is None or parent.type != "redirected_statement" or parent.child_by_field_name("body") != body:
2297
+ return []
2298
+ pieces = []
2299
+ for redirect in parent.children_by_field_name("redirect"):
2300
+ pieces += redirect.children_by_field_name("destination")[1:]
2301
+ pieces += redirect.children_by_field_name("argument")
2302
+ words = []
2303
+ for piece in sorted(pieces, key=lambda n: n.start_byte):
2304
+ source = _node_text(piece)
2305
+ text, causes = lex_word(source)
2306
+ literal = not causes or (causes == {"glob"} and bool(_ELEMENT_FORM.fullmatch(text)))
2307
+ words.append((text, literal, source))
2308
+ return words
2309
+
2310
+
2311
+ def _may_split(source: str) -> bool:
2312
+ """Whether a word as written may become several words: an unquoted expansion (split),
2313
+ `"$@"`, `"${a[@]}"` or `${!p*}` even quoted, or an unquoted brace or glob (`{-v,a[X]}`,
2314
+ `*`). Callers excuse a glob that ``_name_words`` judged an element name."""
2315
+ if _MANY_WORDS.search(source):
2316
+ return True
2317
+ quote = None
2318
+ k = 0
2319
+ while k < len(source):
2320
+ c = source[k]
2321
+ if c == "\\" and quote != "'":
2322
+ k += 2
2323
+ continue
2324
+ if quote == "'":
2325
+ quote = None if c == "'" else quote
2326
+ elif c == '"':
2327
+ quote = None if quote == '"' else '"'
2328
+ elif quote is None and c == "'":
2329
+ quote = "'"
2330
+ elif quote is None and c in "$`{*?[":
2331
+ return True
2332
+ k += 1
2333
+ return False
2334
+
2335
+
2336
+ # A name written plainly, perhaps after an opening double quote (`"PATH=$x"`), up to `=`.
2337
+ _WRITTEN_NAME = re.compile(r'"?[A-Za-z_]\w*(?:\[[^\]$`"\\]*\])?\+?=')
2338
+ _DECLARED_NAME = re.compile(r"[A-Za-z_]\w*(?:\[.*\])?\+?", re.S)
2339
+ _ARRAY_LITERAL = re.compile(r"[A-Za-z_]\w*(?:\[[^\]]*\])?\+?=\(")
2340
+
2341
+
2342
+ def _unproven_declaration(keyword: str, words: list[tuple[str, bool, str]]) -> bool:
2343
+ """A declaration that may make bash evaluate arithmetic: `-i`, an option or name the
2344
+ walker cannot read, a name that is one of bash's integer variables, an element whose
2345
+ index is not literal numbers, or a value bash may reparse as a compound assignment.
2346
+ `declare -p`/`-f` only display. Each word is (text, literal, source as written)."""
2347
+ pairs = [(text, literal) for text, literal, _ in words]
2348
+ if keyword in _INTEGER_DECLARERS and _display_only(pairs):
2349
+ return False
2350
+ options = [text for text, literal in pairs if literal and text.startswith(("-", "+"))]
2351
+ # Bash reparses a value as a compound assignment, whose keys it evaluates, with `-a`
2352
+ # (`declare -a "a=$X"`, also export/readonly), or with declare/typeset/local on a name
2353
+ # that may already be an array (`b=(); declare 'b=([X]=1)'`). Only an unquoted
2354
+ # `name=(` is the grammar's array, whose keys the array rule checks; any other value
2355
+ # that may start with `(` (quoted, or from `$`, `` ` ``, `~` or `{`) is not proven.
2356
+ arrays = keyword in _INTEGER_DECLARERS or any(o[0] == "-" and ("a" in o or "A" in o) for o in options)
2357
+ functions = any(o[0] == "-" and "f" in o for o in options) # `export -f my-func`
2358
+ for text, literal, source in words:
2359
+ if text.startswith(("-", "+")):
2360
+ # `-i` adds the integer attribute; `+i` removes it.
2361
+ if not literal or (keyword in _INTEGER_DECLARERS and text[0] == "-" and "i" in text[1:]):
2362
+ return True # `declare -i x`, `declare "$opt" x`
2363
+ continue
2364
+ if not literal and not _WRITTEN_NAME.match(source):
2365
+ # Judged as written: `export OPT"$s"=…` lexes to `OPT…=` but may name OPTIND.
2366
+ return True # `export "$name=$value"`: the name is not known
2367
+ name, equals, value = text.partition("=")
2368
+ if literal and not functions and not _DECLARED_NAME.fullmatch(name):
2369
+ return True # `declare -a {a='([X]=1)',b}`: brace expansion makes the name
2370
+ if arrays and equals and not _ARRAY_LITERAL.match(source):
2371
+ written = _STATIC_NAME.match(source) # `x[2]=$v`: only the value counts
2372
+ value_literal = not lex_word(source[written.end() :])[1] if written else literal
2373
+ if not value_literal or value.startswith(("$", "`", "(", "~", "{")):
2374
+ return True # `declare -a "a=$X"`, `declare -a 'a=([X]=1)'`
2375
+ name = name.removesuffix("+") # `OPTIND+=…` appends
2376
+ if name.split("[", 1)[0] in _BASH_INTEGERS:
2377
+ return True # `export OPTIND=$X`: bash evaluates the value
2378
+ if not all(_index_pure(i) for i in _element_indexes(name)):
2379
+ return True
2380
+ return False