pi-roundtable 0.22.5 → 0.23.0

package/src/core/modules/host-shell/shell-lexer.ts

@@ -0,0 +1,493 @@
+/**
+ * Splits a bash command line into the simple commands bash would run, for the shell hold rule.
+ *
+ * The rule's one promise is: if bash runs a held command, the line is held. So the lexer reads
+ * the way bash does, and where it cannot (a line too long or too deeply nested, or a construct it
+ * cannot place), it gives up with `unreadable` and the rule holds the whole line. It never guesses
+ * that text is inert. What it must therefore know is where each kind of text ends: single, double
+ * and `$'…'` quotes, `$(…)`, backticks, `${…}`, `$[…]`, comments, here-document bodies and `case`
+ * patterns.
+ */
+
+import { ArithGroups } from "./shell-arithmetic.ts";
+import {
+	COMMAND_PREFIX_WORDS,
+	EXPANDED,
+	KEYWORDS_BEFORE_CASE,
+	type Lexed,
+	type Pending,
+	type Shared,
+} from "./shell-lexer-types.ts";
+import {
+	ansiQuoted,
+	backtickBody,
+	closesGroup,
+	continuedToken,
+	heredocBody,
+	readDelimiter,
+	redirectOperator,
+	Unreadable,
+} from "./shell-quoting.ts";
+
+export { COMMAND_PREFIX_WORDS, EXPANDED, type Lexed };
+
+/** A line longer than this is not read. */
+const MAX_LENGTH = 200_000;
+/** Nested substitutions deeper than this are not read. */
+const MAX_DEPTH = 48;
+
+const isSpace = (ch: string) => /\s/.test(ch);
+
+/** Splits a command line into simple commands of words; quotes are removed, operators separate. */
+export function lexShell(command: string): Lexed {
+	const gone = {
+		commands: [],
+		unreadable: true,
+		arithmetic: false,
+		literalSubstitution: false,
+	};
+	if (command.length > MAX_LENGTH) return gone;
+	// A line continuation inside a token is not read the way bash joins it.
+	if (continuedToken(command)) return gone;
+	const shared = { depth: 0, arithmetic: false, literalSubstitution: false };
+	try {
+		// Bash drops a NUL byte from the script it reads.
+		const lexer = new Lexer(command.replaceAll("\0", ""), shared, false);
+		const commands = lexer.list("eof");
+		return {
+			commands,
+			unreadable: false,
+			arithmetic: shared.arithmetic,
+			literalSubstitution: shared.literalSubstitution,
+		};
+	} catch (error) {
+		// A stack that ran out is a line too deep to read, whatever the cap says.
+		if (error instanceof Unreadable || error instanceof RangeError) return gone;
+		throw error;
+	}
+}
+
+/** Splits a command line into simple commands of words; the commands of a line that cannot be read are none. */
+export function simpleCommands(command: string): string[][] {
+	return lexShell(command).commands;
+}
+
+class Lexer {
+	pos = 0;
+	/** Here-documents whose bodies follow the line being read. */
+	readonly pending: Pending[] = [];
+	#calls = 0;
+
+	constructor(
+		readonly src: string,
+		readonly shared: Shared,
+		/** Whether `<<` is a shift: the text is arithmetic, where there are no here-documents. */
+		readonly arithmetic: boolean,
+	) {}
+
+	#enter() {
+		if (++this.shared.depth > MAX_DEPTH) throw new Unreadable();
+	}
+
+	#leave() {
+		this.shared.depth--;
+	}
+
+	/**
+	 * The commands from here to the end of the text, or, for `paren`, to the `)` that closes the
+	 * substitution that starts here.
+	 */
+	list(end: "eof" | "paren"): string[][] {
+		const id = ++this.#calls;
+		const s = this.src;
+		const out: string[][] = [];
+		let words: string[] = [];
+		let word = "";
+		let inWord = false;
+		// Whether the word was written without a quote, escape or expansion, so it may be a keyword.
+		let plain = true;
+		let parenDepth = 0;
+		let caseOpen = 0;
+		/** Whether the next `)` in a `case` ends a pattern: after `in` and after `;;`. */
+		let expectPattern = false;
+		/** How many words of the current command come before a `case` pattern (`case x in`). */
+		let patternBase = 0;
+		const arith = new ArithGroups(out, s, this.pos, end === "paren");
+		let sawShift = false;
+		const endWord = () => {
+			if (!inWord) return;
+			if (plain && words.every((w) => KEYWORDS_BEFORE_CASE.has(w))) {
+				if (word === "case") {
+					caseOpen++;
+					expectPattern = false;
+				} else if (word === "esac" && caseOpen > 0) {
+					caseOpen--;
+					expectPattern = false;
+				}
+			} else if (plain && word === "in" && caseOpen > 0) {
+				// `case WORD in`: the first pattern follows.
+				const at = words.lastIndexOf("case");
+				if (at !== -1 && words.length - at === 2) {
+					expectPattern = true;
+					patternBase = words.length + 1;
+				}
+			}
+			words.push(word);
+			word = "";
+			inWord = false;
+			plain = true;
+		};
+		const endCommand = () => {
+			endWord();
+			if (words.length > 0) arith.add(words);
+			words = [];
+			patternBase = 0;
+		};
+		const quoted = (text: string) => {
+			word += text;
+			inWord = true;
+			plain = false;
+		};
+		while (this.pos < s.length) {
+			const ch = s[this.pos] as string;
+			const next = s[this.pos + 1];
+			if (ch === "\\") {
+				if (next === undefined) this.pos++;
+				else if (next === "\n") this.pos += 2;
+				else {
+					this.#literal(next, s[this.pos + 2]);
+					quoted(next);
+					this.pos += 2;
+				}
+			} else if (ch === "'") {
+				const close = s.indexOf("'", this.pos + 1);
+				const stop = close === -1 ? s.length : close;
+				const literal = s.slice(this.pos + 1, stop);
+				this.#literal(literal);
+				quoted(literal);
+				this.pos = stop + 1;
+			} else if (ch === '"') {
+				this.pos++;
+				quoted(this.#doubleQuoted(out));
+			} else if (ch === "$") {
+				if (next === "'") quoted(this.#ansiQuoted());
+				else if (next === '"') {
+					this.pos += 2;
+					quoted(this.#doubleQuoted(out));
+				} else if (next === "(") {
+					// The text is dropped and the word keeps the mark, so a value built from it counts as unknown.
+					this.#paren(out);
+					quoted(`$(${EXPANDED}`);
+				} else if (next === "{") quoted(this.#param(out, false));
+				else if (next === "[") quoted(this.#bracket(out));
+				else {
+					word += "$";
+					inWord = true;
+					this.pos++;
+				}
+			} else if (ch === "`") {
+				this.#backtick(out);
+				quoted(`\`${EXPANDED}`);
+			} else if (ch === "\n") {
+				endCommand();
+				this.pos++;
+				if ((this.arithmetic || arith.open) && sawShift) throw new Unreadable();
+				this.#drain(id, out);
+			} else if (isSpace(ch)) {
+				// Bash 5 takes `then` or `do` right after `]]` as a new command.
+				const closes = inWord && plain && word === "]]" && words.includes("[[");
+				endWord();
+				if (closes) endCommand();
+				this.pos++;
+			} else if (
+				ch === "#" &&
+				!inWord &&
+				(this.pos === 0 || " \t\n;&|()<>".includes(s[this.pos - 1] as string))
+			) {
+				// A comment: a `)` or a quote in it means nothing.
+				const stop = s.indexOf("\n", this.pos);
+				const comment = s.slice(this.pos, stop === -1 ? s.length : stop);
+				// Bash 3.2 and 5 differ on a quote in a comment inside a substitution; neither reading is trusted.
+				if (this.shared.depth > 0 && /['"`]/.test(comment))
+					throw new Unreadable();
+				this.pos += comment.length;
+			} else if (ch === "(" && caseOpen > 0 && expectPattern && !inWord) {
+				// The optional `(` before a `case` pattern; its `)` is the one that ends the pattern.
+				this.pos++;
+			} else if (ch === "(") {
+				endCommand();
+				parenDepth++;
+				if (next === "(" || s[this.pos - 1] === "(")
+					arith.push(parenDepth, this.pos);
+				this.pos++;
+			} else if (ch === ")") {
+				endWord();
+				this.pos++;
+				if (parenDepth > 0) {
+					endCommand();
+					if (arith.close(parenDepth, this.pos)) this.shared.arithmetic = true;
+					if (!arith.open && !this.arithmetic) sawShift = false;
+					parenDepth--;
+				} else if (caseOpen > 0) {
+					// The `)` that ends a `case` pattern; the pattern runs nothing. Any other `)` there is
+					// one that bash versions place differently, so the line is not read.
+					// A pattern is one word; anything more is not one bash would accept.
+					if (!expectPattern || words.length - patternBase > 1)
+						throw new Unreadable();
+					expectPattern = false;
+					words = [];
+				} else {
+					endCommand();
+					if (end === "paren") {
+						if (this.pending.some((p) => p.owner === id))
+							throw new Unreadable();
+						arith.flush();
+						return out;
+					}
+				}
+			} else if (
+				";&|".includes(ch) ||
+				(ch === "{" && (next === undefined || isSpace(next))) ||
+				(ch === "}" && closesGroup(this.src, this.pos))
+			) {
+				endCommand();
+				if (ch === ";" && caseOpen > 0 && (next === ";" || next === "&"))
+					expectPattern = true;
+				this.pos++;
+			} else if (
+				(ch === "<" || ch === ">") &&
+				next === "(" &&
+				!this.arithmetic &&
+				!arith.open
+			) {
+				if (s[this.pos + 2] === "(") throw new Unreadable();
+				this.#paren(out);
+				quoted(`${ch}(${EXPANDED}`);
+			} else if (ch === ">" || ch === "<") {
+				// A number or a bash 5 `{name}` right before the operator is its descriptor (`3>&1`, `{fd}>f`), no word of the command.
+				if (inWord && plain && /^(?:\d+|\{[A-Za-z_]\w*\})$/.test(word)) {
+					word = "";
+					inWord = false;
+				} else endWord();
+				if (ch === "<" && next === "<") {
+					if (this.#shiftOrHeredoc(id, words, arith.open)) sawShift = true;
+					continue;
+				}
+				// Redirections become their own words, so their targets can be checked.
+				const found = redirectOperator(s, this.pos);
+				words.push(found.op);
+				if (found.closes) words.push("-");
+				this.pos = found.end;
+			} else {
+				word += ch;
+				inWord = true;
+				this.pos++;
+			}
+		}
+		endCommand();
+		// A group left open is not known to be arithmetic.
+		arith.flush();
+		return out;
+	}
+
+	/**
+	 * `<<<`, a shift in arithmetic, or a here-document, at the `<<` that starts here. Returns
+	 * whether it was a shift, which a newline after it in the same arithmetic makes unreadable.
+	 */
+	#shiftOrHeredoc(id: number, words: string[], inArithmetic: boolean): boolean {
+		const s = this.src;
+		if (s[this.pos + 2] === "<") {
+			words.push("<");
+			this.pos += 3;
+			return false;
+		}
+		if (this.arithmetic || inArithmetic) {
+			words.push("<");
+			this.pos += 2;
+			return true;
+		}
+		this.pos += 2;
+		let stripTabs = false;
+		if (s[this.pos] === "-") {
+			stripTabs = true;
+			this.pos++;
+		}
+		while (s[this.pos] === " " || s[this.pos] === "\t") this.pos++;
+		const { text, quoted, end } = readDelimiter(s, this.pos);
+		this.pos = end;
+		this.pending.push({ owner: id, delimiter: text, quoted, stripTabs });
+		// The delimiter stays a word so the redirection's target is skipped like any other.
+		words.push("<", text);
+		return false;
+	}
+
+	/** Reads the bodies of the here-documents of the line that just ended, and the expansions in them. */
+	#drain(id: number, out: string[][]) {
+		const s = this.src;
+		for (let p = this.pending.shift(); p; p = this.pending.shift()) {
+			// A line ended inside another substitution than the one that opened the here-document.
+			if (p.owner !== id) throw new Unreadable();
+			const start = this.pos;
+			const { bodyEnd, next, terminated } = heredocBody(s, start, p);
+			// Unterminated inside a substitution: bash 3.2 closes it at the first `)`, and 5 does not.
+			if (!terminated && this.shared.depth > 0) throw new Unreadable();
+			if (!p.quoted) {
+				const body = new Lexer(
+					s.slice(start, bodyEnd),
+					this.shared,
+					this.arithmetic,
+				);
+				for (const command of body.expansions()) out.push(command);
+			}
+			this.pos = next;
+		}
+	}
+
+	/** The commands run by the expansions in text that is not split into words (a here-document body). */
+	expansions(): string[][] {
+		const out: string[][] = [];
+		const s = this.src;
+		while (this.pos < s.length) {
+			const ch = s[this.pos] as string;
+			const next = s[this.pos + 1];
+			if (ch === "\\") this.pos += 2;
+			else if (ch === "`") this.#backtick(out);
+			else if (ch === "$" && next === "(") this.#paren(out);
+			else if (ch === "$" && next === "{") this.#param(out, true);
+			else if (ch === "$" && next === "[") this.#bracket(out);
+			else this.pos++;
+		}
+		return out;
+	}
+
+	/** The text of a double-quoted string that starts here, after its opening quote; its substitutions run. */
+	#doubleQuoted(out: string[][]): string {
+		const s = this.src;
+		let text = "";
+		while (this.pos < s.length) {
+			const ch = s[this.pos] as string;
+			const next = s[this.pos + 1];
+			if (ch === '"') {
+				this.pos++;
+				return text;
+			}
+			if (ch === "\\") {
+				// A backslash before a newline joins the lines.
+				if (next !== undefined && next !== "\n") {
+					this.#literal(next, s[this.pos + 2]);
+					text += next;
+				}
+				this.pos += 2;
+			} else if (ch === "`") {
+				this.#backtick(out, true);
+				text += `\`${EXPANDED}`;
+			} else if (ch === "$" && next === "(") {
+				this.#paren(out);
+				text += `$(${EXPANDED}`;
+			} else if (ch === "$" && next === "{") text += this.#param(out, true);
+			else if (ch === "$" && next === "[") text += this.#bracket(out);
+			else {
+				text += ch;
+				this.pos++;
+			}
+		}
+		return text;
+	}
+
+	/** Notes text that holds a `$(` or a backtick no shell expands where it is written; `after` follows a lone character. */
+	#literal(text: string, after?: string) {
+		if (
+			text.includes("`") ||
+			text.includes("$(") ||
+			(text === "$" && after === "(")
+		)
+			this.shared.literalSubstitution = true;
+	}
+
+	/** The text of a `$'…'` string that starts here, with its escapes decoded. */
+	#ansiQuoted(): string {
+		const found = ansiQuoted(this.src, this.pos);
+		this.#literal(found.text);
+		this.pos = found.end;
+		return found.text;
+	}
+
+	/** A `$(…)` that starts here: its commands go to `out`; returns its text. An unterminated one runs to the end. */
+	#paren(out: string[][]): string {
+		const start = this.pos;
+		this.#enter();
+		this.pos += 2;
+		for (const command of this.list("paren")) out.push(command);
+		this.#leave();
+		return this.src.slice(start, this.pos);
+	}
+
+	/** A backtick substitution that starts here: its commands go to `out`; returns its text. */
+	#backtick(out: string[][], inDouble = false): string {
+		const s = this.src;
+		const start = this.pos;
+		const { inner, end } = backtickBody(s, start, inDouble);
+		this.pos = end;
+		this.#enter();
+		const body = new Lexer(inner, this.shared, this.arithmetic);
+		for (const command of body.list("eof")) out.push(command);
+		this.#leave();
+		return s.slice(start, this.pos);
+	}
+
+	/**
+	 * A `${…}` that starts here: the substitutions in it run, and a `}` in a quote or behind a
+	 * backslash does not end it. Returns its text.
+	 */
+	#param(out: string[][], inDouble: boolean): string {
+		return this.#braced(out, "}", undefined, inDouble);
+	}
+
+	/** A `$[…]` that starts here: arithmetic, where there are no here-documents. Returns its text. */
+	#bracket(out: string[][]): string {
+		this.shared.arithmetic = true;
+		return this.#braced(out, "]", "[", false);
+	}
+
+	#braced(
+		out: string[][],
+		close: string,
+		open: string | undefined,
+		inDouble: boolean,
+	): string {
+		const s = this.src;
+		const start = this.pos;
+		this.#enter();
+		this.pos += 2;
+		let depth = 1;
+		while (this.pos < s.length) {
+			const ch = s[this.pos] as string;
+			const next = s[this.pos + 1];
+			if (ch === close && --depth === 0) {
+				this.pos++;
+				break;
+			}
+			if (ch === open) {
+				depth++;
+				this.pos++;
+			} else if (ch === "\\") this.pos += 2;
+			else if (ch === "'" && !inDouble) {
+				const stop = s.indexOf("'", this.pos + 1);
+				this.pos = stop === -1 ? s.length : stop + 1;
+			} else if (ch === '"') {
+				this.pos++;
+				this.#doubleQuoted(out);
+			} else if (ch === "`") this.#backtick(out, inDouble);
+			else if (ch === "$" && next === "(") this.#paren(out);
+			else if (ch === "$" && next === "{") this.#param(out, inDouble);
+			else if (ch === "$" && next === "[") this.#bracket(out);
+			else this.pos++;
+		}
+		this.#leave();
+		this.pos = Math.min(this.pos, s.length);
+		const text = s.slice(start, this.pos);
+		// `${x@P}` expands the prompt escapes in the value of x, and that runs a `$(` in it.
+		if (close === "}" && /@P\}$/.test(text)) throw new Unreadable();
+		return text;
+	}
+}

package/src/core/modules/host-shell/shell-paths.ts

@@ -22,6 +22,10 @@ export interface LineState {
 	vars: Map<string, string | undefined>;
 	/** The working directory, or undefined once a `cd` went somewhere unknown. */
 	cwd: string | undefined;
+	/** The names the line itself writes anywhere, as a bare word: what it assigns, loops over or reads into. */
+	bound: ReadonlySet<string>;
+	/** The files the line has written so far with `>`, `>>` and `tee`, as it names them without a leading `./`. */
+	written: Set<string>;
 	/** Whether `cd` can be followed: false in a line with subshells or `||`, where the last `cd` may not apply. */
 	followsCd: boolean;
 }
@@ -38,15 +42,17 @@ export function lineState(
 		roots: scratchDir === undefined ? [workspace] : [workspace, scratchDir],
 		vars,
 		cwd: workspace,
+		written: new Set(),
+		bound: new Set(bindingCounts(command).keys()),
 		followsCd: !/(^|[^$])\(|\|\|/.test(command),
 	};
 }
 
-const ASSIGNMENT = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s;
+const ASSIGNMENT = /^([A-Za-z_][A-Za-z0-9_]*)(\+?)=(.*)$/s;
 
 /**
  * The word with the line's variables substituted and `~` expanded, or undefined when it holds a
- * command substitution, an unknown variable or `~user`.
+ * command substitution, an unknown variable, a brace expansion or `~user`.
  */
 export function expand(word: string, state: LineState): string | undefined {
 	if (word.includes("`") || word.includes("$(")) return undefined;
@@ -60,6 +66,8 @@ export function expand(word: string, state: LineState): string | undefined {
 		},
 	);
 	if (unknown || substituted.includes("$")) return undefined;
+	// A brace expansion (`{a,b}`, `{1..3}`) names several words.
+	if (/\{[^{}]*(?:,|\.\.)[^{}]*\}/.test(substituted)) return undefined;
 	if (substituted === "~" || substituted.startsWith("~/"))
 		return `${homedir()}${substituted.slice(1)}`;
 	return substituted.startsWith("~") ? undefined : substituted;
@@ -88,8 +96,12 @@ export function follow(argv: readonly string[], state: LineState): void {
 				: [];
 	for (const word of assignments) {
 		const match = ASSIGNMENT.exec(word);
+		// An append (`x+=y`) makes a value the line does not follow.
 		if (match)
-			state.vars.set(match[1] as string, expand(match[2] as string, state));
+			state.vars.set(
+				match[1] as string,
+				match[2] === "" ? expand(match[3] as string, state) : undefined,
+			);
 	}
 	if (head === "cd" || head === "pushd" || head === "popd") {
 		const target = args.find((a) => !a.startsWith("-"));
@@ -100,6 +112,49 @@ export function follow(argv: readonly string[], state: LineState): void {
 	}
 }
 
+/**
+ * `expand` for a word that names a command or a script: a variable the line itself binds is
+ * unknown there, because the line is followed assignment by assignment and bash may skip the one
+ * that matters (`false && x=ls; $x sudo`, `(V=1); s${V}udo`), so the variable is empty. What the
+ * line does not bind, such as `$HOME`, keeps its value.
+ */
+export function expandName(word: string, state: LineState): string | undefined {
+	for (const match of word.matchAll(/\$\{?([A-Za-z_][A-Za-z0-9_]*)/g))
+		if (state.bound.has(match[1] as string)) return undefined;
+	return expand(word, state);
+}
+
+/** How many times a line writes each name where it could bind it: as a bare word, or in `${name:=…}`. */
+export function bindingCounts(command: string): Map<string, number> {
+	const counts = new Map<string, number>();
+	for (const match of command.matchAll(
+		/(?<![$\w{])([A-Za-z_]\w*)(?!\w)|\$\{([A-Za-z_]\w*):?=/g,
+	)) {
+		const name = (match[1] ?? match[2]) as string;
+		counts.set(name, (counts.get(name) ?? 0) + 1);
+	}
+	return counts;
+}
+
+/**
+ * Forgets the value of each variable a command assigns when the line names it more than once. The
+ * line is followed one assignment after another, but bash may bind a name by a loop, `read`,
+ * `declare`, `eval`, a function or a branch the rule does not follow, so a name written twice may
+ * hold either value (`x=ls; for x in sudo; do $x; done`); a name written once holds what it was
+ * given or is unset.
+ */
+export function forgetRebound(
+	words: readonly string[],
+	state: LineState,
+	counts: ReadonlyMap<string, number>,
+): void {
+	for (const word of words) {
+		const name = /^([A-Za-z_][A-Za-z0-9_]*)\+?=/.exec(word)?.[1];
+		if (name !== undefined && (counts.get(name) ?? 0) > 1)
+			state.vars.set(name, undefined);
+	}
+}
+
 /** Whether `path` is `root` or under it, comparing the paths as written. */
 export function under(path: string, root: string): boolean {
 	const rel = relative(resolve(root), resolve(path));
@@ -119,7 +174,7 @@ function realPath(path: string): string {
 }
 
 /** Whether a write to `path` stays inside a root, as written. */
-export function insideRoots(path: string, state: LineState): boolean {
+function insideRoots(path: string, state: LineState): boolean {
 	return state.roots.some((root) => under(path, root));
 }
 
@@ -158,3 +213,29 @@ export function rmHeld(
 	}
 	return undefined;
 }
+
+/**
+ * Whether a write to the word stays inside a scratch root. A word the line cannot resolve is read
+ * as written from the workspace, as before variables were followed.
+ */
+export function writable(word: string, state: LineState): boolean {
+	if (word === "/dev/null") return true;
+	return insideRoots(targetPath(word, state), state);
+}
+
+/** Where a write to the word lands: the word resolved from the line, else from the workspace, else as written. */
+function targetPath(word: string, state: LineState): string {
+	return (
+		resolvePath(word, state) ??
+		resolvePath(word, { ...state, vars: new Map(), cwd: state.workspace }) ??
+		word
+	);
+}
+
+/** Whether a write to the word reaches a device node, such as a disk, rather than `/dev/null`. */
+export function deviceTarget(word: string, state: LineState): boolean {
+	// A target the line cannot expand may be any device, so it is read as one.
+	if (/[$`]/.test(word) && resolvePath(word, state) === undefined) return true;
+	const path = targetPath(word, state);
+	return path !== "/dev/null" && under(path, "/dev");
+}