rooklang 0.0.0-stage → 0.3.1

package/dist/lexer/base.d.ts

@@ -0,0 +1,24 @@
+import type { Diagnostic } from '../diagnostics.ts';
+import { type Mode } from './tables.ts';
+import type { StringKind, Token, TokenKind } from './token.ts';
+export declare abstract class LexerBase {
+    protected readonly text: string;
+    protected pos: number;
+    protected readonly tokens: Token[];
+    protected readonly diagnostics: Diagnostic[];
+    protected readonly modes: Mode[];
+    constructor(text: string);
+    /** Skips whitespace and comments; returns whether a line break was skipped. */
+    protected skipTrivia(): boolean;
+    protected lastEndsStatement(): boolean;
+    /**
+     * Unlike Go, a line starting with `.` / `?.` continues a method chain, and a line starting with
+     * a closing bracket ends a list, so no semicolon is needed before it.
+     */
+    protected nextLineContinues(): boolean;
+    protected insertSemicolon(): void;
+    protected top(): Mode | undefined;
+    protected braces(): number;
+    protected push(kind: Exclude<TokenKind, 'Number' | StringKind>, start: number, end: number): void;
+    protected error(message: string, start: number, end: number): void;
+}

package/dist/lexer/base.js

@@ -0,0 +1,87 @@
+import { ENDS_STATEMENT, isDigit } from "./tables.js";
+// The lexer is a chain of layers: base → literals → markup → style → Lexer (lexer.ts). This layer
+// keeps the position and the modes (code, markup tags, markup content, CSS), skips spaces and
+// comments, and inserts semicolons at the ends of lines.
+export class LexerBase {
+    text;
+    pos = 0;
+    tokens = [];
+    diagnostics = [];
+    modes = [];
+    constructor(text) {
+        this.text = text;
+    }
+    /** Skips whitespace and comments; returns whether a line break was skipped. */
+    skipTrivia() {
+        const t = this.text;
+        let sawNewline = false;
+        while (this.pos < t.length) {
+            const c = t[this.pos];
+            if (c === '\n') {
+                sawNewline = true;
+                this.pos++;
+            }
+            else if (c === ' ' || c === '\t' || c === '\r' || c === '\v' || c === '\f') {
+                this.pos++;
+            }
+            else if (t.startsWith('//', this.pos)) {
+                const end = t.indexOf('\n', this.pos);
+                this.pos = end === -1 ? t.length : end;
+            }
+            else if (t.startsWith('/*', this.pos)) {
+                const end = t.indexOf('*/', this.pos + 2);
+                if (end === -1) {
+                    this.error('unterminated block comment', this.pos, this.pos + 2);
+                    this.pos = t.length;
+                }
+                else {
+                    // A comment spanning several lines counts as a line break.
+                    const newline = t.indexOf('\n', this.pos);
+                    if (newline !== -1 && newline < end)
+                        sawNewline = true;
+                    this.pos = end + 2;
+                }
+            }
+            else {
+                break;
+            }
+        }
+        return sawNewline;
+    }
+    lastEndsStatement() {
+        const last = this.tokens.at(-1);
+        return last !== undefined && ENDS_STATEMENT.has(last.kind);
+    }
+    /**
+     * Unlike Go, a line starting with `.` / `?.` continues a method chain, and a line starting with
+     * a closing bracket ends a list, so no semicolon is needed before it.
+     */
+    nextLineContinues() {
+        const t = this.text;
+        const c = t[this.pos];
+        const next = t[this.pos + 1];
+        if (c === ')' || c === ']' || c === '}')
+            return true;
+        if (c === '.')
+            return next !== '.' && !isDigit(next);
+        return c === '?' && next === '.' && !isDigit(t[this.pos + 2]);
+    }
+    insertSemicolon() {
+        const end = this.tokens.at(-1)?.end ?? 0;
+        this.tokens.push({ kind: ';', text: '', start: end, end });
+    }
+    top() {
+        return this.modes.at(-1);
+    }
+    braces() {
+        const top = this.top();
+        return top?.kind === 'template' || top?.kind === 'expression' ? top.braces : 0;
+    }
+    push(kind, start, end) {
+        this.tokens.push({ kind, text: this.text.slice(start, end), start, end });
+        this.pos = end;
+    }
+    error(message, start, end) {
+        this.diagnostics.push({ message, start, end });
+    }
+}

package/dist/lexer/lexer.d.ts

@@ -0,0 +1,8 @@
+import type { Diagnostic } from '../diagnostics.ts';
+import { type Token } from './token.ts';
+export interface LexResult {
+    /** Always ends with an EOF token. */
+    tokens: Token[];
+    diagnostics: Diagnostic[];
+}
+export declare function tokenize(text: string): LexResult;

package/dist/lexer/lexer.js

@@ -0,0 +1,134 @@
+import { StyleLexer } from "./style.js";
+import { IDENTIFIER, isDigit, isMarkup } from "./tables.js";
+import { isKeyword, isPunctuator } from "./token.js";
+// The lexer of a module; its layers are listed in base.ts.
+class Lexer extends StyleLexer {
+    run() {
+        if (this.text.startsWith(''))
+            this.pos = 1;
+        while (true) {
+            const mode = this.modes.at(-1);
+            if (isMarkup(mode)) {
+                // Markup has its own rules for whitespace and no automatic semicolons.
+                if (this.pos >= this.text.length)
+                    break;
+                if (mode.kind === 'content')
+                    this.scanContent(mode);
+                else
+                    this.scanTag(mode);
+                continue;
+            }
+            if (mode?.kind === 'css') {
+                if (this.pos >= this.text.length)
+                    break;
+                this.scanCss(mode);
+                continue;
+            }
+            const sawNewline = this.skipTrivia();
+            if (sawNewline && this.lastEndsStatement() && !this.nextLineContinues()) {
+                this.insertSemicolon();
+            }
+            if (this.pos >= this.text.length)
+                break;
+            this.scanToken();
+        }
+        if (this.lastEndsStatement())
+            this.insertSemicolon();
+        for (const mode of this.modes) {
+            if (mode.kind === 'template') {
+                this.error('unterminated template literal', mode.start, mode.start + 1);
+            }
+            else if (mode.kind === 'css') {
+                this.error('unterminated style block: close it with "}"', mode.start, mode.start + 5);
+            }
+        }
+        // Nested elements are inside the first one, so it is enough to report that.
+        const element = this.modes.find(isMarkup);
+        if (element)
+            this.error('unterminated element', element.start, element.start + 1);
+        this.tokens.push({ kind: 'EOF', text: '', start: this.pos, end: this.pos });
+        this.diagnostics.sort((a, b) => a.start - b.start);
+        return { tokens: this.tokens, diagnostics: this.diagnostics };
+    }
+    scanToken() {
+        const t = this.text;
+        const start = this.pos;
+        const c = t[start];
+        if (isDigit(c) || (c === '.' && isDigit(t[start + 1]))) {
+            this.scanNumber(start);
+        }
+        else if (c === '"' || c === "'") {
+            this.scanString(start, c);
+        }
+        else if (c === '`') {
+            this.scanTemplate(start, false);
+        }
+        else if (c === '}' && this.top()?.kind === 'template' && this.braces() === 0) {
+            this.scanTemplate(start, true);
+        }
+        else if (c === '}' && this.top()?.kind === 'expression' && this.braces() === 0) {
+            // The end of `{ ... }` inside markup: back to the tag or the element content.
+            this.push('}', start, start + 1);
+            this.modes.pop();
+        }
+        else if (c === '<' && this.startsElement()) {
+            this.push('JsxTagOpen', start, start + 1);
+            this.modes.push({ kind: 'tag', start });
+        }
+        else if (!this.scanWord(start) && !this.scanPunctuator(start)) {
+            const char = String.fromCodePoint(t.codePointAt(start) ?? 0);
+            this.error(`unexpected character "${char}"`, start, start + char.length);
+            this.pos += char.length;
+        }
+    }
+    scanWord(start) {
+        IDENTIFIER.lastIndex = start;
+        const word = IDENTIFIER.exec(this.text)?.[0];
+        if (word === undefined)
+            return false;
+        this.pos = start + word.length;
+        if (word.startsWith('$$')) {
+            this.error('names starting with "$$" are reserved for generated code', start, this.pos);
+        }
+        this.tokens.push({
+            kind: isKeyword(word) ? word : 'Identifier',
+            text: word,
+            start,
+            end: this.pos,
+        });
+        if (word === 'style')
+            this.startStyle(start);
+        return true;
+    }
+    scanPunctuator(start) {
+        const t = this.text;
+        const three = t.slice(start, start + 3);
+        if (three === '===' || three === '!==') {
+            const kind = three === '===' ? '==' : '!=';
+            this.error(`"${three}" is not needed: "${kind}" is already strict`, start, start + 3);
+            this.pos = start + 3;
+            this.tokens.push({ kind, text: three, start, end: this.pos });
+            return true;
+        }
+        for (let length = 4; length > 0; length--) {
+            const candidate = t.slice(start, start + length);
+            // `a ?.5 : b` is a conditional expression, not optional chaining (same rule as in JS).
+            if (!isPunctuator(candidate) || (candidate === '?.' && isDigit(t[start + 2])))
+                continue;
+            this.pos = start + length;
+            this.tokens.push({ kind: candidate, text: candidate, start, end: this.pos });
+            const top = this.top();
+            if (top?.kind === 'template' || top?.kind === 'expression') {
+                if (candidate === '{')
+                    top.braces++;
+                if (candidate === '}')
+                    top.braces--;
+            }
+            return true;
+        }
+        return false;
+    }
+}
+export function tokenize(text) {
+    return new Lexer(text).run();
+}

package/dist/lexer/literals.d.ts

@@ -0,0 +1,18 @@
+import { LexerBase } from './base.ts';
+import { type CharTest } from './tables.ts';
+export declare abstract class LiteralLexer extends LexerBase {
+    protected scanNumber(start: number): void;
+    /** Consumes digits and `_` separators starting at `pos`; returns the end offset. */
+    protected scanDigits(pos: number, isValid: CharTest): number;
+    protected scanString(start: number, quote: '"' | "'"): void;
+    /**
+     * Scans template text up to the closing backtick or the next `${`. Starts either at the opening
+     * backtick or, when `continued`, at the `}` that closes a substitution.
+     */
+    protected scanTemplate(start: number, continued: boolean): void;
+    /** Decodes the escape sequence whose backslash is at `pos`. */
+    protected scanEscape(pos: number): {
+        value: string;
+        end: number;
+    };
+}

package/dist/lexer/literals.js

@@ -0,0 +1,203 @@
+import { LexerBase } from "./base.js";
+import { IDENTIFIER, SIMPLE_ESCAPES, isDigit, isHexDigit, isOctalDigit, isBinaryDigit, } from "./tables.js";
+// Numbers, strings, template strings and escapes.
+export class LiteralLexer extends LexerBase {
+    scanNumber(start) {
+        const t = this.text;
+        let pos = start;
+        const prefix = t[pos] === '0' ? t[pos + 1]?.toLowerCase() : undefined;
+        const radixDigit = prefix === 'x'
+            ? isHexDigit
+            : prefix === 'b'
+                ? isBinaryDigit
+                : prefix === 'o'
+                    ? isOctalDigit
+                    : undefined;
+        if (radixDigit) {
+            pos = this.scanDigits(pos + 2, radixDigit);
+            if (pos === start + 2) {
+                this.error(`expected digits after "${t.slice(start, pos)}"`, start, pos);
+            }
+        }
+        else {
+            pos = this.scanDigits(pos, isDigit);
+            if (/^0[0-9_]/.test(t.slice(start, pos))) {
+                this.error('leading zeros are not allowed (use "0o" for octal numbers)', start, pos);
+            }
+            if (t[pos] === '.' && isDigit(t[pos + 1])) {
+                pos = this.scanDigits(pos + 1, isDigit);
+            }
+            if (t[pos] === 'e' || t[pos] === 'E') {
+                const exponent = pos++;
+                if (t[pos] === '+' || t[pos] === '-')
+                    pos++;
+                if (isDigit(t[pos]))
+                    pos = this.scanDigits(pos, isDigit);
+                else
+                    this.error('expected digits in exponent', exponent, pos);
+            }
+        }
+        // `123abc` is almost certainly a typo, not a number followed by a name.
+        IDENTIFIER.lastIndex = pos;
+        const suffix = IDENTIFIER.exec(t)?.[0];
+        // `10n`, `0xffn`: a bigint, a whole number of any size.
+        if (suffix === 'n') {
+            const text = t.slice(start, pos + 1);
+            if (/^\d[\d_]*[.eE]/.test(text)) {
+                this.error(`a bigint is a whole number, without a fraction or an exponent: e.g. ${text.replace(/[.eE].*/, '')}n`, start, pos + 1);
+            }
+            this.pos = pos + 1;
+            this.tokens.push({ kind: 'BigInt', text, start, end: pos + 1 });
+            return;
+        }
+        if (suffix !== undefined) {
+            this.error('identifier cannot start immediately after a number', pos, pos + suffix.length);
+            pos += suffix.length;
+        }
+        this.pos = pos;
+        const text = t.slice(start, pos);
+        this.tokens.push({
+            kind: 'Number',
+            value: Number(text.replaceAll('_', '')),
+            text,
+            start,
+            end: pos,
+        });
+    }
+    /** Consumes digits and `_` separators starting at `pos`; returns the end offset. */
+    scanDigits(pos, isValid) {
+        const t = this.text;
+        while (true) {
+            if (isValid(t[pos])) {
+                pos++;
+            }
+            else if (t[pos] === '_') {
+                const run = pos;
+                while (t[pos] === '_')
+                    pos++;
+                if (pos - run > 1 || !isValid(t[run - 1]) || !isValid(t[pos])) {
+                    this.error('"_" can only appear between digits', run, pos);
+                }
+            }
+            else {
+                return pos;
+            }
+        }
+    }
+    scanString(start, quote) {
+        const t = this.text;
+        let pos = start + 1;
+        let value = '';
+        while (true) {
+            const c = t[pos];
+            if (c === undefined || c === '\n' || c === '\r') {
+                this.error('unterminated string literal', start, pos);
+                break;
+            }
+            if (c === quote) {
+                pos++;
+                break;
+            }
+            if (c === '\\') {
+                const escape = this.scanEscape(pos);
+                value += escape.value;
+                pos = escape.end;
+            }
+            else {
+                value += c;
+                pos++;
+            }
+        }
+        this.pos = pos;
+        this.tokens.push({ kind: 'String', value, text: t.slice(start, pos), start, end: pos });
+    }
+    /**
+     * Scans template text up to the closing backtick or the next `${`. Starts either at the opening
+     * backtick or, when `continued`, at the `}` that closes a substitution.
+     */
+    scanTemplate(start, continued) {
+        const t = this.text;
+        let pos = start + 1;
+        let value = '';
+        let kind;
+        while (true) {
+            const c = t[pos];
+            if (c === undefined) {
+                const template = continued ? this.modes.pop() : undefined;
+                const templateStart = template?.kind === 'template' ? template.start : start;
+                this.error('unterminated template literal', templateStart, templateStart + 1);
+                kind = continued ? 'TemplateTail' : 'Template';
+                break;
+            }
+            if (c === '`') {
+                pos++;
+                if (continued)
+                    this.modes.pop();
+                kind = continued ? 'TemplateTail' : 'Template';
+                break;
+            }
+            if (c === '$' && t[pos + 1] === '{') {
+                pos += 2;
+                if (!continued)
+                    this.modes.push({ kind: 'template', start, braces: 0 });
+                kind = continued ? 'TemplateMiddle' : 'TemplateHead';
+                break;
+            }
+            if (c === '\\') {
+                const escape = this.scanEscape(pos);
+                value += escape.value;
+                pos = escape.end;
+            }
+            else if (c === '\r') {
+                // Line breaks inside templates are normalized to \n, as in JS.
+                value += '\n';
+                pos += t[pos + 1] === '\n' ? 2 : 1;
+            }
+            else {
+                value += c;
+                pos++;
+            }
+        }
+        this.pos = pos;
+        this.tokens.push({ kind, value, text: t.slice(start, pos), start, end: pos });
+    }
+    /** Decodes the escape sequence whose backslash is at `pos`. */
+    scanEscape(pos) {
+        const t = this.text;
+        const c = t[pos + 1];
+        // A backslash at the end of a line or input: the caller reports the unterminated literal.
+        if (c === undefined || c === '\n' || c === '\r')
+            return { value: '', end: pos + 1 };
+        const simple = SIMPLE_ESCAPES[c];
+        if (simple !== undefined)
+            return { value: simple, end: pos + 2 };
+        if (c === 'x') {
+            const hex = t.slice(pos + 2, pos + 4);
+            if (/^[0-9a-fA-F]{2}$/.test(hex)) {
+                return { value: String.fromCharCode(parseInt(hex, 16)), end: pos + 4 };
+            }
+            this.error('expected two hex digits after "\\x"', pos, pos + 2);
+            return { value: '', end: pos + 2 };
+        }
+        if (c === 'u') {
+            if (t[pos + 2] === '{') {
+                const close = t.indexOf('}', pos + 3);
+                const hex = close === -1 ? '' : t.slice(pos + 3, close);
+                if (/^[0-9a-fA-F]{1,6}$/.test(hex) && parseInt(hex, 16) <= 0x10ffff) {
+                    return { value: String.fromCodePoint(parseInt(hex, 16)), end: close + 1 };
+                }
+            }
+            else {
+                const hex = t.slice(pos + 2, pos + 6);
+                if (/^[0-9a-fA-F]{4}$/.test(hex)) {
+                    return { value: String.fromCharCode(parseInt(hex, 16)), end: pos + 6 };
+                }
+            }
+            this.error('expected "\\uXXXX" or "\\u{X...}" with a valid code point', pos, pos + 2);
+            return { value: '', end: pos + 2 };
+        }
+        const char = String.fromCodePoint(t.codePointAt(pos + 1) ?? 0);
+        this.error(`unknown escape sequence "\\${char}"`, pos, pos + 1 + char.length);
+        return { value: char, end: pos + 1 + char.length };
+    }
+}

package/dist/lexer/markup.d.ts

@@ -0,0 +1,10 @@
+import { LiteralLexer } from './literals.ts';
+import { type MarkupMode } from './tables.ts';
+export declare abstract class MarkupLexer extends LiteralLexer {
+    /** `<` starts markup where a value is expected and is "less than" after a value. */
+    protected startsElement(): boolean;
+    /** Inside `<tag ...>` or `</tag>`: names, `=`, attribute values, `{`, `>` and `/>`. */
+    protected scanTag(mode: MarkupMode): void;
+    /** Element content: text, `{expression}`, child elements and the closing tag. */
+    protected scanContent(mode: MarkupMode): void;
+}

package/dist/lexer/markup.js

@@ -0,0 +1,93 @@
+import { LiteralLexer } from "./literals.js";
+import { ENDS_VALUE, JSX_NAME, decodeEntities } from "./tables.js";
+// Markup: where an element starts, tags with their attributes, and text content.
+export class MarkupLexer extends LiteralLexer {
+    /** `<` starts markup where a value is expected and is "less than" after a value. */
+    startsElement() {
+        const last = this.tokens.at(-1);
+        if (last && ENDS_VALUE.has(last.kind))
+            return false;
+        // `chan<number>(10)` makes a channel: its `<` opens the type of the values.
+        if (last?.kind === 'chan')
+            return false;
+        const next = this.text[this.pos + 1];
+        return next === '>' || (next !== undefined && /^[\p{ID_Start}$_]$/u.test(next));
+    }
+    /** Inside `<tag ...>` or `</tag>`: names, `=`, attribute values, `{`, `>` and `/>`. */
+    scanTag(mode) {
+        const t = this.text;
+        while (/^\s$/.test(t[this.pos] ?? ''))
+            this.pos++;
+        const start = this.pos;
+        const c = t[start];
+        if (c === undefined)
+            return;
+        const opening = mode.kind === 'tag';
+        if (c === '>') {
+            this.push('JsxTagEnd', start, start + 1);
+            // After an opening tag comes the content; a closing tag ends the element.
+            if (opening)
+                this.modes[this.modes.length - 1] = { kind: 'content', start: mode.start };
+            else
+                this.modes.pop();
+        }
+        else if (opening && c === '/' && t[start + 1] === '>') {
+            this.push('JsxSelfClose', start, start + 2);
+            this.modes.pop();
+        }
+        else if (opening && c === '{') {
+            this.push('{', start, start + 1);
+            this.modes.push({ kind: 'expression', braces: 0 });
+        }
+        else if (opening && c === '=') {
+            this.push('=', start, start + 1);
+        }
+        else if (opening && (c === '"' || c === "'")) {
+            // Attribute values are taken as written, as in HTML: no backslash escapes.
+            const close = t.indexOf(c, start + 1);
+            const end = close === -1 ? t.length : close + 1;
+            if (close === -1)
+                this.error('unterminated attribute value', start, end);
+            const text = t.slice(start, end);
+            const raw = text.slice(1, close === -1 ? undefined : -1);
+            this.tokens.push({ kind: 'JsxString', value: decodeEntities(raw), text, start, end });
+            this.pos = end;
+        }
+        else {
+            JSX_NAME.lastIndex = start;
+            const name = JSX_NAME.exec(t)?.[0];
+            if (name !== undefined) {
+                this.push('JsxName', start, start + name.length);
+            }
+            else {
+                this.error(`unexpected character "${c}" in a tag`, start, start + 1);
+                this.pos++;
+            }
+        }
+    }
+    /** Element content: text, `{expression}`, child elements and the closing tag. */
+    scanContent(mode) {
+        const t = this.text;
+        const start = this.pos;
+        if (t.startsWith('</', start)) {
+            this.push('JsxCloseTagOpen', start, start + 2);
+            this.modes[this.modes.length - 1] = { kind: 'closingTag', start: mode.start };
+        }
+        else if (t[start] === '<') {
+            this.push('JsxTagOpen', start, start + 1);
+            this.modes.push({ kind: 'tag', start });
+        }
+        else if (t[start] === '{') {
+            this.push('{', start, start + 1);
+            this.modes.push({ kind: 'expression', braces: 0 });
+        }
+        else {
+            let end = start;
+            while (end < t.length && t[end] !== '<' && t[end] !== '{')
+                end++;
+            const text = t.slice(start, end);
+            this.tokens.push({ kind: 'JsxText', value: decodeEntities(text), text, start, end });
+            this.pos = end;
+        }
+    }
+}

package/dist/lexer/style.d.ts

@@ -0,0 +1,17 @@
+import { MarkupLexer } from './markup.ts';
+import type { Mode } from './tables.ts';
+type CssMode = Extract<Mode, {
+    kind: 'css';
+}>;
+export declare abstract class StyleLexer extends MarkupLexer {
+    /**
+     * `style {` at the start of a statement starts CSS: the identifier `style` has just been read at
+     * `start`. Elsewhere `style` is an ordinary name.
+     */
+    protected startStyle(start: number): void;
+    /** CSS text up to a value from code or to the `}` that closes the block. */
+    protected scanCss(mode: CssMode): void;
+    /** The end of a CSS string that starts at `start`, after its closing quote. */
+    private cssStringEnd;
+}
+export {};