parser-lr 0.0.0 → 0.2.1

package/dist/lib/lexer/lexer-compile.d.ts

@@ -0,0 +1,94 @@
+import type { Grammar } from '../grammar/grammar.js';
+/** Default lexer state when a grammar omits a `states` section. */
+export declare const DEFAULT_LEXER_STATE = "initial";
+/**
+ * Compiled token or skip rule ready for anchored matching.
+ */
+export interface CompiledRule {
+    readonly name: string;
+    readonly skip: boolean;
+    readonly regex: RegExp;
+    readonly states: readonly string[] | null;
+}
+/**
+ * Compiled lexer rules and declared state names for one grammar.
+ */
+export interface CompiledLexerRules {
+    readonly states: readonly string[];
+    readonly initialState: string;
+    readonly rules: readonly CompiledRule[];
+}
+/**
+ * Compiles token and skip rules from a grammar for streaming lexing.
+ *
+ * @param grammar - Grammar supplying lexer rule definitions.
+ * @returns Compiled rules and lexer state metadata.
+ */
+export declare function compileLexerRules(grammar: Grammar): CompiledLexerRules;
+/**
+ * Returns compiled rules active in a lexer state.
+ *
+ * @param compiled - Compiled lexer rules for a grammar.
+ * @param state - Current lexer state name.
+ * @returns Rules that apply in the state, preserving declaration order.
+ */
+export declare function rulesForState(compiled: CompiledLexerRules, state: string): readonly CompiledRule[];
+/**
+ * Resolves declared lexer states, supplying a default when omitted.
+ *
+ * @param states - State names from the grammar `states` section.
+ * @returns Non-empty state name list.
+ */
+export declare function resolveLexerStates(states: readonly string[]): readonly string[];
+/**
+ * Matches a compiled rule at the start of a source slice.
+ *
+ * @param rule - Compiled token or skip rule.
+ * @param slice - Remaining source text from the current offset.
+ * @returns Matched lexeme text, or null when the rule does not match.
+ */
+export declare function matchRule(rule: CompiledRule, slice: string): string | null;
+/**
+ * Returns whether more input could extend the match via a different token rule.
+ *
+ * @param buffer - Buffered source not yet consumed.
+ * @param matchedText - Current best match at the buffer start.
+ * @param rules - Candidate rules for the active lexer state.
+ */
+export declare function hasLongerPossibleMatchCrossRule(buffer: string, matchedText: string, rules: readonly CompiledRule[]): boolean;
+/**
+ * Returns whether a longer match may appear if more input arrives.
+ *
+ * @param buffer - Buffered source not yet consumed.
+ * @param matchedText - Current best match at the buffer start.
+ * @param rules - Candidate rules for the active lexer state.
+ * @param finished - Whether the input stream has ended.
+ */
+export declare function hasLongerPossibleMatch(buffer: string, matchedText: string, rules: readonly CompiledRule[], finished: boolean): boolean;
+/**
+ * Returns whether buffered input may become a token once more data arrives.
+ *
+ * @param buffer - Buffered source not yet consumed.
+ * @param rules - Candidate rules for the active lexer state.
+ */
+export declare function isPrefixOfPotentialMatch(buffer: string, rules: readonly CompiledRule[]): boolean;
+/**
+ * Finds the longest winning rule match at the start of a buffer.
+ *
+ * @param buffer - Buffered source to match from the start.
+ * @param rules - Candidate rules for the active lexer state.
+ * @returns The longest winning match, or null when nothing matches.
+ */
+export declare function findLongestMatch(buffer: string, rules: readonly CompiledRule[]): {
+    name: string;
+    text: string;
+    skip: boolean;
+} | null;
+/**
+ * Validates that a lexer state name is declared in the grammar.
+ *
+ * @param compiled - Compiled lexer rules for a grammar.
+ * @param state - Candidate lexer state name.
+ */
+export declare function assertLexerState(compiled: CompiledLexerRules, state: string): void;
+//# sourceMappingURL=lexer-compile.d.ts.map

package/dist/lib/lexer/lexer-compile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-compile.d.ts","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-compile.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAMrD,mEAAmE;AACnE,eAAO,MAAM,mBAAmB,YAAY,CAAC;AAI7C;;GAEG;AACH,MAAM,WAAW,YAAY;IAEzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAE/B,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAC;CAC3C;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,OAAO,GAAG,kBAAkB,CAYtE;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CACzB,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,EAAE,MAAM,GACd,SAAS,YAAY,EAAE,CAGzB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,MAAM,EAAE,CAQ/E;AA2BD;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAuB1E;AAED;;;;;;GAMG;AACH,wBAAgB,+BAA+B,CAC3C,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,SAAS,YAAY,EAAE,GAC/B,OAAO,CAwBT;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAClC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,SAAS,YAAY,EAAE,EAC9B,QAAQ,EAAE,OAAO,GAClB,OAAO,CAyBT;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,SAAS,YAAY,EAAE,GAC/B,OAAO,CAmBT;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC5B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,SAAS,YAAY,EAAE,GAC/B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAAG,IAAI,CAyBtD;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAMlF"}

package/dist/lib/lexer/lexer-compile.js

@@ -0,0 +1,189 @@
+import { LexerCompileError } from './lexer-compile-error.js';
+import { LexerStateError } from './lexer-state-error.js';
+/** Default lexer state when a grammar omits a `states` section. */
+export const DEFAULT_LEXER_STATE = 'initial';
+const REGEX_LITERAL_TOKEN = 'regex_literal';
+/**
+ * Compiles token and skip rules from a grammar for streaming lexing.
+ *
+ * @param grammar - Grammar supplying lexer rule definitions.
+ * @returns Compiled rules and lexer state metadata.
+ */
+export function compileLexerRules(grammar) {
+    const states = resolveLexerStates(grammar.states);
+    return {
+        states,
+        initialState: states[0] ?? DEFAULT_LEXER_STATE,
+        rules: [
+            ...grammar.tokenRules.map((rule) => compileRule(rule, false)),
+            ...grammar.skipRules.map((rule) => compileRule(rule, true)),
+        ],
+    };
+}
+/**
+ * Returns compiled rules active in a lexer state.
+ *
+ * @param compiled - Compiled lexer rules for a grammar.
+ * @param state - Current lexer state name.
+ * @returns Rules that apply in the state, preserving declaration order.
+ */
+export function rulesForState(compiled, state) {
+    return compiled.rules.filter((rule) => rule.states === null || rule.states.includes(state));
+}
+/**
+ * Resolves declared lexer states, supplying a default when omitted.
+ *
+ * @param states - State names from the grammar `states` section.
+ * @returns Non-empty state name list.
+ */
+export function resolveLexerStates(states) {
+    if (states.length === 0) {
+        return [DEFAULT_LEXER_STATE];
+    }
+    return states;
+}
+/**
+ * Compiles one grammar token or skip rule into an anchored regular expression.
+ *
+ * @param rule - Token rule from the grammar file.
+ * @param skip - Whether matches should be discarded instead of emitted.
+ * @returns A compiled rule for lexing.
+ */
+function compileRule(rule, skip) {
+    try {
+        return {
+            name: rule.name,
+            skip,
+            regex: new RegExp(`^(?:${rule.pattern})`, rule.flags),
+            states: rule.states === undefined ? null : [...rule.states],
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new LexerCompileError(rule.name, message);
+    }
+}
+/**
+ * Matches a compiled rule at the start of a source slice.
+ *
+ * @param rule - Compiled token or skip rule.
+ * @param slice - Remaining source text from the current offset.
+ * @returns Matched lexeme text, or null when the rule does not match.
+ */
+export function matchRule(rule, slice) {
+    const match = rule.regex.exec(slice);
+    if (match === null || match.index !== 0) {
+        return null;
+    }
+    let text = match[0];
+    // Extend regex literals with optional JS flag letters after the closing slash.
+    if (rule.name === REGEX_LITERAL_TOKEN) {
+        const suffix = /^[gimsuy]*/.exec(slice.slice(text.length));
+        if (suffix !== null) {
+            text += suffix[0];
+        }
+    }
+    return text;
+}
+/**
+ * Returns whether more input could extend the match via a different token rule.
+ *
+ * @param buffer - Buffered source not yet consumed.
+ * @param matchedText - Current best match at the buffer start.
+ * @param rules - Candidate rules for the active lexer state.
+ */
+export function hasLongerPossibleMatchCrossRule(buffer, matchedText, rules) {
+    const current = findLongestMatch(buffer, rules);
+    if (current === null || current.text !== matchedText) {
+        return false;
+    }
+    for (const suffix of ['=', '+', '-', '/', '"', '(', '0', 'a', 'X']) {
+        const extendedMatch = findLongestMatch(buffer + suffix, rules);
+        if (extendedMatch !== null
+            && extendedMatch.text.length > current.text.length
+            && extendedMatch.name !== current.name) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Returns whether a longer match may appear if more input arrives.
+ *
+ * @param buffer - Buffered source not yet consumed.
+ * @param matchedText - Current best match at the buffer start.
+ * @param rules - Candidate rules for the active lexer state.
+ * @param finished - Whether the input stream has ended.
+ */
+export function hasLongerPossibleMatch(buffer, matchedText, rules, finished) {
+    if (finished) {
+        return false;
+    }
+    const current = findLongestMatch(buffer, rules);
+    if (current === null || current.text !== matchedText) {
+        return false;
+    }
+    for (const suffix of ['=', '+', '-', '/', '"', '(', '0', 'a', 'X']) {
+        const extendedMatch = findLongestMatch(buffer + suffix, rules);
+        if (extendedMatch !== null && extendedMatch.text.length > current.text.length) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Returns whether buffered input may become a token once more data arrives.
+ *
+ * @param buffer - Buffered source not yet consumed.
+ * @param rules - Candidate rules for the active lexer state.
+ */
+export function isPrefixOfPotentialMatch(buffer, rules) {
+    if (buffer.length === 0) {
+        return true;
+    }
+    for (const suffix of ['=', '+', '-', '/', '"', '(', '0', 'a', 'X']) {
+        const extended = buffer + suffix;
+        const match = findLongestMatch(extended, rules);
+        if (match !== null && match.text.startsWith(buffer) && match.text.length > buffer.length) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Finds the longest winning rule match at the start of a buffer.
+ *
+ * @param buffer - Buffered source to match from the start.
+ * @param rules - Candidate rules for the active lexer state.
+ * @returns The longest winning match, or null when nothing matches.
+ */
+export function findLongestMatch(buffer, rules) {
+    let bestMatch = null;
+    // Try every rule and keep the longest match, breaking ties by rule order.
+    for (const rule of rules) {
+        const matchedText = matchRule(rule, buffer);
+        if (matchedText === null || matchedText.length === 0) {
+            continue;
+        }
+        if (bestMatch === null || matchedText.length > bestMatch.text.length) {
+            bestMatch = {
+                name: rule.name,
+                text: matchedText,
+                skip: rule.skip,
+            };
+        }
+    }
+    return bestMatch;
+}
+/**
+ * Validates that a lexer state name is declared in the grammar.
+ *
+ * @param compiled - Compiled lexer rules for a grammar.
+ * @param state - Candidate lexer state name.
+ */
+export function assertLexerState(compiled, state) {
+    if (!compiled.states.includes(state)) {
+        throw new LexerStateError(state);
+    }
+}
+//# sourceMappingURL=lexer-compile.js.map

package/dist/lib/lexer/lexer-compile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-compile.js","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-compile.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,mEAAmE;AACnE,MAAM,CAAC,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAE7C,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAuB5C;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAgB;IAE9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAElD,OAAO;QACH,MAAM;QACN,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,mBAAmB;QAC9C,KAAK,EAAE;YACH,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7D,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;SAC9D;KACJ,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CACzB,QAA4B,EAC5B,KAAa;IAGb,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;AAChG,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAyB;IAExD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EACvB,CAAC;QACG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,WAAW,CAAC,IAAe,EAAE,IAAa;IAE/C,IACA,CAAC;QACG,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;YACJ,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;YACrD,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;SAC9D,CAAC;IACN,CAAC;IACD,OAAO,KAAK,EACZ,CAAC;QACG,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,IAAkB,EAAE,KAAa;IAEvD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAErC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EACvC,CAAC;QACG,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAEpB,+EAA+E;IAC/E,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,EACrC,CAAC;QACG,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3D,IAAI,MAAM,KAAK,IAAI,EACnB,CAAC;YACG,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,+BAA+B,CAC3C,MAAc,EACd,WAAmB,EACnB,KAA8B;IAG9B,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAEhD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EACpD,CAAC;QACG,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAClE,CAAC;QACG,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC;QAE/D,IACI,aAAa,KAAK,IAAI;eACnB,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM;eAC/C,aAAa,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,EAE1C,CAAC;YACG,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAClC,MAAc,EACd,WAAmB,EACnB,KAA8B,EAC9B,QAAiB;IAGjB,IAAI,QAAQ,EACZ,CAAC;QACG,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAEhD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EACpD,CAAC;QACG,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAClE,CAAC;QACG,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC;QAE/D,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAC7E,CAAC;YACG,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CACpC,MAAc,EACd,KAA8B;IAG9B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EACvB,CAAC;QACG,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAClE,CAAC;QACG,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;QACjC,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEhD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EACxF,CAAC;YACG,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC5B,MAAc,EACd,KAA8B;IAG9B,IAAI,SAAS,GAAyD,IAAI,CAAC;IAE3E,0EAA0E;IAC1E,KAAK,MAAM,IAAI,IAAI,KAAK,EACxB,CAAC;QACG,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAE5C,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EACpD,CAAC;YACG,SAAS;QACb,CAAC;QAED,IAAI,SAAS,KAAK,IAAI,IAAI,WAAW,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,EACpE,CAAC;YACG,SAAS,GAAG;gBACR,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;aAClB,CAAC;QACN,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAA4B,EAAE,KAAa;IAExE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EACpC,CAAC;QACG,MAAM,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;AACL,CAAC"}

package/dist/lib/lexer/lexer-error.d.ts

@@ -0,0 +1,15 @@
+import { ParserLrError } from '../errors/parser-lr-error.js';
+/**
+ * Error thrown when the lexer cannot match input at the current offset.
+ */
+export declare class LexerError extends ParserLrError {
+    readonly offset: number;
+    /**
+     * Creates a lexer error at a source offset.
+     *
+     * @param message - Human-readable failure description.
+     * @param offset - Source offset where lexing stopped.
+     */
+    constructor(message: string, offset: number);
+}
+//# sourceMappingURL=lexer-error.d.ts.map

package/dist/lib/lexer/lexer-error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-error.d.ts","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D;;GAEG;AACH,qBAAa,UAAW,SAAQ,aAAa;aAUrB,MAAM,EAAE,MAAM;IARlC;;;;;OAKG;gBAEC,OAAO,EAAE,MAAM,EACC,MAAM,EAAE,MAAM;CAKrC"}

package/dist/lib/lexer/lexer-error.js

@@ -0,0 +1,18 @@
+import { ParserLrError } from '../errors/parser-lr-error.js';
+/**
+ * Error thrown when the lexer cannot match input at the current offset.
+ */
+export class LexerError extends ParserLrError {
+    offset;
+    /**
+     * Creates a lexer error at a source offset.
+     *
+     * @param message - Human-readable failure description.
+     * @param offset - Source offset where lexing stopped.
+     */
+    constructor(message, offset) {
+        super(message);
+        this.offset = offset;
+    }
+}
+//# sourceMappingURL=lexer-error.js.map

package/dist/lib/lexer/lexer-error.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-error.js","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,aAAa;IAUrB;IARpB;;;;;OAKG;IACH,YACI,OAAe,EACC,MAAc;QAG9B,KAAK,CAAC,OAAO,CAAC,CAAC;QAHC,WAAM,GAAN,MAAM,CAAQ;IAIlC,CAAC;CACJ"}

package/dist/lib/lexer/lexer-input-error.d.ts

@@ -0,0 +1,11 @@
+import { ParserLrError } from '../errors/parser-lr-error.js';
+/**
+ * Thrown when source input is pushed after the lexer has finished.
+ */
+export declare class LexerInputError extends ParserLrError {
+    /**
+     * Creates a lexer input sequencing error.
+     */
+    constructor();
+}
+//# sourceMappingURL=lexer-input-error.d.ts.map

package/dist/lib/lexer/lexer-input-error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-input-error.d.ts","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-input-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D;;GAEG;AACH,qBAAa,eAAgB,SAAQ,aAAa;IAE9C;;OAEG;;CAKN"}

package/dist/lib/lexer/lexer-input-error.js

@@ -0,0 +1,13 @@
+import { ParserLrError } from '../errors/parser-lr-error.js';
+/**
+ * Thrown when source input is pushed after the lexer has finished.
+ */
+export class LexerInputError extends ParserLrError {
+    /**
+     * Creates a lexer input sequencing error.
+     */
+    constructor() {
+        super('Cannot push input after finish()');
+    }
+}
+//# sourceMappingURL=lexer-input-error.js.map

package/dist/lib/lexer/lexer-input-error.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-input-error.js","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-input-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,aAAa;IAE9C;;OAEG;IACH;QAEI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC9C,CAAC;CACJ"}

package/dist/lib/lexer/lexer-state-error.d.ts

@@ -0,0 +1,13 @@
+import { ParserLrError } from '../errors/parser-lr-error.js';
+/**
+ * Thrown when a lexer state name is not declared in the grammar.
+ */
+export declare class LexerStateError extends ParserLrError {
+    /**
+     * Creates an unknown lexer state error.
+     *
+     * @param state - Requested lexer state name.
+     */
+    constructor(state: string);
+}
+//# sourceMappingURL=lexer-state-error.d.ts.map

package/dist/lib/lexer/lexer-state-error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-state-error.d.ts","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-state-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D;;GAEG;AACH,qBAAa,eAAgB,SAAQ,aAAa;IAE9C;;;;OAIG;gBACgB,KAAK,EAAE,MAAM;CAInC"}

package/dist/lib/lexer/lexer-state-error.js

@@ -0,0 +1,15 @@
+import { ParserLrError } from '../errors/parser-lr-error.js';
+/**
+ * Thrown when a lexer state name is not declared in the grammar.
+ */
+export class LexerStateError extends ParserLrError {
+    /**
+     * Creates an unknown lexer state error.
+     *
+     * @param state - Requested lexer state name.
+     */
+    constructor(state) {
+        super(`Unknown lexer state ${JSON.stringify(state)}`);
+    }
+}
+//# sourceMappingURL=lexer-state-error.js.map

package/dist/lib/lexer/lexer-state-error.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer-state-error.js","sourceRoot":"","sources":["../../../src/lib/lexer/lexer-state-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,aAAa;IAE9C;;;;OAIG;IACH,YAAmB,KAAa;QAE5B,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC1D,CAAC;CACJ"}

package/dist/lib/lexer/lexer.d.ts

@@ -0,0 +1,111 @@
+import type { Grammar } from '../grammar/grammar.js';
+import type { Token } from './token.js';
+/**
+ * Stream-based lexer driven by a grammar's `tokens`, `skip`, and `states` sections.
+ *
+ * Push source chunks with {@link push}, signal end-of-input with {@link finish},
+ * then read tokens via {@link next} or iteration. The final token is always `$eof`.
+ */
+export declare class Lexer {
+    private readonly compiled;
+    private readonly queue;
+    private buffer;
+    private offset;
+    private finished;
+    private eofEmitted;
+    private currentState;
+    /**
+     * Creates a lexer from a parsed grammar.
+     *
+     * @param grammar - Grammar supplying token and skip rule definitions.
+     */
+    constructor(grammar: Grammar);
+    /**
+     * Returns the active lexer state name.
+     */
+    get state(): string;
+    /**
+     * Enters a declared lexer state.
+     *
+     * @param stateName - Lexer state to activate.
+     */
+    enterState(stateName: string): void;
+    /**
+     * Appends the next chunk of source text to the scan buffer.
+     *
+     * @param chunk - Source text fragment.
+     */
+    push(chunk: string): void;
+    /**
+     * Signals that no further source chunks will arrive.
+     */
+    finish(): void;
+    /**
+     * Returns the next token, or null when waiting for more input.
+     *
+     * @returns The next token, null while blocked, or `$eof` after input ends.
+     */
+    next(): Token | null;
+    /**
+     * Lexes an entire source string, including a trailing `$eof` token.
+     *
+     * @param source - Input text to tokenize.
+     * @returns Matched tokens in source order.
+     */
+    lex(source: string): readonly Token[];
+    /**
+     * Iterates all tokens until `$eof` after {@link finish} has been called.
+     */
+    [Symbol.iterator](): Iterator<Token>;
+    /**
+     * Scans the buffer and enqueues any tokens ready to emit.
+     *
+     * @param inputFinished - Whether the full input stream has ended.
+     * @param allowChunkBoundary - Whether to commit tokens at a chunk boundary.
+     */
+    private drain;
+    /**
+     * Finds the winning rule match at the start of the scan buffer.
+     *
+     * @param inputFinished - Whether the full input stream has ended.
+     * @param allowChunkBoundary - Whether to commit tokens at a chunk boundary.
+     */
+    private matchAtBufferStart;
+    /**
+     * Resets stream state for a fresh lex run.
+     */
+    private reset;
+}
+/**
+ * Lexes synchronous source chunks into a token stream ending with `$eof`.
+ *
+ * @param grammar - Grammar supplying lexer rules.
+ * @param chunks - Source text fragments in order.
+ * @returns Token iterator including `$eof`.
+ */
+export declare function lexChunks(grammar: Grammar, chunks: Iterable<string>): IterableIterator<Token>;
+/**
+ * Lexes asynchronous source chunks into a token stream ending with `$eof`.
+ *
+ * @param grammar - Grammar supplying lexer rules.
+ * @param chunks - Source text fragments in order.
+ * @returns Async token iterator including `$eof`.
+ */
+export declare function lexChunksAsync(grammar: Grammar, chunks: AsyncIterable<string>): AsyncIterableIterator<Token>;
+/**
+ * Collects all tokens from a synchronous chunk stream.
+ *
+ * @param grammar - Grammar supplying lexer rules.
+ * @param chunks - Source text fragments in order.
+ * @returns Matched tokens including `$eof`.
+ */
+export declare function lexChunkStream(grammar: Grammar, chunks: Iterable<string>): readonly Token[];
+/**
+ * Collects all tokens from an asynchronous chunk stream.
+ *
+ * @param grammar - Grammar supplying lexer rules.
+ * @param chunks - Source text fragments in order.
+ * @returns Matched tokens including `$eof`.
+ */
+export declare function lexChunkStreamAsync(grammar: Grammar, chunks: AsyncIterable<string>): Promise<readonly Token[]>;
+//# sourceMappingURL=lexer.d.ts.map

package/dist/lib/lexer/lexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer.d.ts","sourceRoot":"","sources":["../../../src/lib/lexer/lexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAcrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC;;;;;GAKG;AACH,qBAAa,KAAK;IAEd,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAS;IAE7B;;;;OAIG;gBACgB,OAAO,EAAE,OAAO;IAMnC;;OAEG;IACH,IAAW,KAAK,IAAI,MAAM,CAGzB;IAED;;;;OAIG;IACI,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAM1C;;;;OAIG;IACI,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAWhC;;OAEG;IACI,MAAM,IAAI,IAAI;IAMrB;;;;OAIG;IACI,IAAI,IAAI,KAAK,GAAG,IAAI;IA6B3B;;;;;OAKG;IACI,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,KAAK,EAAE;IAkB5C;;OAEG;IACI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC;IAuB3C;;;;;OAKG;IACH,OAAO,CAAC,KAAK;IAmCb;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAuC1B;;OAEG;IACH,OAAO,CAAC,KAAK;CAQhB;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CACrB,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,GACzB,gBAAgB,CAAC,KAAK,CAAC,CAKzB;AAED;;;;;;GAMG;AACH,wBAAuB,cAAc,CACjC,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,GAC9B,qBAAqB,CAAC,KAAK,CAAC,CA0B9B;AAsCD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC1B,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,GACzB,SAAS,KAAK,EAAE,CAGlB;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,SAAS,KAAK,EAAE,CAAC,CAU3B"}