yukigo-haskell-parser 0.1.0 → 0.1.1

package/src/parser/lexer.ts

@@ -1,77 +1,353 @@
-import moo, { Lexer } from "moo";
-import { keywords } from "../utils/types.js";
-import { makeLexer } from "moo-ignore";
-import IndentationLexer from "moo-indentation-lexer";
-
-export const HaskellLexerConfig = {
-  NL: { match: /\r?\n/, lineBreaks: true },
-  anonymousVariable: "_",
-  WS: / |\t/,
-  comment: /--.*?$|{-[\s\S]*?-}/,
-  number:
-    /0[xX][0-9a-fA-F]+|0[bB][01]+|0[oO][0-7]+|(?:\d*\.\d+|\d+)(?:[eE][+-]?\d+)?/,
-  char: /'(?:\\['\\bfnrtv0]|\\u[0-9a-fA-F]{4}|[^'\\\n\r])?'/,
-  string: /"(?:\\["\\bfnrtv0]|\\u[0-9a-fA-F]{4}|[^"\\\n\r])*"/,
-  //template: /`(?:\\[\s\S]|[^\\`])*`/,
-  backtick: "`",
-  lparen: "(",
-  rparen: ")",
-  lbracket: "{",
-  rbracket: "}",
-  lsquare: "[",
-  rsquare: "]",
-  rangeOperator: "..",
-  semicolon: ";",
-  typeArrow: "->",
-  leftArrow: "<-",
-  typeEquals: "::",
-  question: "?",
-  arrow: "=>",
-  bool: {
-    match: ["True", "False"],
-  },
-  op: /,|>>=|>>|\\\\|\\|\.|\+\+|\+|\-|\*\*|\*|===|!==|==|\/=|<=|>=|<|>|&&|\/|\|\||\.\.|\$|\^\^|\^|#|@|~|!!|!|%|\?|:|&|\||`/,
-  assign: "=",
-  constructor: {
-    match: /[A-Z][a-zA-Z0-9']*/,
-    type: moo.keywords({
-      typeClass: [
-        "Foldable",
-        "Bounded",
-        "Enum",
-        "Eq",
-        "Floating",
-        "Fractional",
-        "Functor",
-        "Integral",
-        "Ix",
-        "Monad",
-        "MonadPlus",
-        "Num",
-        "Ord",
-        "Random",
-        "RandomGen",
-        "Read",
-        "Real",
-        "RealFloat",
-        "RealFrac",
-        "Show",
-      ],
-    }),
-  },
-  variable: {
-    match: /[a-z_][a-zA-Z0-9_']*/,
-    type: moo.keywords({
-      otherwise: "otherwise",
-      error: "error",
-      keyword: keywords,
-    }),
-  },
-};
-
-export const HSLexer: Lexer = makeLexer(HaskellLexerConfig, []);
-export const PreprocessorLexer: Lexer = new IndentationLexer({
-  lexer: moo.compile(HaskellLexerConfig),
-  indentationType: "WS",
-  newlineType: 'NL',
-});
+import moo, { Lexer, LexerState, Token } from "moo";
+import { keywords } from "../utils/types.js";
+declare module "moo" {
+  interface Lexer {
+    index: number; // Add the missing property to the type definition
+  }
+}
+
+interface LayoutState {
+  stack: number[]; // indentation stack (follows moo naming)
+  tokenBuffer: Token[];
+  expectingBlock: boolean;
+  firstTokenProcessed: boolean;
+}
+type FullState = {
+  layoutState: LayoutState;
+  mooIndex?: number;
+} & LexerState;
+
+export const HaskellLexerConfig = {
+  comment: { match: /--.*?$|{-[\s\S]*?-}/, lineBreaks: true },
+
+  number:
+    /0[xX][0-9a-fA-F]+|0[bB][01]+|0[oO][0-7]+|(?:\d*\.\d+|\d+)(?:[eE][+-]?\d+)?/,
+  char: /'(?:\\['\\bfnrtv0]|\\u[0-9a-fA-F]{4}|[^'\\\n\r])?'/,
+  string: { match: /"(?:\\[\s\S]|[^"\\])*"/, lineBreaks: true },
+
+  lparen: "(",
+  rparen: ")",
+  lbracket: "{",
+  rbracket: "}",
+  lsquare: "[",
+  rsquare: "]",
+  semicolon: ";",
+
+  typeArrow: "->",
+  leftArrow: "<-",
+  typeEquals: "::",
+  fatArrow: "=>",
+  op: {
+    match:
+    /,|>>=|>>|\\\\|\\|\.\.|\.|\+\+|\+|\-|\*\*|\*|===|!==|==|\/=|<=|>=|<|>|&&|\/|\|\||\$!|\$|\^\^|\^|#|@|~|!!|!|%|\?|:|&|`/,
+    lineBreaks: false,
+  },
+  assign: "=",
+  backslash: "\\",
+  pipe: "|",
+  at: "@",
+  tilde: "~",
+
+  anonymousVariable: "_",
+
+  constructor: {
+    match: /[A-Z][a-zA-Z0-9']*/,
+    type: moo.keywords({ bool: ["True", "False"] }),
+  },
+  variable: {
+    match: /[a-z_][a-zA-Z0-9_']*/,
+    type: moo.keywords({ otherwise: "otherwise", keyword: keywords }),
+  },
+
+  NL: { match: /\r?\n/, lineBreaks: true },
+  WS: { match: /[ \t]+/, lineBreaks: false },
+};
+
+export class HaskellLayoutLexer implements Lexer {
+  public index: number;
+  private mooLexer: Lexer;
+  private state: LayoutState;
+
+  private readonly layoutTriggers = new Set(["where", "let", "do", "of"]);
+  private readonly noSemicolonKeywords = new Set(["in", "then", "else", "of"]);
+  private readonly alwaysNewDeclarationKeywords = new Set([
+    "type",
+    "data",
+    "class",
+    "instance",
+    "describe",
+    "it",
+    "module",
+    "import",
+  ]);
+
+  constructor() {
+    this.mooLexer = moo.compile(HaskellLexerConfig);
+    this.state = {
+      stack: [1],
+      tokenBuffer: [],
+      expectingBlock: false,
+      firstTokenProcessed: false,
+    };
+  }
+
+  [Symbol.iterator](): Iterator<Token> {
+    return {
+      next: () => {
+        const value = this.next();
+        return value
+          ? { value, done: false }
+          : { value: undefined, done: true };
+      },
+    };
+  }
+
+  reset(chunk?: string, state?: FullState): this {
+    if (state && state.layoutState) {
+      const fullState = state;
+      this.state = {
+        stack: [...fullState.layoutState.stack],
+        tokenBuffer: [...fullState.layoutState.tokenBuffer],
+        expectingBlock: fullState.layoutState.expectingBlock,
+        firstTokenProcessed: fullState.layoutState.firstTokenProcessed,
+      };
+      this.mooLexer.reset(chunk, fullState);
+      if (typeof fullState.mooIndex === "number") {
+        this.mooLexer.index = fullState.mooIndex;
+      }
+    } else {
+      this.state = {
+        stack: [1],
+        tokenBuffer: [],
+        expectingBlock: false,
+        firstTokenProcessed: false,
+      };
+      this.mooLexer.reset(chunk);
+    }
+    return this;
+  }
+
+  save(): FullState {
+    return {
+      ...this.mooLexer.save(),
+      mooIndex: this.mooLexer.index,
+      layoutState: {
+        stack: [...this.state.stack],
+        tokenBuffer: [...this.state.tokenBuffer],
+        expectingBlock: this.state.expectingBlock,
+        firstTokenProcessed: this.state.firstTokenProcessed,
+      },
+    };
+  }
+
+  pushState(state: string) {
+    this.mooLexer.pushState(state);
+  }
+  popState() {
+    return this.mooLexer.popState();
+  }
+  setState(state: string) {
+    this.mooLexer.setState(state);
+  }
+  formatError(token: Token, message?: string) {
+    return this.mooLexer.formatError(token, message);
+  }
+  has(tokenType: string) {
+    return true; // deprecated in moo but Lexer interface requires it.
+  }
+
+  next(): Token | undefined {
+    let token: Token | undefined;
+
+    if (this.state.tokenBuffer.length > 0) {
+      token = this.state.tokenBuffer.shift();
+    } else {
+      token = this.fetchAndProcessNextToken();
+    }
+
+    if (token) {
+      this.checkLayoutTrigger(token);
+      if (!this.state.firstTokenProcessed && token.col > 1) {
+        this.state.stack[0] = token.col;
+      }
+      this.state.firstTokenProcessed = true;
+    }
+
+    return token;
+  }
+
+  private fetchAndProcessNextToken(): Token | undefined {
+    let { token: rawToken, crossedNewline } =
+      this.advanceSkippingWhitespace();
+
+    if (!rawToken) return this.emitEOF();
+
+    
+    if (this.state.expectingBlock) return this.handleBlockStart(rawToken);
+
+    if (this.isInToken(rawToken)) return this.handleInKeyword(rawToken);
+
+    if (crossedNewline) return this.handleIndentationChange(rawToken);
+
+    return rawToken;
+  }
+
+  private advanceSkippingWhitespace(): {
+    token: Token | undefined;
+    crossedNewline: boolean;
+  } {
+    let token = this.mooLexer.next();
+    let crossedNewline = false;
+
+    while (token && this.isSkippableToken(token)) {
+      if (this.isNewlineToken(token)) crossedNewline = true;
+      token = this.mooLexer.next();
+    }
+    return { token, crossedNewline };
+  }
+
+  private closeBlocksUntil(targetIndent: number, sourceToken: Token): Token {
+    const ops: Token[] = [];
+
+    while (
+      this.state.stack.length > 1 &&
+      this.state.stack[this.state.stack.length - 1] > targetIndent
+    ) {
+      this.state.stack.pop();
+      ops.push(this.createVirtualToken("rbracket", "}", sourceToken));
+    }
+
+    const isAlwaysNewDeclaration =
+      this.state.stack.length === 1 &&
+      sourceToken.type === "keyword" &&
+      this.alwaysNewDeclarationKeywords.has(sourceToken.value);
+
+    const shouldInjectSemicolon =
+      !sourceToken.value || !this.noSemicolonKeywords.has(sourceToken.value);
+
+    if (
+      this.state.stack.length > 1 &&
+      this.state.stack[this.state.stack.length - 1] === targetIndent &&
+      this.isInToken(sourceToken)
+    ) {
+      this.state.stack.pop();
+      ops.push(this.createVirtualToken("rbracket", "}", sourceToken));
+    } else if (
+      (this.state.stack.length > 0 &&
+        this.state.stack[this.state.stack.length - 1] === targetIndent &&
+        shouldInjectSemicolon) ||
+      isAlwaysNewDeclaration
+    ) {
+      ops.push(this.createVirtualToken("semicolon", ";", sourceToken));
+    }
+
+    this.state.tokenBuffer.unshift(...ops);
+
+    return this.state.tokenBuffer.shift();
+  }
+
+  private emitEOF(): Token | undefined {
+    if (this.state.stack.length > 1) {
+      const dummy = {
+        line: 0,
+        col: 0,
+        text: "",
+        value: "",
+        offset: 0,
+        lineBreaks: 0,
+      };
+      const ops: Token[] = [];
+      while (this.state.stack.length > 1) {
+        this.state.stack.pop();
+        ops.push(this.createVirtualToken("rbracket", "}", dummy));
+      }
+      this.state.tokenBuffer.push(...ops);
+      return this.state.tokenBuffer.shift();
+    }
+    return undefined;
+  }
+
+  private enqueue(token: Token) {
+    this.state.tokenBuffer.push(token);
+  }
+
+  private handleBlockStart(token: Token) {
+    this.state.expectingBlock = false;
+    if (token.type === "lbracket") return token;
+
+    this.state.stack.push(token.col);
+    this.enqueue(token);
+    return this.createVirtualToken("lbracket", "{", token);
+  }
+  private handleInKeyword(token: Token) {
+    const stackTop = this.state.stack[this.state.stack.length - 1];
+    if (token.col < stackTop) {
+      this.enqueue(token);
+      return this.closeBlocksUntil(token.col, token);
+    }
+    if (this.state.stack.length > 1) {
+      this.state.stack.pop();
+      this.enqueue(token);
+      return this.createVirtualToken("rbracket", "}", token);
+    }
+    return token;
+  }
+  private handleIndentationChange(token: Token) {
+    const currentIndent = token.col;
+    const stackTop = this.state.stack[this.state.stack.length - 1];
+
+    const isAlwaysNewDeclaration =
+      this.state.stack.length === 1 &&
+      token.type === "keyword" &&
+      this.alwaysNewDeclarationKeywords.has(token.value);
+
+    if (currentIndent === stackTop || isAlwaysNewDeclaration) {
+      const isContinuationKeyword =
+        token.value && this.noSemicolonKeywords.has(token.value);
+      if (this.state.firstTokenProcessed && !isContinuationKeyword) {
+        this.enqueue(token);
+        return this.createVirtualToken("semicolon", ";", token);
+      }
+      return token;
+    } else if (currentIndent < stackTop) {
+      this.enqueue(token);
+      return this.closeBlocksUntil(currentIndent, token);
+    }
+    return token;
+  }
+
+  private checkLayoutTrigger(token: Token) {
+    if (token && token.value && this.layoutTriggers.has(token.value))
+      this.state.expectingBlock = true;
+  }
+  private isSkippableToken(token: Token) {
+    return (
+      token.type === "WS" ||
+      this.isNewlineToken(token) ||
+      token.type === "comment"
+    );
+  }
+  private isNewlineToken(token: Token) {
+    return token.type === "NL";
+  }
+  private isInToken(token: Token) {
+    return token.type === "keyword" && token.value === "in";
+  }
+
+  private createVirtualToken(
+    type: string,
+    value: string,
+    source: Token
+  ): Token {
+    return {
+      type,
+      value,
+      text: value,
+      line: source.line,
+      col: source.col,
+      offset: source.offset,
+      lineBreaks: 0,
+      toString: () => value,
+    };
+  }
+}