xbintsc 0.1.0

package/src/lexer/token.ts

@@ -0,0 +1,266 @@
+/**
+ * Token taxonomy for the xbintsc lexer. Kept close to the TypeScript grammar so
+ * the parser can stay a direct recursive-descent transcription of the spec.
+ */
+
+export enum TokenKind {
+  EndOfFile = "EndOfFile",
+  Identifier = "Identifier",
+  PrivateIdentifier = "PrivateIdentifier",
+  NumericLiteral = "NumericLiteral",
+  BigIntLiteral = "BigIntLiteral",
+  StringLiteral = "StringLiteral",
+  NoSubstitutionTemplateLiteral = "NoSubstitutionTemplateLiteral",
+  TemplateHead = "TemplateHead",
+  TemplateMiddle = "TemplateMiddle",
+  TemplateTail = "TemplateTail",
+  RegularExpressionLiteral = "RegularExpressionLiteral",
+
+  // Punctuation
+  OpenBrace = "{",
+  CloseBrace = "}",
+  OpenParen = "(",
+  CloseParen = ")",
+  OpenBracket = "[",
+  CloseBracket = "]",
+  Dot = ".",
+  DotDotDot = "...",
+  Semicolon = ";",
+  Comma = ",",
+  LessThan = "<",
+  GreaterThan = ">",
+  LessThanEquals = "<=",
+  GreaterThanEquals = ">=",
+  EqualsEquals = "==",
+  ExclamationEquals = "!=",
+  EqualsEqualsEquals = "===",
+  ExclamationEqualsEquals = "!==",
+  EqualsGreaterThan = "=>",
+  Plus = "+",
+  Minus = "-",
+  Asterisk = "*",
+  AsteriskAsterisk = "**",
+  Slash = "/",
+  Percent = "%",
+  PlusPlus = "++",
+  MinusMinus = "--",
+  LessThanLessThan = "<<",
+  GreaterThanGreaterThan = ">>",
+  GreaterThanGreaterThanGreaterThan = ">>>",
+  Ampersand = "&",
+  Bar = "|",
+  Caret = "^",
+  Exclamation = "!",
+  Tilde = "~",
+  AmpersandAmpersand = "&&",
+  BarBar = "||",
+  Question = "?",
+  Colon = ":",
+  Equals = "=",
+  PlusEquals = "+=",
+  MinusEquals = "-=",
+  AsteriskEquals = "*=",
+  AsteriskAsteriskEquals = "**=",
+  SlashEquals = "/=",
+  PercentEquals = "%=",
+  LessThanLessThanEquals = "<<=",
+  GreaterThanGreaterThanEquals = ">>=",
+  GreaterThanGreaterThanGreaterThanEquals = ">>>=",
+  AmpersandEquals = "&=",
+  BarEquals = "|=",
+  CaretEquals = "^=",
+  AmpersandAmpersandEquals = "&&=",
+  BarBarEquals = "||=",
+  QuestionQuestionEquals = "??=",
+  QuestionQuestion = "??",
+  QuestionDot = "?.",
+  At = "@",
+  Hash = "#",
+  Backtick = "`",
+
+  // Keywords (subset that xbintsc understands today, but the full set is tokenized
+  // so the parser can produce good "unsupported" diagnostics rather than
+  // confusing "unexpected token" errors).
+  AbstractKeyword = "abstract",
+  AnyKeyword = "any",
+  AsKeyword = "as",
+  AssertsKeyword = "asserts",
+  AsyncKeyword = "async",
+  AwaitKeyword = "await",
+  BigIntKeyword = "bigint",
+  BooleanKeyword = "boolean",
+  BreakKeyword = "break",
+  CaseKeyword = "case",
+  CatchKeyword = "catch",
+  ClassKeyword = "class",
+  ConstKeyword = "const",
+  ConstructorKeyword = "constructor",
+  ContinueKeyword = "continue",
+  DebuggerKeyword = "debugger",
+  DeclareKeyword = "declare",
+  DefaultKeyword = "default",
+  DeleteKeyword = "delete",
+  DoKeyword = "do",
+  ElseKeyword = "else",
+  EnumKeyword = "enum",
+  ExportKeyword = "export",
+  ExtendsKeyword = "extends",
+  FalseKeyword = "false",
+  FinallyKeyword = "finally",
+  ForKeyword = "for",
+  FromKeyword = "from",
+  FunctionKeyword = "function",
+  GetKeyword = "get",
+  IfKeyword = "if",
+  ImplementsKeyword = "implements",
+  ImportKeyword = "import",
+  InKeyword = "in",
+  InferKeyword = "infer",
+  InstanceOfKeyword = "instanceof",
+  InterfaceKeyword = "interface",
+  IsKeyword = "is",
+  KeyOfKeyword = "keyof",
+  LetKeyword = "let",
+  ModuleKeyword = "module",
+  NamespaceKeyword = "namespace",
+  NeverKeyword = "never",
+  NewKeyword = "new",
+  NullKeyword = "null",
+  NumberKeyword = "number",
+  ObjectKeyword = "object",
+  PackageKeyword = "package",
+  PrivateKeyword = "private",
+  ProtectedKeyword = "protected",
+  PublicKeyword = "public",
+  ReadonlyKeyword = "readonly",
+  ReturnKeyword = "return",
+  SatisfiesKeyword = "satisfies",
+  SetKeyword = "set",
+  StaticKeyword = "static",
+  StringKeyword = "string",
+  SuperKeyword = "super",
+  SwitchKeyword = "switch",
+  SymbolKeyword = "symbol",
+  ThisKeyword = "this",
+  ThrowKeyword = "throw",
+  TrueKeyword = "true",
+  TryKeyword = "try",
+  TypeKeyword = "type",
+  TypeOfKeyword = "typeof",
+  UndefinedKeyword = "undefined",
+  UniqueKeyword = "unique",
+  UnknownKeyword = "unknown",
+  VarKeyword = "var",
+  VoidKeyword = "void",
+  WhileKeyword = "while",
+  WithKeyword = "with",
+  YieldKeyword = "yield",
+}
+
+export interface Token {
+  readonly kind: TokenKind;
+  readonly start: number;
+  readonly end: number;
+  /** Raw source text of the token. */
+  readonly text: string;
+  /** Decoded value for string/number/template tokens. */
+  readonly value?: string | number | bigint;
+  /** True when a newline appeared between the previous token and this one. */
+  readonly precededByLineBreak: boolean;
+}
+
+export const KEYWORDS = new Map<string, TokenKind>([
+  ["abstract", TokenKind.AbstractKeyword],
+  ["any", TokenKind.AnyKeyword],
+  ["as", TokenKind.AsKeyword],
+  ["asserts", TokenKind.AssertsKeyword],
+  ["async", TokenKind.AsyncKeyword],
+  ["await", TokenKind.AwaitKeyword],
+  ["bigint", TokenKind.BigIntKeyword],
+  ["boolean", TokenKind.BooleanKeyword],
+  ["break", TokenKind.BreakKeyword],
+  ["case", TokenKind.CaseKeyword],
+  ["catch", TokenKind.CatchKeyword],
+  ["class", TokenKind.ClassKeyword],
+  ["const", TokenKind.ConstKeyword],
+  ["constructor", TokenKind.ConstructorKeyword],
+  ["continue", TokenKind.ContinueKeyword],
+  ["debugger", TokenKind.DebuggerKeyword],
+  ["declare", TokenKind.DeclareKeyword],
+  ["default", TokenKind.DefaultKeyword],
+  ["delete", TokenKind.DeleteKeyword],
+  ["do", TokenKind.DoKeyword],
+  ["else", TokenKind.ElseKeyword],
+  ["enum", TokenKind.EnumKeyword],
+  ["export", TokenKind.ExportKeyword],
+  ["extends", TokenKind.ExtendsKeyword],
+  ["false", TokenKind.FalseKeyword],
+  ["finally", TokenKind.FinallyKeyword],
+  ["for", TokenKind.ForKeyword],
+  ["from", TokenKind.FromKeyword],
+  ["function", TokenKind.FunctionKeyword],
+  ["get", TokenKind.GetKeyword],
+  ["if", TokenKind.IfKeyword],
+  ["implements", TokenKind.ImplementsKeyword],
+  ["import", TokenKind.ImportKeyword],
+  ["in", TokenKind.InKeyword],
+  ["infer", TokenKind.InferKeyword],
+  ["instanceof", TokenKind.InstanceOfKeyword],
+  ["interface", TokenKind.InterfaceKeyword],
+  ["is", TokenKind.IsKeyword],
+  ["keyof", TokenKind.KeyOfKeyword],
+  ["let", TokenKind.LetKeyword],
+  ["module", TokenKind.ModuleKeyword],
+  ["namespace", TokenKind.NamespaceKeyword],
+  ["never", TokenKind.NeverKeyword],
+  ["new", TokenKind.NewKeyword],
+  ["null", TokenKind.NullKeyword],
+  ["number", TokenKind.NumberKeyword],
+  ["object", TokenKind.ObjectKeyword],
+  ["package", TokenKind.PackageKeyword],
+  ["private", TokenKind.PrivateKeyword],
+  ["protected", TokenKind.ProtectedKeyword],
+  ["public", TokenKind.PublicKeyword],
+  ["readonly", TokenKind.ReadonlyKeyword],
+  ["return", TokenKind.ReturnKeyword],
+  ["satisfies", TokenKind.SatisfiesKeyword],
+  ["set", TokenKind.SetKeyword],
+  ["static", TokenKind.StaticKeyword],
+  ["string", TokenKind.StringKeyword],
+  ["super", TokenKind.SuperKeyword],
+  ["switch", TokenKind.SwitchKeyword],
+  ["symbol", TokenKind.SymbolKeyword],
+  ["this", TokenKind.ThisKeyword],
+  ["throw", TokenKind.ThrowKeyword],
+  ["true", TokenKind.TrueKeyword],
+  ["try", TokenKind.TryKeyword],
+  ["type", TokenKind.TypeKeyword],
+  ["typeof", TokenKind.TypeOfKeyword],
+  ["undefined", TokenKind.UndefinedKeyword],
+  ["unique", TokenKind.UniqueKeyword],
+  ["unknown", TokenKind.UnknownKeyword],
+  ["var", TokenKind.VarKeyword],
+  ["void", TokenKind.VoidKeyword],
+  ["while", TokenKind.WhileKeyword],
+  ["with", TokenKind.WithKeyword],
+  ["yield", TokenKind.YieldKeyword],
+]);
+
+/**
+ * Keywords that are allowed in identifier position in some contexts and should
+ * therefore not be treated as hard errors when used as a name.
+ */
+export const CONTEXTUAL_KEYWORDS: ReadonlySet<TokenKind> = new Set([
+  TokenKind.AsKeyword,
+  TokenKind.SatisfiesKeyword,
+  TokenKind.FromKeyword,
+  TokenKind.TypeKeyword,
+  TokenKind.GetKeyword,
+  TokenKind.SetKeyword,
+  TokenKind.AsyncKeyword,
+]);
+
+/** True when the token text is the contextual `of` keyword. */
+export function isOfKeyword(token: Token): boolean {
+  return token.kind === TokenKind.Identifier && token.text === "of";
+}