spex-parser 0.1.0

package/src/lexer.ts

@@ -0,0 +1,121 @@
+import { createToken, Lexer } from 'chevrotain'
+
+export const WhiteSpace = createToken({
+  name: 'WhiteSpace',
+  pattern: /\s+/,
+  group: Lexer.SKIPPED,
+})
+
+// Keywords (case-insensitive)
+export const CreateTok = createToken({
+  name: 'CreateTok',
+  pattern: /create\b/i,
+})
+export const AsTok = createToken({
+  name: 'AsTok',
+  pattern: /as\b/i,
+})
+export const FromTok = createToken({
+  name: 'FromTok',
+  pattern: /from\b/i,
+})
+export const SelectTok = createToken({
+  name: 'SelectTok',
+  pattern: /select\b/i,
+})
+export const GenerateTok = createToken({
+  name: 'GenerateTok',
+  pattern: /generate\b/i,
+})
+export const ImportTok = createToken({
+  name: 'ImportTok',
+  pattern: /import\b/i,
+})
+export const ExportTok = createToken({
+  name: 'ExportTok',
+  pattern: /export\b/i,
+})
+
+// Symbols
+export const ArrowTok = createToken({ name: 'ArrowTok', pattern: /->/ })
+export const LCurly = createToken({ name: 'LCurly', pattern: /{/ })
+export const RCurly = createToken({ name: 'RCurly', pattern: /}/ })
+export const LBracket = createToken({ name: 'LBracket', pattern: /\[/ })
+export const RBracket = createToken({ name: 'RBracket', pattern: /\]/ })
+export const LParen = createToken({ name: 'LParen', pattern: /\(/ })
+export const RParen = createToken({ name: 'RParen', pattern: /\)/ })
+export const Colon = createToken({ name: 'Colon', pattern: /:/ })
+export const Comma = createToken({ name: 'Comma', pattern: /,/ })
+export const Semicolon = createToken({ name: 'Semicolon', pattern: /;/ })
+export const Dot = createToken({ name: 'Dot', pattern: /\./ })
+
+// Brace text block (for SELECT { ... })
+export const SelectBlock = createToken({
+  name: 'SelectBlock',
+  pattern: /\{[^}]+\}/,
+})
+
+// Literals
+export const PathLiteral = createToken({
+  name: 'PathLiteral',
+  pattern: /"([^"\\]|\\.)*"/,
+})
+
+// Basic objects (native types)
+export const StringTok = createToken({
+  name: 'StringTok',
+  pattern: /string\b/i,
+})
+export const NumberTok = createToken({
+  name: 'NumberTok',
+  pattern: /number\b/i,
+})
+export const BoolTok = createToken({
+  name: 'BoolTok',
+  pattern: /bool\b/i,
+})
+export const UnitTok = createToken({
+  name: 'UnitTok',
+  pattern: /unit\b/i,
+})
+
+// Identifiers
+export const Identifier = createToken({
+  name: 'Identifier',
+  pattern: /[a-zA-Z_][a-zA-Z0-9_]*/,
+})
+
+export const allTokens = [
+  WhiteSpace,
+
+  CreateTok,
+  AsTok,
+  FromTok,
+  SelectTok,
+  GenerateTok,
+  ImportTok,
+  ExportTok,
+
+  ArrowTok,
+  SelectBlock,
+  LCurly,
+  RCurly,
+  LBracket,
+  RBracket,
+  LParen,
+  RParen,
+  Colon,
+  Comma,
+  Semicolon,
+  Dot,
+  PathLiteral,
+
+  StringTok,
+  NumberTok,
+  BoolTok,
+  UnitTok,
+
+  Identifier,
+]
+
+export const SpexLexer = new Lexer(allTokens)

package/src/parser.ts

@@ -0,0 +1,170 @@
+import { CstParser } from 'chevrotain'
+import {
+  allTokens,
+  CreateTok,
+  AsTok,
+  FromTok,
+  SelectTok,
+  GenerateTok,
+  ImportTok,
+  ExportTok,
+  ArrowTok,
+  SelectBlock,
+  LBracket,
+  RBracket,
+  LParen,
+  RParen,
+  Colon,
+  Comma,
+  Semicolon,
+  Dot,
+  Identifier,
+  PathLiteral,
+  StringTok,
+  NumberTok,
+  BoolTok,
+  UnitTok,
+} from './lexer.js'
+
+export class SpexParser extends CstParser {
+  constructor() {
+    super(allTokens)
+    this.performSelfAnalysis()
+  }
+
+  public spexFile = this.RULE('spexFile', () => {
+    this.MANY(() => {
+      this.SUBRULE(this.declaration)
+    })
+  })
+
+  private declaration = this.RULE('declaration', () => {
+    this.OR([
+      {
+        GATE: this.BACKTRACK(this.objectDeclaration),
+        ALT: () => this.SUBRULE(this.objectDeclaration),
+      },
+      {
+        GATE: this.BACKTRACK(this.importDeclaration),
+        ALT: () => this.SUBRULE(this.importDeclaration),
+      },
+      {
+        GATE: this.BACKTRACK(this.exportDeclaration),
+        ALT: () => this.SUBRULE(this.exportDeclaration),
+      },
+      {
+        ALT: () => this.SUBRULE(this.generateDeclaration),
+      },
+    ])
+  })
+
+  private objectDeclaration = this.RULE('objectDeclaration', () => {
+    this.CONSUME(CreateTok)
+    this.CONSUME(Identifier)
+    this.CONSUME(AsTok)
+    this.SUBRULE(this.objectExpression)
+    this.CONSUME(Semicolon)
+  })
+
+  private objectExpression = this.RULE('objectExpression', () => {
+    this.SUBRULE(this.objectOperand, { LABEL: 'base' })
+    this.OPTION(() => {
+      this.CONSUME(ArrowTok)
+      this.SUBRULE2(this.objectExpression, { LABEL: 'exponent' })
+    })
+  })
+
+  private objectOperand = this.RULE('objectOperand', () => {
+    this.OR([
+      {
+        GATE: this.BACKTRACK(this.subObject),
+        ALT: () => this.SUBRULE(this.subObject),
+      },
+      {
+        GATE: this.BACKTRACK(this.productObject),
+        ALT: () => this.SUBRULE(this.productObject),
+      },
+      {
+        ALT: () => this.SUBRULE(this.namedObject),
+      },
+    ])
+    this.MANY(() => {
+      this.CONSUME(LBracket)
+      this.CONSUME(RBracket)
+    })
+  })
+
+  private namedObject = this.RULE('namedObject', () => {
+    this.OR([
+      { ALT: () => this.CONSUME(Identifier) },
+      { ALT: () => this.CONSUME(StringTok) },
+      { ALT: () => this.CONSUME(NumberTok) },
+      { ALT: () => this.CONSUME(BoolTok) },
+      { ALT: () => this.CONSUME(UnitTok) },
+    ])
+    this.MANY(() => {
+      this.CONSUME(Dot)
+      this.CONSUME2(Identifier)
+    })
+  })
+
+  private productObject = this.RULE('productObject', () => {
+    this.CONSUME(LParen)
+    this.MANY(() => {
+      this.CONSUME(Identifier)
+      this.CONSUME(Colon)
+      this.SUBRULE(this.objectExpression)
+      this.OPTION(() => this.CONSUME(Comma))
+    })
+    this.CONSUME(RParen)
+  })
+
+  private subObject = this.RULE('subObject', () => {
+    this.CONSUME(FromTok)
+    this.SUBRULE(this.objectExpression, { LABEL: 'base' })
+    this.CONSUME(SelectTok)
+    this.CONSUME(SelectBlock)
+  })
+
+  private importDeclaration = this.RULE('importDeclaration', () => {
+    this.CONSUME(ImportTok)
+    this.OR([
+      {
+        GATE: this.BACKTRACK(this.namedImport),
+        ALT: () => this.SUBRULE(this.namedImport),
+      },
+      {
+        ALT: () => this.SUBRULE(this.moduleImport),
+      },
+    ])
+    this.CONSUME(Semicolon)
+  })
+
+  private namedImport = this.RULE('namedImport', () => {
+    this.CONSUME(Identifier)
+    this.CONSUME(FromTok)
+    this.CONSUME(PathLiteral)
+    this.OPTION(() => {
+      this.CONSUME(AsTok)
+      this.CONSUME2(Identifier)
+    })
+  })
+
+  private moduleImport = this.RULE('moduleImport', () => {
+    this.CONSUME(PathLiteral)
+    this.CONSUME(AsTok)
+    this.CONSUME(Identifier)
+  })
+
+  private exportDeclaration = this.RULE('exportDeclaration', () => {
+    this.CONSUME(ExportTok)
+    this.CONSUME(Identifier)
+    this.CONSUME(Semicolon)
+  })
+
+  private generateDeclaration = this.RULE('generateDeclaration', () => {
+    this.CONSUME(GenerateTok)
+    this.CONSUME(Identifier)
+    this.CONSUME(Semicolon)
+  })
+}

package/src/visitor.ts

@@ -0,0 +1,179 @@
+import type { ICstVisitor } from 'chevrotain'
+import type {
+  SpexFile,
+  Declaration,
+  ObjectDeclaration,
+  ImportDeclaration,
+  ExportDeclaration,
+  GenerateDeclaration,
+  ObjectExpression,
+  NamedObject,
+  ProductObject,
+  SubObject,
+  ArrayObject,
+} from './ast.js'
+import { SpexLexer } from './lexer.js'
+import { SpexParser } from './parser.js'
+
+const parserInstance = new SpexParser()
+const BaseSpexVisitor = parserInstance.getBaseCstVisitorConstructor()
+
+export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<any, any> {
+  constructor() {
+    super()
+    this.validateVisitor()
+  }
+
+  spexFile(ctx: any): SpexFile {
+    const declarations = ctx.declaration.map((decl: any) => this.visit(decl))
+    return { kind: 'SpexFile', declarations }
+  }
+
+  declaration(ctx: any): Declaration {
+    if (ctx.objectDeclaration) {
+      return this.visit(ctx.objectDeclaration)
+    }
+    if (ctx.importDeclaration) {
+      return this.visit(ctx.importDeclaration)
+    }
+    if (ctx.exportDeclaration) {
+      return this.visit(ctx.exportDeclaration)
+    }
+    return this.visit(ctx.generateDeclaration)
+  }
+
+  objectDeclaration(ctx: any): ObjectDeclaration {
+    return {
+      kind: 'ObjectDeclaration',
+      name: ctx.Identifier[0].image,
+      object: this.visit(ctx.objectExpression),
+    }
+  }
+
+  objectExpression(ctx: any): ObjectExpression {
+    if (ctx.base) {
+      const exponent = this.visit(ctx.base)
+      if (ctx.exponent) {
+        return {
+          kind: 'ExponentialObject',
+          base: this.visit(ctx.exponent),
+          exponent,
+        }
+      }
+      return exponent
+    }
+    throw new Error('Invalid object expression')
+  }
+
+  objectOperand(ctx: any): ObjectExpression {
+    let expr: ObjectExpression
+    if (ctx.subObject) {
+      expr = this.visit(ctx.subObject)
+    } else if (ctx.productObject) {
+      expr = this.visit(ctx.productObject)
+    } else {
+      expr = this.visit(ctx.namedObject)
+    }
+    if (ctx.LBracket) {
+      for (let i = 0; i < ctx.LBracket.length; i++) {
+        expr = { kind: 'ArrayObject', base: expr } as ArrayObject
+      }
+    }
+    return expr
+  }
+
+  namedObject(ctx: any): NamedObject {
+    let parts: string[]
+    if (ctx.StringTok) {
+      parts = [ctx.StringTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
+    } else if (ctx.NumberTok) {
+      parts = [ctx.NumberTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
+    } else if (ctx.BoolTok) {
+      parts = [ctx.BoolTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
+    } else if (ctx.UnitTok) {
+      parts = [ctx.UnitTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
+    } else {
+      parts = ctx.Identifier.map((id: any) => id.image)
+    }
+    return {
+      kind: 'NamedObject',
+      name: parts.join('.'),
+    }
+  }
+
+  productObject(ctx: any): ProductObject {
+    const fields: Record<string, ObjectExpression> = {}
+    for (let i = 0; i < ctx.Identifier.length; i++) {
+      const name = ctx.Identifier[i].image
+      fields[name] = this.visit(ctx.objectExpression[i])
+    }
+    return { kind: 'ProductObject', fields }
+  }
+
+  subObject(ctx: any): SubObject {
+    const rawText: string = ctx.SelectBlock[0].image
+    const constraint = rawText.slice(1, -1).trim()
+    return {
+      kind: 'SubObject',
+      base: this.visit(ctx.base),
+      constraint,
+    }
+  }
+
+  importDeclaration(ctx: any): ImportDeclaration {
+    if (ctx.namedImport) {
+      return this.visit(ctx.namedImport)
+    }
+    return this.visit(ctx.moduleImport)
+  }
+
+  namedImport(ctx: any): ImportDeclaration {
+    const name = ctx.Identifier[0].image
+    const source = ctx.PathLiteral[0].image.slice(1, -1)
+    const alias = ctx.Identifier[1] ? ctx.Identifier[1].image : null
+    return {
+      kind: 'ImportDeclaration',
+      name,
+      source,
+      alias,
+    }
+  }
+
+  moduleImport(ctx: any): ImportDeclaration {
+    const source = ctx.PathLiteral[0].image.slice(1, -1)
+    const alias = ctx.Identifier[0].image
+    return {
+      kind: 'ImportDeclaration',
+      name: null,
+      source,
+      alias,
+    }
+  }
+
+  exportDeclaration(ctx: any): ExportDeclaration {
+    return {
+      kind: 'ExportDeclaration',
+      name: ctx.Identifier[0].image,
+    }
+  }
+
+  generateDeclaration(ctx: any): GenerateDeclaration {
+    return {
+      kind: 'GenerateDeclaration',
+      name: ctx.Identifier[0].image,
+    }
+  }
+}
+
+export function parseToAst(text: string): SpexFile {
+  const lexingResult = SpexLexer.tokenize(text)
+  parserInstance.input = lexingResult.tokens
+  const cst = parserInstance.spexFile()
+
+  if (parserInstance.errors.length > 0) {
+    throw new Error(`Parsing errors: ${JSON.stringify(parserInstance.errors, null, 2)}`)
+  }
+
+  const visitor = new SpexParserVisitor()
+  return visitor.visit(cst)
+}

package/tests/e2e.test.ts

@@ -0,0 +1,17 @@
+import { describe, it, expect } from 'vitest'
+import { readFileSync } from 'fs'
+import { join, dirname } from 'path'
+import { fileURLToPath } from 'url'
+import { parseToAst } from '../src/visitor.js'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+describe('end-to-end', () => {
+  it('should parse todo.spex file', () => {
+    const code = readFileSync(join(__dirname, 'props/todo.spex'), 'utf-8')
+    const ast = parseToAst(code)
+    expect(ast.kind).toBe('SpexFile')
+    expect(ast.declarations.length).toBe(14)
+  })
+})

package/tests/lexer.test.ts

@@ -0,0 +1,123 @@
+import { describe, it, expect } from 'vitest'
+import { SpexLexer } from '../src/lexer.js'
+
+describe('SpexLexer', () => {
+  describe('tokenization', () => {
+    it('should tokenize keywords', () => {
+      const result = SpexLexer.tokenize('create as from select generate import export')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
+        'CreateTok',
+        'AsTok',
+        'FromTok',
+        'SelectTok',
+        'GenerateTok',
+        'ImportTok',
+        'ExportTok',
+      ])
+    })
+
+    it('should tokenize keywords case-insensitively', () => {
+      const result = SpexLexer.tokenize('CREATE AS FROM SELECT GENERATE IMPORT EXPORT')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
+        'CreateTok',
+        'AsTok',
+        'FromTok',
+        'SelectTok',
+        'GenerateTok',
+        'ImportTok',
+        'ExportTok',
+      ])
+    })
+
+    it('should tokenize symbols', () => {
+      const result = SpexLexer.tokenize('->{}[]():;,.')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
+        'ArrowTok',
+        'LCurly',
+        'RCurly',
+        'LBracket',
+        'RBracket',
+        'LParen',
+        'RParen',
+        'Colon',
+        'Semicolon',
+        'Comma',
+        'Dot',
+      ])
+    })
+
+    it('should tokenize identifiers', () => {
+      const result = SpexLexer.tokenize('foo bar _test _123 ABC')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual(Array(5).fill('Identifier'))
+      expect(result.tokens.map((t) => t.image)).toEqual(['foo', 'bar', '_test', '_123', 'ABC'])
+    })
+
+    it('should skip whitespace', () => {
+      const result = SpexLexer.tokenize('foo   bar\t\nbaz')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual(Array(3).fill('Identifier'))
+    })
+
+    it('should handle mixed input', () => {
+      const result = SpexLexer.tokenize('CREATE Foo as ( name: string )')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
+        'CreateTok',
+        'Identifier',
+        'AsTok',
+        'LParen',
+        'Identifier',
+        'Colon',
+        'StringTok',
+        'RParen',
+      ])
+    })
+
+    it('should handle keywords with word boundary', () => {
+      const result = SpexLexer.tokenize(
+        'createfoo foocreate asfoo fooas fooselect selectfoo foofrom fromfoo generatefoo foogenerate importfoo fooimport exportfoo fooexport'
+      )
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual(Array(14).fill('Identifier'))
+    })
+
+    it('should tokenize the text between braces', () => {
+      const result = SpexLexer.tokenize('{hello\nworld}')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens).toHaveLength(1)
+      expect(result.tokens[0]?.tokenType.name).toBe('SelectBlock')
+      expect(result.tokens[0]?.image).toBe('{hello\nworld}')
+    })
+
+    it('should tokenize path literals', () => {
+      const result = SpexLexer.tokenize('"types.spex"')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens).toHaveLength(1)
+      expect(result.tokens[0]?.tokenType.name).toBe('PathLiteral')
+      expect(result.tokens[0]?.image).toBe('"types.spex"')
+    })
+
+    it('should tokenize array brackets', () => {
+      const result = SpexLexer.tokenize('string[]')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
+        'StringTok',
+        'LBracket',
+        'RBracket',
+      ])
+      expect(result.tokens.map((t) => t.image)).toEqual(['string', '[', ']'])
+    })
+  })
+
+  describe('error handling', () => {
+    it('should return empty tokens for empty input', () => {
+      const result = SpexLexer.tokenize('')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens).toHaveLength(0)
+    })
+  })
+})