spex-parser 0.1.4 → 0.6.1

package/src/lexer.ts

@@ -1,4 +1,4 @@
-import { createToken, Lexer } from 'chevrotain'
+import { createToken, Lexer, type CustomPatternMatcherReturn } from 'chevrotain'
 
 export const WhiteSpace = createToken({
   name: 'WhiteSpace',
@@ -6,6 +6,18 @@ export const WhiteSpace = createToken({
   group: Lexer.SKIPPED,
 })
 
+// Comments (SQL-style)
+export const LineComment = createToken({
+  name: 'LineComment',
+  pattern: /--[^\r\n]*/,
+  group: Lexer.SKIPPED,
+})
+export const BlockComment = createToken({
+  name: 'BlockComment',
+  pattern: /\/\*[\s\S]*?\*\//,
+  group: Lexer.SKIPPED,
+})
+
 // Keywords (case-insensitive)
 export const CreateTok = createToken({
   name: 'CreateTok',
@@ -47,9 +59,42 @@ export const ModuleTok = createToken({
   name: 'ModuleTok',
   pattern: /module\b/i,
 })
+export const EnumTok = createToken({
+  name: 'EnumTok',
+  pattern: /enum\b/i,
+})
+export const UnionTok = createToken({
+  name: 'UnionTok',
+  pattern: /union\b/i,
+})
+export const IntersectTok = createToken({
+  name: 'IntersectTok',
+  pattern: /intersect\b/i,
+})
+export const ExceptTok = createToken({
+  name: 'ExceptTok',
+  pattern: /except\b/i,
+})
+export const RealizeTok = createToken({
+  name: 'RealizeTok',
+  pattern: /realize\b/i,
+})
+export const InTok = createToken({
+  name: 'InTok',
+  pattern: /in\b/i,
+})
+export const IncludeTok = createToken({
+  name: 'IncludeTok',
+  pattern: /include\b/i,
+})
+export const LambdaTok = createToken({
+  name: 'LambdaTok',
+  pattern: /lambda\b/i,
+})
 
 // Symbols
 export const ArrowTok = createToken({ name: 'ArrowTok', pattern: /->/ })
+export const PipeTok = createToken({ name: 'PipeTok', pattern: /\|/ })
 export const LCurly = createToken({ name: 'LCurly', pattern: /{/ })
 export const RCurly = createToken({ name: 'RCurly', pattern: /}/ })
 export const LBracket = createToken({ name: 'LBracket', pattern: /\[/ })
@@ -64,13 +109,100 @@ export const Dot = createToken({ name: 'Dot', pattern: /\./ })
 // Brace text block (for SELECT { ... })
 export const SelectBlock = createToken({
   name: 'SelectBlock',
-  pattern: /\{[^}]+\}/,
+  line_breaks: true,
+  pattern: (text: string, startOffset: number): CustomPatternMatcherReturn | null => {
+    let i = startOffset
+    if (text[i] !== '{') return null
+    for (i++; i < text.length; i++) {
+      if (text[i] === '\\') {
+        i++
+        continue
+      }
+      if (text[i] === '}') {
+        return [text.slice(startOffset, i + 1)]
+      }
+    }
+    return null
+  },
+})
+
+// Regex literal (JS-style): /pattern/flags
+export const PatternLiteral = createToken({
+  name: 'PatternLiteral',
+  line_breaks: true,
+  pattern: (text: string, startOffset: number): CustomPatternMatcherReturn | null => {
+    if (text[startOffset] !== '/') return null
+    let inClass = false
+    let i = startOffset + 1
+    while (i < text.length) {
+      const ch = text[i]
+      if (ch === '\\') {
+        i += 2
+        continue
+      }
+      if (inClass) {
+        if (ch === ']') inClass = false
+      } else if (ch === '[') {
+        inClass = true
+      } else if (ch === '/') {
+        let j = i + 1
+        while (j < text.length && /[a-z]/.test(text[j] ?? '')) j++
+        return [text.slice(startOffset, j)]
+      }
+      i++
+    }
+    return null
+  },
+})
+
+// Code block: ```language\n...content...```
+export const CodeBlock = createToken({
+  name: 'CodeBlock',
+  line_breaks: true,
+  pattern: (text: string, startOffset: number): CustomPatternMatcherReturn | null => {
+    if (text[startOffset] !== '`' || text[startOffset + 1] !== '`' || text[startOffset + 2] !== '`')
+      return null
+    let i = startOffset + 3
+    // require at least one language identifier character
+    if (i >= text.length || !/[a-zA-Z0-9_]/.test(text[i]!)) return null
+    const langStart = i
+    while (i < text.length && text[i] !== '\n' && text[i] !== '\r') i++
+    if (i === langStart) return null // no language identifier
+    if (i >= text.length) return null
+    // skip line ending
+    if (text[i] === '\r' && text[i + 1] === '\n') i += 2
+    else i++
+    // find closing ```
+    while (i < text.length - 2) {
+      if (text[i] === '\\') {
+        i += 2
+        continue
+      }
+      if (text[i] === '`' && text[i + 1] === '`' && text[i + 2] === '`') {
+        return [text.slice(startOffset, i + 3)]
+      }
+      i++
+    }
+    return null
+  },
 })
 
 // Literals
-export const PathLiteral = createToken({
-  name: 'PathLiteral',
-  pattern: /"([^"\\]|\\.)*"/,
+export const StringLiteral = createToken({
+  name: 'StringLiteral',
+  pattern: /'([^'\\]|\\.)*'|"([^"\\]|\\.)*"/,
+})
+export const NumberLiteral = createToken({
+  name: 'NumberLiteral',
+  pattern: /\d+(\.\d+)?/,
+})
+export const TrueTok = createToken({
+  name: 'TrueTok',
+  pattern: /true\b/i,
+})
+export const FalseTok = createToken({
+  name: 'FalseTok',
+  pattern: /false\b/i,
 })
 
 // Basic objects (native types)
@@ -90,6 +222,14 @@ export const UnitTok = createToken({
   name: 'UnitTok',
   pattern: /unit\b/i,
 })
+export const ConceptTok = createToken({
+  name: 'ConceptTok',
+  pattern: /concept\b/i,
+})
+export const EnvironmentTok = createToken({
+  name: 'EnvironmentTok',
+  pattern: /environment\b/i,
+})
 
 // Identifiers
 export const Identifier = createToken({
@@ -99,6 +239,8 @@ export const Identifier = createToken({
 
 export const allTokens = [
   WhiteSpace,
+  LineComment,
+  BlockComment,
 
   CreateTok,
   AsTok,
@@ -110,8 +252,17 @@ export const allTokens = [
   PackageTok,
   ExecutableTok,
   ModuleTok,
+  EnumTok,
+  UnionTok,
+  IntersectTok,
+  ExceptTok,
+  RealizeTok,
+  InTok,
+  IncludeTok,
+  LambdaTok,
 
   ArrowTok,
+  PipeTok,
   SelectBlock,
   LCurly,
   RCurly,
@@ -123,12 +274,19 @@ export const allTokens = [
   Comma,
   Semicolon,
   Dot,
-  PathLiteral,
+  StringLiteral,
+  NumberLiteral,
+  TrueTok,
+  FalseTok,
+  PatternLiteral,
+  CodeBlock,
 
   StringTok,
   NumberTok,
   BoolTok,
   UnitTok,
+  ConceptTok,
+  EnvironmentTok,
 
   Identifier,
 ]

package/src/parser.ts

@@ -11,8 +11,18 @@ import {
   PackageTok,
   ExecutableTok,
   ModuleTok,
+  EnumTok,
+  UnionTok,
+  IntersectTok,
+  ExceptTok,
+  RealizeTok,
+  InTok,
+  IncludeTok,
+  LambdaTok,
   ArrowTok,
+  PipeTok,
   SelectBlock,
+  CodeBlock,
   LBracket,
   RBracket,
   LParen,
@@ -22,11 +32,17 @@ import {
   Semicolon,
   Dot,
   Identifier,
-  PathLiteral,
+  StringLiteral,
+  NumberLiteral,
+  TrueTok,
+  FalseTok,
+  PatternLiteral,
   StringTok,
   NumberTok,
   BoolTok,
   UnitTok,
+  ConceptTok,
+  EnvironmentTok,
 } from './lexer.js'
 
 export class SpexParser extends CstParser {
@@ -43,6 +59,10 @@ export class SpexParser extends CstParser {
 
   private declaration = this.RULE('declaration', () => {
     this.OR([
+      {
+        GATE: this.BACKTRACK(this.realizeDeclaration),
+        ALT: () => this.SUBRULE(this.realizeDeclaration),
+      },
       {
         GATE: this.BACKTRACK(this.packageDeclaration),
         ALT: () => this.SUBRULE(this.packageDeclaration),
@@ -59,6 +79,10 @@ export class SpexParser extends CstParser {
         GATE: this.BACKTRACK(this.exportDeclaration),
         ALT: () => this.SUBRULE(this.exportDeclaration),
       },
+      {
+        GATE: this.BACKTRACK(this.includeDeclaration),
+        ALT: () => this.SUBRULE(this.includeDeclaration),
+      },
       {
         ALT: () => this.SUBRULE(this.generateDeclaration),
       },
@@ -69,10 +93,50 @@ export class SpexParser extends CstParser {
     this.CONSUME(CreateTok)
     this.CONSUME(Identifier)
     this.CONSUME(AsTok)
-    this.SUBRULE(this.objectExpression)
+    this.SUBRULE(this.setObject)
     this.CONSUME(Semicolon)
   })
 
+  private setObject = this.RULE('setObject', () => {
+    this.SUBRULE(this.coproductObject)
+    this.MANY(() => {
+      this.OR([
+        { ALT: () => this.CONSUME(UnionTok, { LABEL: 'op' }) },
+        { ALT: () => this.CONSUME(IntersectTok, { LABEL: 'op' }) },
+        { ALT: () => this.CONSUME(ExceptTok, { LABEL: 'op' }) },
+      ])
+      this.SUBRULE2(this.coproductObject)
+    })
+  })
+
+  private coproductObject = this.RULE('coproductObject', () => {
+    this.SUBRULE(this.objectExpression)
+    this.MANY(() => {
+      this.CONSUME(PipeTok, { LABEL: 'op' })
+      this.SUBRULE2(this.objectExpression)
+    })
+  })
+
+  private enumObject = this.RULE('enumObject', () => {
+    this.CONSUME(EnumTok)
+    this.CONSUME(LParen)
+    this.CONSUME(StringLiteral)
+    this.MANY(() => {
+      this.CONSUME(Comma)
+      this.CONSUME2(StringLiteral)
+    })
+    this.CONSUME(RParen)
+  })
+
+  private literalObject = this.RULE('literalObject', () => {
+    this.OR([
+      { ALT: () => this.CONSUME(StringLiteral) },
+      { ALT: () => this.CONSUME(NumberLiteral) },
+      { ALT: () => this.CONSUME(TrueTok) },
+      { ALT: () => this.CONSUME(FalseTok) },
+    ])
+  })
+
   private objectExpression = this.RULE('objectExpression', () => {
     this.SUBRULE(this.objectOperand, { LABEL: 'base' })
     this.OPTION(() => {
@@ -83,10 +147,27 @@ export class SpexParser extends CstParser {
 
   private objectOperand = this.RULE('objectOperand', () => {
     this.OR([
+      {
+        ALT: () => this.SUBRULE(this.literalObject),
+      },
+      {
+        ALT: () => this.SUBRULE(this.enumObject),
+      },
+      {
+        ALT: () => this.SUBRULE(this.patternObject),
+      },
+      {
+        GATE: this.BACKTRACK(this.lambdaObject),
+        ALT: () => this.SUBRULE(this.lambdaObject),
+      },
       {
         GATE: this.BACKTRACK(this.subObject),
         ALT: () => this.SUBRULE(this.subObject),
       },
+      {
+        GATE: this.BACKTRACK(this.parenthesizedObject),
+        ALT: () => this.SUBRULE(this.parenthesizedObject),
+      },
       {
         GATE: this.BACKTRACK(this.productObject),
         ALT: () => this.SUBRULE(this.productObject),
@@ -101,6 +182,16 @@ export class SpexParser extends CstParser {
     })
   })
 
+  private parenthesizedObject = this.RULE('parenthesizedObject', () => {
+    this.CONSUME(LParen)
+    this.SUBRULE(this.setObject)
+    this.CONSUME(RParen)
+  })
+
+  private patternObject = this.RULE('patternObject', () => {
+    this.CONSUME(PatternLiteral)
+  })
+
   private namedObject = this.RULE('namedObject', () => {
     this.OR([
       { ALT: () => this.CONSUME(Identifier) },
@@ -108,6 +199,8 @@ export class SpexParser extends CstParser {
       { ALT: () => this.CONSUME(NumberTok) },
       { ALT: () => this.CONSUME(BoolTok) },
       { ALT: () => this.CONSUME(UnitTok) },
+      { ALT: () => this.CONSUME(ConceptTok) },
+      { ALT: () => this.CONSUME(EnvironmentTok) },
     ])
     this.MANY(() => {
       this.CONSUME(Dot)
@@ -120,7 +213,7 @@ export class SpexParser extends CstParser {
     this.MANY(() => {
       this.CONSUME(Identifier)
       this.CONSUME(Colon)
-      this.SUBRULE(this.objectExpression)
+      this.SUBRULE(this.setObject)
       this.OPTION(() => this.CONSUME(Comma))
     })
     this.CONSUME(RParen)
@@ -128,7 +221,7 @@ export class SpexParser extends CstParser {
 
   private subObject = this.RULE('subObject', () => {
     this.CONSUME(FromTok)
-    this.SUBRULE(this.objectExpression, { LABEL: 'base' })
+    this.SUBRULE(this.setObject, { LABEL: 'base' })
     this.CONSUME(SelectTok)
     this.CONSUME(SelectBlock)
   })
@@ -150,7 +243,7 @@ export class SpexParser extends CstParser {
   private namedImport = this.RULE('namedImport', () => {
     this.CONSUME(Identifier)
     this.CONSUME(FromTok)
-    this.CONSUME(PathLiteral)
+    this.CONSUME(StringLiteral)
     this.OPTION(() => {
       this.CONSUME(AsTok)
       this.CONSUME2(Identifier)
@@ -158,7 +251,7 @@ export class SpexParser extends CstParser {
   })
 
   private moduleImport = this.RULE('moduleImport', () => {
-    this.CONSUME(PathLiteral)
+    this.CONSUME(StringLiteral)
     this.CONSUME(AsTok)
     this.CONSUME(Identifier)
   })
@@ -180,7 +273,35 @@ export class SpexParser extends CstParser {
     this.OR([{ ALT: () => this.CONSUME(ExecutableTok) }, { ALT: () => this.CONSUME(ModuleTok) }])
     this.CONSUME(Identifier)
     this.CONSUME(AsTok)
-    this.SUBRULE(this.objectExpression)
+    this.SUBRULE(this.setObject)
     this.CONSUME(Semicolon)
   })
+
+  private realizeDeclaration = this.RULE('realizeDeclaration', () => {
+    this.CONSUME(RealizeTok)
+    this.SUBRULE(this.setObject, { LABEL: 'object' })
+    this.CONSUME(AsTok)
+    this.SUBRULE2(this.setObject, { LABEL: 'target' })
+    this.OPTION(() => {
+      this.CONSUME(InTok)
+      this.SUBRULE3(this.setObject, { LABEL: 'environment' })
+    })
+    this.CONSUME(Semicolon)
+  })
+
+  private includeDeclaration = this.RULE('includeDeclaration', () => {
+    this.CONSUME(IncludeTok)
+    this.CONSUME(StringLiteral)
+    this.CONSUME(AsTok)
+    this.CONSUME(Identifier)
+    this.CONSUME(Semicolon)
+  })
+
+  private lambdaObject = this.RULE('lambdaObject', () => {
+    this.CONSUME(LambdaTok)
+    this.SUBRULE(this.objectOperand, { LABEL: 'base' })
+    this.CONSUME(ArrowTok)
+    this.SUBRULE(this.setObject, { LABEL: 'exponent' })
+    this.CONSUME(CodeBlock)
+  })
 }