spex-parser 0.6.2 → 0.7.0

package/src/parser.ts

@@ -7,18 +7,12 @@ import {
   SelectTok,
   GenerateTok,
   ImportTok,
-  ExportTok,
-  PackageTok,
-  ExecutableTok,
-  ModuleTok,
-  EnumTok,
   UnionTok,
   IntersectTok,
   ExceptTok,
   RealizeTok,
   InTok,
   IncludeTok,
-  LambdaTok,
   ArrowTok,
   PipeTok,
   SelectBlock,
@@ -41,6 +35,7 @@ import {
   NumberTok,
   BoolTok,
   UnitTok,
+  ArtifactTok,
   ConceptTok,
   EnvironmentTok,
 } from './lexer.js'
@@ -63,10 +58,6 @@ export class SpexParser extends CstParser {
         GATE: this.BACKTRACK(this.realizeDeclaration),
         ALT: () => this.SUBRULE(this.realizeDeclaration),
       },
-      {
-        GATE: this.BACKTRACK(this.packageDeclaration),
-        ALT: () => this.SUBRULE(this.packageDeclaration),
-      },
       {
         GATE: this.BACKTRACK(this.objectDeclaration),
         ALT: () => this.SUBRULE(this.objectDeclaration),
@@ -75,10 +66,6 @@ export class SpexParser extends CstParser {
         GATE: this.BACKTRACK(this.importDeclaration),
         ALT: () => this.SUBRULE(this.importDeclaration),
       },
-      {
-        GATE: this.BACKTRACK(this.exportDeclaration),
-        ALT: () => this.SUBRULE(this.exportDeclaration),
-      },
       {
         GATE: this.BACKTRACK(this.includeDeclaration),
         ALT: () => this.SUBRULE(this.includeDeclaration),
@@ -117,17 +104,6 @@ export class SpexParser extends CstParser {
     })
   })
 
-  private enumObject = this.RULE('enumObject', () => {
-    this.CONSUME(EnumTok)
-    this.CONSUME(LParen)
-    this.CONSUME(StringLiteral)
-    this.MANY(() => {
-      this.CONSUME(Comma)
-      this.CONSUME2(StringLiteral)
-    })
-    this.CONSUME(RParen)
-  })
-
   private literalObject = this.RULE('literalObject', () => {
     this.OR([
       { ALT: () => this.CONSUME(StringLiteral) },
@@ -150,16 +126,9 @@ export class SpexParser extends CstParser {
       {
         ALT: () => this.SUBRULE(this.literalObject),
       },
-      {
-        ALT: () => this.SUBRULE(this.enumObject),
-      },
       {
         ALT: () => this.SUBRULE(this.patternObject),
       },
-      {
-        GATE: this.BACKTRACK(this.lambdaObject),
-        ALT: () => this.SUBRULE(this.lambdaObject),
-      },
       {
         GATE: this.BACKTRACK(this.subObject),
         ALT: () => this.SUBRULE(this.subObject),
@@ -199,6 +168,7 @@ export class SpexParser extends CstParser {
       { ALT: () => this.CONSUME(NumberTok) },
       { ALT: () => this.CONSUME(BoolTok) },
       { ALT: () => this.CONSUME(UnitTok) },
+      { ALT: () => this.CONSUME(ArtifactTok) },
       { ALT: () => this.CONSUME(ConceptTok) },
       { ALT: () => this.CONSUME(EnvironmentTok) },
     ])
@@ -223,7 +193,7 @@ export class SpexParser extends CstParser {
     this.CONSUME(FromTok)
     this.SUBRULE(this.setObject, { LABEL: 'base' })
     this.CONSUME(SelectTok)
-    this.CONSUME(SelectBlock)
+    this.OR([{ ALT: () => this.CONSUME(SelectBlock) }, { ALT: () => this.CONSUME(CodeBlock) }])
   })
 
   private importDeclaration = this.RULE('importDeclaration', () => {
@@ -256,26 +226,13 @@ export class SpexParser extends CstParser {
     this.CONSUME(Identifier)
   })
 
-  private exportDeclaration = this.RULE('exportDeclaration', () => {
-    this.CONSUME(ExportTok)
-    this.CONSUME(Identifier)
-    this.CONSUME(Semicolon)
-  })
-
   private generateDeclaration = this.RULE('generateDeclaration', () => {
     this.CONSUME(GenerateTok)
     this.CONSUME(Identifier)
-    this.CONSUME(Semicolon)
-  })
-
-  private packageDeclaration = this.RULE('packageDeclaration', () => {
-    this.CONSUME(PackageTok)
-    this.OR([{ ALT: () => this.CONSUME(ExecutableTok) }, { ALT: () => this.CONSUME(ModuleTok) }])
-    this.CONSUME(Identifier)
-    this.CONSUME(AsTok)
-    this.SUBRULE(this.setObject)
-    this.CONSUME(InTok)
-    this.SUBRULE2(this.setObject, { LABEL: 'environment' })
+    this.OPTION(() => {
+      this.CONSUME(InTok)
+      this.SUBRULE(this.setObject, { LABEL: 'environment' })
+    })
     this.CONSUME(Semicolon)
   })
 
@@ -298,12 +255,4 @@ export class SpexParser extends CstParser {
     this.CONSUME(Identifier)
     this.CONSUME(Semicolon)
   })
-
-  private lambdaObject = this.RULE('lambdaObject', () => {
-    this.CONSUME(LambdaTok)
-    this.SUBRULE(this.objectOperand, { LABEL: 'base' })
-    this.CONSUME(ArrowTok)
-    this.SUBRULE(this.setObject, { LABEL: 'exponent' })
-    this.CONSUME(CodeBlock)
-  })
 }

package/src/visitor.ts

@@ -4,21 +4,17 @@ import type {
   Declaration,
   ObjectDeclaration,
   ImportDeclaration,
-  ExportDeclaration,
   GenerateDeclaration,
-  PackageDeclaration,
   RealizeDeclaration,
   IncludeDeclaration,
-  EnumObject,
   LiteralObject,
   SetObject,
   CoproductObject,
   PatternLiteralObject,
-  ExponentialPattern,
-  PatternBlock,
   ObjectExpression,
   NamedObject,
   SubObject,
+  SubObjectConstraint,
   ArrayObject,
   Constraint,
   ConstraintPart,
@@ -49,7 +45,7 @@ export function parseConstraint(raw: string): Constraint {
     if (match.index > lastIndex) {
       parts.push({ kind: 'ConstraintText', text: raw.slice(lastIndex, match.index) })
     }
-    parts.push({ kind: 'ConstraintReference', name: match[1]! })
+    parts.push({ kind: 'ReferenceDirective', name: match[1]! })
     lastIndex = match.index + match[0].length
   }
 
@@ -60,6 +56,35 @@ export function parseConstraint(raw: string): Constraint {
   return { raw, parts }
 }
 
+function extractCodeLanguage(image: string): string {
+  const langEnd = image.indexOf('\n')
+  if (langEnd === -1) return ''
+  return image.slice(3, langEnd).trim()
+}
+
+function extractCodeBody(image: string): string {
+  const firstNewline = image.indexOf('\n')
+  return image.slice(firstNewline + 1, -3).trimEnd()
+}
+
+function codeConstraint(image: string): SubObjectConstraint {
+  const language = extractCodeLanguage(image)
+  const body = extractCodeBody(image)
+  if (language === '') {
+    return {
+      type: 'Structured',
+      raw: body,
+      parts: parseConstraint(body).parts,
+    }
+  }
+  return {
+    type: 'Code',
+    language,
+    body,
+    parts: parseConstraint(body).parts,
+  }
+}
+
 export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<any, any> {
   constructor() {
     super()
@@ -75,31 +100,18 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
     if (ctx.realizeDeclaration) {
       return this.visit(ctx.realizeDeclaration)
     }
-    if (ctx.packageDeclaration) {
-      return this.visit(ctx.packageDeclaration)
-    }
     if (ctx.objectDeclaration) {
       return this.visit(ctx.objectDeclaration)
     }
     if (ctx.importDeclaration) {
       return this.visit(ctx.importDeclaration)
     }
-    if (ctx.exportDeclaration) {
-      return this.visit(ctx.exportDeclaration)
-    }
     if (ctx.includeDeclaration) {
       return this.visit(ctx.includeDeclaration)
     }
     return this.visit(ctx.generateDeclaration)
   }
 
-  enumObject(ctx: any): EnumObject {
-    return {
-      kind: 'EnumObject',
-      values: ctx.StringLiteral.map((s: any) => stringLiteralValue(s.image)),
-    }
-  }
-
   literalObject(ctx: any): LiteralObject {
     if (ctx.StringLiteral) {
       return {
@@ -169,12 +181,8 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
       expr = this.visit(ctx.parenthesizedObject)
     } else if (ctx.productObject) {
       expr = this.visit(ctx.productObject)
-    } else if (ctx.enumObject) {
-      expr = this.visit(ctx.enumObject)
     } else if (ctx.patternObject) {
       expr = this.visit(ctx.patternObject)
-    } else if (ctx.lambdaObject) {
-      expr = this.visit(ctx.lambdaObject)
     } else if (ctx.literalObject) {
       expr = this.visit(ctx.literalObject)
     } else {
@@ -212,6 +220,8 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
       parts = [ctx.BoolTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
     } else if (ctx.UnitTok) {
       parts = [ctx.UnitTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
+    } else if (ctx.ArtifactTok) {
+      parts = [ctx.ArtifactTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
     } else if (ctx.ConceptTok) {
       parts = [ctx.ConceptTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
     } else if (ctx.EnvironmentTok) {
@@ -243,12 +253,24 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
   }
 
   subObject(ctx: any): SubObject {
+    const base = this.visit(ctx.base)
+    if (ctx.CodeBlock) {
+      return {
+        kind: 'SubObject',
+        base,
+        constraint: codeConstraint(ctx.CodeBlock[0].image),
+      }
+    }
     const rawText: string = ctx.SelectBlock[0].image
     const rawConstraint = unescapeConstraint(rawText.slice(1, -1).trim())
     return {
       kind: 'SubObject',
-      base: this.visit(ctx.base),
-      constraint: parseConstraint(rawConstraint),
+      base,
+      constraint: {
+        type: 'NaturalLanguage',
+        raw: rawConstraint,
+        parts: parseConstraint(rawConstraint).parts,
+      },
     }
   }
 
@@ -282,28 +304,13 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
     }
   }
 
-  exportDeclaration(ctx: any): ExportDeclaration {
-    return {
-      kind: 'ExportDeclaration',
-      name: ctx.Identifier[0].image,
-    }
-  }
-
-  packageDeclaration(ctx: any): PackageDeclaration {
-    const packageType = ctx.ExecutableTok ? ('EXECUTABLE' as const) : ('MODULE' as const)
-    return {
-      kind: 'PackageDeclaration',
-      packageType,
-      name: ctx.Identifier[0].image,
-      objectName: this.visit(ctx.setObject),
-      environment: this.visit(ctx.environment),
-    }
-  }
-
   generateDeclaration(ctx: any): GenerateDeclaration {
     return {
       kind: 'GenerateDeclaration',
       name: ctx.Identifier[0].image,
+      environment: ctx.environment
+        ? this.visit(ctx.environment)
+        : { kind: 'NamedObject', name: 'environment' },
     }
   }
 
@@ -327,52 +334,6 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
       address,
     }
   }
-
-  lambdaObject(ctx: any): ExponentialPattern {
-    const image: string = ctx.CodeBlock[0].image
-    const firstNewline = image.indexOf('\n')
-    const langEnd = image.indexOf('\n')
-    const language = image.slice(3, langEnd)
-    const body = image.slice(firstNewline + 1, -3).trimEnd()
-
-    const patterns: PatternBlock[] = []
-    let i = 0
-    while (i < body.length) {
-      if (body[i] === '@' && body[i + 1] === '{') {
-        const start = i
-        let depth = 1
-        i += 2
-        while (i < body.length && depth > 0) {
-          if (body[i] === '\\') {
-            i += 2
-            continue
-          }
-          if (body[i] === '{') depth++
-          if (body[i] === '}') depth--
-          i++
-        }
-        const end = i
-        const raw = body.slice(start + 2, end - 1).trim()
-        patterns.push({
-          raw,
-          parts: parseConstraint(raw).parts,
-          start,
-          end,
-        })
-      } else {
-        i++
-      }
-    }
-
-    return {
-      kind: 'ExponentialPattern',
-      base: this.visit(ctx.exponent),
-      exponent: this.visit(ctx.base),
-      language,
-      body,
-      patterns,
-    }
-  }
 }
 
 export function parseToAst(text: string): SpexFile {

package/tests/constants.test.ts

@@ -74,10 +74,6 @@ describe('constantOf', () => {
     expect(constantOf(firstObject('create K as from string select { are positive };'))).toBeNull()
   })
 
-  it('should reject enums', () => {
-    expect(constantOf(firstObject("create K as enum ('a', 'b');"))).toBeNull()
-  })
-
   it('should reject arrays', () => {
     expect(constantOf(firstObject('create K as string[];'))).toBeNull()
   })

package/tests/e2e.test.ts

@@ -8,15 +8,15 @@ const __filename = fileURLToPath(import.meta.url)
 const __dirname = dirname(__filename)
 
 const fixtures: [string, number][] = [
-  ['todo.spex', 32],
-  ['python_cli_env.spex', 4],
-  ['typescript_cli_env.spex', 4],
-  ['flask_web_env.spex', 4],
-  ['express_web_env.spex', 4],
-  ['python_todo_cli.spex', 16],
-  ['typescript_todo_cli.spex', 16],
-  ['flask_todo_web.spex', 22],
-  ['express_todo_web.spex', 22],
+  ['todo.spex', 17],
+  ['python_cli_env.spex', 3],
+  ['typescript_cli_env.spex', 3],
+  ['flask_web_env.spex', 3],
+  ['express_web_env.spex', 3],
+  ['python_todo_cli.spex', 15],
+  ['typescript_todo_cli.spex', 15],
+  ['flask_todo_web.spex', 21],
+  ['express_todo_web.spex', 21],
 ]
 
 describe('end-to-end', () => {

package/tests/lexer.test.ts

@@ -5,7 +5,7 @@ describe('SpexLexer', () => {
   describe('tokenization', () => {
     it('should tokenize keywords', () => {
       const result = SpexLexer.tokenize(
-        'create as from select generate import export package executable module enum realize in'
+        'create as from select generate import realize in'
       )
       expect(result.errors).toHaveLength(0)
       expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
@@ -15,11 +15,6 @@ describe('SpexLexer', () => {
         'SelectTok',
         'GenerateTok',
         'ImportTok',
-        'ExportTok',
-        'PackageTok',
-        'ExecutableTok',
-        'ModuleTok',
-        'EnumTok',
         'RealizeTok',
         'InTok',
       ])
@@ -27,7 +22,7 @@ describe('SpexLexer', () => {
 
     it('should tokenize keywords case-insensitively', () => {
       const result = SpexLexer.tokenize(
-        'CREATE AS FROM SELECT GENERATE IMPORT EXPORT PACKAGE EXECUTABLE MODULE ENUM UNION INTERSECT EXCEPT REALIZE IN'
+        'CREATE AS FROM SELECT GENERATE IMPORT UNION INTERSECT EXCEPT REALIZE IN'
       )
       expect(result.errors).toHaveLength(0)
       expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
@@ -37,11 +32,6 @@ describe('SpexLexer', () => {
         'SelectTok',
         'GenerateTok',
         'ImportTok',
-        'ExportTok',
-        'PackageTok',
-        'ExecutableTok',
-        'ModuleTok',
-        'EnumTok',
         'UnionTok',
         'IntersectTok',
         'ExceptTok',
@@ -50,23 +40,6 @@ describe('SpexLexer', () => {
       ])
     })
 
-    it('should tokenize enum object declarations', () => {
-      const result = SpexLexer.tokenize("create myEnum as enum ('v1', 'v2');")
-      expect(result.errors).toHaveLength(0)
-      expect(result.tokens.map((t) => t.tokenType.name)).toEqual([
-        'CreateTok',
-        'Identifier',
-        'AsTok',
-        'EnumTok',
-        'LParen',
-        'StringLiteral',
-        'Comma',
-        'StringLiteral',
-        'RParen',
-        'Semicolon',
-      ])
-    })
-
     it('should tokenize symbols', () => {
       const result = SpexLexer.tokenize('->[]():;,.|')
       expect(result.errors).toHaveLength(0)
@@ -120,10 +93,10 @@ describe('SpexLexer', () => {
 
     it('should handle keywords with word boundary', () => {
       const result = SpexLexer.tokenize(
-        'createfoo foocreate asfoo fooas fooselect selectfoo foofrom fromfoo generatefoo foogenerate importfoo fooimport exportfoo fooexport packagefoo fopackage executablefoo foexecutable modulefoo fomodule conceptfoo fooconcept environmentfoo fooenvironment realizefoo foorealize info fooin'
+        'createfoo foocreate asfoo fooas fooselect selectfoo foofrom fromfoo generatefoo foogenerate importfoo fooimport artifactfoo fooartifact conceptfoo fooconcept environmentfoo fooenvironment realizefoo foorealize info fooin'
       )
       expect(result.errors).toHaveLength(0)
-      expect(result.tokens.map((t) => t.tokenType.name)).toEqual(Array(28).fill('Identifier'))
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual(Array(22).fill('Identifier'))
     })
 
     it('should tokenize the concept and environment base types', () => {
@@ -135,6 +108,12 @@ describe('SpexLexer', () => {
       ])
     })
 
+    it('should tokenize the artifact base type', () => {
+      const result = SpexLexer.tokenize('artifact')
+      expect(result.errors).toHaveLength(0)
+      expect(result.tokens.map((t) => t.tokenType.name)).toEqual(['ArtifactTok'])
+    })
+
     it('should tokenize the text between braces', () => {
       const result = SpexLexer.tokenize('{hello\nworld}')
       expect(result.errors).toHaveLength(0)