spex-parser 0.1.5 → 0.6.2

package/src/visitor.ts

@@ -7,14 +7,21 @@ import type {
   ExportDeclaration,
   GenerateDeclaration,
   PackageDeclaration,
+  RealizeDeclaration,
+  IncludeDeclaration,
+  EnumObject,
+  LiteralObject,
+  SetObject,
+  CoproductObject,
+  PatternLiteralObject,
+  ExponentialPattern,
+  PatternBlock,
   ObjectExpression,
   NamedObject,
-  ProductObject,
   SubObject,
   ArrayObject,
   Constraint,
   ConstraintPart,
-  ConstraintReference,
 } from './ast.js'
 import { SpexLexer } from './lexer.js'
 import { SpexParser } from './parser.js'
@@ -24,6 +31,15 @@ const BaseSpexVisitor = parserInstance.getBaseCstVisitorConstructor()
 
 export const REFERENCE_PATTERN = /@([a-zA-Z_][\w]*(?:\.[a-zA-Z_][\w]*)*)/g
 
+function unescapeConstraint(text: string): string {
+  return text.replace(/\\([\\{}])/g, '$1')
+}
+
+function stringLiteralValue(image: string): string {
+  const inner = image.slice(1, -1)
+  return inner.replace(/\\([\\'"])/g, '$1')
+}
+
 export function parseConstraint(raw: string): Constraint {
   const parts: ConstraintPart[] = []
   let lastIndex = 0
@@ -56,6 +72,9 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
   }
 
   declaration(ctx: any): Declaration {
+    if (ctx.realizeDeclaration) {
+      return this.visit(ctx.realizeDeclaration)
+    }
     if (ctx.packageDeclaration) {
       return this.visit(ctx.packageDeclaration)
     }
@@ -68,17 +87,65 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
     if (ctx.exportDeclaration) {
       return this.visit(ctx.exportDeclaration)
     }
+    if (ctx.includeDeclaration) {
+      return this.visit(ctx.includeDeclaration)
+    }
     return this.visit(ctx.generateDeclaration)
   }
 
+  enumObject(ctx: any): EnumObject {
+    return {
+      kind: 'EnumObject',
+      values: ctx.StringLiteral.map((s: any) => stringLiteralValue(s.image)),
+    }
+  }
+
+  literalObject(ctx: any): LiteralObject {
+    if (ctx.StringLiteral) {
+      return {
+        kind: 'StringLiteralObject',
+        value: stringLiteralValue(ctx.StringLiteral[0].image),
+      }
+    }
+    if (ctx.NumberLiteral) {
+      return { kind: 'NumberLiteralObject', value: ctx.NumberLiteral[0].image }
+    }
+    return { kind: 'BoolLiteralObject', value: ctx.TrueTok ? true : false }
+  }
+
   objectDeclaration(ctx: any): ObjectDeclaration {
     return {
       kind: 'ObjectDeclaration',
       name: ctx.Identifier[0].image,
-      object: this.visit(ctx.objectExpression),
+      object: this.visit(ctx.setObject),
     }
   }
 
+  setObject(ctx: any): SetObject {
+    const operands = ctx.coproductObject.map((expr: any) => this.visit(expr))
+    let result: ObjectExpression = operands[0]
+    for (let i = 1; i < operands.length; i++) {
+      const op = ctx.op[i - 1].tokenType.name
+      if (op === 'UnionTok') {
+        result = { kind: 'SetUnionObject', left: result, right: operands[i] }
+      } else if (op === 'IntersectTok') {
+        result = { kind: 'SetIntersectionObject', left: result, right: operands[i] }
+      } else {
+        result = { kind: 'SetDifferenceObject', left: result, right: operands[i] }
+      }
+    }
+    return result as SetObject
+  }
+
+  coproductObject(ctx: any): CoproductObject {
+    const operands = ctx.objectExpression.map((expr: any) => this.visit(expr))
+    let result: ObjectExpression = operands[0]
+    for (let i = 1; i < operands.length; i++) {
+      result = { kind: 'CoproductObject', left: result, right: operands[i] }
+    }
+    return result as CoproductObject
+  }
+
   objectExpression(ctx: any): ObjectExpression {
     if (ctx.base) {
       const exponent = this.visit(ctx.base)
@@ -98,8 +165,18 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
     let expr: ObjectExpression
     if (ctx.subObject) {
       expr = this.visit(ctx.subObject)
+    } else if (ctx.parenthesizedObject) {
+      expr = this.visit(ctx.parenthesizedObject)
     } else if (ctx.productObject) {
       expr = this.visit(ctx.productObject)
+    } else if (ctx.enumObject) {
+      expr = this.visit(ctx.enumObject)
+    } else if (ctx.patternObject) {
+      expr = this.visit(ctx.patternObject)
+    } else if (ctx.lambdaObject) {
+      expr = this.visit(ctx.lambdaObject)
+    } else if (ctx.literalObject) {
+      expr = this.visit(ctx.literalObject)
     } else {
       expr = this.visit(ctx.namedObject)
     }
@@ -111,6 +188,20 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
     return expr
   }
 
+  parenthesizedObject(ctx: any): ObjectExpression {
+    return this.visit(ctx.setObject)
+  }
+
+  patternObject(ctx: any): PatternLiteralObject {
+    const image: string = ctx.PatternLiteral[0].image
+    const lastSlash = image.lastIndexOf('/')
+    return {
+      kind: 'PatternLiteralObject',
+      source: image.slice(1, lastSlash),
+      flags: image.slice(lastSlash + 1),
+    }
+  }
+
   namedObject(ctx: any): NamedObject {
     let parts: string[]
     if (ctx.StringTok) {
@@ -121,6 +212,10 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
       parts = [ctx.BoolTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
     } else if (ctx.UnitTok) {
       parts = [ctx.UnitTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
+    } else if (ctx.ConceptTok) {
+      parts = [ctx.ConceptTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
+    } else if (ctx.EnvironmentTok) {
+      parts = [ctx.EnvironmentTok[0].image, ...(ctx.Identifier ?? []).map((id: any) => id.image)]
     } else {
       parts = ctx.Identifier.map((id: any) => id.image)
     }
@@ -130,18 +225,26 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
     }
   }
 
-  productObject(ctx: any): ProductObject {
+  productObject(ctx: any): ObjectExpression {
+    const names = ctx.Identifier ?? []
     const fields: Record<string, ObjectExpression> = {}
-    for (let i = 0; i < ctx.Identifier.length; i++) {
-      const name = ctx.Identifier[i].image
-      fields[name] = this.visit(ctx.objectExpression[i])
+    for (let i = 0; i < names.length; i++) {
+      const name = names[i].image
+      const value = this.visit(ctx.setObject[i])
+      if (value.kind === 'NamedObject' && value.name === 'unit') {
+        continue
+      }
+      fields[name] = value
+    }
+    if (Object.keys(fields).length === 0) {
+      return { kind: 'NamedObject', name: 'unit' }
     }
     return { kind: 'ProductObject', fields }
   }
 
   subObject(ctx: any): SubObject {
     const rawText: string = ctx.SelectBlock[0].image
-    const rawConstraint = rawText.slice(1, -1).trim()
+    const rawConstraint = unescapeConstraint(rawText.slice(1, -1).trim())
     return {
       kind: 'SubObject',
       base: this.visit(ctx.base),
@@ -158,7 +261,7 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
 
   namedImport(ctx: any): ImportDeclaration {
     const name = ctx.Identifier[0].image
-    const source = ctx.PathLiteral[0].image.slice(1, -1)
+    const source = stringLiteralValue(ctx.StringLiteral[0].image)
     const alias = ctx.Identifier[1] ? ctx.Identifier[1].image : null
     return {
       kind: 'ImportDeclaration',
@@ -169,7 +272,7 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
   }
 
   moduleImport(ctx: any): ImportDeclaration {
-    const source = ctx.PathLiteral[0].image.slice(1, -1)
+    const source = stringLiteralValue(ctx.StringLiteral[0].image)
     const alias = ctx.Identifier[0].image
     return {
       kind: 'ImportDeclaration',
@@ -192,7 +295,8 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
       kind: 'PackageDeclaration',
       packageType,
       name: ctx.Identifier[0].image,
-      objectName: this.visit(ctx.objectExpression),
+      objectName: this.visit(ctx.setObject),
+      environment: this.visit(ctx.environment),
     }
   }
 
@@ -202,10 +306,91 @@ export class SpexParserVisitor extends BaseSpexVisitor implements ICstVisitor<an
       name: ctx.Identifier[0].image,
     }
   }
+
+  realizeDeclaration(ctx: any): RealizeDeclaration {
+    return {
+      kind: 'RealizeDeclaration',
+      object: this.visit(ctx.object),
+      target: this.visit(ctx.target),
+      environment: ctx.environment
+        ? this.visit(ctx.environment)
+        : { kind: 'NamedObject', name: 'environment' },
+    }
+  }
+
+  includeDeclaration(ctx: any): IncludeDeclaration {
+    const address = stringLiteralValue(ctx.StringLiteral[0].image)
+    const name = ctx.Identifier[0].image
+    return {
+      kind: 'IncludeDeclaration',
+      name,
+      address,
+    }
+  }
+
+  lambdaObject(ctx: any): ExponentialPattern {
+    const image: string = ctx.CodeBlock[0].image
+    const firstNewline = image.indexOf('\n')
+    const langEnd = image.indexOf('\n')
+    const language = image.slice(3, langEnd)
+    const body = image.slice(firstNewline + 1, -3).trimEnd()
+
+    const patterns: PatternBlock[] = []
+    let i = 0
+    while (i < body.length) {
+      if (body[i] === '@' && body[i + 1] === '{') {
+        const start = i
+        let depth = 1
+        i += 2
+        while (i < body.length && depth > 0) {
+          if (body[i] === '\\') {
+            i += 2
+            continue
+          }
+          if (body[i] === '{') depth++
+          if (body[i] === '}') depth--
+          i++
+        }
+        const end = i
+        const raw = body.slice(start + 2, end - 1).trim()
+        patterns.push({
+          raw,
+          parts: parseConstraint(raw).parts,
+          start,
+          end,
+        })
+      } else {
+        i++
+      }
+    }
+
+    return {
+      kind: 'ExponentialPattern',
+      base: this.visit(ctx.exponent),
+      exponent: this.visit(ctx.base),
+      language,
+      body,
+      patterns,
+    }
+  }
 }
 
 export function parseToAst(text: string): SpexFile {
   const lexingResult = SpexLexer.tokenize(text)
+
+  if (lexingResult.errors.length > 0) {
+    const details = lexingResult.errors
+      .map((e) => {
+        const where =
+          e.line !== undefined && e.column !== undefined
+            ? ` (line ${e.line}, column ${e.column})`
+            : ''
+        return e.message + where
+      })
+      .join('; ')
+    throw new Error(`Lexing errors: ${details}`)
+  }
+
   parserInstance.input = lexingResult.tokens
   const cst = parserInstance.spexFile()
 

package/tests/constants.test.ts

@@ -0,0 +1,102 @@
+import { describe, it, expect } from 'vitest'
+import { parseToAst } from '../src/visitor.js'
+import { constantOf } from '../src/constants.js'
+import type { ObjectDeclaration } from '../src/ast.js'
+
+function firstObject(text: string) {
+  const ast = parseToAst(text)
+  return (ast.declarations[0] as ObjectDeclaration).object
+}
+
+describe('constantOf', () => {
+  it('should identify string literals as constants', () => {
+    expect(constantOf(firstObject('create K as "SpexFile";'))).toEqual({
+      kind: 'StringConstant',
+      value: 'SpexFile',
+    })
+  })
+
+  it('should identify number literals as constants', () => {
+    expect(constantOf(firstObject('create K as 42;'))).toEqual({
+      kind: 'NumberConstant',
+      value: '42',
+    })
+  })
+
+  it('should identify bool literals as constants', () => {
+    expect(constantOf(firstObject('create K as true;'))).toEqual({
+      kind: 'BoolConstant',
+      value: true,
+    })
+    expect(constantOf(firstObject('create K as false;'))).toEqual({
+      kind: 'BoolConstant',
+      value: false,
+    })
+  })
+
+  it('should identify products of constants as constants', () => {
+    expect(constantOf(firstObject('create K as (name: "John", age: 42);'))).toEqual({
+      kind: 'ProductConstant',
+      fields: {
+        name: { kind: 'StringConstant', value: 'John' },
+        age: { kind: 'NumberConstant', value: '42' },
+      },
+    })
+  })
+
+  it('should reject products with non-constant fields', () => {
+    expect(constantOf(firstObject('create K as (name: "John", age: number);'))).toBeNull()
+  })
+
+  it('should identify exponentials of constants as constants', () => {
+    expect(constantOf(firstObject('create K as "a" -> "b";'))).toEqual({
+      kind: 'ExponentialConstant',
+      base: { kind: 'StringConstant', value: 'b' },
+      exponent: { kind: 'StringConstant', value: 'a' },
+    })
+  })
+
+  it('should reject exponentials with non-constant sides', () => {
+    expect(constantOf(firstObject('create K as number -> "b";'))).toBeNull()
+  })
+
+  it('should reject nested non-constants', () => {
+    expect(
+      constantOf(firstObject('create K as (a: (b: "x", c: string));'))
+    ).toBeNull()
+  })
+
+  it('should reject named types', () => {
+    expect(constantOf(firstObject('create K as string;'))).toBeNull()
+  })
+
+  it('should reject subobjects', () => {
+    expect(constantOf(firstObject('create K as from string select { are positive };'))).toBeNull()
+  })
+
+  it('should reject enums', () => {
+    expect(constantOf(firstObject("create K as enum ('a', 'b');"))).toBeNull()
+  })
+
+  it('should reject arrays', () => {
+    expect(constantOf(firstObject('create K as string[];'))).toBeNull()
+  })
+
+  it('should reject set objects', () => {
+    expect(constantOf(firstObject('create K as "a" UNION "b";'))).toBeNull()
+    expect(constantOf(firstObject('create K as string EXCEPT "root";'))).toBeNull()
+  })
+
+  it('should treat the empty product the same as unit', () => {
+    expect(constantOf(firstObject('create K as ();'))).toBeNull()
+    expect(constantOf(firstObject('create K as unit;'))).toBeNull()
+  })
+
+  it('should reject patterns', () => {
+    expect(constantOf(firstObject('create K as /\\d+/;'))).toBeNull()
+  })
+
+  it('should reject coproducts', () => {
+    expect(constantOf(firstObject('create K as "a" | "b";'))).toBeNull()
+  })
+})

package/tests/e2e.test.ts

@@ -7,11 +7,23 @@ import { parseToAst } from '../src/visitor.js'
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = dirname(__filename)
 
+const fixtures: [string, number][] = [
+  ['todo.spex', 32],
+  ['python_cli_env.spex', 4],
+  ['typescript_cli_env.spex', 4],
+  ['flask_web_env.spex', 4],
+  ['express_web_env.spex', 4],
+  ['python_todo_cli.spex', 16],
+  ['typescript_todo_cli.spex', 16],
+  ['flask_todo_web.spex', 22],
+  ['express_todo_web.spex', 22],
+]
+
 describe('end-to-end', () => {
-  it('should parse todo.spex file', () => {
-    const code = readFileSync(join(__dirname, 'props/todo.spex'), 'utf-8')
+  it.each(fixtures)('should parse %s', (file, expectedDeclarations) => {
+    const code = readFileSync(join(__dirname, 'props', file), 'utf-8')
     const ast = parseToAst(code)
     expect(ast.kind).toBe('SpexFile')
-    expect(ast.declarations.length).toBe(15)
+    expect(ast.declarations.length).toBe(expectedDeclarations)
   })
 })