wesl 0.6.0-pre10

package/src/parse/WeslStream.ts

@@ -0,0 +1,279 @@
+import {
+  CachingStream,
+  MatchersStream,
+  matchOneOf,
+  ParseError,
+  RegexMatchers,
+  Stream,
+  TypedToken,
+  withStreamAction,
+} from "mini-parse";
+import { keywords, reservedWords } from "./Keywords";
+export type WeslTokenKind = "word" | "keyword" | "number" | "symbol";
+
+export type WeslToken<Kind extends WeslTokenKind = WeslTokenKind> =
+  TypedToken<Kind>;
+
+// https://www.w3.org/TR/WGSL/#blankspace-and-line-breaks
+/** Whitespaces including new lines */
+const blankspaces = /[ \t\n\v\f\r\u{0085}\u{200E}\u{200F}\u{2028}\u{2029}]+/u;
+const symbolSet =
+  "& && -> @ / ! [ ] { } :: : , == = != >>= >> >= > <<= << <= < % - --" +
+  " . + ++ | || ( ) ; * ~ ^ // /* */ += -= *= /= %= &= |= ^=" +
+  // For the _ = expr; syntax
+  " _";
+
+const ident =
+  /(?:(?:[_\p{XID_Start}][\p{XID_Continue}]+)|(?:[\p{XID_Start}]))/u;
+
+/** Checks if a word is a valid WGSL ident, and not a keyword */
+export function isIdent(text: string): boolean {
+  if (text.match(ident)?.[0] !== text) {
+    return false;
+  }
+  if (keywordOrReserved.has(text)) {
+    return false;
+  }
+  return true;
+}
+
+const keywordOrReserved = new Set(keywords.concat(reservedWords));
+
+const digits = new RegExp(
+  // decimal_float_literal
+  /(?:0[fh])|(?:[1-9][0-9]*[fh])/.source +
+    /|(?:[0-9]*\.[0-9]+(?:[eE][+-]?[0-9]+)?[fh]?)/.source +
+    /|(?:[0-9]+\.[0-9]*(?:[eE][+-]?[0-9]+)?[fh]?)/.source +
+    /|(?:[0-9]+[eE][+-]?[0-9]+[fh]?)/.source +
+    // hex_float_literal
+    /|(?:0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+(?:[pP][+-]?[0-9]+[fh]?)?)/.source +
+    /|(?:0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*(?:[pP][+-]?[0-9]+[fh]?)?)/.source +
+    /|(?:0[xX][0-9a-fA-F]+[pP][+-]?[0-9]+[fh]?)/.source +
+    // hex_int_literal
+    /|(?:0[xX][0-9a-fA-F]+[iu]?)/.source +
+    // decimal_int_literal
+    /|(?:0[iu]?)|(?:[1-9][0-9]*[iu]?)/.source,
+);
+
+const commentStart = /\/\/|\/\*/;
+
+type InternalTokenKind =
+  | "word"
+  | "number"
+  | "blankspaces"
+  | "commentStart"
+  | "symbol"
+  | "invalid";
+const weslMatcher = new RegexMatchers<InternalTokenKind>({
+  word: ident,
+  number: digits,
+  blankspaces,
+  commentStart,
+  symbol: matchOneOf(symbolSet),
+  invalid: /[^]/,
+});
+
+/** To mark parts of the grammar implementation that are WESL specific extensions */
+export function weslExtension<T>(combinator: T): T {
+  return combinator;
+}
+
+export class WeslStream implements Stream<WeslToken> {
+  private stream: Stream<TypedToken<InternalTokenKind>>;
+  /** New line */
+  private eolPattern = /[\n\v\f\u{0085}\u{2028}\u{2029}]|\r\n?/gu;
+  /** Block comments */
+  private blockCommentPattern = /\/\*|\*\//g;
+  constructor(public src: string) {
+    this.stream = new CachingStream(new MatchersStream(src, weslMatcher));
+  }
+  checkpoint(): number {
+    return this.stream.checkpoint();
+  }
+  reset(position: number): void {
+    this.stream.reset(position);
+  }
+  nextToken(): WeslToken | null {
+    while (true) {
+      const token = this.stream.nextToken();
+      if (token === null) return null;
+
+      const kind = token.kind;
+      if (kind === "blankspaces") {
+        continue;
+      } else if (kind === "commentStart") {
+        // SAFETY: The underlying streams can be seeked to any position
+        if (token.text === "//") {
+          this.stream.reset(this.skipToEol(token.span[1]));
+        } else {
+          this.stream.reset(this.skipBlockComment(token.span[1]));
+        }
+      } else if (kind === "word") {
+        let returnToken = token as TypedToken<typeof kind | "keyword">;
+        if (keywordOrReserved.has(token.text)) {
+          returnToken.kind = "keyword";
+        }
+        return returnToken;
+      } else if (kind === "invalid") {
+        throw new ParseError(
+          "Invalid token " + token.text,
+          this.stream.checkpoint(),
+        );
+      } else {
+        return token as TypedToken<typeof kind>;
+      }
+    }
+  }
+
+  private skipToEol(position: number): number {
+    this.eolPattern.lastIndex = position;
+    const result = this.eolPattern.exec(this.src);
+    if (result === null) {
+      // We reached the end of the file
+      return this.src.length;
+    } else {
+      // Move forward
+      return this.eolPattern.lastIndex;
+    }
+  }
+
+  private skipBlockComment(position: number): number {
+    while (true) {
+      this.blockCommentPattern.lastIndex = position;
+      const result = this.blockCommentPattern.exec(this.src);
+      if (result === null) {
+        throw new ParseError("Unclosed block comment!", position);
+      } else if (result[0] === "*/") {
+        // Close block
+        return this.blockCommentPattern.lastIndex;
+      } else if (result[0] === "/*") {
+        // Open block
+        position = this.skipBlockComment(this.blockCommentPattern.lastIndex);
+      } else {
+        throw new Error("Unreachable, invalid block comment pattern");
+      }
+    }
+  }
+
+  /**
+   * Only matches the `<` token if it is a template
+   * Precondition: An ident was parsed right before this.
+   * Runs the [template list discovery algorithm](https://www.w3.org/TR/WGSL/#template-list-discovery).
+   */
+  nextTemplateStartToken(): (WeslToken & { kind: "symbol" }) | null {
+    const startPosition = this.stream.checkpoint();
+    const token: WeslToken | null = this.nextToken();
+    this.stream.reset(startPosition);
+    if (token === null) return null;
+
+    if (token.kind !== "symbol") {
+      return null;
+    }
+
+    //<<= << <= cannot be templates, so we match the entire token text
+    if (token.text === "<") {
+      if (this.isTemplateStart(token.span[1])) {
+        this.stream.reset(token.span[1]);
+        return token as WeslToken & { kind: typeof token.kind };
+      } else {
+        this.stream.reset(startPosition);
+        return null;
+      }
+    } else {
+      return null;
+    }
+  }
+
+  nextTemplateEndToken(): (WeslToken & { kind: "symbol" }) | null {
+    const startPosition = this.stream.checkpoint();
+    const token: WeslToken | null = this.nextToken();
+    this.stream.reset(startPosition);
+    if (token === null) return null;
+
+    // template closing can also match a >= or >>, so we split the token
+    if (token.kind === "symbol" && token.text[0] === ">") {
+      // SAFETY: The underlying streams implementations can be reset to any position.
+      const tokenPosition = token.span[0];
+      this.stream.reset(tokenPosition + 1);
+      return {
+        kind: "symbol",
+        span: [tokenPosition, tokenPosition + 1],
+        text: ">",
+      };
+    } else {
+      return null;
+    }
+  }
+
+  isTemplateStart(afterToken: number): boolean {
+    // Skip over <
+    this.stream.reset(afterToken);
+    // We start with a < token
+    let pendingCounter = 1;
+    while (true) {
+      const nextToken = this.stream.nextToken();
+      if (nextToken === null) return false;
+      if (nextToken.kind !== "symbol") continue;
+      if (nextToken.text === "<") {
+        // Start a nested template
+        pendingCounter += 1;
+      } else if (nextToken.text[0] === ">") {
+        if (nextToken.text === ">" || nextToken.text == ">=") {
+          pendingCounter -= 1;
+        } else if (nextToken.text === ">>=" || nextToken.text === ">>") {
+          pendingCounter -= 2;
+        } else {
+          throw new Error(
+            "This case should never be reached, looks like we forgot one of the tokens that start with >",
+          );
+        }
+        if (pendingCounter <= 0) {
+          return true;
+        }
+      } else if (nextToken.text === "(") {
+        this.skipBracketsTo(")");
+      } else if (nextToken.text === "[") {
+        this.skipBracketsTo("]");
+      } else if (
+        nextToken.text === "==" ||
+        nextToken.text === "!=" ||
+        nextToken.text === ";" ||
+        nextToken.text === "{" ||
+        nextToken.text === ":" ||
+        nextToken.text === "&&" ||
+        nextToken.text === "||"
+      ) {
+        return false;
+      }
+    }
+  }
+
+  /**
+   * Call this after consuming an opening bracket.
+   * Skips until a closing bracket. This also consumes the closing bracket.
+   */
+  skipBracketsTo(closingBracket: string) {
+    while (true) {
+      const nextToken = this.stream.nextToken();
+      if (nextToken === null)
+        throw new ParseError("Unclosed bracket!", this.stream.checkpoint());
+
+      if (nextToken.kind !== "symbol") continue;
+      if (nextToken.text === "(") {
+        this.skipBracketsTo(")");
+      } else if (nextToken.text === "[") {
+        this.skipBracketsTo("]");
+      } else if (nextToken.text === closingBracket) {
+        // We're done!
+        return;
+      }
+    }
+  }
+}
+
+export const templateOpen = withStreamAction(stream => {
+  return (stream as WeslStream).nextTemplateStartToken();
+});
+export const templateClose = withStreamAction(stream => {
+  return (stream as WeslStream).nextTemplateEndToken();
+});

package/src/test/BindWESL.test.ts

@@ -0,0 +1,57 @@
+import { expect, test } from "vitest";
+import { scopeToStringLong } from "../debug/ScopeToString.ts";
+import { bindTest } from "./TestUtil.ts";
+
+test("nested scopes binding", () => {
+  const src = `
+    fn main() {
+      let bar = 72;
+      if (true) {
+        if (true) {
+          let new_bar = bar; // Should be 72!
+        }
+        let bar = 5;
+      }
+    }
+  `;
+
+  const { registry } = bindTest(src);
+  const ast = registry.modules["package::test"];
+
+  expect(scopeToStringLong(ast.rootScope)).toMatchInlineSnapshot(`
+  "{ 
+    -{ %main(main) #1  
+      { %bar(bar) #3  
+        { 
+          { %new_bar #5  bar #7 -> %bar(bar) #3  } #5
+          %bar #9 
+        } #4
+      } #2
+    } #1
+  } #0"
+`);
+});
+
+test("@location attribute const", () => {
+  const src = `
+    const pos = 0;
+
+    @fragment
+    fn fragmentMain(@location(0) pos : vec3f) -> @location(pos) vec4f { 
+      let x = pos;
+    }
+  `;
+  const { registry } = bindTest(src);
+  const ast = registry.modules["package::test"];
+  expect(scopeToStringLong(ast.rootScope)).toMatchInlineSnapshot(`
+    "{ %pos(pos) #1  
+      {  } #1
+      -{ %fragmentMain(fragmentMain) #3  pos #9 -> %pos(pos) #1 
+         
+        { %pos(pos) #5  vec3f #7 -> undefined 
+          vec4f #11 -> undefined %x #13  pos #15 -> %pos(pos) #5 }
+           #3
+      } #2
+    } #0"
+  `);
+});

package/src/test/ConditionLinking.test.ts

@@ -0,0 +1,91 @@
+import { test } from "vitest";
+import { testLink } from "./TestLink.ts";
+
+test("conditional statement", async () => {
+  const app = `
+    fn main() {
+      @if(false) let x = 1;
+    }
+  `;
+
+  const expectWgsl = `
+    fn main() {
+    }
+  `;
+  await testLink({ app: app }, "app", expectWgsl);
+});
+
+test("conditional compound statement", async () => {
+  const app = `
+    fn main() {
+      @if(false) {
+        let x = 1;
+      }
+    }
+  `;
+
+  const expectWgsl = `
+    fn main() {
+    }
+  `;
+  await testLink({ app: app }, "app", expectWgsl);
+});
+
+test("conditional local variables", async () => {
+  const app = `
+    fn main() {
+      @if(true) var x = 1;
+      @if(false) var x = 2 ;
+    }
+  `;
+
+  const expectWgsl = `
+    fn main() {
+       var x = 1;
+    }
+  `;
+  await testLink({ app: app }, "app", expectWgsl);
+});
+
+test("conditional binding ", async () => {
+  const app = `
+    fn main() {
+      @if(true) var x = 1;
+      @if(false) var x = 2;
+      x += 1;
+    }
+  `;
+
+  const expectWgsl = `
+    fn main() {
+       var x = 1;
+      x += 1;
+    }
+  `;
+  await testLink({ app: app }, "app", expectWgsl);
+});
+
+test("conditional binding references", async () => {
+  const app = `
+    import package::file1::b;
+
+    fn main() {
+      @if(true) var x = b;
+      @if(false) var x = 2;
+      x += 1;
+    }
+  `;
+
+  const file1 = `
+    const b = 9;
+  `;
+
+  const expectWgsl = `
+    fn main() {
+       var x = b;
+      x += 1;
+    }
+const b = 9;
+  `;
+  await testLink({ app, file1 }, "app", expectWgsl);
+});

package/src/test/ConditionalTranslationCases.test.ts

@@ -0,0 +1,56 @@
+import { afterAll, test, TestContext } from "vitest";
+import { conditionalTranslationCases } from "wesl-testsuite";
+import { testFromCase, verifyCaseCoverage } from "./TestLink.js";
+
+const examplesByName = new Map(
+  conditionalTranslationCases.map(t => [t.name, t]),
+);
+
+async function caseTest(ctx: TestContext): Promise<void> {
+  return testFromCase(ctx.task.name, examplesByName);
+}
+
+// requires code stripping to be disabled (which wesl-js doesn't support)
+const weslJsTests = conditionalTranslationCases.filter(
+  c => c.name !== "conditional declaration shadowing",
+);
+afterAll(verifyCaseCoverage(weslJsTests));
+
+test("@if on diagnostic directive", ctx => caseTest(ctx));
+test("@if on enable directive", ctx => caseTest(ctx));
+test("@if on requires directive", ctx => caseTest(ctx));
+test("@if on global const declaration", ctx => caseTest(ctx));
+test("@if on global override declaration", ctx => caseTest(ctx));
+test("@if on global variable declaration", ctx => caseTest(ctx));
+test("@if on type alias", ctx => caseTest(ctx));
+test("@if on module scope const_assert", ctx => caseTest(ctx));
+test("@if on function declaration", ctx => caseTest(ctx));
+test("@if on function formal parameter", ctx => caseTest(ctx));
+test("@if on structure declaration", ctx => caseTest(ctx));
+test("@if on structure member", ctx => caseTest(ctx));
+test("@if on compound statement", ctx => caseTest(ctx));
+test("@if on if statement", ctx => caseTest(ctx));
+test("@if on switch statement", ctx => caseTest(ctx));
+test("@if on switch clause", ctx => caseTest(ctx));
+test("@if on loop statement", ctx => caseTest(ctx));
+test("@if on for statement", ctx => caseTest(ctx));
+test("@if on while statement", ctx => caseTest(ctx));
+test("@if on break statement", ctx => caseTest(ctx));
+test("@if on break-if statement", ctx => caseTest(ctx));
+test("@if on continue statement", ctx => caseTest(ctx));
+test("@if on continuing statement", ctx => caseTest(ctx));
+test("@if on return statement", ctx => caseTest(ctx));
+test("@if on discard statement", ctx => caseTest(ctx));
+test("@if on call statement", ctx => caseTest(ctx));
+test("@if on function-scope const_assert", ctx => caseTest(ctx));
+test("@if short-circuiting OR", ctx => caseTest(ctx));
+test("@if short-circuiting AND", ctx => caseTest(ctx));
+test("@if logical NOT", ctx => caseTest(ctx));
+test("@if parentheses", ctx => caseTest(ctx));
+
+test("declaration shadowing", ctx => caseTest(ctx));
+
+test("conditional import of const_assert", ctx => caseTest(ctx));
+test("double conditional import of const_assert", ctx => caseTest(ctx));
+test("conditional transitive const", ctx => caseTest(ctx));
+test("conditional transitive fn", ctx => caseTest(ctx));

package/src/test/ErrorLogging.test.ts

@@ -0,0 +1,30 @@
+import { expect, test } from "vitest";
+import { linkWithLogQuietly } from "./TestUtil.ts";
+
+test("unresolved identifier", async () => {
+  const src = `
+    fn main() { x = 7; }
+    `;
+  const { log } = await linkWithLogQuietly(src);
+  expect(log).toMatchInlineSnapshot(`
+    "unresolved identifier 'x' in file: ./test.wesl
+        fn main() { x = 7; }   Ln 2
+                    ^^"
+  `);
+});
+
+test("conditionally empty struct", async () => {
+  const src = `
+    struct Empty {
+      @if(false) u: u32,
+    }
+    `;
+  const { log } = await linkWithLogQuietly(src);
+  expect(log).toMatchInlineSnapshot(
+    `
+    "struct Empty in ./test.wesl has no members (with current conditions)
+        struct Empty {   Ln 2
+        ^"
+  `,
+  );
+});

package/src/test/Expression.test.ts

@@ -0,0 +1,22 @@
+import { eof, seq } from "mini-parse";
+import { expect, test } from "vitest";
+import { expression } from "../parse/WeslExpression.ts";
+import { testAppParse } from "./TestUtil.ts";
+
+test("parse number", () => {
+  const src = `3`;
+  const { parsed } = testAppParse(seq(expression, eof), src);
+  expect(parsed).not.toBeNull();
+});
+
+test("parse comparisons with && ||", () => {
+  const src = `a<3   &&   4>(5)`;
+  const { parsed } = testAppParse(seq(expression, eof), src);
+  expect(parsed).not.toBeNull();
+});
+
+test("parse vec templated type", () => {
+  const src = `vec2<f32>`;
+  const { parsed } = testAppParse(seq(expression, eof), src);
+  expect(parsed).not.toBeNull();
+});

package/src/test/FlattenTreeImport.test.ts

@@ -0,0 +1,74 @@
+import { expect, test } from "vitest";
+import { ImportCollection, ImportStatement } from "../AbstractElems.ts";
+import { flattenTreeImport } from "../FlattenTreeImport.ts";
+
+test("complex tree import", () => {
+  const list: ImportCollection = {
+    kind: "import-collection",
+    subtrees: [
+      {
+        kind: "import-statement",
+        segments: [],
+        finalSegment: { kind: "import-item", name: "foo", as: "bar" },
+      },
+      {
+        kind: "import-statement",
+        segments: [],
+        finalSegment: { kind: "import-item", name: "doh" },
+      },
+      {
+        kind: "import-statement",
+        segments: [{ kind: "import-segment", name: "bib" }],
+        finalSegment: { kind: "import-item", name: "bog" },
+      },
+    ],
+  };
+
+  const tree: ImportStatement = {
+    kind: "import-statement",
+    segments: [
+      {
+        kind: "import-segment",
+        name: "zap",
+      },
+    ],
+    finalSegment: list,
+  };
+  const flattened = flattenTreeImport(tree);
+  expect(flattened).toMatchInlineSnapshot(`
+    [
+      {
+        "importPath": [
+          "zap",
+          "bar",
+        ],
+        "modulePath": [
+          "zap",
+          "foo",
+        ],
+      },
+      {
+        "importPath": [
+          "zap",
+          "doh",
+        ],
+        "modulePath": [
+          "zap",
+          "doh",
+        ],
+      },
+      {
+        "importPath": [
+          "zap",
+          "bib",
+          "bog",
+        ],
+        "modulePath": [
+          "zap",
+          "bib",
+          "bog",
+        ],
+      },
+    ]
+  `);
+});

package/src/test/ImportCases.test.ts

@@ -0,0 +1,56 @@
+import { afterAll, test, TestContext } from "vitest";
+import { importCases } from "wesl-testsuite";
+import { testFromCase, verifyCaseCoverage } from "./TestLink.js";
+
+// wgsl example src, indexed by name
+const examplesByName = new Map(importCases.map(t => [t.name, t]));
+
+async function caseTest(ctx: TestContext): Promise<void> {
+  return testFromCase(ctx.task.name, examplesByName);
+}
+
+afterAll(verifyCaseCoverage(importCases));
+
+test("import package::bar::foo;", ctx => caseTest(ctx));
+test("main has other root elements", ctx => caseTest(ctx));
+test("import foo as bar", ctx => caseTest(ctx));
+test("import twice doesn't get two copies", ctx => caseTest(ctx));
+test("imported fn calls support fn with root conflict", ctx => caseTest(ctx));
+test("import twice with two as names", ctx => caseTest(ctx));
+test("import transitive conflicts with main", ctx => caseTest(ctx));
+test("multiple exports from the same module", ctx => caseTest(ctx));
+test("import and resolve conflicting support function", ctx => caseTest(ctx));
+test("import support fn that references another import", ctx => caseTest(ctx));
+test("import support fn from two exports", ctx => caseTest(ctx));
+test("import a struct", ctx => caseTest(ctx));
+test("struct referenced by a fn param", ctx => caseTest(ctx));
+test("import fn with support struct constructor", ctx => caseTest(ctx));
+test("import a transitive struct", ctx => caseTest(ctx));
+test("'import as' a struct", ctx => caseTest(ctx));
+test("import a struct with name conflicting support struct", ctx =>
+  caseTest(ctx));
+test("copy alias to output", ctx => caseTest(ctx));
+test("copy diagnostics to output", ctx => caseTest(ctx));
+test("const referenced by imported fn", ctx => caseTest(ctx));
+test("fn call with a separator", ctx => caseTest(ctx));
+test("local var to struct", ctx => caseTest(ctx));
+test("global var to struct", ctx => caseTest(ctx));
+test("return type of function", ctx => caseTest(ctx));
+test("import a const", ctx => caseTest(ctx));
+test("import an alias", ctx => caseTest(ctx));
+test("alias f32", ctx => caseTest(ctx));
+test("fn f32()", ctx => caseTest(ctx));
+test("circular import", ctx => caseTest(ctx));
+test("inline package reference", ctx => caseTest(ctx));
+test("inline super:: reference", ctx => caseTest(ctx));
+test("import super::file1", ctx => caseTest(ctx));
+test("declaration after subscope", ctx => caseTest(ctx));
+
+// test(, ctx =>
+//   linkTest2(ctx.task.name, {
+//     linked: `
+//     `,
+//   });
+// });
+
+// TODO add case for diagnostic in non-root module (should fail?)

package/src/test/ImportSyntaxCases.test.ts

@@ -0,0 +1,24 @@
+import { withLogSpy } from "mini-parse/test-util";
+import { expect, test } from "vitest";
+import { importSyntaxCases } from "wesl-testsuite";
+import { weslImports } from "../parse/ImportGrammar.js";
+import { testAppParse } from "./TestUtil.js";
+
+function expectParseFail(src: string): void {
+  withLogSpy(() => {
+    expect(() => testAppParse(weslImports, src)).toThrow();
+  });
+}
+
+function expectParses(src: string): void {
+  const result = testAppParse(weslImports, src);
+  expect(result.stable.imports.length).toBeGreaterThan(0);
+}
+
+importSyntaxCases.forEach(c => {
+  if (c.fails) {
+    test(c.src, () => expectParseFail(c.src));
+  } else {
+    test(c.src, () => expectParses(c.src));
+  }
+});