wesl 0.6.0-pre10

package/src/test/TestLink.ts

@@ -0,0 +1,82 @@
+import { expectTrimmedMatch, trimSrc } from "mini-parse/vitest-util";
+import { expect, RunnerTestSuite } from "vitest";
+import { WgslTestSrc } from "wesl-testsuite";
+import { link } from "../Linker.js";
+import { ManglerFn, underscoreMangle } from "../Mangler.ts";
+import { resetScopeIds } from "../Scope.ts";
+
+/**
+ * Link wesl sources and compare the linked wgsl vs expectations.
+ * Ignores blank lines and initial blank columns.
+ *
+ * (for tests)
+ */
+export async function testLink(
+  weslSrc: Record<string, string>,
+  rootModuleName: string,
+  expectedWgsl: string,
+  mangler?: ManglerFn,
+): Promise<void> {
+  resetScopeIds();
+  /* -- link -- */
+  const stdResultMap = await link({
+    weslSrc,
+    rootModuleName,
+    mangler,
+  });
+  const stdResult = stdResultMap.dest;
+
+  /* -- trim and verify results line by line -- */
+  expectTrimmedMatch(stdResult, expectedWgsl);
+}
+
+type TestCaseMap = Map<string, WgslTestSrc>;
+
+/**
+ * Test link one test case from one a shared test suite
+ *  (ImportCases, ConditionalTranslationCases, etc.)
+ */
+export async function testFromCase(
+  name: string,
+  cases: TestCaseMap,
+): Promise<void> {
+  /* -- find and trim source texts -- */
+  const caseFound = cases.get(name);
+  if (!caseFound) {
+    throw new Error(`Skipping test "${name}"\nNo example found.`);
+  }
+
+  const {
+    weslSrc,
+    expectedWgsl = "",
+    underscoreWgsl = expectedWgsl,
+  } = caseFound;
+
+  const srcEntries = Object.entries(weslSrc).map(([name, wgsl]) => {
+    const trimmedSrc = trimSrc(wgsl);
+    return [name, trimmedSrc] as [string, string];
+  });
+
+  const trimmedWesl = Object.fromEntries(srcEntries);
+
+  const rootName = srcEntries[0][0];
+  await testLink(trimmedWesl, rootName, expectedWgsl);
+  await testLink(trimmedWesl, rootName, underscoreWgsl, underscoreMangle);
+}
+
+/**
+ * for afterAll(), to verify that all cases are covered from one of the shared test suites
+ */
+export function verifyCaseCoverage(
+  caseList: WgslTestSrc[],
+): (suite: RunnerTestSuite) => void {
+  return function verifyCases(suite: RunnerTestSuite) {
+    const testNameSet = new Set(suite.tasks.map(t => t.name));
+    const caseNames = caseList.map(c => c.name);
+    const missing = caseNames.filter(name => !testNameSet.has(name));
+    if (missing.length) {
+      console.error("Missing tests for cases:", missing);
+      expect("missing test: " + missing.toString()).toBe("");
+    }
+  };
+}

package/src/test/TestSetup.ts

@@ -0,0 +1,4 @@
+import { enableTracing } from "mini-parse";
+
+// enable parser tracing features
+enableTracing();

package/src/test/TestUtil.ts

@@ -0,0 +1,126 @@
+import { Parser, Stream, withLoggerAsync } from "mini-parse";
+import {
+  expectNoLog,
+  logCatch,
+  TestParseResult,
+  testParseWithStream,
+} from "mini-parse/test-util";
+import {
+  BoundAndTransformed,
+  ParsedRegistry,
+  parsedRegistry,
+  parseIntoRegistry,
+} from "wesl";
+import { bindAndTransform, link, LinkParams } from "../Linker.js";
+import { WeslStream, WeslToken } from "../parse/WeslStream.js";
+import { parseWESL, syntheticWeslParseState, WeslAST } from "../ParseWESL.js";
+import { resetScopeIds } from "../Scope.js";
+
+export function testAppParse<T>(
+  parser: Parser<Stream<WeslToken>, T>,
+  src: string,
+): TestParseResult<T, WeslAST> {
+  const appState = syntheticWeslParseState();
+  const stream = new WeslStream(src);
+  return testParseWithStream(parser, stream, appState);
+}
+
+/** Convenience wrapper to link wgsl for tests.
+ * The first module is named "./test.wesl",
+ * subsequent modules are named "./file1.wesl", "./file2.wesl", etc.
+ */
+export async function linkTest(...rawWgsl: string[]): Promise<string> {
+  return linkTestOpts({}, ...rawWgsl);
+}
+
+export type LinkTestOpts = Pick<
+  LinkParams,
+  "conditions" | "libs" | "config" | "virtualLibs" | "constants" | "mangler"
+>;
+
+export async function linkTestOpts(
+  opts: LinkTestOpts,
+  ...rawWgsl: string[]
+): Promise<string> {
+  const weslSrc = makeTestBundle(rawWgsl);
+
+  const rootModuleName = "test";
+  const srcMap = await link({ weslSrc, rootModuleName, ...opts });
+  return srcMap.dest;
+}
+
+/** Link wesl for tests, and return the console log as well */
+export async function linkWithLog(...rawWgsl: string[]): Promise<{
+  log: string;
+  result: string;
+}> {
+  return linkWithLogInternal(rawWgsl);
+}
+
+/** Link wesl for tests, and return the console log as well.
+ * Quietly swallow any exceptions thrown */
+export async function linkWithLogQuietly(...rawWgsl: string[]): Promise<{
+  log: string;
+  result: string;
+}> {
+  return linkWithLogInternal(rawWgsl, true);
+}
+
+/** link wesl for tests */
+async function linkWithLogInternal(
+  rawWgsl: string[],
+  quiet = false,
+): Promise<{
+  log: string;
+  result: string;
+}> {
+  const { log, logged } = logCatch();
+  let result = "??";
+  try {
+    result = await withLoggerAsync(log, async () => linkTest(...rawWgsl));
+  } catch (e) {
+    if (!quiet) console.error(e);
+  }
+  return { result, log: logged() };
+}
+
+/** parse wesl for testing, and return the AST */
+export function parseTest(src: string): WeslAST {
+  return expectNoLog(() => parseTestRaw(src));
+}
+
+/** test w/o any log collection, to not confuse debugging */
+export function parseTestRaw(src: string) {
+  return parseWESL(src, undefined);
+}
+
+interface BindTestResult {
+  bound: BoundAndTransformed;
+  registry: ParsedRegistry;
+}
+
+/**
+ * Test parsing and binding some wesl src.
+ * @return both the bound result and the internal registry.
+ * (since binding mutates the AST, it's useful for tests to review)
+ */
+export function bindTest(...rawWesl: string[]): BindTestResult {
+  resetScopeIds();
+  const weslSrc = makeTestBundle(rawWesl);
+
+  const registry = parsedRegistry();
+  parseIntoRegistry(weslSrc, registry, "package", "test");
+  const bound = bindAndTransform({ rootModuleName: "test", registry });
+  return { bound, registry };
+}
+
+/** @return a weslSrc record for tests by synthesizing file names
+ * The root module is named ./test.wesl
+ */
+function makeTestBundle(rawWgsl: string[]): Record<string, string> {
+  const [root, ...rest] = rawWgsl;
+  const restWgsl = Object.fromEntries(
+    rest.map((src, i) => [`./file${i + 1}.wesl`, src]),
+  );
+  return { "./test.wesl": root, ...restWgsl };
+}

package/src/test/Tokenizer.test.ts

@@ -0,0 +1,135 @@
+import { expect, test } from "vitest";
+import { WeslStream, WeslToken } from "../parse/WeslStream";
+
+test("tokenize empty string", () => {
+  const tokenizer = new WeslStream("");
+  expect(tokenizer.nextToken()).toEqual(null);
+});
+
+test("parse fn foo() { }", () => {
+  const src = "fn foo() { }";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "keyword",
+    text: "fn",
+    span: [0, 2],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "word",
+    text: "foo",
+    span: [3, 6],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: "(",
+    span: [6, 7],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: ")",
+    span: [7, 8],
+  });
+});
+
+test("parse var<storage> lights : vec3<f32>", () => {
+  const src = "var<storage> lights : vec3<f32>";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "keyword",
+    text: "var",
+    span: [0, 3],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: "<",
+    span: [3, 4],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "word",
+    text: "storage",
+    span: [4, 11],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: ">",
+    span: [11, 12],
+  });
+  expect(tokenizer.nextToken()?.text).toEqual("lights");
+  expect(tokenizer.nextToken()?.text).toEqual(":");
+  expect(tokenizer.nextToken()?.text).toEqual("vec3");
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: "<",
+    span: [26, 27],
+  });
+  expect(tokenizer.nextToken()?.text).toEqual("f32");
+  expect(tokenizer.nextToken()?.text).toEqual(">");
+});
+
+test("parse >>", () => {
+  const src = ">>";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: ">>",
+    span: [0, 2],
+  });
+});
+
+test("parse >> as template", () => {
+  const src = "array<foo >>";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "word",
+    text: "array",
+    span: [0, 5],
+  });
+  expect(tokenizer.nextTemplateStartToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: "<",
+    span: [5, 6],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "word",
+    text: "foo",
+    span: [6, 9],
+  });
+  expect(tokenizer.nextTemplateEndToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: ">",
+    span: [10, 11],
+  });
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "symbol",
+    text: ">",
+    span: [11, 12],
+  });
+  expect(tokenizer.nextToken()).toBe(null);
+});
+
+test("parse skip block comment", () => {
+  const src = "/* /* // */ */vec3<f32>";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "word",
+    text: "vec3",
+    span: [14, 18],
+  });
+});
+
+test("parse skip line comment", () => {
+  const src = "// vec3<f32> */ a\nvec3";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual(<WeslToken>{
+    kind: "word",
+    text: "vec3",
+    span: [18, 22],
+  });
+});
+
+test("parse skip line without newline", () => {
+  const src = "// foo bar";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toBe(null);
+  expect(tokenizer.checkpoint()).toBe(src.length);
+});

package/src/test/TransformBindingStructs.test.ts

@@ -0,0 +1,230 @@
+import { SrcMapBuilder } from "mini-parse";
+import { expectTrimmedMatch } from "mini-parse/vitest-util";
+import { expect, test } from "vitest";
+import { bindIdents } from "../BindIdents.ts";
+import { astToString } from "../debug/ASTtoString.ts";
+import { lowerAndEmit } from "../LowerAndEmit.ts";
+import { parsedRegistry } from "../ParsedRegistry.ts";
+import {
+  bindingStructsPlugin,
+  findRefsToBindingStructs,
+  lowerBindingStructs,
+  markBindingStructs,
+  transformBindingReference,
+  transformBindingStruct,
+} from "../TransformBindingStructs.ts";
+import { linkTestOpts, parseTest } from "./TestUtil.ts";
+
+test("markBindingStructs true", () => {
+  const src = `
+    struct Bindings {
+      @group(0) @binding(0) particles: ptr<storage, array<f32>, read_write>, 
+    }
+  `;
+
+  const ast = parseTest(src);
+  const structs = markBindingStructs(ast.moduleElem);
+  expect(structs.length).toBe(1);
+  expect(structs[0].bindingStruct).toBe(true);
+});
+
+test("markBindingStructs false", () => {
+  const src = `
+    struct Bindings {
+      particles: ptr<storage, array<f32>, read_write>, 
+    }
+  `;
+
+  const ast = parseTest(src);
+  const structs = markBindingStructs(ast.moduleElem);
+  expect(structs.length).toBe(0);
+});
+
+test("transformBindingStruct", () => {
+  const src = `
+    struct Bindings {
+      @group(0) @binding(0) particles: ptr<storage, array<f32>, read_write>, 
+    }
+  `;
+
+  const rootAst = parseTest(src);
+  bindIdents({ rootAst, registry: parsedRegistry() });
+  const bindingStruct = markBindingStructs(rootAst.moduleElem)[0];
+  const newVars = transformBindingStruct(bindingStruct, new Set());
+
+  const srcBuilder = new SrcMapBuilder({ text: rootAst.srcModule.src });
+  lowerAndEmit(srcBuilder, newVars, {});
+  const linked = SrcMapBuilder.build([srcBuilder]).dest.text;
+  expect(linked).toMatchInlineSnapshot(
+    `
+    "@group(0) @binding(0) var<storage, read_write> particles : array<f32>;
+    "
+  `,
+  );
+});
+
+test("findRefsToBindingStructs", () => {
+  const src = `
+    struct Bindings {
+      @group(0) @binding(0) particles: ptr<storage, array<f32>, read_write>, 
+    }
+    struct NotBindings { a: i32 }
+    var y: NotBindings;
+
+    fn main(b: Bindings) {
+      let x = b.particles;
+      let z = y.a;
+    }
+  `;
+
+  const rootAst = parseTest(src);
+  bindIdents({ rootAst, registry: parsedRegistry() });
+  markBindingStructs(rootAst.moduleElem)[0];
+  const found = findRefsToBindingStructs(rootAst.moduleElem);
+  expect(found.length).toBe(1);
+  const foundAst = astToString(found[0].memberRef);
+  expect(foundAst).toMatchInlineSnapshot(`
+    "memberRef b.particles
+      ref b
+      text '.'
+      name particles"
+  `);
+});
+
+test("transformBindingReference", () => {
+  const src = `
+    struct Bindings {
+      @group(0) @binding(0) particles: ptr<storage, array<f32>, read_write>, 
+    }
+    fn main(b: Bindings) {
+      let x = b.particles;
+    }
+  `;
+
+  const rootAst = parseTest(src);
+  bindIdents({ rootAst, registry: parsedRegistry() });
+  const bindingStruct = markBindingStructs(rootAst.moduleElem)[0];
+  transformBindingStruct(bindingStruct, new Set());
+  const found = findRefsToBindingStructs(rootAst.moduleElem);
+  expect(found.length).toBe(1);
+  const { memberRef, struct } = found[0];
+  const synthElem = transformBindingReference(memberRef, struct);
+  const synthAst = astToString(synthElem);
+  expect(synthAst).toMatchInlineSnapshot(`"synthetic 'particles'"`);
+});
+
+test("lower binding structs", () => {
+  const src = `
+    struct Bindings {
+      @group(0) @binding(0) particles: ptr<storage, array<f32>, read_write>, 
+    }
+    fn main(b: Bindings) {
+      let x = b.particles;
+    }
+  `;
+
+  const expected = `
+@group(0) @binding(0) var<storage, read_write> particles : array<f32>;
+       
+    fn main() {
+      let x = particles;
+    }
+  `;
+  const rootAst = parseTest(src);
+  const { globalNames } = bindIdents({ rootAst, registry: parsedRegistry() });
+  const tAst = { ...rootAst, globalNames, notableElems: {} };
+  const lowered = lowerBindingStructs(tAst);
+
+  const loweredAst = astToString(lowered.moduleElem);
+  expect(loweredAst).toMatchInlineSnapshot(`
+    "module
+      synthetic '@group(0) @binding(0) var<storage, read_write> particles : array<f32>;
+    '
+      text '
+        '
+      text '
+        '
+      fn main(b: Bindings)
+        decl %main
+        param
+        statement
+          text '{
+          let '
+          typeDecl %x
+            decl %x
+          text ' = '
+          memberRef b.particles
+            synthetic 'particles'
+          text ';
+        }'
+      text '
+      '"
+  `);
+
+  const srcBuilder = new SrcMapBuilder({ text: lowered.srcModule.src });
+  lowerAndEmit(srcBuilder, [lowered.moduleElem], {}, false);
+  const linked = SrcMapBuilder.build([srcBuilder]).dest.text;
+  expectTrimmedMatch(linked, expected);
+});
+
+test("lower binding structs with conflicting root name", async () => {
+  const src = `
+    struct Bindings {
+      @group(0) @binding(0) particles: ptr<storage, array<f32>, read_write>, 
+    }
+    const particles = 7;
+    fn main(b: Bindings) {
+      let x = b.particles;
+    }
+  `;
+
+  const expected = `
+@group(0) @binding(0) var<storage, read_write> particles0 : array<f32>;
+       
+    const particles = 7;
+    fn main() {
+      let x = particles0;
+    }
+  `;
+
+  const opts = { config: { plugins: [bindingStructsPlugin()] } };
+  const linked = await linkTestOpts(opts, src);
+  expectTrimmedMatch(linked, expected);
+});
+
+test("lower 5 bindings", async () => {
+  const src = `
+    struct Uniforms {
+      foo: u32
+    }
+
+    struct MyBindings {
+      @group(0) @binding(0) particles: ptr<storage, array<u32>, read_write>, 
+      @group(0) @binding(1) uniforms: ptr<uniform, Uniforms>, 
+      @group(0) @binding(2) tex: texture_2d<rgba8unorm>,
+      @group(0) @binding(3) samp: sampler,
+      @group(0) @binding(4) stTex: texture_storage_2d<rgba8unorm, read>,
+    }
+
+    @compute fn main(b: MyBindings) {
+      b.particles[0] = b.uniforms.foo;
+    }
+  `;
+
+  const expected = `
+@group(0) @binding(0) var<storage, read_write> particles : array<u32>;
+@group(0) @binding(1) var<uniform> uniforms : Uniforms;
+@group(0) @binding(2) var tex : texture_2d<rgba8unorm>;
+@group(0) @binding(3) var samp : sampler;
+@group(0) @binding(4) var stTex : texture_storage_2d<rgba8unorm, read>;
+
+    struct Uniforms { foo: u32 }
+    @compute fn main() {
+      particles[0] = uniforms.foo;
+    }
+`;
+
+  const opts = { config: { plugins: [bindingStructsPlugin()] } };
+  const linked = await linkTestOpts(opts, src);
+  expectTrimmedMatch(linked, expected);
+});

package/src/test/Util.test.ts

@@ -0,0 +1,22 @@
+import { expect, test } from "vitest";
+import { overlapTail, scan } from "../Util.js";
+
+test("scan", () => {
+  const result = scan([1, 2, 1], (a, b: string) => b.slice(a), "foobar");
+  expect(result).toEqual(["foobar", "oobar", "bar", "ar"]);
+});
+
+test("overlap 0", () => {
+  const result = overlapTail([2, 3], [4, 5]);
+  expect(result).toBeUndefined();
+});
+
+test("overlap 1", () => {
+  const result = overlapTail([2, 3], [3, 4, 5]);
+  expect(result).toEqual([4, 5]);
+});
+
+test("overlap 2", () => {
+  const result = overlapTail([2, 3], [2, 3]);
+  expect(result).toEqual([]);
+});

package/src/test/VirtualModules.test.ts

@@ -0,0 +1,37 @@
+import { expectTrimmedMatch } from "mini-parse/vitest-util";
+import { test } from "vitest";
+import { linkTestOpts } from "./TestUtil.ts";
+
+test("simple virtual module", async () => {
+  const src = `
+    import virt::Uniforms;
+    @binding(0) @group(0) var<uniform> u: Uniforms;
+  `;
+  const result = await linkTestOpts(
+    { virtualLibs: { virt: () => "struct Uniforms { foo: u32 }" } },
+    src,
+  );
+  const expected = `
+    @binding(0) @group(0) var<uniform> u: Uniforms;
+    struct Uniforms { foo: u32 }
+  `;
+  expectTrimmedMatch(result, expected);
+});
+
+test("virtual constants", async () => {
+  const src = `
+    import constants::num_lights;
+    fn main() {
+      for (var i = 0; i < num_lights; i++) { }
+    }
+  `;
+  const result = await linkTestOpts({ constants: { num_lights: 4 } }, src);
+  const expected = `
+    fn main() {
+      for (var i = 0; i < num_lights; i++) { }
+    }
+
+    const num_lights = 4;
+  `;
+  expectTrimmedMatch(result, expected);
+});