wesl 0.7.26 → 0.7.28

package/src/test/StatementEmit.test.ts

@@ -0,0 +1,143 @@
+import { expect, test } from "vitest";
+import { linkTest } from "./TestUtil.ts";
+import { expectTrimmedMatch } from "./TrimmedMatch.ts";
+
+// Statements are emitted structurally from typed AST fields (not by copying
+// source text), so these tests lock the canonical layout and confirm that
+// comments attached to statements survive linking.
+
+test("linked WGSL keeps leading and trailing body comments", async () => {
+  const src = `
+    fn main() {
+      // leading comment
+      let x = 1; // trailing comment
+      let y = x;
+    }
+  `;
+  const result = await linkTest(src);
+  expect(result).toContain("// leading comment");
+  expect(result).toContain("// trailing comment");
+  // leading comment on its own line above the statement, trailing kept inline
+  expectTrimmedMatch(result, src);
+});
+
+test("linked WGSL keeps a comment in an otherwise empty block", async () => {
+  const src = `
+    fn foo() {
+      // fooImpl
+    }
+  `;
+  const result = await linkTest(src);
+  expect(result).toContain("// fooImpl");
+  expectTrimmedMatch(result, src);
+});
+
+test("keeps attributes on compound statement bodies", async () => {
+  const src = `
+    fn main() {
+      for (var i = 0; i < 4; i++) @diagnostic(off, derivative_uniformity) {
+        let x = i;
+      }
+      switch 0 @diagnostic(off, derivative_uniformity) {
+        default: {
+          let y = 1;
+        }
+      }
+    }
+  `;
+  const result = await linkTest(src);
+  expectTrimmedMatch(result, src);
+});
+
+test("structural emit of control flow", async () => {
+  const src = `
+    fn main() {
+      var i = 0;
+      if i < 10 {
+        i = i + 1;
+      } else if i < 20 {
+        i += 2;
+      } else {
+        i = 0;
+      }
+      for (var j = 0; j < 4; j++) {
+        i = i + j;
+      }
+      while i > 0 {
+        i = i - 1;
+      }
+    }
+  `;
+  const result = await linkTest(src);
+  expectTrimmedMatch(result, src);
+});
+
+test("structural emit of loop, continuing, and switch", async () => {
+  const src = `
+    fn main() {
+      var i = 0;
+      loop {
+        i = i + 1;
+        continuing {
+          break if i > 100;
+        }
+      }
+      switch i {
+        case 0, 1: {
+          i = 5;
+        }
+        default: {
+          i = 9;
+        }
+      }
+    }
+  `;
+  const result = await linkTest(src);
+  expectTrimmedMatch(result, src);
+});
+
+test("linked WGSL keeps an inline comment before a call argument", async () => {
+  const src = `
+    fn main() {
+      let y = max(1, /* big */ 2);
+    }
+  `;
+  const result = await linkTest(src);
+  expect(result).toContain("/* big */");
+  expectTrimmedMatch(result, src);
+});
+
+test("linked WGSL keeps an inline comment after a call argument", async () => {
+  const src = `
+    fn main() {
+      let y = max(1 /* big */, 2);
+    }
+  `;
+  const result = await linkTest(src);
+  expectTrimmedMatch(result, src);
+});
+
+test("linked WGSL keeps an inline comment between binary operands", async () => {
+  const src = `
+    fn main() {
+      let y = 1 + /* mid */ 2;
+    }
+  `;
+  const result = await linkTest(src);
+  expectTrimmedMatch(result, src);
+});
+
+test("a line comment inside an expression stays on its own line", async () => {
+  const src = `
+    fn main() {
+      let y = max(1, // big
+      2);
+    }
+  `;
+  const result = await linkTest(src);
+  expect(result).toContain("// big");
+  // the line comment must be followed by a newline, or it would swallow the
+  // rest of the expression (`2);`) into the comment
+  expect(result).not.toContain("// big 2");
+  expect(result).toContain("2)");
+});

package/src/test/StripWesl.ts

@@ -6,29 +6,50 @@ import { WeslStream } from "../parse/WeslStream.ts";
  *  . extra whitespace,
  *  . comments,
  *  . trailing commas in brackets, paren, and array containers
+ *  . redundant module-scope `;` (an empty global declaration, which structural
+ *    emit canonicalizes away)
  */
 export function stripWesl(text: string): string {
   const stream = new WeslStream(text);
   const firstToken = stream.nextToken();
   if (firstToken === null) return "";
 
+  let depth = firstToken.text === "{" ? 1 : 0;
+  let prev = firstToken.text;
   let result = firstToken.text;
   while (true) {
     const token = stream.nextToken();
     if (token === null) return result;
 
+    // A `;` at module scope following a `}` (or another such `;`) is an empty
+    // global declaration; the structural emitter drops it, so drop it here too.
+    if (token.text === ";" && depth === 0 && (prev === "}" || prev === ";")) {
+      continue;
+    }
+
     if (token.text === ",") {
       const nextToken = stream.nextToken();
       const nextText = nextToken?.text;
       if (nextText === "}" || nextText === "]" || nextText === ")") {
         // Ignore trailing comma
-        result += " ";
-        result += nextText;
+        result += " " + nextText;
       } else {
-        result += ", " + (nextToken?.text ?? "");
+        result += ", " + (nextText ?? "");
+      }
+      if (nextText !== undefined) {
+        depth += braceDelta(nextText);
+        prev = nextText;
       }
     } else {
       result += " " + token.text;
+      depth += braceDelta(token.text);
+      prev = token.text;
     }
   }
 }
+
+function braceDelta(text: string): number {
+  if (text === "{") return 1;
+  if (text === "}") return -1;
+  return 0;
+}

package/src/test/TestLink.ts

@@ -3,9 +3,15 @@ import type { WgslTestSrc } from "wesl-testsuite";
 import { link } from "../Linker.ts";
 import { type ManglerFn, underscoreMangle } from "../Mangler.ts";
 import { mapValues } from "../Util.ts";
-import { expectTrimmedMatch, trimSrc } from "./TrimmedMatch.ts";
+import { expectTokenMatch } from "./TestUtil.ts";
+import { trimSrc } from "./TrimmedMatch.ts";
 
-/** Link wesl sources and compare linked wgsl vs expectations (ignores whitespace). */
+/**
+ * Link wesl sources and compare the linked wgsl against expectations.
+ * These cases verify linking (imports, mangling, conditional compilation,
+ * tree-shaking), not layout, so the comparison is token-based: whitespace and
+ * comments are ignored, leaving line layout to the emit-specific tests.
+ */
 export async function testLink(
   weslSrc: Record<string, string>,
   rootModuleName: string,
@@ -13,7 +19,7 @@ export async function testLink(
   mangler?: ManglerFn,
 ): Promise<void> {
   const resultMap = await link({ weslSrc, rootModuleName, mangler });
-  expectTrimmedMatch(resultMap.dest, expectedWgsl);
+  expectTokenMatch(resultMap.dest, expectedWgsl);
 }
 
 type CaseMap = Map<string, WgslTestSrc>;
@@ -43,6 +49,16 @@ const knownFormattingDifferences: Record<string, string> = {
   "@if on break statement": `
     fn foo() { while true {  break; } }
     fn bar() { while true {  } }`,
+  "@if on switch statement": `
+    fn func() {
+      switch 0 { default: { let foo = 10; } }
+    }`,
+  "@if on switch clause": `
+    fn func() {
+      switch 0 {
+        default: { let foo = 10; }
+      }
+    }`,
 };
 
 /** Test linking a single case from a shared test suite (ImportCases, etc.) */

package/src/test/TestUtil.ts

@@ -2,13 +2,23 @@ import { expect } from "vitest";
 import { type BoundAndTransformed, RecordResolver, type SrcModule } from "wesl";
 import { bindAndTransform, type LinkParams, link } from "../Linker.ts";
 import { withLoggerAsync } from "../Logging.ts";
-import { parseSrcModule, type WeslAST } from "../ParseWESL.ts";
+import {
+  type ParseOptions,
+  parseSrcModule,
+  type WeslAST,
+} from "../ParseWESL.ts";
 import { expectNoLog, logCatch } from "./LogCatcher.ts";
 import { stripWesl } from "./StripWesl.ts";
 
 export type LinkTestOpts = Pick<
   LinkParams,
-  "conditions" | "libs" | "config" | "virtualLibs" | "constants" | "mangler"
+  | "conditions"
+  | "libs"
+  | "config"
+  | "virtualLibs"
+  | "constants"
+  | "mangler"
+  | "weslExtensions"
 >;
 
 interface BindTestResult {
@@ -22,13 +32,13 @@ export function expectTokenMatch(actual: string, expected: string): void {
 }
 
 /** Parse a single wesl file. */
-export function parseWESL(src: string): WeslAST {
+export function parseWESL(src: string, options?: ParseOptions): WeslAST {
   const srcModule: SrcModule = {
     modulePath: "package::test",
     debugFilePath: "./test.wesl",
     src,
   };
-  return parseSrcModule(srcModule);
+  return parseSrcModule(srcModule, options);
 }
 
 /** Link wesl for tests. First module is ./test.wesl, rest are ./file1.wesl, etc. */
@@ -74,13 +84,13 @@ async function linkWithLogInternal(
 }
 
 /** Parse wesl for testing, ensuring no logged warnings. */
-export function parseTest(src: string): WeslAST {
-  return expectNoLog(() => parseWESL(src));
+export function parseTest(src: string, options?: ParseOptions): WeslAST {
+  return expectNoLog(() => parseWESL(src, options));
 }
 
 /** Parse wesl without log collection (for debugging). */
-export function parseTestRaw(src: string): WeslAST {
-  return parseWESL(src);
+export function parseTestRaw(src: string, options?: ParseOptions): WeslAST {
+  return parseWESL(src, options);
 }
 
 /** Parse and bind wesl source for testing. Returns bound result and resolver. */

package/src/test/Tokenizer.test.ts

@@ -107,6 +107,24 @@ test("parse >> as template", () => {
   expect(tokenizer.nextToken()).toBe(null);
 });
 
+test("template discovery ignores > inside comments", () => {
+  const src = "a < b /* > */ ;";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()?.text).toBe("a");
+  expect(tokenizer.nextTemplateStartToken()).toBe(null);
+});
+
+test("template discovery sees comments inside template lists", () => {
+  const src = "array< /* len */ f32, 4 >";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()?.text).toBe("array");
+  expect(tokenizer.nextTemplateStartToken()).toEqual({
+    kind: "symbol",
+    text: "<",
+    span: [5, 6],
+  } as WeslToken);
+});
+
 test("parse skip block comment", () => {
   const src = "/* /* // */ */vec3<f32>";
   const tokenizer = new WeslStream(src);
@@ -133,3 +151,81 @@ test("parse skip line without newline", () => {
   expect(tokenizer.nextToken()).toBe(null);
   expect(tokenizer.checkpoint()).toBe(src.length);
 });
+
+test("unicode mid-word falls back with correct span", () => {
+  const src = "réflexion x";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual({
+    kind: "word",
+    text: "réflexion",
+    span: [0, 9],
+  } as WeslToken);
+  expect(tokenizer.nextToken()?.span).toEqual([10, 11]);
+});
+
+test("surrogate-pair ident spans count UTF-16 units", () => {
+  const src = "𐰓𐰏𐰇 x";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()).toEqual({
+    kind: "word",
+    text: "𐰓𐰏𐰇",
+    span: [0, 6],
+  } as WeslToken);
+  expect(tokenizer.nextToken()?.span).toEqual([7, 8]);
+});
+
+test("underscore boundary: _é word, _x word, bare _ symbol", () => {
+  const under = new WeslStream("_é _x _ x");
+  expect(under.nextToken()).toEqual({
+    kind: "word",
+    text: "_é",
+    span: [0, 2],
+  } as WeslToken);
+  expect(under.nextToken()).toEqual({
+    kind: "word",
+    text: "_x",
+    span: [3, 5],
+  } as WeslToken);
+  expect(under.nextToken()).toEqual({
+    kind: "symbol",
+    text: "_",
+    span: [6, 7],
+  } as WeslToken);
+});
+
+test("unicode blankspace separates tokens", () => {
+  const src = "a\u{2028}b\u{0085}c";
+  const tokenizer = new WeslStream(src);
+  expect(tokenizer.nextToken()?.span).toEqual([0, 1]);
+  expect(tokenizer.nextToken()?.span).toEqual([2, 3]);
+  expect(tokenizer.nextToken()?.span).toEqual([4, 5]);
+  expect(tokenizer.nextToken()).toBe(null);
+});
+
+test("leading-dot float and 3-char symbols", () => {
+  const src = "x >>= .5 <<= 0x1p4";
+  const tokenizer = new WeslStream(src);
+  tokenizer.nextToken(); // x
+  expect(tokenizer.nextToken()).toEqual({
+    kind: "symbol",
+    text: ">>=",
+    span: [2, 5],
+  } as WeslToken);
+  expect(tokenizer.nextToken()).toEqual({
+    kind: "number",
+    text: ".5",
+    span: [6, 8],
+  } as WeslToken);
+  expect(tokenizer.nextToken()?.text).toBe("<<=");
+  expect(tokenizer.nextToken()).toEqual({
+    kind: "number",
+    text: "0x1p4",
+    span: [13, 18],
+  } as WeslToken);
+});
+
+test("invalid character throws", () => {
+  const tokenizer = new WeslStream("a # b");
+  tokenizer.nextToken();
+  expect(() => tokenizer.nextToken()).toThrow(/Invalid token #/);
+});

package/src/test/__snapshots__/ParseWeslV2.test.ts.snap

@@ -2,66 +2,34 @@
 
 exports[`parse unicode ident 1`] = `
 "module
-  text '
-  '
   fn Δέλτα()
     decl %Δέλτα
-    statement
-      text '{}'
-  text ' 
-  '
+    block
   fn réflexion()
     decl %réflexion
-    statement
-      text '{}'
-  text ' 
-  '
+    block
   fn Кызыл()
     decl %Кызыл
-    statement
-      text '{}'
-  text ' 
-  '
+    block
   fn 𐰓𐰏𐰇()
     decl %𐰓𐰏𐰇
-    statement
-      text '{}'
-  text ' 
-  '
+    block
   fn 朝焼け()
     decl %朝焼け
-    statement
-      text '{}'
-  text '
-  '
+    block
   fn سلام()
     decl %سلام
-    statement
-      text '{}'
-  text ' 
-  '
+    block
   fn 검정()
     decl %검정
-    statement
-      text '{}'
-  text ' 
-  '
+    block
   fn שָׁלוֹם()
     decl %שָׁלוֹם
-    statement
-      text '{}'
-  text '
-  '
+    block
   fn गुलाबी()
     decl %गुलाबी
-    statement
-      text '{}'
-  text '
-  '
+    block
   fn փիրուզ()
     decl %փիրուզ
-    statement
-      text '{}'
-  text '
-  '"
+    block"
 `;

package/src/RawEmit.ts

@@ -1,103 +0,0 @@
-import type {
-  AttributeElem,
-  NameElem,
-  StuffElem,
-  TranslateTimeExpressionElem,
-  TypeRefElem,
-  TypeTemplateParameter,
-  UnknownExpressionElem,
-} from "./AbstractElems.ts";
-import { assertUnreachable } from "./Assertions.ts";
-import {
-  diagnosticControlToString,
-  expressionToString,
-  findDecl,
-} from "./LowerAndEmit.ts";
-import type { RefIdent } from "./Scope.ts";
-
-// LATER DRY emitting elements like this with LowerAndEmit?
-
-export function attributeToString(e: AttributeElem): string {
-  const { kind } = e.attribute;
-  // LATER emit more precise source map info by making use of all the spans
-  // Like the first case does
-  if (kind === "@attribute") {
-    const { params } = e.attribute;
-    if (params === undefined || params.length === 0) {
-      return "@" + e.attribute.name;
-    } else {
-      return `@${e.attribute.name}(${params
-        .map(param => contentsToString(param))
-        .join(", ")})`;
-    }
-  } else if (kind === "@builtin") {
-    return "@builtin(" + e.attribute.param.name + ")";
-  } else if (kind === "@diagnostic") {
-    return (
-      "@diagnostic" +
-      diagnosticControlToString(e.attribute.severity, e.attribute.rule)
-    );
-  } else if (kind === "@if") {
-    return `@if(${expressionToString(e.attribute.param.expression)})`;
-  } else if (kind === "@elif") {
-    return `@elif(${expressionToString(e.attribute.param.expression)})`;
-  } else if (kind === "@else") {
-    return "@else";
-  } else if (kind === "@interpolate") {
-    return `@interpolate(${e.attribute.params.map(v => v.name).join(", ")})`;
-  } else {
-    assertUnreachable(kind);
-  }
-}
-
-export function typeListToString(params: TypeTemplateParameter[]): string {
-  return `<${params.map(typeParamToString).join(", ")}>`;
-}
-
-export function typeParamToString(param?: TypeTemplateParameter): string {
-  if (param === undefined) return "?";
-  if (param.kind === "type") return typeRefToString(param);
-  return expressionToString(param);
-}
-
-export function typeRefToString(t?: TypeRefElem): string {
-  if (!t) return "?";
-  const { name, templateParams } = t;
-  const params = templateParams ? typeListToString(templateParams) : "";
-  return `${refToString(name)}${params}`;
-}
-
-function refToString(ref: RefIdent | string): string {
-  if (typeof ref === "string") return ref;
-  if (ref.std) return ref.originalName;
-  const decl = findDecl(ref);
-  return decl.mangledName || decl.originalName;
-}
-
-export function contentsToString(
-  elem:
-    | TranslateTimeExpressionElem
-    | UnknownExpressionElem
-    | NameElem
-    | StuffElem,
-): string {
-  if (elem.kind === "translate-time-expression") {
-    throw new Error("Not supported");
-  } else if (elem.kind === "expression" || elem.kind === "stuff") {
-    const parts = elem.contents.map(c => {
-      const { kind } = c;
-      if (kind === "text") {
-        return c.srcModule.src.slice(c.start, c.end);
-      } else if (kind === "ref") {
-        return refToString(c.ident);
-      } else {
-        return `?${c.kind}?`;
-      }
-    });
-    return parts.join(" ");
-  } else if (elem.kind === "name") {
-    return elem.name;
-  } else {
-    assertUnreachable(elem);
-  }
-}