wesl 0.6.49 → 0.7.1

package/src/parse/ParseWesl.ts

@@ -0,0 +1,51 @@
+import type { ModuleElem } from "../AbstractElems.ts";
+import { ParseError } from "../ParseError.ts";
+import type { WeslAST, WeslParseState } from "../ParseWESL.ts";
+import { WeslParseError } from "../ParseWESL.ts";
+import type { SrcModule } from "../Scope.ts";
+import { emptyScope } from "../Scope.ts";
+import { beginElem, finishContents } from "./ContentsHelpers.ts";
+import { parseModule } from "./ParseModule.ts";
+import { ParsingContext } from "./ParsingContext.ts";
+import { WeslStream } from "./WeslStream.ts";
+
+/** Parse a WESL source module into an AST. */
+export function parseWesl(srcModule: SrcModule): WeslAST {
+  const { ctx, state } = createParseState(srcModule);
+  try {
+    beginElem(ctx, "module");
+    parseModule(ctx);
+    const moduleElem = state.stable.moduleElem;
+    moduleElem.contents = finishContents(ctx, 0, moduleElem.end);
+    return state.stable;
+  } catch (e) {
+    if (e instanceof ParseError) {
+      throw new WeslParseError({ cause: e, src: srcModule });
+    }
+    // unexpected error (bug in parser), wrap for user-friendly reporting
+    const message = e instanceof Error ? e.message : String(e);
+    const parseError = new ParseError(message, [0, 0]);
+    throw new WeslParseError({ cause: parseError, src: srcModule });
+  }
+}
+
+/** Initialize parse state: token stream, root scope, and module element. */
+function createParseState(srcModule: SrcModule): {
+  ctx: ParsingContext;
+  state: WeslParseState;
+} {
+  const stream = new WeslStream(srcModule.src);
+  const rootScope = emptyScope(null);
+  const moduleElem: ModuleElem = {
+    kind: "module",
+    contents: [],
+    start: 0,
+    end: srcModule.src.length,
+  };
+  const state: WeslParseState = {
+    context: { scope: rootScope, openElems: [] },
+    stable: { srcModule, moduleElem, rootScope, imports: [] },
+  };
+  const ctx = new ParsingContext(stream, state);
+  return { ctx, state };
+}

package/src/parse/ParsingContext.ts

@@ -0,0 +1,93 @@
+import type { AbstractElem } from "../AbstractElems.ts";
+import type { WeslParseContext, WeslParseState } from "../ParseWESL.ts";
+import {
+  type DeclIdent,
+  emptyScope,
+  type Ident,
+  nextIdentId,
+  type RefIdent,
+  type Scope,
+  type SrcModule,
+} from "../Scope.ts";
+import type { WeslStream } from "./WeslStream.ts";
+
+/** Context for parsers to build AST and manage scopes. */
+export class ParsingContext {
+  src: string;
+  srcModule: SrcModule;
+  stream: WeslStream;
+  state: WeslParseState;
+
+  constructor(stream: WeslStream, state: WeslParseState) {
+    this.stream = stream;
+    this.state = state;
+    this.srcModule = state.stable.srcModule;
+    this.src = this.srcModule.src;
+  }
+
+  position(): number {
+    return this.stream.checkpoint();
+  }
+
+  currentScope(): Scope {
+    return this.state.context.scope;
+  }
+
+  addElem(elem: AbstractElem): void {
+    const { openElems } = this.state.context;
+    if (openElems.length > 0) {
+      const open = openElems[openElems.length - 1];
+      open.contents.push(elem);
+    }
+  }
+
+  pushScope(kind: Scope["kind"] = "scope"): void {
+    const { scope } = this.state.context;
+    const newScope = emptyScope(scope, kind);
+    scope.contents.push(newScope);
+    this.state.context.scope = newScope;
+  }
+
+  popScope(): Scope {
+    const weslContext = this.state.context as WeslParseContext;
+    const completedScope = weslContext.scope;
+    if (completedScope.parent) {
+      weslContext.scope = completedScope.parent;
+    }
+    return completedScope;
+  }
+
+  isModuleScope(): boolean {
+    let scope = this.currentScope();
+    while (scope.kind === "partial" && scope.parent) {
+      scope = scope.parent;
+    }
+    return scope.parent === null;
+  }
+
+  createRefIdent(name: string): RefIdent {
+    return {
+      kind: "ref",
+      originalName: name,
+      ast: this.state.stable,
+      id: nextIdentId(),
+      refIdentElem: null as any, // linked by caller
+    };
+  }
+
+  createDeclIdent(name: string, isGlobal = false): DeclIdent {
+    return {
+      kind: "decl",
+      originalName: name,
+      containingScope: this.state.context.scope,
+      isGlobal,
+      id: nextIdentId(),
+      srcModule: this.srcModule,
+      declElem: null as any, // linked by caller
+    };
+  }
+
+  saveIdent(ident: Ident): void {
+    this.state.context.scope.contents.push(ident);
+  }
+}

package/src/parse/WeslStream.ts

@@ -1,14 +1,9 @@
-import {
-  CachingStream,
-  MatchersStream,
-  matchOneOf,
-  ParseError,
-  RegexMatchers,
-  type Stream,
-  type TypedToken,
-  withStreamAction,
-} from "mini-parse";
+import { ParseError } from "../ParseError.ts";
+import type { Stream, TypedToken } from "../Stream.ts";
 import { keywords, reservedWords } from "./Keywords.ts";
+import { CachingStream } from "./stream/CachingStream.ts";
+import { MatchersStream, RegexMatchers } from "./stream/MatchersStream.ts";
+import { matchOneOf } from "./stream/RegexHelpers.ts";
 export type WeslTokenKind = "word" | "keyword" | "number" | "symbol";
 
 export type WeslToken<Kind extends WeslTokenKind = WeslTokenKind> =
@@ -84,7 +79,6 @@ export class WeslStream implements Stream<WeslToken> {
   private stream: Stream<TypedToken<InternalTokenKind>>;
   /** New line */
   private eolPattern = /[\n\v\f\u{0085}\u{2028}\u{2029}]|\r\n?/gu;
-  /** Block comments */
   private blockCommentPattern = /\/\*|\*\//g;
   public src: string;
   constructor(src: string) {
@@ -126,6 +120,62 @@ export class WeslStream implements Stream<WeslToken> {
     }
   }
 
+  /** Peek at the next token without consuming it */
+  peek(): WeslToken | null {
+    const pos = this.checkpoint();
+    const token = this.nextToken();
+    this.reset(pos);
+    return token;
+  }
+
+  /** Consume token if text matches, otherwise leave position unchanged */
+  matchText(text: string): WeslToken | null {
+    const token = this.peek();
+    if (token?.text === text) {
+      this.nextToken();
+      return token;
+    }
+    return null;
+  }
+
+  /** Consume token if kind matches (and optionally text), otherwise leave position unchanged */
+  matchKind<K extends WeslTokenKind>(
+    kind: K,
+    text?: string,
+  ): WeslToken<K> | null {
+    const token = this.peek();
+    if (token?.kind === kind && (!text || token.text === text)) {
+      this.nextToken();
+      return token as WeslToken<K>;
+    }
+    return null;
+  }
+
+  /** Consume token if predicate matches, otherwise leave position unchanged */
+  nextIf(predicate: (token: WeslToken) => boolean): WeslToken | null {
+    const token = this.peek();
+    if (token && predicate(token)) {
+      this.nextToken();
+      return token;
+    }
+    return null;
+  }
+
+  /** Match a sequence of tokens by text. Resets and returns null if any fails. */
+  matchSequence(...texts: string[]): WeslToken[] | null {
+    const startPos = this.checkpoint();
+    const tokens: WeslToken[] = [];
+    for (const text of texts) {
+      const token = this.matchText(text);
+      if (!token) {
+        this.reset(startPos);
+        return null;
+      }
+      tokens.push(token);
+    }
+    return tokens;
+  }
+
   private skipToEol(position: number): number {
     this.eolPattern.lastIndex = position;
     const result = this.eolPattern.exec(this.src);
@@ -207,7 +257,7 @@ export class WeslStream implements Stream<WeslToken> {
     }
   }
 
-  isTemplateStart(afterToken: number): boolean {
+  private isTemplateStart(afterToken: number): boolean {
     // Skip over <
     this.stream.reset(afterToken);
     // We start with a < token
@@ -254,7 +304,7 @@ export class WeslStream implements Stream<WeslToken> {
    * Call this after consuming an opening bracket.
    * Skips until a closing bracket. This also consumes the closing bracket.
    */
-  skipBracketsTo(closingBracket: string) {
+  private skipBracketsTo(closingBracket: string): void {
     while (true) {
       const nextToken = this.stream.nextToken();
       if (nextToken === null) {
@@ -273,10 +323,3 @@ export class WeslStream implements Stream<WeslToken> {
     }
   }
 }
-
-export const templateOpen = withStreamAction(stream => {
-  return (stream as WeslStream).nextTemplateStartToken();
-});
-export const templateClose = withStreamAction(stream => {
-  return (stream as WeslStream).nextTemplateEndToken();
-});

package/src/parse/stream/CachingStream.ts

@@ -0,0 +1,48 @@
+import type { Stream, Token } from "../../Stream.ts";
+
+export class CachingStream<T extends Token> implements Stream<T> {
+  private cache = new Cache<number, { token: T | null; checkpoint: number }>(5);
+  private inner: Stream<T>;
+  constructor(inner: Stream<T>) {
+    this.inner = inner;
+  }
+  checkpoint(): number {
+    return this.inner.checkpoint();
+  }
+  reset(position: number): void {
+    this.inner.reset(position);
+  }
+  nextToken(): T | null {
+    const startPos = this.checkpoint();
+    const cachedValue = this.cache.get(startPos);
+    if (cachedValue !== undefined) {
+      this.reset(cachedValue.checkpoint);
+      return cachedValue.token;
+    } else {
+      const token = this.inner.nextToken();
+      const checkpoint = this.checkpoint();
+      this.cache.set(startPos, { token, checkpoint });
+      return token;
+    }
+  }
+  get src(): string {
+    return this.inner.src;
+  }
+}
+
+/** size limited key value cache */
+class Cache<K, V> extends Map<K, V> {
+  private readonly max: number;
+  constructor(max: number) {
+    super();
+    this.max = max;
+  }
+
+  set(k: K, v: V): this {
+    if (this.size > this.max) {
+      const first = this.keys().next().value;
+      if (first) this.delete(first);
+    }
+    return super.set(k, v);
+  }
+}

package/src/parse/stream/MatchersStream.ts

@@ -0,0 +1,85 @@
+import type { Span } from "../../Span.ts";
+import type { Stream, TypedToken } from "../../Stream.ts";
+import { toRegexSource } from "./RegexHelpers.ts";
+
+/** Runs a `RegexMatchers` on an input string */
+export class MatchersStream<Kind extends string>
+  implements Stream<TypedToken<Kind>>
+{
+  private position = 0;
+  public text: string;
+  private matchers: RegexMatchers<Kind>;
+
+  constructor(text: string, matchers: RegexMatchers<Kind>) {
+    this.text = text;
+    this.matchers = matchers;
+  }
+
+  checkpoint(): number {
+    return this.position;
+  }
+  reset(position: number): void {
+    this.position = position;
+  }
+  nextToken(): TypedToken<Kind> | null {
+    const result = this.matchers.execAt(this.text, this.position);
+    if (result === null) return null;
+    this.position = result.span[1];
+    return result;
+  }
+  get src(): string {
+    return this.text;
+  }
+}
+
+/**
+ * The matchers passed to this object must follow certain rules:
+ * - They must use non-capturing groups: `(?:...)`
+ * - They must NOT use `^` or `$`
+ */
+export class RegexMatchers<Kind extends string> {
+  private groups: Kind[];
+  private exp: RegExp;
+  constructor(matchers: Record<Kind, string | RegExp>) {
+    this.groups = Object.keys(matchers) as Kind[];
+    const expParts = Object.entries(matchers as Record<string, string | RegExp>)
+      .map(toRegexSource)
+      .join("|");
+    // d = return substrings of each match
+    // y = sticky, only match at the start of the string
+    // u = unicode aware
+    this.exp = new RegExp(expParts, "dyu");
+  }
+
+  execAt(text: string, position: number): TypedToken<Kind> | null {
+    this.exp.lastIndex = position;
+    const matches = this.exp.exec(text);
+    const matchedIndex = findGroupDex(matches?.indices);
+
+    if (matchedIndex) {
+      const { span, groupDex } = matchedIndex;
+      const kind = this.groups[groupDex];
+      return { kind, span, text: text.slice(span[0], span[1]) };
+    } else {
+      return null;
+    }
+  }
+}
+
+interface MatchedIndex {
+  span: Span;
+  groupDex: number;
+}
+
+function findGroupDex(
+  indices: RegExpIndicesArray | undefined,
+): MatchedIndex | undefined {
+  if (indices !== undefined) {
+    for (let i = 1; i < indices.length; i++) {
+      const span = indices[i];
+      if (span !== undefined) {
+        return { span, groupDex: i - 1 };
+      }
+    }
+  }
+}

package/src/parse/stream/RegexHelpers.ts

@@ -0,0 +1,38 @@
+export function toRegexSource(nameExp: [string, RegExp | string]): string {
+  const [name, e] = nameExp;
+  if (typeof e === "string") {
+    const expSrc = `(${escapeRegex(e)})`;
+    verifyNonCapturing(name, new RegExp(expSrc));
+    return expSrc;
+  } else {
+    verifyNonCapturing(name, e);
+    return `(${e.source})`;
+  }
+}
+
+function verifyNonCapturing(name: string, exp: RegExp): void {
+  const willMatch = new RegExp("|" + exp.source);
+  const result = willMatch.exec("")!;
+  if (result.length > 1) {
+    throw new Error(
+      `match expression groups must be non-capturing: ${name}: /${exp.source}/. Use (?:...) instead.`,
+    );
+  }
+}
+
+const regexSpecials = /[$+*.?|(){}[\]\\/^]/g;
+
+function escapeRegex(s: string): string {
+  return s.replace(regexSpecials, "\\$&");
+}
+
+/**
+ * @return a regexp to match any of the space separated tokens in the provided string.
+ * Regex special characters are escaped, and the matchers are sorted by length
+ * so that longer matches are preferred.
+ */
+export function matchOneOf(syms: string): RegExp {
+  const symbolList = syms.split(/\s+/).sort((a, b) => b.length - a.length);
+  const escaped = symbolList.filter(s => s).map(escapeRegex);
+  return new RegExp(escaped.join("|"));
+}

package/src/test/BevyLink.test.ts

@@ -0,0 +1,100 @@
+import fs from "node:fs/promises";
+import { expect, test } from "vitest";
+import { fetchBulkTest } from "wesl-testsuite/fetch-bulk-tests";
+import { link } from "../Linker.ts";
+import type { Conditions } from "../Scope.ts";
+import { expectNoLogAsync } from "./LogCatcher.ts";
+
+const fixturesDir = new URL("../../fixtures/", import.meta.url);
+
+const bevyBulkTest = {
+  name: "Bevy",
+  baseDir: "bevy-wgsl",
+  git: {
+    url: "https://github.com/wgsl-tooling-wg/bevy-wgsl.git",
+    revision: "84977ff025eaf8d92e56a9c35b815fae70eb4af0",
+  },
+};
+
+await fetchBulkTest(bevyBulkTest, fixturesDir);
+
+// Constants needed by various Bevy modules
+const bevyConstants = {
+  MAX_CASCADES_PER_LIGHT: 4,
+  MAX_DIRECTIONAL_LIGHTS: 4,
+  PER_OBJECT_BUFFER_BATCH_SIZE: 64,
+  TONEMAPPING_LUT_TEXTURE_BINDING_INDEX: 26,
+  TONEMAPPING_LUT_SAMPLER_BINDING_INDEX: 27,
+};
+
+// Files that need specific conditions to link or produce non-empty output
+const conditionalFiles: Record<string, Conditions> = {
+  "./pbr/ssr.wesl": { DEPTH_PREPASS: true, DEFERRED_PREPASS: true },
+  "./pbr/raymarch.wesl": { DEPTH_PREPASS: true },
+  "./pbr/prepass_utils.wesl": { DEPTH_PREPASS: true },
+  "./core_pipeline/oit.wesl": { OIT_ENABLED: true },
+  "./pbr/decal/clustered.wesl": { CLUSTERED_DECALS_ARE_USABLE: true },
+  "./pbr/decal/forward.wesl": { DEPTH_PREPASS: true },
+  "./pbr/morph.wesl": { MORPH_TARGETS: true },
+};
+
+// LATER: binding_array is a WGSL extension not yet supported
+const skipFiles = [
+  "./render/bindless.wesl", // uses binding_array directly
+  "./pbr/decal/clustered.wesl", // imports mesh_view_bindings which uses binding_array
+];
+
+// Files that only contain imports (re-export modules) - always produce empty output
+const importOnlyFiles = ["./sprite/mesh2d_view_types.wesl"];
+
+async function loadBevyBundle(): Promise<Record<string, string>> {
+  const bevyDir = new URL(
+    "src/shaders/bevy/",
+    new URL(bevyBulkTest.baseDir + "/", fixturesDir),
+  );
+  const bundle: Record<string, string> = {};
+  await loadDir(bevyDir, "", bundle);
+  return bundle;
+}
+
+async function loadDir(
+  baseUrl: URL,
+  relPath: string,
+  bundle: Record<string, string>,
+): Promise<void> {
+  const dirUrl = new URL(relPath, baseUrl);
+  const entries = await fs.readdir(dirUrl, { withFileTypes: true });
+  for (const entry of entries) {
+    const entryPath = relPath ? `${relPath}${entry.name}` : entry.name;
+    if (entry.isDirectory()) {
+      await loadDir(baseUrl, entryPath + "/", bundle);
+    } else if (entry.name.endsWith(".wesl")) {
+      const src = await fs.readFile(new URL(entryPath, baseUrl), "utf8");
+      bundle["./" + entryPath] = src;
+    }
+  }
+}
+
+const weslSrc = await loadBevyBundle();
+const allFiles = Object.keys(weslSrc)
+  .filter(f => !skipFiles.includes(f))
+  .sort();
+
+allFiles.forEach(file => {
+  test(`bevy: link ${file}`, async () => {
+    const conditions = conditionalFiles[file] ?? {};
+    const constants = bevyConstants;
+    const rootModuleName = file;
+
+    const result = await expectNoLogAsync(() =>
+      link({ weslSrc, rootModuleName, conditions, constants }),
+    );
+
+    const wgsl = result.dest;
+    if (importOnlyFiles.includes(file)) {
+      expect(wgsl).toBe("");
+    } else {
+      expect(wgsl.length).toBeGreaterThan(0);
+    }
+  });
+});

package/src/test/BindStdTypes.test.ts

@@ -0,0 +1,110 @@
+import { expect, test } from "vitest";
+import { link } from "../Linker.ts";
+
+test("bind standard types in local contexts", async () => {
+  const result = await link({
+    weslSrc: {
+      "main.wesl": `
+        struct Vertex { position: vec3f, normal: vec3f }
+        const SCALE: vec4f = vec4f(0.1, 0.2, 0.3, 0.4);
+
+        fn identity() -> mat3x3f {
+          return mat3x3f(1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0);
+        }
+
+        fn main() {
+          let v = Vertex(vec3f(0.0), vec3f(1.0, 0.0, 0.0));
+          let m = identity();
+        }
+      `,
+    },
+    rootModuleName: "main.wesl",
+  });
+
+  expect(result.dest).toContain("struct Vertex");
+  expect(result.dest).toContain("vec3f");
+  expect(result.dest).toContain("vec4f");
+  expect(result.dest).toContain("mat3x3f");
+  expect(result.dest).toContain("const SCALE");
+});
+
+test("bind types in @if conditionals", async () => {
+  const result = await link({
+    weslSrc: {
+      "main.wesl": `
+        @if(USE_VEC4)
+        fn process() -> vec4f { return vec4f(1.0); }
+
+        @if(USE_MAT)
+        fn transform() -> mat4x4f { return mat4x4f(); }
+
+        fn main() {
+          let x = process();
+          let y = transform();
+        }
+      `,
+    },
+    rootModuleName: "main.wesl",
+    conditions: { USE_VEC4: true, USE_MAT: true },
+  });
+
+  expect(result.dest).toContain("vec4f");
+  expect(result.dest).toContain("mat4x4f");
+  expect(result.dest).toContain("fn process()");
+  expect(result.dest).toContain("fn transform()");
+});
+
+test("bind types in cross-module imports with initializers", async () => {
+  const result = await link({
+    weslSrc: {
+      "main.wesl": `
+        import package::lib::SCALE;
+        import package::lib::color;
+
+        fn main() {
+          let s = SCALE.x;
+          let c = color.rgb;
+        }
+      `,
+      "lib.wesl": `
+        override SCALE: vec4f = vec4f(0.1, 0.2, 0.3, 0.4);
+        var<private> color: vec4f = vec4f(1.0, 0.0, 0.0, 1.0);
+      `,
+    },
+    rootModuleName: "main.wesl",
+  });
+
+  expect(result.dest).toContain("vec4f");
+  expect(result.dest).toContain("override SCALE");
+  expect(result.dest).toContain("var<private> color");
+});
+
+test("import function and struct from same module", async () => {
+  const result = await link({
+    weslSrc: {
+      "main.wesl": `
+        import package::space::bracketing::bracketing;
+        import package::space::bracketing::BracketingResult;
+
+        fn main() {
+          let r: BracketingResult = bracketing(vec2f(1.0, 0.0));
+        }
+      `,
+      "space/bracketing.wesl": `
+        struct BracketingResult {
+          vAxis0: vec2f,
+          vAxis1: vec2f,
+          blendAlpha: f32,
+        }
+
+        fn bracketing(dir: vec2f) -> BracketingResult {
+          return BracketingResult(dir, dir, 0.5);
+        }
+      `,
+    },
+    rootModuleName: "main.wesl",
+  });
+
+  expect(result.dest).toContain("struct BracketingResult");
+  expect(result.dest).toContain("fn bracketing");
+});