wesl 0.7.26 → 0.7.28

package/src/Mangler.ts

@@ -25,7 +25,7 @@ export type ManglerFn = (
 /**
  * Construct a globally unique name based on the declaration
  * module path separated by underscores.
- * Corresponds to "Underscore-count mangling" from [NameMangling.md](https://github.com/wgsl-tooling-wg/wesl-spec/blob/main/NameMangling.md)
+ * Corresponds to "Underscore-count mangling" from [NameMangling.md](https://github.com/webgpu-tools/wesl-spec/blob/main/NameMangling.md)
  */
 export function underscoreMangle(
   decl: DeclIdent,

package/src/ModuleResolver.ts

@@ -1,5 +1,9 @@
 import { moduleToRelativePath, normalizeDebugRoot } from "./ModulePathUtil.ts";
-import { parseSrcModule, type WeslAST } from "./ParseWESL.ts";
+import {
+  parseSrcModule,
+  type WeslAST,
+  type WeslExtensions,
+} from "./ParseWESL.ts";
 import { normalize, noSuffix } from "./PathUtil.ts";
 import type { WeslBundle } from "./WeslBundle.ts";
 
@@ -20,6 +24,8 @@ export interface RecordResolverOptions {
   packageName?: string;
   /** Debug path prefix for error messages */
   debugWeslRoot?: string;
+  /** Enable experimental, not-yet-spec'd syntax extensions while parsing. */
+  weslExtensions?: WeslExtensions;
 }
 
 const libRegex = /^lib\.w[eg]sl$/i;
@@ -30,15 +36,17 @@ export class RecordResolver implements BatchModuleResolver {
   readonly sources: Record<string, string>;
   readonly packageName: string;
   readonly debugWeslRoot: string;
+  readonly weslExtensions?: WeslExtensions;
 
   constructor(
     sources: Record<string, string>,
     options: RecordResolverOptions = {},
   ) {
-    const { packageName = "package", debugWeslRoot } = options;
+    const { packageName = "package", debugWeslRoot, weslExtensions } = options;
     this.sources = sources;
     this.packageName = packageName;
     this.debugWeslRoot = normalizeDebugRoot(debugWeslRoot);
+    this.weslExtensions = weslExtensions;
   }
 
   resolveModule(modulePath: string): WeslAST | undefined {
@@ -49,7 +57,10 @@ export class RecordResolver implements BatchModuleResolver {
     if (source === undefined) return undefined;
 
     const debugFilePath = this.modulePathToDebugPath(modulePath);
-    const ast = parseSrcModule({ modulePath, debugFilePath, src: source });
+    const ast = parseSrcModule(
+      { modulePath, debugFilePath, src: source },
+      { weslExtensions: this.weslExtensions },
+    );
     this.astCache.set(modulePath, ast);
     return ast;
   }
@@ -171,7 +182,7 @@ export function freshResolver(inner: ModuleResolver): ModuleResolver {
       if (cached) return cached;
       const ast = inner.resolveModule(modulePath);
       if (!ast) return undefined;
-      const fresh = parseSrcModule(ast.srcModule);
+      const fresh = parseSrcModule(ast.srcModule, ast.parseOptions);
       cache.set(modulePath, fresh);
       return fresh;
     },

package/src/ParseWESL.ts

@@ -1,27 +1,18 @@
 import type {
   ConstAssertElem,
-  ContainerElem,
-  ImportElem,
   ImportStatement,
   ModuleElem,
 } from "./AbstractElems.ts";
 import { filterValidElements } from "./Conditions.ts";
 import { type FlatImport, flattenTreeImport } from "./FlattenTreeImport.ts";
+import { declsOfKind } from "./LinkerUtil.ts";
 import type { ParseError } from "./ParseError.ts";
 import { parseWesl } from "./parse/ParseWesl.ts";
-import type { ParseOptions } from "./parse/ParsingContext.ts";
+import type { ParseOptions, WeslExtensions } from "./parse/ParsingContext.ts";
 import type { Conditions, Scope, SrcModule } from "./Scope.ts";
 import type { Span } from "./Span.ts";
 import { errorHighlight, offsetToLineNumber } from "./Util.ts";
 
-export type { ParseOptions };
-
-/** Partial element being constructed during parsing. */
-export type OpenElem<T extends ContainerElem = ContainerElem> = Pick<
-  T,
-  "kind" | "contents"
->;
-
 /**
  * Result of parsing one WESL module (e.g., one .wesl file).
  *
@@ -41,6 +32,8 @@ export interface WeslAST {
   imports: ImportStatement[];
   /** Module level const_assert statements. */
   moduleAsserts?: ConstAssertElem[];
+  /** Parse options used to produce this AST (so re-parsing preserves them). */
+  parseOptions?: ParseOptions;
 }
 
 /** Extended AST with cached flattened imports. */
@@ -61,22 +54,25 @@ export type StableState = WeslAST;
 /** Unstable values used during parse collection. */
 export interface WeslParseContext {
   scope: Scope; // current scope (points somewhere in rootScope)
-  openElems: OpenElem[]; // elems that are collecting their contents
 }
 
+export type { ParseOptions, WeslExtensions };
+
 /** Human-readable error when parsing WESL fails. */
 export class WeslParseError extends Error {
   span: Span;
   src: SrcModule;
   constructor(opts: { cause: ParseError; src: SrcModule }) {
-    const source = opts.src.src;
-    const [lineNum, linePos] = offsetToLineNumber(opts.cause.span[0], source);
-    let message = `${opts.src.debugFilePath}:${lineNum}:${linePos}`;
-    message += ` error: ${opts.cause.message}\n`;
-    message += errorHighlight(source, opts.cause.span).join("\n");
-    super(message, { cause: opts.cause });
-    this.span = opts.cause.span;
-    this.src = opts.src;
+    const { cause, src } = opts;
+    const source = src.src;
+    const [lineNum, linePos] = offsetToLineNumber(cause.span[0], source);
+    const highlight = errorHighlight(source, cause.span).join("\n");
+    const message =
+      `${src.debugFilePath}:${lineNum}:${linePos}` +
+      ` error: ${cause.message}\n${highlight}`;
+    super(message, { cause });
+    this.span = cause.span;
+    this.src = src;
   }
 }
 
@@ -96,22 +92,14 @@ export function flatImports(
   // TODO cache per condition set?
   if (ast._flatImports && !conditions) return ast._flatImports;
 
-  // Get ImportElem elements from moduleElem contents
-  const importElems = ast.moduleElem.contents.filter(
-    (elem): elem is ImportElem => elem.kind === "import",
-  );
-
-  // Filter based on conditions if provided
+  const importElems = declsOfKind(ast.moduleElem, "import");
   const validImportElems = conditions
     ? filterValidElements(importElems, conditions)
     : importElems;
 
-  // Extract ImportStatement from valid ImportElem elements
-  const importStatements = validImportElems.map(elem => elem.imports);
-
-  const flat = importStatements.flatMap(flattenTreeImport);
-  if (!conditions) {
-    ast._flatImports = flat;
-  }
+  const flat = validImportElems.flatMap(elem =>
+    flattenTreeImport(elem.imports),
+  );
+  if (!conditions) ast._flatImports = flat;
   return flat;
 }

package/src/SrcMap.ts

@@ -88,14 +88,8 @@ export class SrcMap {
       const { src, position: srcStart } = this.destToSrc(e.srcStart);
       const { src: endSrc, position: srcEnd } = this.destToSrc(e.srcEnd);
       if (endSrc !== src) throw new Error("NYI, need to split");
-      const newEntry: SrcMapEntry = {
-        src,
-        srcStart,
-        srcEnd,
-        destStart: e.destStart,
-        destEnd: e.destEnd,
-      };
-      return newEntry;
+      const { destStart, destEnd } = e;
+      return { src, srcStart, srcEnd, destStart, destEnd };
     });
 
     const otherSources = other.entries.filter(
@@ -122,11 +116,6 @@ export class SrcMap {
   }
 }
 
-/** sort entries in place by src start position */
-function sortSrc(entries: SrcMapEntry[]): void {
-  entries.sort((a, b) => a.srcStart - b.srcStart);
-}
-
 /** Incrementally append to a string, tracking source references */
 export class SrcMapBuilder {
   #fragments: string[] = [];
@@ -191,12 +180,6 @@ export class SrcMapBuilder {
     this.add("\n", srcStart, srcEnd);
   }
 
-  /** copy a string fragment from the src to the destination string */
-  addCopy(srcStart: number, srcEnd: number): void {
-    const fragment = this.source.text.slice(srcStart, srcEnd);
-    this.add(fragment, srcStart, srcEnd);
-  }
-
   /** return a SrcMap */
   static build(builders: SrcMapBuilder[]): SrcMap {
     const map = new SrcMap(
@@ -207,3 +190,8 @@ export class SrcMapBuilder {
     return map;
   }
 }
+
+/** sort entries in place by src start position */
+function sortSrc(entries: SrcMapEntry[]): void {
+  entries.sort((a, b) => a.srcStart - b.srcStart);
+}

package/src/StandardTypes.ts

@@ -1,5 +1,5 @@
 // From https://www.w3.org/TR/WGSL/#predeclared
-// Use https://github.com/wgsl-tooling-wg/wgsl-spec to regenerate these list in the future
+// Use https://github.com/webgpu-tools/wgsl-spec to regenerate these list in the future
 
 export const stdFns = `bitcast all any select arrayLength 
   abs acos acosh asin asinh atan atanh atan2 ceil clamp cos cosh 

package/src/WeslDevice.ts

@@ -89,6 +89,7 @@ export function makeWeslDevice(device: GPUDevice): WeslDevice {
                   lineNumber: message.lineNum,
                   lineColumn: message.linePos,
                   length: message.length,
+                  offset: message.offset,
                   error: new Error(message.type + ": " + message.message),
                 });
               }

package/src/debug/ASTtoString.ts

@@ -2,15 +2,18 @@ import type {
   AbstractElem,
   Attribute,
   AttributeElem,
+  CommentElem,
   DirectiveElem,
+  DoBlockElem,
   FnElem,
-  StuffElem,
+  FnParamElem,
   TypedDeclElem,
   TypeRefElem,
   TypeTemplateParameter,
   UnknownExpressionElem,
 } from "../AbstractElems.ts";
 import { assertUnreachable } from "../Assertions.ts";
+import { childElems } from "../LinkerUtil.ts";
 import {
   diagnosticControlToString,
   expressionToString,
@@ -25,34 +28,38 @@ export function astToString(elem: AbstractElem, indent = 0): string {
   const str = new LineWrapper(indent, maxLineLength);
   str.add(kind);
   addElemFields(elem, str);
-  let childStrings: string[] = [];
-  if ("contents" in elem) {
-    childStrings = elem.contents.map(e => astToString(e, indent + 2));
-  }
-  if (childStrings.length) {
+  addCommentFields(elem, str);
+  const children = childElems(elem);
+  if (children.length) {
     str.nl();
+    const childStrings = children.map(e => astToString(e, indent + 2));
     str.addBlock(childStrings.join("\n"), false);
   }
 
   return str.result;
 }
 
+/** @return string representation of an attribute (for test/debug) */
+export function attributeToString(attr: Attribute): string {
+  const str = new LineWrapper(0, maxLineLength);
+  addAttributeFields(attr, str);
+  return str.result;
+}
+
 // LATER rewrite to be shorter and easier to read
 function addElemFields(elem: AbstractElem, str: LineWrapper): void {
   const { kind } = elem;
   // no-op kinds (handled elsewhere or no additional fields)
   if (
     kind === "assert" ||
+    kind === "expression" ||
     kind === "module" ||
     kind === "param" ||
-    kind === "stuff" ||
     kind === "switch-clause"
   ) {
     return;
   }
-  if (kind === "text") {
-    str.add(` '${elem.srcModule.src.slice(elem.start, elem.end)}'`);
-  } else if (
+  if (
     kind === "var" ||
     kind === "let" ||
     kind === "gvar" ||
@@ -68,13 +75,10 @@ function addElemFields(elem: AbstractElem, str: LineWrapper): void {
     str.add(` ${elem.name.name}: ${typeRefElemToString(elem.typeRef)}`);
   } else if (kind === "name") {
     str.add(" " + elem.name);
-  } else if (kind === "memberRef") {
-    const extra = elem.extraComponents
-      ? debugContentsToString(elem.extraComponents)
-      : "";
-    str.add(` ${elem.name.ident.originalName}.${elem.member.name}${extra}`);
   } else if (kind === "fn") {
     addFnFields(elem, str);
+  } else if (kind === "do") {
+    addDoFields(elem, str);
   } else if (kind === "alias") {
     const prefix = elem.name.ident.kind === "decl" ? "%" : "";
     str.add(
@@ -82,20 +86,8 @@ function addElemFields(elem: AbstractElem, str: LineWrapper): void {
     );
   } else if (kind === "attribute") {
     addAttributeFields(elem.attribute, str);
-  } else if (kind === "expression") {
-    const contents = elem.contents
-      .map(e =>
-        e.kind === "text"
-          ? `'${e.srcModule.src.slice(e.start, e.end)}'`
-          : astToString(e),
-      )
-      .join(" ");
-    str.add(" " + contents);
   } else if (kind === "type") {
-    const nameStr =
-      typeof elem.name === "string" ? elem.name : elem.name.originalName;
-    const params = elem.templateParams?.map(templateParamToString).join(", ");
-    str.add(params ? ` ${nameStr}<${params}>` : ` ${nameStr}`);
+    str.add(" " + typeRefElemToString(elem));
   } else if (kind === "synthetic") {
     str.add(` '${elem.text}'`);
   } else if (kind === "import") {
@@ -109,7 +101,25 @@ function addElemFields(elem: AbstractElem, str: LineWrapper): void {
     str.add(" %" + elem.ident.originalName);
   } else if (kind === "directive") {
     addDirective(elem, str);
-  } else if (kind === "statement" || kind === "continuing") {
+  } else if (
+    // statement kinds: the debug printer shows only their attributes
+    kind === "block" ||
+    kind === "if" ||
+    kind === "for" ||
+    kind === "while" ||
+    kind === "loop" ||
+    kind === "continuing" ||
+    kind === "switch" ||
+    kind === "return" ||
+    kind === "break" ||
+    kind === "continue" ||
+    kind === "discard" ||
+    kind === "assign" ||
+    kind === "increment" ||
+    kind === "decrement" ||
+    kind === "call" ||
+    kind === "empty"
+  ) {
     listAttributeElems(elem.attributes, str);
   } else if (kind === "literal") {
     str.add(` literal(${elem.value})`);
@@ -130,6 +140,18 @@ function addElemFields(elem: AbstractElem, str: LineWrapper): void {
   }
 }
 
+/** Show attached leading/trailing comments to verify comment attachment. */
+function addCommentFields(elem: AbstractElem, str: LineWrapper): void {
+  // SyntheticElem has no source position and so carries no comments
+  if (!("start" in elem)) return;
+  const { commentsBefore, commentsAfter } = elem;
+  if (commentsBefore?.length)
+    str.add(commentsToString("before", commentsBefore));
+  if (commentsAfter?.length) str.add(commentsToString("after", commentsAfter));
+  if ("innerComments" in elem && elem.innerComments?.length)
+    str.add(commentsToString("inner", elem.innerComments));
+}
+
 function addAttributeFields(attr: Attribute, str: LineWrapper) {
   const { kind } = attr;
   if (kind === "@attribute") {
@@ -155,13 +177,6 @@ function addAttributeFields(attr: Attribute, str: LineWrapper) {
   }
 }
 
-/** @return string representation of an attribute (for test/debug) */
-export function attributeToString(attr: Attribute): string {
-  const str = new LineWrapper(0, maxLineLength);
-  addAttributeFields(attr, str);
-  return str.result;
-}
-
 function addTypedDeclIdent(elem: TypedDeclElem, str: LineWrapper) {
   const { decl, typeRef } = elem;
   str.add(" %" + decl.ident.originalName);
@@ -170,17 +185,6 @@ function addTypedDeclIdent(elem: TypedDeclElem, str: LineWrapper) {
   }
 }
 
-function addFnFields(elem: FnElem, str: LineWrapper) {
-  const { name, params, returnType, attributes } = elem;
-  const paramStrs = params.map(p => {
-    const { originalName } = p.name.decl.ident;
-    return `${originalName}: ${typeRefElemToString(p.name.typeRef!)}`;
-  });
-  str.add(` ${name.ident.originalName}(${paramStrs.join(", ")})`);
-  listAttributeElems(attributes, str);
-  if (returnType) str.add(" -> " + typeRefElemToString(returnType));
-}
-
 /** show attribute names in short form to verify collection */
 function listAttributeElems(
   attrs: AttributeElem[] | undefined,
@@ -191,8 +195,25 @@ function listAttributeElems(
   });
 }
 
-function attributeName(attr: Attribute): string {
-  return attr.kind === "@attribute" ? "@" + attr.name : attr.kind;
+function typeRefElemToString(elem: TypeRefElem): string {
+  if (!elem) return "?type?";
+  const nameStr =
+    typeof elem.name === "string" ? elem.name : elem.name.originalName;
+  const params = elem.templateParams?.map(templateParamToString).join(", ");
+  return params ? `${nameStr}<${params}>` : nameStr;
+}
+
+function addFnFields(elem: FnElem, str: LineWrapper) {
+  const { name, params, returnType, attributes } = elem;
+  str.add(` ${name.ident.originalName}(${paramListToString(params)})`);
+  listAttributeElems(attributes, str);
+  if (returnType) str.add(" -> " + typeRefElemToString(returnType));
+}
+
+function addDoFields(elem: DoBlockElem, str: LineWrapper) {
+  const { name, params, attributes } = elem;
+  str.add(` ${name.name}(${paramListToString(params)})`);
+  listAttributeElems(attributes, str);
 }
 
 function addDirective(elem: DirectiveElem, str: LineWrapper) {
@@ -210,39 +231,34 @@ function addDirective(elem: DirectiveElem, str: LineWrapper) {
   listAttributeElems(attributes, str);
 }
 
-// LATER Temp hack while I clean up the expression parsing
+function commentsToString(label: string, comments: CommentElem[]): string {
+  const texts = comments.map(c => {
+    const text = c.srcModule.src.slice(c.start, c.end);
+    return c.blankBefore ? `'${text}'(blank)` : `'${text}'`;
+  });
+  return ` ${label}[${texts.join(", ")}]`;
+}
+
 function unknownExpressionToString(elem: UnknownExpressionElem): string {
-  if (!("contents" in elem)) return astToString(elem);
-  return elem.contents
-    .map(e => {
-      if (e.kind === "text")
-        return `'${e.srcModule.src.slice(e.start, e.end)}'`;
-      return astToString(e);
-    })
-    .join(" ");
+  return astToString(elem.expression);
+}
+
+function attributeName(attr: Attribute): string {
+  return attr.kind === "@attribute" ? "@" + attr.name : attr.kind;
 }
 
 function templateParamToString(p: TypeTemplateParameter): string {
   if (typeof p === "string") return p;
   if (p.kind === "type") return typeRefElemToString(p);
-  // ExpressionElem - use astToString for expression elements
-  return astToString(p);
+  // template params render inline inside <...>, so keep expressions on one line
+  return expressionToString(p);
 }
 
-function typeRefElemToString(elem: TypeRefElem): string {
-  if (!elem) return "?type?";
-  const nameStr =
-    typeof elem.name === "string" ? elem.name : elem.name.originalName;
-  const params = elem.templateParams?.map(templateParamToString).join(", ");
-  return params ? `${nameStr}<${params}>` : nameStr;
-}
-
-export function debugContentsToString(elem: StuffElem): string {
-  return elem.contents
-    .map(c => {
-      if (c.kind === "text") return c.srcModule.src.slice(c.start, c.end);
-      if (c.kind === "ref") return c.ident.originalName; // not using mapped decl name for debug
-      return `?${c.kind}?`;
-    })
-    .join(" ");
+/** @return "name: type, ..." for a fn/do parameter list. */
+function paramListToString(params: FnParamElem[]): string {
+  const paramStrs = params.map(p => {
+    const { originalName } = p.name.decl.ident;
+    return `${originalName}: ${typeRefElemToString(p.name.typeRef!)}`;
+  });
+  return paramStrs.join(", ");
 }

package/src/index.ts

@@ -7,6 +7,7 @@ export * from "./discovery/FindUnboundIdents.ts";
 export * from "./discovery/PackageNameUtils.ts";
 export * from "./LinkedWesl.ts";
 export * from "./Linker.ts";
+export * from "./LinkerUtil.ts";
 export * from "./LiveDeclarations.ts";
 export { debug, log, srcLog, validation, withLoggerAsync } from "./Logging.ts";
 export * from "./Mangler.ts";
@@ -20,7 +21,6 @@ export * from "./Scope.ts";
 export * from "./Span.ts";
 export * from "./SrcMap.ts";
 export * from "./StandardTypes.ts";
-export * from "./TransformBindingStructs.ts";
 export * from "./Util.ts";
 export * from "./WeslBundle.ts";
 export * from "./WeslDevice.ts";