wesl 0.6.0-pre10

package/src/ParsedRegistry.ts

@@ -0,0 +1,120 @@
+import { WeslBundle } from "wesl";
+import { parseSrcModule, parseWESL, WeslAST } from "./ParseWESL.ts";
+import { normalize, noSuffix } from "./PathUtil.ts";
+import { resetScopeIds, SrcModule } from "./Scope.ts";
+
+export interface ParsedRegistry {
+  modules: Record<string, WeslAST>; // key is module path, e.g. "rand_pkg::foo::bar"
+}
+
+export function parsedRegistry(): ParsedRegistry {
+  resetScopeIds(); // for debug
+  return { modules: {} };
+}
+
+/** for debug */
+export function registryToString(registry: ParsedRegistry): string {
+  return `modules: ${[...Object.keys(registry.modules)]}`;
+}
+
+/**
+ * Parse WESL each src module (file) into AST elements and a Scope tree.
+ * @param src keys are module paths, values are wesl src strings
+ */
+export function parseWeslSrc(src: Record<string, string>): ParsedRegistry {
+  const parsedEntries = Object.entries(src).map(([path, src]) => {
+    const weslAST = parseWESL(src);
+    return [path, weslAST];
+  });
+  return { modules: Object.fromEntries(parsedEntries) };
+}
+
+/** Look up a module with a flexible selector.
+ *    :: separated module path,   package::util
+ *    / separated file path       ./util.wesl (or ./util)
+ *          - note: a file path should not include a weslRoot prefix, e.g. not ./shaders/util.wesl
+ *    simpleName                  util
+ */
+export function selectModule(
+  parsed: ParsedRegistry,
+  selectPath: string,
+  packageName = "package",
+): WeslAST | undefined {
+  // dlog({reg: [...Object.keys(parsed.modules)]});
+  let modulePath: string;
+  if (selectPath.includes("::")) {
+    modulePath = selectPath;
+  } else if (
+    selectPath.includes("/") ||
+    selectPath.endsWith(".wesl") ||
+    selectPath.endsWith(".wgsl")
+  ) {
+    modulePath = fileToModulePath(selectPath, packageName);
+  } else {
+    modulePath = packageName + "::" + selectPath;
+  }
+
+  return parsed.modules[modulePath];
+}
+
+/**
+ * @param srcFiles    map of source strings by file path
+ *                    key is '/' separated relative path (relative to srcRoot, not absolute file path )
+ *                    value is wesl source string
+ * @param registry    add parsed modules to this registry
+ * @param packageName name of package
+ */
+export function parseIntoRegistry(
+  srcFiles: Record<string, string>,
+  registry: ParsedRegistry,
+  packageName: string = "package",
+  debugWeslRoot?: string,
+): void {
+  if (debugWeslRoot === undefined) {
+    debugWeslRoot = "";
+  } else if (!debugWeslRoot.endsWith("/")) {
+    debugWeslRoot += "/";
+  }
+  const srcModules: SrcModule[] = Object.entries(srcFiles).map(
+    ([filePath, src]) => {
+      const modulePath = fileToModulePath(filePath, packageName);
+      return { modulePath, debugFilePath: debugWeslRoot + filePath, src };
+    },
+  );
+  srcModules.forEach(mod => {
+    const parsed = parseSrcModule(mod, undefined);
+    if (registry.modules[mod.modulePath]) {
+      throw new Error(`duplicate module path: '${mod.modulePath}'`);
+    }
+    registry.modules[mod.modulePath] = parsed;
+  });
+}
+
+export function parseLibsIntoRegistry(
+  libs: WeslBundle[],
+  registry: ParsedRegistry,
+): void {
+  libs.forEach(({ modules, name }) =>
+    parseIntoRegistry(modules, registry, name),
+  );
+}
+
+const libRegex = /^lib\.w[eg]sl$/i;
+
+/** convert a file path (./foo/bar.wesl)
+ *  to a module path (package::foo::bar) */
+function fileToModulePath(filePath: string, packageName: string): string {
+  if (filePath.includes("::")) {
+    // already a module path
+    return filePath;
+  }
+  if (packageName !== "package" && libRegex.test(filePath)) {
+    // special case for lib.wesl files in external packages
+    return packageName;
+  }
+
+  const strippedPath = noSuffix(normalize(filePath));
+  const moduleSuffix = strippedPath.replaceAll("/", "::");
+  const modulePath = packageName + "::" + moduleSuffix;
+  return modulePath;
+}

package/src/PathUtil.ts

@@ -0,0 +1,31 @@
+/** simplistic path manipulation utilities */
+
+/** return path with ./ and foo/.. elements removed */
+export function normalize(path: string): string {
+  const segments = path.split("/");
+  const noDots = segments.filter(s => s !== ".");
+  const noDbl: string[] = [];
+
+  noDots.forEach(s => {
+    if (s !== "") {
+      if (s === ".." && noDbl.length && noDbl[noDbl.length - 1] !== "..") {
+        noDbl.pop();
+      } else {
+        noDbl.push(s);
+      }
+    }
+  });
+
+  return noDbl.join("/");
+}
+
+/** return path w/o a suffix.
+ * e.g. /foo/bar.wgsl => /foo/bar */
+export function noSuffix(path: string): string {
+  const lastSlash = path.lastIndexOf("/");
+  const lastStart = lastSlash === -1 ? 0 : lastSlash + 1;
+
+  const suffix = path.indexOf(".", lastStart);
+  const suffixStart = suffix === -1 ? path.length : suffix;
+  return path.slice(0, suffixStart);
+}

package/src/RawEmit.ts

@@ -0,0 +1,102 @@
+import {
+  AttributeElem,
+  NameElem,
+  StuffElem,
+  TranslateTimeExpressionElem,
+  TypeRefElem,
+  TypeTemplateParameter,
+  UnknownExpressionElem,
+} from "./AbstractElems.ts";
+import { assertUnreachable } from "./Assertions.ts";
+import {
+  diagnosticControlToString,
+  expressionToString,
+  findDecl,
+} from "./LowerAndEmit.ts";
+import { RefIdent } from "./Scope.ts";
+
+// LATER DRY emitting elements like this with LowerAndEmit?
+
+export function attributeToString(e: AttributeElem): string {
+  const { kind } = e.attribute;
+  // LATER emit more precise source map info by making use of all the spans
+  // Like the first case does
+  if (kind === "@attribute") {
+    const { params } = e.attribute;
+    if (params === undefined || params.length === 0) {
+      return "@" + e.attribute.name;
+    } else {
+      return `@${e.attribute.name}(${params
+        .map(param => contentsToString(param))
+        .join(", ")})`;
+    }
+  } else if (kind === "@builtin") {
+    return "@builtin(" + e.attribute.param.name + ")";
+  } else if (kind === "@diagnostic") {
+    return (
+      "@diagnostic" +
+      diagnosticControlToString(e.attribute.severity, e.attribute.rule)
+    );
+  } else if (kind === "@if") {
+    return `@if(${expressionToString(e.attribute.param.expression)})`;
+  } else if (kind === "@interpolate") {
+    return `@interpolate(${e.attribute.params.map(v => v.name).join(", ")})`;
+  } else {
+    assertUnreachable(kind);
+  }
+}
+
+export function typeListToString(params: TypeTemplateParameter[]): string {
+  return `<${params.map(typeParamToString).join(", ")}>`;
+}
+
+export function typeParamToString(param?: TypeTemplateParameter): string {
+  if (param === undefined) return "?";
+  if (typeof param === "string") return param;
+
+  if (param.kind === "expression") return contentsToString(param);
+  if (param.kind === "type") return typeRefToString(param);
+  assertUnreachable(param);
+}
+
+export function typeRefToString(t?: TypeRefElem): string {
+  if (!t) return "?";
+  const { name, templateParams } = t;
+  const params = templateParams ? typeListToString(templateParams) : "";
+  return `${refToString(name)}${params}`;
+}
+
+function refToString(ref: RefIdent | string): string {
+  if (typeof ref === "string") return ref;
+  if (ref.std) return ref.originalName;
+  const decl = findDecl(ref);
+  return decl.mangledName || decl.originalName;
+}
+
+export function contentsToString(
+  elem:
+    | TranslateTimeExpressionElem
+    | UnknownExpressionElem
+    | NameElem
+    | StuffElem,
+): string {
+  if (elem.kind === "translate-time-expression") {
+    throw new Error("Not supported");
+  } else if (elem.kind === "expression" || elem.kind === "stuff") {
+    const parts = elem.contents.map(c => {
+      const { kind } = c;
+      if (kind === "text") {
+        return c.srcModule.src.slice(c.start, c.end);
+      } else if (kind === "ref") {
+        return refToString(c.ident);
+      } else {
+        return `?${c.kind}?`;
+      }
+    });
+    return parts.join(" ");
+  } else if (elem.kind === "name") {
+    return elem.name;
+  } else {
+    assertUnreachable(elem);
+  }
+}

package/src/Reflection.ts

@@ -0,0 +1,334 @@
+import { matchOneOf } from "mini-parse";
+import {
+  BindingStructElem,
+  NameElem,
+  StructMemberElem,
+  TextElem,
+  TranslateTimeExpressionElem,
+  TypeRefElem,
+  UnknownExpressionElem,
+} from "./AbstractElems.ts";
+import { assertThat } from "./Assertions.ts";
+import { TransformedAST, WeslJsPlugin } from "./Linker.ts";
+import { identElemLog } from "./LinkerUtil.ts";
+import { RefIdent } from "./Scope.ts";
+import {
+  multisampledTextureTypes,
+  sampledTextureTypes,
+  textureStorageTypes,
+} from "./StandardTypes.ts";
+import { findMap } from "./Util.ts";
+
+export type BindingStructReportFn = (structs: BindingStructElem[]) => void;
+export const textureStorage = matchOneOf(textureStorageTypes);
+
+export function reportBindingStructsPlugin(
+  fn: BindingStructReportFn,
+): WeslJsPlugin {
+  return {
+    transform: reportBindingStructs(fn),
+  };
+}
+/** 
+ * Linker plugin that generates TypeScript strings for GPUBindingGroupLayouts
+ * based on the binding structs in the WESL source
+ * 
+ * requires the enableBindingStructs() transform to be enabled
+ * 
+ * @param fn a function that will be called with the binding structs
+ * (Normally the caller will pass a function that uses bindingGroupLayoutTs() 
+ * to generate the TypeScript)
+ *
+ * The generated TypeScript looks looks roughly like this 
+
+  export function MyBindingLayout(device: GPUDevice): GPUBindGroupLayout {
+    return device.createBindGroupLayout({
+      entries: [
+        {
+          binding: 0,
+          visibility: GPUShaderStage.COMPUTE,
+          buffer: {
+            type: "storage",
+          },
+        },
+      ],
+    });
+  }
+*/
+export function reportBindingStructs(
+  fn: BindingStructReportFn,
+): (ast: TransformedAST) => TransformedAST {
+  return (ast: TransformedAST) => {
+    const structs = ast.notableElems.bindingStructs as BindingStructElem[];
+    fn(structs);
+    return ast;
+  };
+}
+
+function firstLetterLower(s: string): string {
+  return s[0].toLowerCase() + s.slice(1);
+}
+
+/**
+ * @return a string containing a generated TypeScript function that creates
+ * a GPUBindingGroupLayout instance to align with the binding structures
+ * in wesl source.
+ */
+export function bindingGroupLayoutTs(
+  struct: BindingStructElem,
+  typeScript = true,
+): string {
+  if (!struct) {
+    console.log("no struct!???");
+    return "";
+  }
+  const structName = firstLetterLower(struct.name.ident.mangledName!);
+  const visibility = shaderVisiblity(struct);
+  const entries = struct.members
+    .map(m => memberToLayoutEntry(m, visibility))
+    .join(",");
+
+  const fnName = `${structName}Layout`;
+  const entriesName = `${structName}Entries`;
+
+  const fnParams =
+    typeScript ? `(device: GPUDevice): GPUBindGroupLayout` : `(device)`;
+
+  const src = `
+const ${entriesName} = [ ${entries} ];
+function ${fnName}${fnParams} {
+  return device.createBindGroupLayout({
+    entries: ${entriesName} 
+  });
+}
+
+export const layoutFunctions = { ${fnName} };
+export const layoutEntries = { ${entriesName} };
+  `;
+  return src;
+}
+
+/** return the shader stage visibility for a binding struct, based on
+ * the shader entry function that has the binding struct as a parameter.
+ *
+ * The shader entry function is attached to the binding struct
+ * by the enableBindingStructs() transform.
+ */
+function shaderVisiblity(struct: BindingStructElem): string {
+  const { entryFn } = struct;
+  if (!entryFn) {
+    identElemLog(struct.name, "missing entry function for binding struct");
+  } else {
+    const { attributes = [] } = entryFn;
+    if (
+      attributes.find(
+        ({ attribute: a }) => a.kind === "@attribute" && a.name === "compute",
+      )
+    ) {
+      return "GPUShaderStage.COMPUTE";
+    }
+    if (
+      attributes.find(
+        ({ attribute: a }) => a.kind === "@attribute" && a.name === "vertex",
+      )
+    ) {
+      return "GPUShaderStage.VERTEX";
+    }
+    if (
+      attributes.find(
+        ({ attribute: a }) => a.kind === "@attribute" && a.name === "fragment",
+      )
+    ) {
+      return "GPUShaderStage.FRAGMENT";
+    }
+  }
+  identElemLog(struct.name, "unknown entry point type for binding struct");
+  return "GPUShaderStage.COMPUTE";
+}
+
+/**
+ * @return a GPUBindGroupLayoutEntry corresponding to one member
+ * of a WESL binding struct.
+ */
+function memberToLayoutEntry(
+  member: StructMemberElem,
+  visibility: string,
+): string {
+  const bindingParam = findMap(member.attributes ?? [], ({ attribute: a }) =>
+    a.kind === "@attribute" && a.name === "binding" ? a : undefined,
+  )?.params?.[0];
+  const binding = bindingParam ? paramText(bindingParam) : "?";
+
+  const src = `
+      {
+        binding: ${binding},
+        visibility: ${visibility},
+        ${layoutEntry(member)}
+      }`;
+  return src;
+}
+
+/** @return the guts of the GPUBindGroupLayoutEntry for this binding struct member.
+ * ptr references to storage arrays become 'buffer' GPUBufferBindingLayout intances,
+ * references to WGSL samplers become 'sampler' GPUSamplerBindingLayout instances, etc.
+ */
+function layoutEntry(member: StructMemberElem): string {
+  const { typeRef } = member;
+  let entry: string | undefined;
+  const { name: typeName } = typeRef;
+  entry = ptrLayoutEntry(typeRef) ?? storageTextureLayoutEntry(typeRef);
+  if (!entry && typeof typeName !== "string" && typeName.std) {
+    entry =
+      samplerLayoutEntry(typeRef) ??
+      textureLayoutEntry(typeRef) ??
+      externalTextureLayoutEntry(typeRef);
+  }
+  if (!entry) {
+    console.error(`unhandled type`, typeName);
+    entry = `{ }`;
+  }
+  return entry;
+}
+
+function ptrLayoutEntry(typeRef: TypeRefElem): string | undefined {
+  if (typeRef.name.originalName === "ptr") {
+    const param1 = typeRef.templateParams?.[0];
+    const param3 = typeRef.templateParams?.[2];
+    if (param1?.kind === "type" && param1.name.originalName === "uniform") {
+      return `buffer: { type: "uniform" }`;
+    } else if (
+      param1?.kind === "type" &&
+      param1.name.originalName === "storage"
+    ) {
+      if (param3?.kind === "type" && param3.name.originalName === "read") {
+        return `buffer: { type: "read-only-storage" }`;
+      } else {
+        return `buffer: { type: "storage" }`;
+      }
+      // LATER what do we do with the element type (2nd parameter)
+      // LATER should there be an ability to set hasDynamicOffset?
+    }
+  }
+}
+
+function samplerLayoutEntry(typeRef: TypeRefElem): string | undefined {
+  const { originalName } = typeRef.name as RefIdent;
+  if (originalName === "sampler") {
+    // LATER how do we set: type GPUSamplerBindingType = | "filtering" | "non-filtering";
+    // (just assuming filtering as a placeholder for now)
+    return `sampler: { type: "filtering" }`;
+  }
+  if (originalName === "sampler_comparison") {
+    return `sampler: { type: "comparison" }`;
+  }
+}
+
+const textureTypes = matchOneOf(sampledTextureTypes);
+const multiNames = matchOneOf(multisampledTextureTypes);
+
+function textureLayoutEntry(typeRef: TypeRefElem): string | undefined {
+  const { originalName } = typeRef.name as RefIdent;
+  const multisampled =
+    multiNames.test(originalName) ? ", multisampled: true, " : "";
+  if (multisampled || textureTypes.test(originalName)) {
+    // LATER viewDimension
+    const sampleType = getSampleType(typeRef);
+    return `texture: { sampleType: "${sampleType}"${multisampled} }`;
+  }
+  return undefined;
+
+  function getSampleType(typeRef: TypeRefElem): GPUTextureSampleType {
+    const firstParam = typeRef.templateParams?.[0] as TypeRefElem;
+    const texelType = (firstParam.name as RefIdent)
+      .originalName as WgslTexelType;
+    const sampleType = texelTypeToSampleType(texelType);
+    return sampleType;
+  }
+}
+
+function storageTextureLayoutEntry(typeRef: TypeRefElem): string | undefined {
+  if (textureStorage.test(typeRef.name.originalName)) {
+    const firstParam = typeRef.templateParams?.[0];
+    const secondParam = typeRef.templateParams?.[1];
+    assertThat(firstParam?.kind === "type"); // LATER: Temp hack
+    assertThat(secondParam?.kind === "type"); // LATER: Temp hack
+    const sampleType = formatToTextureSampleType(
+      firstParam.name.originalName as GPUTextureFormat,
+    );
+    const access = accessMode(secondParam.name.originalName);
+    return `storageTexture: { format: "${firstParam.name.originalName}", sampleType: "${sampleType}", access: "${access}" }`;
+  }
+  return undefined;
+}
+
+function externalTextureLayoutEntry(typeRef: TypeRefElem): string | undefined {
+  const { originalName } = typeRef.name as RefIdent;
+  if (originalName === "texture_external") {
+    // LATER. how would we set the required source: HTMLVideoElement or VideoFrame?
+  }
+  return undefined;
+}
+
+function paramText(
+  expression: UnknownExpressionElem | NameElem | TranslateTimeExpressionElem,
+): string {
+  assertThat(
+    expression.kind === "expression",
+    "Only expression elements are supported in this position",
+  );
+  const text = expression.contents[0] as TextElem;
+  return text.srcModule.src.slice(expression.start, expression.end);
+}
+
+export function formatToTextureSampleType(
+  format: GPUTextureFormat,
+  float32Filterable = false,
+): GPUTextureSampleType {
+  if (format.includes("32float")) {
+    return float32Filterable ? "float" : "unfilterable-float";
+  }
+  if (format.includes("float") || format.includes("unorm")) {
+    return "float";
+  }
+  if (format.includes("uint")) {
+    return "uint";
+  }
+  if (format.includes("sint")) {
+    return "sint";
+  }
+  throw new Error(`native sample type unknwon for texture format ${format}`);
+}
+
+export type WgslTexelType = "f32" | "u32" | "i32";
+
+/** return the wgsl element type for a given texture format */
+export function formatToTexelType(format: GPUTextureFormat): WgslTexelType {
+  if (format.includes("float")) return "f32";
+  if (format.includes("unorm")) return "f32";
+  if (format.includes("uint")) return "u32";
+  if (format.includes("sint")) return "i32";
+  throw new Error(`unknown format ${format}`);
+}
+
+/** @return the webgpu GPUTextureSampleType from the wgsl texel type */
+export function texelTypeToSampleType(
+  type: WgslTexelType,
+): GPUTextureSampleType {
+  if (type === "f32") return "float";
+  if (type === "u32") return "uint";
+  if (type === "i32") return "sint";
+  throw new Error(`unknown texel type ${type}`);
+}
+
+export function accessMode(access: string): GPUStorageTextureAccess {
+  if (access === "read") {
+    return "read-only";
+  }
+  if (access === "write") {
+    return "write-only";
+  }
+  if (access === "read_write") {
+    return "read-write";
+  }
+  throw new Error(`unknown access mode: ${access}`);
+}