wesl 0.6.0-pre10

package/src/LowerAndEmit.ts

@@ -0,0 +1,413 @@
+import { srcLog, SrcMapBuilder } from "mini-parse";
+import {
+  AbstractElem,
+  AttributeElem,
+  ContainerElem,
+  DeclIdentElem,
+  DirectiveElem,
+  ElemWithAttributes,
+  ExpressionElem,
+  FnElem,
+  NameElem,
+  RefIdentElem,
+  StructElem,
+  SyntheticElem,
+  TextElem,
+} from "./AbstractElems.ts";
+import {
+  assertThatDebug,
+  assertUnreachable,
+  assertUnreachableSilent,
+} from "./Assertions.ts";
+import { isGlobal } from "./BindIdents.ts";
+import { elementValid } from "./Conditions.ts";
+import { identToString } from "./debug/ScopeToString.ts";
+import { Conditions, DeclIdent, Ident } from "./Scope.ts";
+
+/** passed to the emitters */
+interface EmitContext {
+  srcBuilder: SrcMapBuilder; // constructing the linked output
+  conditions: Conditions; // settings for conditional compilation
+  extracting: boolean; // are we extracting or copying the root module
+}
+
+/** traverse the AST, starting from root elements, emitting wgsl for each */
+export function lowerAndEmit(
+  srcBuilder: SrcMapBuilder,
+  rootElems: AbstractElem[],
+  conditions: Conditions,
+  extracting = true,
+): void {
+  const emitContext: EmitContext = { conditions, srcBuilder, extracting };
+  // rootElems.forEach(r => console.log(astToString(r) + "\n"));
+  rootElems.forEach(e => lowerAndEmitElem(e, emitContext));
+}
+
+export function lowerAndEmitElem(e: AbstractElem, ctx: EmitContext): void {
+  if (!conditionsValid(e, ctx.conditions)) return;
+
+  switch (e.kind) {
+    // import statements are dropped from from emitted text
+    case "import":
+      return;
+
+    // terminal elements copy strings to the output
+    case "text":
+      return emitText(e, ctx);
+    case "name":
+      return emitName(e, ctx);
+    case "synthetic":
+      return emitSynthetic(e, ctx);
+
+    // identifiers are copied to the output, but with potentially mangled names
+    case "ref":
+      return emitRefIdent(e, ctx);
+    case "decl":
+      return emitDeclIdent(e, ctx);
+
+    // container elements just emit their child elements
+    case "param":
+    case "var":
+    case "typeDecl":
+    case "let":
+    case "module":
+    case "member":
+    case "memberRef":
+    case "expression":
+    case "type":
+    case "statement":
+    case "stuff":
+    case "switch-clause":
+      return emitContents(e, ctx);
+
+    // root level container elements get some extra newlines to make the output prettier
+    case "override":
+    case "const":
+    case "assert":
+    case "alias":
+    case "gvar":
+      emitRootElemNl(ctx);
+      return emitContents(e, ctx);
+
+    case "fn":
+      emitRootElemNl(ctx);
+      return emitFn(e, ctx);
+
+    case "struct":
+      emitRootElemNl(ctx);
+      return emitStruct(e, ctx);
+
+    case "attribute":
+      return emitAttribute(e, ctx);
+    case "directive":
+      return emitDirective(e, ctx);
+
+    default:
+      assertUnreachable(e);
+  }
+}
+
+/** emit root elems with a blank line inbetween */
+function emitRootElemNl(ctx: EmitContext): void {
+  if (ctx.extracting) {
+    ctx.srcBuilder.addNl();
+    ctx.srcBuilder.addNl();
+  }
+}
+
+export function emitText(e: TextElem, ctx: EmitContext): void {
+  ctx.srcBuilder.addCopy(e.start, e.end);
+}
+
+export function emitName(e: NameElem, ctx: EmitContext): void {
+  ctx.srcBuilder.add(e.name, e.start, e.end);
+}
+
+/** emit function explicitly so we can control commas between conditional parameters */
+export function emitFn(e: FnElem, ctx: EmitContext): void {
+  const { attributes, name, params, returnAttributes, returnType, body } = e;
+  const { conditions, srcBuilder: builder } = ctx;
+
+  emitAttributes(attributes, ctx);
+
+  builder.add("fn ", name.start - 3, name.start);
+  emitDeclIdent(name, ctx);
+
+  builder.appendNext("(");
+  const validParams = params.filter(p => conditionsValid(p, conditions));
+  validParams.forEach((p, i) => {
+    emitContentsNoWs(p, ctx);
+    if (i < validParams.length - 1) {
+      builder.appendNext(", ");
+    }
+  });
+  builder.appendNext(") ");
+
+  if (returnType) {
+    builder.appendNext("-> ");
+    emitAttributes(returnAttributes, ctx);
+    emitContents(returnType, ctx);
+    builder.appendNext(" ");
+  }
+
+  emitContents(body, ctx);
+}
+
+function emitAttributes(
+  attributes: AttributeElem[] | undefined,
+  ctx: EmitContext,
+): void {
+  attributes?.forEach(a => {
+    emitAttribute(a, ctx);
+    ctx.srcBuilder.add(" ", a.start, a.end);
+  });
+}
+
+/** emit structs explicitly so we can control commas between conditional members */
+export function emitStruct(e: StructElem, ctx: EmitContext): void {
+  const { name, members, start, end } = e;
+  const { srcBuilder } = ctx;
+
+  const validMembers = members.filter(m => conditionsValid(m, ctx.conditions));
+  const validLength = validMembers.length;
+
+  if (validLength === 0) {
+    warnEmptyStruct(e);
+    return;
+  }
+
+  srcBuilder.add("struct ", start, name.start);
+  emitDeclIdent(name, ctx);
+
+  if (validLength === 1) {
+    srcBuilder.add(" { ", name.end, members[0].start);
+    emitContentsNoWs(validMembers[0], ctx);
+    srcBuilder.add(" }\n", end - 1, end);
+  } else {
+    srcBuilder.add(" {\n", name.end, members[0].start);
+
+    validMembers.forEach(m => {
+      srcBuilder.add("  ", m.start - 1, m.start);
+      emitContentsNoWs(m, ctx);
+      srcBuilder.add(",", m.end, m.end + 1);
+      srcBuilder.addNl();
+    });
+
+    srcBuilder.add("}\n", end - 1, end);
+  }
+}
+
+function warnEmptyStruct(e: StructElem): void {
+  const { name, members } = e;
+  const condStr = members.length ? "(with current conditions)" : "";
+  const { debugFilePath: filePath } = name.srcModule;
+  srcLog(
+    name.srcModule.src,
+    e.start,
+    `struct ${name.ident.originalName} in ${filePath} has no members ${condStr}`,
+  );
+}
+
+export function emitSynthetic(e: SyntheticElem, ctx: EmitContext): void {
+  const { text } = e;
+  ctx.srcBuilder.addSynthetic(text, text, 0, text.length);
+}
+
+export function emitContents(elem: ContainerElem, ctx: EmitContext): void {
+  elem.contents.forEach(e => lowerAndEmitElem(e, ctx));
+}
+
+/** emit contents w/o white space */
+function emitContentsNoWs(elem: ContainerElem, ctx: EmitContext): void {
+  elem.contents.forEach(e => {
+    if (e.kind === "text") {
+      const { srcModule, start, end } = e;
+      const text = srcModule.src.slice(start, end);
+      if (text.trim() === "") {
+        return;
+      }
+    }
+    lowerAndEmitElem(e, ctx);
+  });
+}
+
+export function emitRefIdent(e: RefIdentElem, ctx: EmitContext): void {
+  if (e.ident.std) {
+    ctx.srcBuilder.add(e.ident.originalName, e.start, e.end);
+  } else {
+    const declIdent = findDecl(e.ident);
+    const mangledName = displayName(declIdent);
+    ctx.srcBuilder.add(mangledName!, e.start, e.end);
+  }
+}
+
+export function emitDeclIdent(e: DeclIdentElem, ctx: EmitContext): void {
+  const mangledName = displayName(e.ident);
+  ctx.srcBuilder.add(mangledName!, e.start, e.end);
+}
+
+function emitAttribute(e: AttributeElem, ctx: EmitContext): void {
+  const { kind } = e.attribute;
+  // LATER emit more precise source map info by making use of all the spans
+  // Like the first case does
+  if (kind === "@attribute") {
+    const { params } = e.attribute;
+    if (!params || params.length === 0) {
+      ctx.srcBuilder.add("@" + e.attribute.name, e.start, e.end);
+    } else {
+      ctx.srcBuilder.add(
+        "@" + e.attribute.name + "(",
+        e.start,
+        params[0].start,
+      );
+      for (let i = 0; i < params.length; i++) {
+        emitContents(params[i], ctx);
+        if (i < params.length - 1) {
+          ctx.srcBuilder.add(",", params[i].end, params[i + 1].start);
+        }
+      }
+      ctx.srcBuilder.add(")", params[params.length - 1].end, e.end);
+    }
+  } else if (kind === "@builtin") {
+    ctx.srcBuilder.add(
+      "@builtin(" + e.attribute.param.name + ")",
+      e.start,
+      e.end,
+    );
+  } else if (kind === "@diagnostic") {
+    ctx.srcBuilder.add(
+      "@diagnostic" +
+        diagnosticControlToString(e.attribute.severity, e.attribute.rule),
+      e.start,
+      e.end,
+    );
+  } else if (kind === "@if") {
+    // (@if is wesl only, dropped from wgsl)
+  } else if (kind === "@interpolate") {
+    ctx.srcBuilder.add(
+      `@interpolate(${e.attribute.params.map(v => v.name).join(", ")})`,
+      e.start,
+      e.end,
+    );
+  } else {
+    assertUnreachable(kind);
+  }
+}
+
+export function diagnosticControlToString(
+  severity: NameElem,
+  rule: [NameElem, NameElem | null],
+): string {
+  const ruleStr = rule[0].name + (rule[1] !== null ? "." + rule[1].name : "");
+  return `(${severity.name}, ${ruleStr})`;
+}
+
+export function expressionToString(elem: ExpressionElem): string {
+  const { kind } = elem;
+  if (kind === "binary-expression") {
+    return `${expressionToString(elem.left)} ${elem.operator.value} ${expressionToString(elem.right)}`;
+  } else if (kind === "unary-expression") {
+    return `${elem.operator.value}${expressionToString(elem.expression)}`;
+  } else if (kind === "ref") {
+    return elem.ident.originalName;
+  } else if (kind === "literal") {
+    return elem.value;
+  } else if (kind === "translate-time-feature") {
+    return elem.name;
+  } else if (kind === "parenthesized-expression") {
+    return `(${expressionToString(elem.expression)})`;
+  } else if (kind === "component-expression") {
+    return `${expressionToString(elem.base)}[${elem.access}]`;
+  } else if (kind === "component-member-expression") {
+    return `${expressionToString(elem.base)}.${elem.access}`;
+  } else if (kind === "call-expression") {
+    return `${elem.function.ident.originalName}(${elem.arguments.map(expressionToString).join(", ")})`;
+  } else {
+    assertUnreachable(kind);
+  }
+}
+
+function emitDirective(e: DirectiveElem, ctx: EmitContext): void {
+  const { directive } = e;
+  const { kind } = directive;
+  if (kind === "diagnostic") {
+    ctx.srcBuilder.add(
+      `diagnostic${diagnosticControlToString(directive.severity, directive.rule)};`,
+      e.start,
+      e.end,
+    );
+  } else if (kind === "enable") {
+    ctx.srcBuilder.add(
+      `enable ${directive.extensions.map(v => v.name).join(", ")};`,
+      e.start,
+      e.end,
+    );
+  } else if (kind === "requires") {
+    ctx.srcBuilder.add(
+      `requires ${directive.extensions.map(v => v.name).join(", ")};`,
+      e.start,
+      e.end,
+    );
+  } else {
+    assertUnreachable(kind);
+  }
+}
+
+function displayName(declIdent: DeclIdent): string {
+  if (isGlobal(declIdent)) {
+    assertThatDebug(
+      declIdent.mangledName,
+      `ERR: mangled name not found for decl ident ${identToString(declIdent)}`,
+    );
+    // mangled name was set in binding step
+    return declIdent.mangledName!;
+  }
+
+  return declIdent.mangledName || declIdent.originalName;
+}
+
+/** trace through refersTo links in reference Idents until we find the declaration
+ * expects that bindIdents has filled in all refersTo: links
+ */
+export function findDecl(ident: Ident): DeclIdent {
+  let i: Ident | undefined = ident;
+  do {
+    if (i.kind === "decl") {
+      return i;
+    }
+    i = i.refersTo;
+  } while (i);
+
+  // TODO show source position if this can happen in a non buggy linker.
+  throw new Error(`unresolved identifer: ${ident.originalName}`);
+}
+
+/** check if the element is visible with the current current conditional compilation settings */
+export function conditionsValid(
+  elem: AbstractElem,
+  conditions: Conditions,
+): true | false | undefined {
+  const attrElem = elem as ElemWithAttributes;
+  const { kind } = attrElem;
+
+  switch (kind) {
+    case "alias":
+    case "assert":
+    case "const":
+    case "directive":
+    case "member":
+    case "var":
+    case "let":
+    case "statement":
+    case "switch-clause":
+    case "override":
+    case "gvar":
+    case "fn":
+    case "struct":
+    case "param":
+      return elementValid(attrElem, conditions);
+    default:
+      assertUnreachableSilent(kind);
+  }
+  return true;
+}

package/src/Mangler.ts

@@ -0,0 +1,94 @@
+import { DeclIdent, SrcModule } from "./Scope.ts";
+/**
+ * A function for constructing a unique identifier name for a global declaration.
+ * Global names must be unique in the linked wgsl.
+ *
+ * Three manglers are currently available:
+ * . minimalMangle preserves original source names where possible
+ * . underscoreMangle constructs long but predictable names for every declaration
+ * . lengthPrefixMangle constructs long but predictable names for every declaration
+ */
+export type ManglerFn = (
+  /** global declaration that needs a name */
+  decl: DeclIdent,
+
+  /** module that contains the declaration */
+  srcModule: SrcModule,
+
+  /** name at use site (possibly import as renamed from declaration) */
+  proposedName: string,
+
+  /** current set of mangled root level names for the linked result (read only) */
+  globalNames: Set<string>,
+) => string;
+
+/**
+ * Construct a globally unique name based on the declaration
+ * module path separated by underscores.
+ * Corresponds to "Underscore-count mangling" from [NameMangling.md](https://github.com/wgsl-tooling-wg/wesl-spec/blob/main/NameMangling.md)
+ */
+export function underscoreMangle(
+  decl: DeclIdent,
+  srcModule: SrcModule,
+): string {
+  const { modulePath } = srcModule;
+  return [...modulePath.split("::"), decl.originalName]
+    .map(v => {
+      const underscoreCount = (v.match(/_/g) ?? []).length;
+      if (underscoreCount > 0) {
+        return "_" + underscoreCount + v;
+      } else {
+        return v;
+      }
+    })
+    .join("_");
+}
+
+/**
+ * Construct a globally unique name based on the declaration
+ */
+export function lengthPrefixMangle(
+  decl: DeclIdent,
+  srcModule: SrcModule,
+): string {
+  function codepointCount(text: string): number {
+    return [...text].length;
+  }
+  const qualifiedIdent = [
+    ...srcModule.modulePath.split("::"),
+    decl.originalName,
+  ];
+  return "_" + qualifiedIdent.map(v => codepointCount(v) + v).join("");
+}
+
+/**
+ * ManglerFn to construct a globally unique name
+ * using the requested name plus a uniquing number suffix if necessary
+ */
+export function minimalMangle(
+  _d: DeclIdent,
+  _s: SrcModule,
+  proposedName: string,
+  globalNames: Set<string>,
+): string {
+  return minimallyMangledName(proposedName, globalNames);
+}
+
+/**
+ * Construct a globally unique name by using the requested name if possible
+ * and appending a number suffix necessary
+ */
+export function minimallyMangledName(
+  proposedName: string,
+  globalNames: Set<string>,
+): string {
+  let renamed = proposedName;
+  let conflicts = 0;
+
+  // create a unique name
+  while (globalNames.has(renamed)) {
+    renamed = proposedName + conflicts++;
+  }
+
+  return renamed;
+}

package/src/ParseWESL.ts

@@ -0,0 +1,157 @@
+import { AppState, ParseError, ParserInit, SrcMap } from "mini-parse";
+import {
+  ConstAssertElem,
+  ImportStatement,
+  ModuleElem,
+} from "./AbstractElems.ts";
+import { FlatImport, flattenTreeImport } from "./FlattenTreeImport.ts";
+import { weslRoot } from "./parse/WeslGrammar.ts";
+import { WeslStream } from "./parse/WeslStream.ts";
+import { emptyScope, Scope, SrcModule } from "./Scope.ts";
+import { errorHighlight, offsetToLineNumber } from "./Util.ts";
+import { OpenElem } from "./WESLCollect.ts";
+import { throwClickableError } from "./WeslDevice.ts";
+
+/** result of a parse for one wesl module (e.g. one .wesl file)
+ *
+ * The parser constructs the AST constructed into three sections
+ * for convenient access by the binding stage.
+ *  - import statements
+ *  - language elements (fn, struct, etc)
+ *  - scopes
+ *
+ */
+export interface WeslAST {
+  /** source text for this module */
+  srcModule: SrcModule;
+
+  /** root module element */
+  moduleElem: ModuleElem;
+
+  /** root scope for this module */
+  rootScope: Scope;
+
+  /** imports found in this module */
+  imports: ImportStatement[];
+
+  /** module level const_assert statements */
+  moduleAsserts?: ConstAssertElem[];
+}
+
+/** an extended version of the AST */
+export interface BindingAST extends WeslAST {
+  /* a flattened version of the import statements constructed on demand from import trees, and cached here */
+  _flatImports?: FlatImport[];
+}
+
+/** stable and unstable state used during parsing */
+export interface WeslParseState
+  extends AppState<WeslParseContext, StableState> {
+  context: WeslParseContext;
+  stable: StableState;
+}
+
+/** stable values used or accumulated during parsing */
+export type StableState = WeslAST;
+
+/** unstable values used during parse collection */
+export interface WeslParseContext {
+  scope: Scope; // current scope (points somewhere in rootScope)
+  openElems: OpenElem[]; // elems that are collecting their contents
+}
+
+/**
+ * An error when parsing WESL fails. Designed to be human-readable.
+ */
+export class WeslParseError extends Error {
+  position: number;
+  src: SrcModule;
+  constructor(opts: { cause: ParseError; src: SrcModule }) {
+    const source = opts.src.src;
+    const [lineNum, linePos] = offsetToLineNumber(opts.cause.position, source);
+    let message = `${opts.src.debugFilePath}:${lineNum}:${linePos}`;
+    message += ` error: ${opts.cause.message}\n`;
+    message += errorHighlight(source, [
+      opts.cause.position,
+      opts.cause.position + 1,
+    ]).join("\n");
+    super(message, {
+      cause: opts.cause,
+    });
+    this.position = opts.cause.position;
+    this.src = opts.src;
+  }
+}
+
+/** Parse a WESL file. Throws on error. */
+export function parseSrcModule(srcModule: SrcModule, srcMap?: SrcMap): WeslAST {
+  const stream = new WeslStream(srcModule.src);
+
+  const appState = blankWeslParseState(srcModule);
+
+  const init: ParserInit = { stream, appState };
+  try {
+    const parseResult = weslRoot.parse(init);
+    if (parseResult === null) {
+      throw new Error("parseWESL failed");
+    }
+  } catch (e) {
+    if (e instanceof ParseError) {
+      const [lineNumber, lineColumn] = offsetToLineNumber(
+        e.position,
+        srcModule.src,
+      );
+      const error = new WeslParseError({ cause: e, src: srcModule });
+      throwClickableError({
+        url: srcModule.debugFilePath,
+        text: srcModule.src,
+        error,
+        lineNumber,
+        lineColumn,
+        length: 1,
+      });
+    } else {
+      throw e;
+    }
+  }
+
+  return appState.stable as WeslAST;
+}
+
+export function parseWESL(src: string, srcMap?: SrcMap): WeslAST {
+  const srcModule: SrcModule = {
+    modulePath: "package::test", // TODO this ought not be used outside of tests
+    debugFilePath: "./test.wesl",
+    src,
+  };
+
+  return parseSrcModule(srcModule, srcMap);
+}
+
+export function blankWeslParseState(srcModule: SrcModule): WeslParseState {
+  const rootScope = emptyScope(null);
+  const moduleElem = null as any; // we'll fill this in later
+  return {
+    context: { scope: rootScope, openElems: [] },
+    stable: { srcModule, imports: [], rootScope, moduleElem },
+  };
+}
+
+export function syntheticWeslParseState(): WeslParseState {
+  const srcModule: SrcModule = {
+    modulePath: "package::test",
+    debugFilePath: "./test.wesl",
+    src: "",
+  };
+
+  return blankWeslParseState(srcModule);
+}
+
+/** @return a flattened form of the import tree for convenience in binding idents. */
+export function flatImports(ast: BindingAST): FlatImport[] {
+  if (ast._flatImports) return ast._flatImports;
+
+  const flat = ast.imports.flatMap(flattenTreeImport);
+  ast._flatImports = flat;
+  return flat;
+}