wesl 0.6.0-pre2

package/src/Linker.ts

@@ -0,0 +1,151 @@
+import { SrcMap, SrcMapBuilder, tracing } from "mini-parse";
+import { bindIdents } from "./BindIdents.ts";
+import { lowerAndEmit } from "./LowerAndEmit.ts";
+import {
+  parsedRegistry,
+  ParsedRegistry,
+  parseIntoRegistry,
+  parseLibsIntoRegistry,
+  parseWeslSrc,
+  selectModule,
+} from "./ParsedRegistry.ts";
+import { Conditions } from "./Scope.ts";
+import { WgslBundle } from "./WgslBundle.ts";
+
+/* --- Overview: Plan for Linking WESL --- */
+
+/* 
+This is a bit of a rework/reshuffling from the 'legacy' version described in Internals.md.
+
+It expects the parser to identify three types of idents: 
+  global declarations, local declarations, references
+  (the legacy version distingished between type and variable idents, and more)
+
+It tracks scopes, and keeps them independently from the AST. It uses the scopes trees for
+binding references to declarations. (The legacy linker used combination
+of naming tricks and AST traversal to bind references to declarations.)
+
+Binding idents is more generic, which should simplify the code 
+and extend more easily to for importing elements beyond structs and functions.
+
+It asks less of the grammar, a complete WGSL grammar is easier to maintain 
+if it can match the WGSL spec.
+
+It replaces an AST pass with scope table pass, which should be a little faster. 
+
+It's much more friendly to future parallel execution and incremental rebuilding, 
+which should make things a lot faster when we go there.
+
+The architecture allows conditional compilation from the AST rather than from the src text.
+
+The AST is now immutable, mutation is confined to the Idents and Scopes. 
+*/
+
+/**
+ * Link a set of WESL source modules (typically the text from .wesl files) into a single WGSL string.
+ * Linking starts with a specified 'root' source module, and recursively incorporates code
+ * referenced from other modules (in local files or libraries).
+ *
+ * Unreferenced (dead) code outside the root module is not included in the output WGSL.
+ * Additionally the caller can specify conditions for to control conditional compilation.
+ * Only code that is valid with the current conditions is included in the output.
+ *
+ * @param weslSrc map of wesl source strings (aka modules) by scoped path
+ *                key is module path or file path
+ *                  `package::foo::bar`, or './foo/bar.wesl', or './foo/bar'
+ *                value is wesl src
+ *                (inludes both library and local WESL modules)
+ * @param rootModuleName name or module path of the root module
+ * @param conditions runtime conditions for conditional compilation
+ */
+export function linkWesl(
+  weslSrc: Record<string, string>,
+  rootModuleName: string = "main",
+  conditions: Conditions = {},
+): SrcMap {
+  /* --- Step #1   Parsing WESL --- */
+  // parse all source modules in both app and libraries,
+  // producing Scope tree and AST elements for each module
+  const parsed: ParsedRegistry = parseWeslSrc(weslSrc);
+
+  return linkRegistry(parsed, rootModuleName, conditions);
+}
+
+// TODO DRY entry points
+export function linkWeslFiles(
+  weslSrc: Record<string, string>,
+  rootModuleName: string = "main",
+  conditions: Conditions = {},
+  /** record of file names and wgsl text for modules */
+  libs: WgslBundle[] = [],
+  maxParseCount?: number,
+): SrcMap {
+  const registry = parsedRegistry();
+  parseIntoRegistry(weslSrc, registry, "package", maxParseCount);
+  parseLibsIntoRegistry(libs, registry);
+  return linkRegistry(registry, rootModuleName, conditions);
+}
+
+export function linkRegistry(
+  parsed: ParsedRegistry,
+  rootModuleName: string = "main",
+  conditions: Conditions = {},
+): SrcMap {
+  // get a reference to the root module
+  const found = selectModule(parsed, rootModuleName);
+  if (!found) {
+    if (tracing) {
+      console.log(`parsed modules: ${Object.keys(parsed.modules)}`); 
+      console.log(`root module not found: ${rootModuleName}`); 
+    }
+    throw new Error(`Root module not found: ${rootModuleName}`);
+  }
+  const { moduleElem: rootModule } = found;
+  
+  /* --- Step #2   Binding Idents --- */
+  // link active Ident references to declarations, and uniquify global declarations
+  const newDecls = bindIdents(found, parsed, conditions);
+
+  /* --- Step #3   Writing WGSL --- */
+  // traverse the AST and emit WGSL (doesn't need scopes)
+  const srcBuilder = new SrcMapBuilder();
+  lowerAndEmit(srcBuilder, [rootModule], conditions, false); // emit the entire root module
+  lowerAndEmit(srcBuilder, newDecls, conditions); // emit referenced declarations from other modules
+  return srcBuilder.build();
+}
+
+/* ---- Commentary on present and future features ---- */
+/*
+
+TODO 
+- distinguish between global and local declaration idents (only global ones need be uniquified)
+
+Conditions
+- conditions are attached to the AST elements where they are defined
+  - only conditionally valid elements are emitted
+- consolidated conditions are attached to Idents
+  - only conditionally valid ref Idents are bound, and only to conditionaly valid declarations
+  - a condition stack (akin to the scope stack) is maintained while parsing to attach consolidated conditions to Idents
+- re-linking with new conditions, conservatively 
+  - clear all mutated Ident fields (refersTo and mangled links) 
+  - re-bind Idents, re-emit 
+
+Generics & specialization
+- attach generic parameters to ref and decl Idents, effectively creating a new Ident for each specialization
+- generate specialized elements at emit time, by checking the generic parameters of the decl ident
+
+Incrementally rebuilding
+- unchanged files don't need to be reparsed, only reparse dirty files.
+- support reflection only mode? no need to bind idents or emit for e.g. vite/IDE plugin generating reflection types 
+
+Parallel Processing (coarse grained via webworkers)
+- Parsing each module can be done in parallel
+- binding could be done partially in parallel? (esbuild doesn't parallelize here though)
+  - finding the declaration for each local ident could be done in parallel by module
+  - matching 
+- Emitting could be easily modified to be done in partially in parallel
+  - traversing the AST to list the top level elements to emit could be done serially
+  - the text for each top level element could be emitted in parallel (presumably the bulk of the work)
+  - the merged text can be assembled serially
+
+*/

package/src/LowerAndEmit.ts

@@ -0,0 +1,143 @@
+import { SrcMapBuilder, tracing } from "mini-parse";
+import {
+  AbstractElem,
+  DeclIdentElem,
+  RefIdentElem,
+  NameElem,
+  TextElem,
+} from "./AbstractElems.ts";
+import { isGlobal } from "./BindIdents.ts";
+import { Conditions, DeclIdent, Ident, RefIdent } from "./Scope.ts";
+import { identToString } from "./debug/ScopeToString.ts";
+
+/** passed to the emitters */
+interface EmitContext {
+  srcBuilder: SrcMapBuilder; // constructing the linked output
+  conditions: Conditions; // settings for conditional compilation
+  extracting: boolean; // are we extracting or copying the root module
+}
+
+/** traverse the AST, starting from root elements, emitting wgsl for each */
+export function lowerAndEmit(
+  srcBuilder: SrcMapBuilder,
+  rootElems: AbstractElem[],
+  conditions: Conditions,
+  extracting = true,
+): void {
+  const emitContext: EmitContext = { conditions, srcBuilder, extracting };
+  lowerAndEmitRecursive(rootElems, emitContext);
+}
+
+function lowerAndEmitRecursive(
+  elems: AbstractElem[],
+  emitContext: EmitContext,
+): void {
+  const validElems = elems.filter(e =>
+    conditionsValid(e, emitContext.conditions),
+  );
+  validElems.forEach(e => lowerAndEmitElem(e, emitContext));
+}
+
+export function lowerAndEmitElem(e: AbstractElem, ctx: EmitContext): void {
+  switch (e.kind) {
+    case "import":
+      return; // drop imports statements from emitted text
+    case "fn":
+    case "struct":
+    case "override":
+    case "const":
+    case "assert":
+    case "alias":
+    case "gvar":
+      if (ctx.extracting) {
+        ctx.srcBuilder.addNl();
+        ctx.srcBuilder.addNl();
+      }
+      return emitContents(e, ctx);
+    case "text":
+      return emitText(e, ctx);
+    case "ref":
+    case "decl":
+      return emitIdent(e, ctx);
+    case "param":
+    case "var":
+    case "module":
+    case "member":
+      return emitContents(e, ctx);
+    case "name":
+      return emitName(e, ctx);
+    default:
+      const kind = (e as any).kind;
+      console.log("NYI for emit, elem kind:", kind);
+      throw new Error(`NYI emit elem kind: ${kind}`);
+  }
+}
+
+export function emitText(e: TextElem, ctx: EmitContext): void {
+  ctx.srcBuilder.addCopy(e.srcModule.src, e.start, e.end);
+}
+
+export function emitName(e: NameElem, ctx: EmitContext): void {
+  ctx.srcBuilder.add(e.name, e.srcModule.src, e.start, e.end);
+}
+
+export function emitContents(
+  elem: AbstractElem & { contents: AbstractElem[] },
+  ctx: EmitContext,
+): void {
+  elem.contents.forEach(e => lowerAndEmitElem(e, ctx));
+}
+
+export function emitIdent(
+  e: RefIdentElem | DeclIdentElem,
+  ctx: EmitContext,
+): void {
+  if ((e.ident as RefIdent).std) {
+    ctx.srcBuilder.add(e.ident.originalName, e.srcModule.src, e.start, e.end);
+  } else {
+    const declIdent = findDecl(e.ident);
+    const mangledName = displayName(declIdent);
+    ctx.srcBuilder.add(mangledName!, e.srcModule.src, e.start, e.end);
+  }
+}
+
+function displayName(declIdent: DeclIdent): string {
+  if (declIdent.declElem && isGlobal(declIdent.declElem)) {
+    // mangled name was set in binding step
+    const mangledName = declIdent.mangledName;
+    if (tracing && !mangledName) {
+      console.log(
+        "ERR: mangled name not found for decl ident",
+        identToString(declIdent),
+      );
+    }
+    return mangledName!;
+  }
+
+  return declIdent.mangledName || declIdent.originalName;
+}
+
+/** trace through refersTo links in reference Idents until we find the declaration
+ * expects that bindIdents has filled in all refersTo: links
+ */
+export function findDecl(ident: Ident): DeclIdent {
+  let i: Ident | undefined = ident;
+  do {
+    if (i.kind === "decl") {
+      return i;
+    }
+    i = i.refersTo;
+  } while (i);
+
+  throw new Error(
+    `unresolved ident: ${ident.originalName} (bug in bindIdents?)`,
+  );
+}
+
+/** check if the element is visible with the current current conditional compilation settings */
+export function conditionsValid(
+  elem: AbstractElem,
+  conditions: Conditions,
+): boolean {
+  return true;
+}

package/src/ParseWESL.ts

@@ -0,0 +1,106 @@
+import { AppState, matchingLexer, ParserInit, SrcMap } from "mini-parse";
+import { ModuleElem } from "./AbstractElems.ts";
+import { ImportTree } from "./ImportTree.ts";
+import { mainTokens } from "./WESLTokens.ts";
+import { emptyScope, resetScopeIds, Scope, SrcModule } from "./Scope.ts";
+import { OpenElem } from "./WESLCollect.ts";
+import { weslRoot } from "./WESLGrammar.ts";
+import { FlatImport, flattenTreeImport } from "./FlattenTreeImport.ts";
+
+/** result of a parse */
+export interface WeslAST {
+  /** source text for this module */
+  srcModule: SrcModule;
+
+  /** root module element */
+  moduleElem: ModuleElem;
+
+  /** root scope for this module */
+  rootScope: Scope;
+
+  /** imports found in this module */
+  imports: ImportTree[];
+
+  /* constructed on demand from import trees, and cached */
+  _flatImports?: FlatImport[];
+}
+
+/** stable and unstable state used during parsing */
+export interface WeslParseState
+  extends AppState<WeslParseContext, StableState> {
+  context: WeslParseContext;
+  stable: StableState;
+}
+
+/** stable values used or accumulated during parsing */
+export type StableState = WeslAST;
+
+/** unstable values used during parse collection */
+export interface WeslParseContext {
+  scope: Scope; // current scope (points somewhere in rootScope)
+  openElems: OpenElem[]; // elems that are collecting their contents
+}
+
+export function parseSrcModule(
+  srcModule: SrcModule,
+  srcMap?: SrcMap,
+  maxParseCount: number | undefined = undefined,
+): WeslAST {
+  // TODO allow returning undefined for failure, or throw?
+
+  resetScopeIds();
+  const lexer = matchingLexer(srcModule.src, mainTokens);
+
+  const appState = blankWeslParseState(srcModule);
+
+  const init: ParserInit = { lexer, appState, srcMap, maxParseCount };
+  const parseResult = weslRoot.parse(init);
+  if (parseResult === null) {
+    throw new Error("parseWESL failed");
+  }
+
+  return appState.stable as WeslAST;
+}
+
+// for tests. TODO rename
+export function parseWESL(
+  src: string,
+  srcMap?: SrcMap,
+  maxParseCount: number | undefined = undefined,
+): WeslAST {
+  const srcModule: SrcModule = {
+    modulePath: "package::test",
+    filePath: "./test.wesl",
+    src,
+  };
+
+  return parseSrcModule(srcModule, srcMap, maxParseCount);
+}
+
+export function blankWeslParseState(srcModule: SrcModule): WeslParseState {
+  const rootScope = emptyScope("module-scope");
+  const moduleElem = null as any; // we'll fill this in later
+  return {
+    context: { scope: rootScope, openElems: [] },
+    stable: { srcModule, imports: [], rootScope, moduleElem },
+  };
+}
+
+export function syntheticWeslParseState(): WeslParseState {
+  const srcModule: SrcModule = {
+    modulePath: "package::test",
+    filePath: "./test.wesl",
+    src: "",
+  };
+
+  return blankWeslParseState(srcModule);
+}
+
+/** @return a flattened form of the import tree for convenience in binding idents. */
+export function flatImports(ast: WeslAST): FlatImport[] {
+  if (ast._flatImports) return ast._flatImports;
+
+  const flat = ast.imports.flatMap(flattenTreeImport);
+  ast._flatImports = flat;
+  return flat;
+}

package/src/ParsedRegistry.ts

@@ -0,0 +1,97 @@
+import { WgslBundle } from "random_wgsl";
+import { parseSrcModule, parseWESL, WeslAST } from "./ParseWESL.ts";
+import { normalize, noSuffix } from "./PathUtil.ts";
+import { SrcModule } from "./Scope.ts";
+
+export interface ParsedRegistry {
+  modules: Record<string, WeslAST>; // key is module path, e.g. "rand_pkg::foo::bar"
+}
+
+export function parsedRegistry(): ParsedRegistry {
+  return { modules: {} };
+}
+
+/**
+ * Parse WESL each src module (file) into AST elements and a Scope tree.
+ * @param src keys are module paths, values are wesl src strings
+ */
+export function parseWeslSrc(src: Record<string, string>): ParsedRegistry {
+  const parsedEntries = Object.entries(src).map(([path, src]) => {
+    const weslAST = parseWESL(src);
+    return [path, weslAST];
+  });
+  return { modules: Object.fromEntries(parsedEntries) };
+}
+
+/** Look up a module with a flexible selector.
+ *    :: separated module path,   package::util
+ *    / separated file path       ./util.wesl (or ./util)
+ *    simpleName                  util
+ */
+export function selectModule(
+  parsed: ParsedRegistry,
+  selectPath: string,
+  packageName = "package",
+): WeslAST | undefined {
+  let modulePath: string;
+  if (selectPath.includes("::")) {
+    modulePath = selectPath;
+  } else if (selectPath.includes("/")) {
+    modulePath = fileToModulePath(selectPath, packageName);
+  } else {
+    modulePath = packageName + "::" + selectPath;
+  }
+
+  return parsed.modules[modulePath];
+}
+
+/**
+ * @param srcFiles    map of source strings by file path
+ *                    key is '/' separated relative path (relative to srcRoot, not absolute file path )
+ *                    value is wesl source string
+ * @param registry    add parsed modules to this registry
+ * @param packageName name of package
+ */
+export function parseIntoRegistry(
+  srcFiles: Record<string, string>,
+  registry: ParsedRegistry,
+  packageName: string = "package",
+  maxParseCount?: number,
+): void {
+  const srcModules: SrcModule[] = Object.entries(srcFiles).map(
+    ([filePath, src]) => {
+      const modulePath = fileToModulePath(filePath, packageName);
+      return { modulePath, filePath, src };
+    },
+  );
+  srcModules.forEach(mod => {
+    const parsed = parseSrcModule(mod, undefined, maxParseCount);
+    if (registry.modules[mod.modulePath]) {
+      throw new Error(`duplicate module path: '${mod.modulePath}'`);
+    }
+    registry.modules[mod.modulePath] = parsed;
+  });
+}
+
+export function parseLibsIntoRegistry(
+  libs: WgslBundle[],
+  registry: ParsedRegistry,
+): void {
+  libs.forEach(({ modules, name }) =>
+    parseIntoRegistry(modules, registry, name),
+  );
+}
+
+const libExp = /^lib\.w[eg]sl$/i;
+
+/** convert a relative file path (./foo/bar.wesl) to a module path (package::foo::bar) */
+function fileToModulePath(filePath: string, packageName: string): string {
+  if (packageName !== "package" && libExp.test(filePath)) {
+    // special case for lib.wesl files in external packages
+    return packageName;
+  }
+  const strippedPath = noSuffix(normalize(filePath));
+  const moduleSuffix = strippedPath.replaceAll("/", "::");
+  const modulePath = packageName + "::" + moduleSuffix;
+  return modulePath;
+}

package/src/PathUtil.ts

@@ -0,0 +1,52 @@
+/** simplistic path manipulation utilities */
+
+export function relativePath(
+  srcPath: string | undefined,
+  reqPath: string,
+): string {
+  if (!srcPath) return reqPath;
+  const srcDir = dirname(srcPath);
+  const relative = join(srcDir, reqPath);
+  return relative;
+}
+
+export function dirname(path: string): string {
+  const lastSlash = path.lastIndexOf("/");
+  if (lastSlash === -1) return ".";
+  return path.slice(0, lastSlash);
+}
+
+export function join(a: string, b: string): string {
+  const joined = b.startsWith("/") ? a + b : a + "/" + b;
+  return normalize(joined);
+}
+
+/** return path with ./ and foo/.. elements removed */
+export function normalize(path: string): string {
+  const segments = path.split("/");
+  const noDots = segments.filter(s => s !== ".");
+  const noDbl: string[] = [];
+
+  noDots.forEach(s => {
+    if (s !== "") {
+      if (s === ".." && noDbl.length && noDbl[noDbl.length - 1] !== "..") {
+        noDbl.pop();
+      } else {
+        noDbl.push(s);
+      }
+    }
+  });
+
+  return noDbl.join("/");
+}
+
+/** return path w/o a suffix.
+ * e.g. /foo/bar.wgsl => /foo/bar */
+export function noSuffix(path: string): string {
+  const lastSlash = path.lastIndexOf("/");
+  const lastStart = lastSlash === -1 ? 0 : lastSlash + 1;
+
+  const suffix = path.indexOf(".", lastStart);
+  const suffixStart = suffix === -1 ? path.length : suffix;
+  return path.slice(0, suffixStart);
+}

package/src/Scope.ts

@@ -0,0 +1,100 @@
+import { DeclarationElem, RefIdentElem } from "./AbstractElems.ts";
+import { WeslAST } from "./ParseWESL.ts";
+
+export interface SrcModule {
+  /** module path "rand_pkg::sub::foo", or "package::main" */
+  modulePath: string; // TODO drop this?
+
+  /** file path to the module for user error reporting e.g "rand_pkg:sub/foo.wesl", or "./sub/foo.wesl" */
+  filePath: string;
+
+  /** original src for module */
+  src: string;
+}
+
+/** a src declaration or reference to an ident */
+export type Ident = DeclIdent | RefIdent;
+
+export type Conditions = Record<string, boolean>;
+
+interface IdentBase {
+  originalName: string; // name in the source code for ident matching (may be mangled in the output)
+  conditions?: Conditions; // conditions under which this ident is valid (combined from all containing elems)
+  id?: number; // for debugging
+}
+
+export interface RefIdent extends IdentBase {
+  kind: "ref";
+  refersTo?: Ident; // import or decl ident in scope to which this ident refers. undefined before binding
+  std?: true; // true if this is a standard wgsl identifier (like sin, or u32)
+  ast: WeslAST; // AST from module that contains this ident (to find imports during decl binding)
+  scope: Scope; // scope containing this reference (bind to decls starting from this scope)
+  refIdentElem?: RefIdentElem; // for error reporting
+}
+
+export interface DeclIdent extends IdentBase {
+  kind: "decl";
+  mangledName?: string; // name in the output code
+  declElem?: DeclarationElem; // link to AST so that we can traverse scopes and know what elems to emit // TODO make separate GlobalDecl kind with this required
+  scope: Scope; // scope for the references within this declaration
+}
+
+export type ScopeKind =
+  | "module-scope" // root scope for a module (file)
+  | "body-scope"; // a scope inside the module (fn body, nested block, etc.)
+
+/** tree of ident references, organized by lexical scope */
+export interface Scope {
+  id?: number; // for debugging
+  idents: Ident[]; // idents found in lexical order in this scope
+  parent: Scope | null; // null for root scope in a module
+  children: Scope[];
+  kind: ScopeKind;
+}
+
+export function resetScopeIds() {
+  // for debugging
+  scopeId = 0;
+}
+
+let scopeId = 0; // for debugging
+
+/** make a new Scope object */
+export function makeScope(s: Omit<Scope, "id">): Scope {
+  return { ...s, id: scopeId++ };
+}
+
+export interface RootAndScope {
+  rootScope: Scope;
+  scope: Scope;
+}
+
+export function emptyScope(kind: ScopeKind): Scope {
+  return makeScope({ idents: [], parent: null, children: [], kind });
+}
+
+export function emptyBodyScope(parent: Scope): Scope {
+  return makeScope({ kind: "body-scope", idents: [], parent, children: [] });
+}
+
+/** For debugging,
+ * @return true if a scope is in the rootScope tree somewhere */
+export function containsScope(rootScope: Scope, scope: Scope): boolean {
+  if (scope === rootScope) {
+    return true;
+  }
+  for (const child of rootScope.children) {
+    if (containsScope(child, scope)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export function exportDecl(scope: Scope, name: string): DeclIdent | undefined {
+  for (const ident of scope.idents) {
+    if (ident.originalName === name && ident.kind === "decl") {
+      return ident;
+    }
+  }
+}