wesl 0.6.0-pre2

package/src/AbstractElems.ts

@@ -0,0 +1,148 @@
+import { ImportTree } from "./ImportTree.ts";
+import { DeclIdent, RefIdent, SrcModule } from "./Scope.ts";
+
+export type AbstractElem =
+  | AliasElem
+  | ConstElem
+  | ImportElem
+  | ConstAssertElem
+  | FnElem
+  | RefIdentElem
+  | DeclIdentElem
+  | ModuleElem
+  | NameElem
+  | OverrideElem
+  | ParamElem
+  | StructElem
+  | StructMemberElem
+  | TextElem
+  | GlobalVarElem
+  | VarElem;
+
+export type DeclarationElem =
+  | AliasElem
+  | ConstElem
+  | OverrideElem
+  | ParamElem
+  | FnElem
+  | StructElem
+  | GlobalVarElem
+  | VarElem;
+
+export interface AbstractElemBase {
+  kind: string;
+  start: number;
+  end: number;
+}
+
+export interface ElemWithContents extends AbstractElemBase {
+  contents: AbstractElem[];
+}
+
+export interface ImportElem extends ElemWithContents {
+  kind: "import";
+  imports: ImportTree;
+}
+
+/** an identifier in WESL source */
+export interface RefIdentElem extends AbstractElemBase {
+  kind: RefIdent["kind"];
+  ident: RefIdent;
+  srcModule: SrcModule;
+}
+
+/** an identifier in WESL source */
+export interface DeclIdentElem extends AbstractElemBase {
+  kind: DeclIdent["kind"];
+  ident: DeclIdent;
+  srcModule: SrcModule;
+}
+
+/** a raw bit of text in WESL source that's typically copied to the linked WGSL. 
+ e.g. a keyword  like 'var' or '@diagnostic(off,derivative_uniformity)'
+*/
+export interface TextElem extends AbstractElemBase {
+  kind: "text";
+  srcModule: SrcModule;
+}
+
+/** a parameter in a function declaration */
+export interface ParamElem extends ElemWithContents {
+  kind: "param";
+  name: DeclIdentElem;
+  typeRef: RefIdentElem;
+}
+
+/** a variable declaration */
+export interface VarElem extends ElemWithContents {
+  kind: "var";
+  name: DeclIdentElem;
+  typeRef?: RefIdentElem;
+}
+
+/** a global variable declaration (at the root level) */
+export interface GlobalVarElem extends ElemWithContents {
+  kind: "gvar";
+  name: DeclIdentElem;
+  typeRef?: RefIdentElem;
+}
+
+/** a const declaration */
+export interface ConstElem extends ElemWithContents {
+  kind: "const";
+  name: DeclIdentElem;
+  typeRef?: RefIdentElem;
+}
+
+/** an override declaration */
+export interface OverrideElem extends ElemWithContents {
+  kind: "override";
+  name: DeclIdentElem;
+  typeRef?: RefIdentElem;
+}
+
+/** an entire file */
+export interface ModuleElem extends ElemWithContents {
+  kind: "module";
+}
+
+/** an alias statement */
+export interface AliasElem extends ElemWithContents {
+  kind: "alias";
+  name: DeclIdentElem;
+  typeRef: RefIdentElem;
+}
+
+/** a const_assert statement */
+export interface ConstAssertElem extends ElemWithContents {
+  kind: "assert";
+}
+
+/** a struct declaration */
+export interface StructElem extends ElemWithContents {
+  kind: "struct";
+  name: DeclIdentElem;
+  members: StructMemberElem[];
+}
+
+/** a member of a struct declaration */
+export interface StructMemberElem extends ElemWithContents {
+  kind: "member";
+  name: NameElem;
+  typeRef: RefIdentElem;
+}
+
+/** a name (e.g. a struct member name) that doesn't need to be an Ident */
+export interface NameElem extends AbstractElemBase {
+  kind: "name";
+  name: string;
+  srcModule: SrcModule;
+}
+
+/** a function declaration */
+export interface FnElem extends ElemWithContents {
+  kind: "fn";
+  name: DeclIdentElem;
+  params: ParamElem[];
+  returnType?: RefIdentElem;
+}

package/src/BindIdents.ts

@@ -0,0 +1,277 @@
+import { debugNames, srcLog } from "mini-parse";
+import { DeclarationElem } from "./AbstractElems.ts";
+import { FlatImport } from "./FlattenTreeImport.ts";
+import { ParsedRegistry } from "./ParsedRegistry.ts";
+import { flatImports, WeslAST } from "./ParseWESL.ts";
+import { DeclIdent, exportDecl, RefIdent, Scope } from "./Scope.ts";
+import { identToString } from "./debug/ScopeToString.ts";
+import { stdFn, stdType } from "./StandardTypes.ts";
+import { last, overlapTail } from "./Util.ts";
+
+/**
+ * Bind active reference idents to declaration Idents by mutating the refersTo: field
+ * Also in this pass, set the mangledName: field for all active global declaration idents.
+ *
+ * @param parsed
+ * @param conditions  only bind to/from idents that are valid with the current condition set
+ * @return any new declaration elements found (they will need to be emitted)
+ */
+export function bindIdents(
+  ast: WeslAST,
+  parsed: ParsedRegistry,
+  conditions: Record<string, any>,
+): DeclarationElem[] {
+  /* 
+    For each module's scope, search through the scope tree to find all ref idents
+      - For each ref ident, search up the scope tree to find a matching decl ident
+      - If no local match is found, check for partial matches with import statements
+        - combine ident with import statement to match a decl in exporting module
+
+    As global decl idents are found, mutate their mangled name to be globally unique.
+*/
+  const { rootScope } = ast;
+
+  const globalNames = new Set<string>();
+  const knownDecls = new Set<DeclIdent>();
+  rootScope.idents.forEach(ident => {
+    if (ident.kind === "decl") {
+      ident.mangledName = ident.originalName;
+      globalNames.add(ident.originalName);
+      knownDecls.add(ident);
+    }
+  });
+
+  const bindContext = {
+    parsed,
+    conditions,
+    knownDecls,
+    foundScopes: new Set<Scope>(),
+    globalNames,
+  };
+  const decls = bindIdentsRecursive(rootScope, bindContext);
+  return decls.flatMap(d =>
+    d.declElem && isGlobal(d.declElem) ? [d.declElem] : [],
+  );
+}
+
+interface BindContext {
+  parsed: ParsedRegistry;
+  conditions: Record<string, any>;
+  knownDecls: Set<DeclIdent>; // decl idents discovered so far
+  foundScopes: Set<Scope>; // save work by not processing scopes multiple times
+  globalNames: Set<string>; // root level names  used so far
+}
+
+/**
+ * Recursively bind references to declarations in this scope and
+ * any child scopes referenced by these declarations.
+ * Uses a hash set of found declarations to avoid duplication
+ * @ return any new declarations found
+ */
+function bindIdentsRecursive(
+  scope: Scope,
+  bindContext: BindContext,
+): DeclIdent[] {
+  // early exist if we've processed this scope before
+  const { foundScopes } = bindContext;
+  if (foundScopes.has(scope)) return [];
+  foundScopes.add(scope);
+
+  // dlog(scopeIdentTree(scope));
+
+  const { parsed, conditions } = bindContext;
+  const { globalNames, knownDecls } = bindContext;
+  const newDecls: DeclIdent[] = []; // new decl idents to process (and return)
+
+  scope.idents.forEach(ident => {
+    // dlog(`--- considering ident ${identToString(ident)}`);
+    if (ident.kind === "ref") {
+      if (!ident.refersTo && !ident.std) {
+        if (stdWgsl(ident.originalName)) {
+          ident.std = true;
+        } else {
+          let foundDecl =
+            findDeclInModule(ident.scope, ident) ??
+            findDeclImport(ident, parsed);
+
+          if (foundDecl) {
+            ident.refersTo = foundDecl;
+            if (!knownDecls.has(foundDecl)) {
+              knownDecls.add(foundDecl);
+              setDisplayName(ident.originalName, foundDecl, globalNames);
+              if (foundDecl.declElem && isGlobal(foundDecl.declElem)) {
+                newDecls.push(foundDecl);
+              }
+              // dlog(`  > found new decl: ${identToString(foundDecl)}`);
+            }
+          } else {
+            const { refIdentElem } = ident;
+            if (refIdentElem) {
+              const { srcModule, start, end } = refIdentElem;
+              const { filePath } = srcModule;
+              const msg = `unresolved identifier in file: ${filePath}`;
+              srcLog(srcModule.src, [start, end], msg);
+            }
+          }
+        }
+      }
+    }
+  });
+
+  // follow references from child scopes
+  const newFromChildren = scope.children.flatMap(child => {
+    // dlog("to newFromChildren", { childScope: scopeIdentTree(child) });
+    return bindIdentsRecursive(child, bindContext);
+  });
+  // console.log(
+  //   "new from children",
+  //   newFromChildren.map(d => identToString(d)),
+  // );
+
+  // follow references from referenced declarations
+  const newFromRefs = newDecls.flatMap(decl => {
+    // dlog("to newFromRefs", {
+    //   decl: identToString(decl),
+    //   declScope: scopeIdentTree(decl.scope),
+    // });
+    if (debugNames && !decl.scope) {
+      console.log(`--- decl ${identToString(decl)} has no scope`);
+      return [];
+    }
+    return bindIdentsRecursive(decl.scope, bindContext);
+  });
+  // console.log(
+  //   "new from refs",
+  //   newFromRefs.map(d => identToString(d)),
+  // );
+
+  return [newDecls, newFromChildren, newFromRefs].flat();
+}
+
+function setDisplayName(
+  proposedName: string,
+  decl: DeclIdent,
+  globalNames: Set<string>,
+): void {
+  if (!decl.mangledName) {
+    // if (!decl.declElem) {
+    //   console.log(
+    //     `--- decl ident ${identToString(decl)} has no declElem attached`,)
+    // } else
+    if (decl.declElem && isGlobal(decl.declElem)) {
+      decl.mangledName = declUniqueName(proposedName, globalNames);
+      // dlog(`  > mangle global decl: ${identToString(decl)}`);
+    } else {
+      // dlog(`  > no-mangle local decl: ${identToString(decl)}`);
+      decl.mangledName = decl.originalName;
+    }
+  }
+}
+
+function stdWgsl(name: string): boolean {
+  return stdType(name) || stdFn(name);
+}
+
+/** search earlier in the scope and in parent scopes to find a matching decl ident */
+function findDeclInModule(
+  scope: Scope,
+  ident: RefIdent,
+): DeclIdent | undefined {
+  const { parent } = scope;
+  const { originalName } = ident;
+
+  const found = scope.idents.find(
+    i => i.kind === "decl" && i.originalName === originalName,
+  );
+  if (found) return found as DeclIdent;
+
+  // recurse to check all idents in parent scope
+  if (parent) {
+    // dlog("checking parent scope", {
+    //   ident: identToString(ident),
+    // });
+    // console.log(scopeIdentTree(parent));
+    return findDeclInModule(parent, ident);
+  }
+}
+
+/** Match a reference identifier to a declaration in
+ * another module via an import statement */
+function findDeclImport(
+  refIdent: RefIdent,
+  parsed: ParsedRegistry,
+): DeclIdent | undefined {
+  // dlog(identToString(refIdent), { ast: !!refIdent.ast });
+  const flatImps = flatImports(refIdent.ast);
+
+  // find module path by combining identifer reference with import statement
+  const modulePathParts = matchingImport(refIdent, flatImps); // module path in array form
+
+  if (modulePathParts) {
+    return findExport(modulePathParts, parsed);
+  }
+}
+
+/** using the flattened import array, find an import that matches a provided identifier */
+function matchingImport(
+  ident: RefIdent,
+  flatImports: FlatImport[],
+): string[] | undefined {
+  const identParts = ident.originalName.split("::");
+  for (const flat of flatImports) {
+    const impTail = overlapTail(flat.importPath, identParts);
+    if (impTail) {
+      return [...flat.modulePath, ...impTail];
+    }
+  }
+}
+
+/** @return an exported root element for the provided path */
+function findExport(
+  modulePathParts: string[],
+  parsed: ParsedRegistry,
+): DeclIdent | undefined {
+  const legacyConvert = modulePathParts.map(p => (p === "." ? "package" : p)); // TODO rm after we update grammar
+  const modulePath = legacyConvert.slice(0, -1).join("::");
+  const module = parsed.modules[modulePath];
+  if (!module) {
+    // TODO show error with source location
+    console.log(
+      `ident ${modulePathParts.join("::")} in import statement, but module not found`,
+    );
+  }
+
+  return exportDecl(module.rootScope, last(modulePathParts)!);
+}
+
+/** return mangled name for decl ident,
+ *  mutating the Ident to remember mangled name if it hasn't yet been determined */
+export function declUniqueName(
+  proposedName: string,
+  rootNames: Set<string>,
+): string {
+  const displayName = uniquifyName(proposedName, rootNames);
+  rootNames.add(displayName);
+
+  return displayName;
+}
+
+/** construct global unique name for use in the output */
+function uniquifyName(proposedName: string, rootNames: Set<string>): string {
+  let renamed = proposedName;
+  let conflicts = 0;
+
+  // create a unique name
+  while (rootNames.has(renamed)) {
+    renamed = proposedName + conflicts++;
+  }
+
+  // dlog({ proposedName, renamed });
+  return renamed;
+}
+
+export function isGlobal(elem: DeclarationElem): boolean {
+  return ["alias", "const", "override", "fn", "struct", "gvar"].includes(
+    elem.kind,
+  );
+}

package/src/CommentsGrammar.ts

@@ -0,0 +1,44 @@
+import {
+  any,
+  anyNot,
+  anyThrough,
+  disablePreParse,
+  makeEolf,
+  or,
+  Parser,
+  repeat,
+  req,
+  resultLog,
+  seq,
+  tokens,
+} from "mini-parse";
+import { argsTokens, lineCommentTokens, mainTokens } from "./WESLTokens.ts";
+
+/* Basic parsing functions for comment handling . */
+
+/** currently unused */
+export const unknown = any().map(r => {
+  const { kind, text } = r.value;
+  const deepName = r.ctx._debugNames.join(" > ");
+
+  resultLog(r, `??? ${kind}: '${text}'  ${deepName}`);
+  // throw new Error("Fail fast");
+});
+
+const eolf: Parser<any> = disablePreParse(
+  makeEolf(argsTokens, argsTokens.ws),
+);
+
+const skipToEol = tokens(lineCommentTokens, anyThrough(eolf));
+
+export const blockComment: Parser<any> = seq(
+  "/*",
+  repeat(or(() => blockComment, anyNot("*/"))),
+  req("*/"),
+);
+
+export const lineComment = seq(tokens(mainTokens, "//"), skipToEol);
+
+export const comment = or(() => lineComment, blockComment).trace({
+  hide: true,
+});

package/src/FlattenTreeImport.ts

@@ -0,0 +1,59 @@
+import { tracing } from "mini-parse";
+import {
+  ImportTree,
+  PathSegment,
+  SegmentList,
+  SimpleSegment,
+} from "./ImportTree.js";
+
+export interface FlatImport {
+  importPath: string[];
+  modulePath: string[];
+}
+
+/**
+ * Simplify importTree into a flattened map from import paths to module paths.
+ *
+ * @return map from import path (with 'as' renaming) to module Path
+ */
+export function flattenTreeImport(imp: ImportTree): FlatImport[] {
+  return recursiveResolve([], [], imp.segments);
+
+  /** recurse through segments of path, producing  */
+  function recursiveResolve(
+    resolvedImportPath: string[],
+    resolvedExportPath: string[],
+    remainingPath: PathSegment[],
+  ): FlatImport[] {
+    const [segment, ...rest] = remainingPath;
+    if (segment === undefined) {
+      throw new Error(`undefined segment ${imp.segments}`);
+    }
+    if (segment instanceof SimpleSegment) {
+      const importPath = [...resolvedImportPath, segment.as || segment.name];
+      const modulePath = [...resolvedExportPath, segment.name];
+      if (rest.length) {
+        // we're in the middle of the path so keep recursing
+        return recursiveResolve(importPath, modulePath, rest);
+      } else {
+        return [{ importPath, modulePath }];
+      }
+    }
+    if (segment instanceof SegmentList) {
+      // resolve path with each element in the list
+      return segment.list.flatMap(elem => {
+        const rPath = [elem, ...rest];
+        return recursiveResolve(resolvedImportPath, resolvedExportPath, rPath);
+      });
+    } else if (segment instanceof ImportTree) {
+      return recursiveResolve(
+        resolvedImportPath,
+        resolvedExportPath,
+        segment.segments,
+      );
+    }
+
+    if (tracing) console.log("unknown segment type", segment); // should be impossible
+    return [];
+  }
+}

package/src/ImportGrammar.ts

@@ -0,0 +1,142 @@
+import {
+  disablePreParse,
+  kind,
+  makeEolf,
+  matchOneOf,
+  NoTags,
+  opt,
+  or,
+  Parser,
+  repeat,
+  repeatPlus,
+  seq,
+  setTraceName,
+  TagRecord,
+  tagScope,
+  tokenMatcher,
+  tokens,
+  tokenSkipSet,
+  tracing,
+  withSepPlus
+} from "mini-parse";
+import {
+  importElem,
+  importList,
+  importSegment,
+  importTree,
+} from "./WESLCollect.js";
+import { digits, eol, ident, } from "./WESLTokens.js";
+
+
+// TODO now that ';' is required, special ws and eol handling is probably not needed.
+const skipWsSet = new Set(["ws"]);
+function skipWs<V, T extends TagRecord>(p: Parser<V, T>): Parser<V, T> {
+  return tokenSkipSet(skipWsSet, p);
+}
+function noSkipWs<V, T extends TagRecord>(p: Parser<V, T>): Parser<V, T> {
+  return tokenSkipSet(null, p);
+}
+
+const importSymbolSet = "/ { } , ( ) .. . * ; @ #"; // Had to add @ and # here to get the parsing tests to work. Weird.
+const importSymbol = matchOneOf(importSymbolSet);
+
+// TODO reconsider whether we need a separate token set for import statements vs wgsl/wesl
+export const importTokens = tokenMatcher({
+  ws: /\s+/,
+  importSymbol,
+  ident, // TODO allow '-' in pkg names?
+  digits,
+});
+
+export const eolTokens = tokenMatcher({
+  ws: /[ \t]+/, // don't include \n, for eolf
+  eol,
+});
+
+const eolf = disablePreParse(makeEolf(eolTokens, importTokens.ws));
+const wordToken = kind(importTokens.ident);
+
+// forward references for mutual recursion
+let packagePath: Parser<any, NoTags> = null as any;
+
+const simpleSegment = tagScope(
+  wordToken.ptag("segment").collect(importSegment),
+);
+
+/** last simple segment is allowed to have an 'as' rename */
+const lastSimpleSegment = tagScope(
+  seq(
+    wordToken.ptag("segment"),
+    skipWs(opt(seq("as", wordToken.ptag("as")))),
+  ).collect(importSegment),
+);
+
+/** an item an a collection list {a, b} */
+const collectionItem = or(
+  tagScope(or(() => packagePath).collect(importTree)),
+  lastSimpleSegment,
+);
+
+const importCollection = tagScope(
+  seq(
+    "{",
+    skipWs(
+      seq(
+        withSepPlus(",", () => collectionItem.ctag("list")),
+        "}",
+      ),
+    ),
+  ).collect(importList),
+);
+
+/** a relative path element like "./" or "../" */
+const relativeSegment = tagScope(
+  seq(or(".", "..").ptag("segment"), "/").collect(importSegment),
+).ctag("p");
+
+const lastSegment = or(lastSimpleSegment, importCollection);
+
+const packageTail = seq(
+  repeat(seq(simpleSegment.ctag("p"), "/")),
+  lastSegment.ctag("p"),
+);
+
+/** a module path starting with ../ or ./ */
+const relativePath = seq(repeatPlus(relativeSegment), packageTail);
+
+const packagePrefix = tagScope(
+  seq(wordToken.ptag("segment"), "/").collect(importSegment),
+).ctag("p");
+
+/** a module path, starting with a simple element */
+packagePath = seq(packagePrefix, packageTail);
+
+const fullPath = noSkipWs(
+  seq(kind(importTokens.ws), or(relativePath, packagePath)),
+);
+
+/** parse a WESL style wgsl import statement. */
+export const weslImport = tagScope(
+  tokens(
+    importTokens,
+    seq("import", fullPath, opt(";"), eolf).collect(importElem()),
+  ),
+);
+
+if (tracing) {
+  const names: Record<string, Parser<unknown, TagRecord>> = {
+    simpleSegment,
+    lastSimpleSegment,
+    importCollection,
+    relativeSegment,
+    relativePath,
+    packagePrefix,
+    packagePath,
+    fullPath,
+    weslImport,
+  };
+
+  Object.entries(names).forEach(([name, parser]) => {
+    setTraceName(parser, name);
+  });
+}

package/src/ImportTree.ts

@@ -0,0 +1,19 @@
+export class ImportTree {
+  /** segments in path order */
+  constructor(public segments: PathSegment[]) {}
+}
+
+export type PathSegment = SimpleSegment | ImportTree | SegmentList;
+
+export class SimpleSegment {
+  constructor(
+    public name: string,
+    public as?: string,
+    public args?: string[], // generic args (only allowed on final segment). TODO drop
+  ) {}
+}
+
+/** or choices for this path segment */
+export class SegmentList {
+  constructor(public list: PathSegment[]) {}
+}