wesl 0.6.48 → 0.7.0

package/src/ModuleResolver.ts

@@ -1,30 +1,17 @@
+import { moduleToRelativePath, normalizeDebugRoot } from "./ModulePathUtil.ts";
 import { parseSrcModule, type WeslAST } from "./ParseWESL.ts";
 import { normalize, noSuffix } from "./PathUtil.ts";
 import type { WeslBundle } from "./WeslBundle.ts";
 
-const libRegex = /^lib\.w[eg]sl$/i;
-
-/**
- * Resolves module paths to parsed ASTs.
- *
- * Implementations may cache ASTs or load them lazily.
- */
+/** Resolves module paths to parsed ASTs. Implementations may cache or load lazily. */
 export interface ModuleResolver {
-  /**
-   * Resolve a module path to its parsed AST.
-   *
-   * @param modulePath - Module path in :: format (e.g., "package::foo::bar")
-   * @returns Parsed AST or undefined if module not found
-   */
+  /** Resolve module path (e.g., "package::foo::bar") to AST, or undefined if not found. */
   resolveModule(modulePath: string): WeslAST | undefined;
 }
 
 /** Module resolver that supports batch enumeration of all modules. */
 export interface BatchModuleResolver extends ModuleResolver {
-  /**
-   * Return all modules, parsing them on-demand if needed.
-   * Implementations should ensure all sources are parsed before returning.
-   */
+  /** Return all modules, parsing them on-demand if needed. */
   allModules(): Iterable<[string, WeslAST]>;
 }
 
@@ -35,6 +22,8 @@ export interface RecordResolverOptions {
   debugWeslRoot?: string;
 }
 
+const libRegex = /^lib\.w[eg]sl$/i;
+
 /** Module resolver for in-memory source records. Lazy by default. */
 export class RecordResolver implements BatchModuleResolver {
   readonly astCache = new Map<string, WeslAST>();
@@ -57,48 +46,40 @@ export class RecordResolver implements BatchModuleResolver {
     if (cached) return cached;
 
     const source = this.findSource(modulePath);
-    if (!source) return undefined;
+    if (source === undefined) return undefined;
 
     const debugFilePath = this.modulePathToDebugPath(modulePath);
-
-    const ast = parseSrcModule({
-      modulePath,
-      debugFilePath,
-      src: source,
-    });
+    const ast = parseSrcModule({ modulePath, debugFilePath, src: source });
     this.astCache.set(modulePath, ast);
     return ast;
   }
 
   private findSource(modulePath: string): string | undefined {
-    if (this.sources[modulePath]) return this.sources[modulePath];
+    if (this.sources[modulePath] !== undefined) return this.sources[modulePath];
 
     const filePath = this.moduleToFilePath(modulePath);
+    if (filePath === undefined) return undefined;
     return findInVariants(this.sources, filePath);
   }
 
-  private moduleToFilePath(modulePath: string): string {
-    const parts = modulePath.split("::");
-    if (parts[0] !== this.packageName && parts[0] !== "package") {
-      return modulePath;
-    }
-
-    const pathParts = parts.slice(1);
-    return pathParts.join("/");
+  /** Convert module path to file path, or undefined if not local. */
+  private moduleToFilePath(modulePath: string): string | undefined {
+    return moduleToRelativePath(modulePath, this.packageName);
   }
 
   private modulePathToDebugPath(modulePath: string): string {
-    const filePath = this.moduleToFilePath(modulePath);
+    const filePath = this.moduleToFilePath(modulePath) ?? modulePath;
     return this.debugWeslRoot + filePath + ".wesl";
   }
 
-  /** Return all modules, parsing them on-demand if needed. */
+  /** Parse all modules and return entries. */
   allModules(): Iterable<[string, WeslAST]> {
     for (const filePath of Object.keys(this.sources)) {
+      const treatLibAsRoot = this.packageName !== "package";
       const modulePath = fileToModulePath(
         filePath,
         this.packageName,
-        this.packageName !== "package",
+        treatLibAsRoot,
       );
       this.resolveModule(modulePath);
     }
@@ -137,10 +118,8 @@ export class BundleResolver implements ModuleResolver {
   }
 
   resolveModule(modulePath: string): WeslAST | undefined {
-    if (
-      modulePath !== this.packageName &&
-      !modulePath.startsWith(this.packageName + "::")
-    ) {
+    const pkgPrefix = this.packageName + "::";
+    if (modulePath !== this.packageName && !modulePath.startsWith(pkgPrefix)) {
       return undefined;
     }
 
@@ -151,11 +130,7 @@ export class BundleResolver implements ModuleResolver {
     if (!source) return undefined;
 
     const debugFilePath = this.modulePathToDebugPath(modulePath);
-    const ast = parseSrcModule({
-      modulePath,
-      debugFilePath,
-      src: source,
-    });
+    const ast = parseSrcModule({ modulePath, debugFilePath, src: source });
     this.astCache.set(modulePath, ast);
     return ast;
   }
@@ -173,11 +148,7 @@ export class BundleResolver implements ModuleResolver {
   }
 
   private moduleToFilePath(modulePath: string): string {
-    const parts = modulePath.split("::");
-    if (parts[0] !== this.packageName) {
-      return modulePath;
-    }
-    return parts.slice(1).join("/");
+    return moduleToRelativePath(modulePath, this.packageName) ?? modulePath;
   }
 
   private modulePathToDebugPath(modulePath: string): string {
@@ -186,8 +157,8 @@ export class BundleResolver implements ModuleResolver {
   }
 }
 
-/** Convert file path to module path (e.g., "foo/bar.wesl" -> "package::foo::bar"). */
-function fileToModulePath(
+/** Convert file path to module path (e.g., "foo/bar.wesl" to "package::foo::bar"). */
+export function fileToModulePath(
   filePath: string,
   packageName: string,
   treatLibAsRoot: boolean,
@@ -204,14 +175,7 @@ function fileToModulePath(
   return packageName + "::" + moduleSuffix;
 }
 
-/** Normalize debug root to end with / or be empty. */
-function normalizeDebugRoot(debugWeslRoot?: string): string {
-  if (debugWeslRoot === undefined) return "./";
-  if (debugWeslRoot === "") return "";
-  return debugWeslRoot.endsWith("/") ? debugWeslRoot : debugWeslRoot + "/";
-}
-
-/** Try path variants with and without ./ prefix and extension suffixes. */
+/** Try path variants with/without ./ prefix and extension suffixes. */
 function findInVariants(
   sources: Record<string, string>,
   basePath: string,
@@ -221,11 +185,11 @@ function findInVariants(
 
   for (const prefix of prefixes) {
     const path = prefix + basePath;
-    if (sources[path]) return sources[path];
+    if (sources[path] !== undefined) return sources[path];
 
     for (const ext of extensions) {
       const withExt = `${path}.${ext}`;
-      if (sources[withExt]) return sources[withExt];
+      if (sources[withExt] !== undefined) return sources[withExt];
     }
   }
 

package/src/ParseError.ts

@@ -0,0 +1,9 @@
+import type { Span } from "./Span.ts";
+
+export class ParseError extends Error {
+  span: Span;
+  constructor(msg: string, span: Span) {
+    super(msg);
+    this.span = span;
+  }
+}

package/src/ParseWESL.ts

@@ -1,80 +1,67 @@
-import {
-  type AppState,
-  ParseError,
-  type ParserInit,
-  type Span,
-} from "mini-parse";
 import type {
   ConstAssertElem,
+  ContainerElem,
   ImportElem,
   ImportStatement,
   ModuleElem,
 } from "./AbstractElems.ts";
-import { throwClickableError } from "./ClickableError.ts";
 import { filterValidElements } from "./Conditions.ts";
 import { type FlatImport, flattenTreeImport } from "./FlattenTreeImport.ts";
-import { weslRoot } from "./parse/WeslGrammar.ts";
-import { WeslStream } from "./parse/WeslStream.ts";
-import {
-  type Conditions,
-  emptyScope,
-  type Scope,
-  type SrcModule,
-} from "./Scope.ts";
+import type { ParseError } from "./ParseError.ts";
+import { parseWesl } from "./parse/ParseWesl.ts";
+import type { Conditions, Scope, SrcModule } from "./Scope.ts";
+import type { Span } from "./Span.ts";
 import { errorHighlight, offsetToLineNumber } from "./Util.ts";
-import type { OpenElem } from "./WESLCollect.ts";
 
-/** result of a parse for one wesl module (e.g. one .wesl file)
+/** Partial element being constructed during parsing. */
+export type OpenElem<T extends ContainerElem = ContainerElem> = Pick<
+  T,
+  "kind" | "contents"
+>;
+
+/**
+ * Result of parsing one WESL module (e.g., one .wesl file).
  *
- * The parser constructs the AST constructed into three sections
- * for convenient access by the binding stage.
+ * The AST is constructed into three sections for the binding stage:
  *  - import statements
  *  - language elements (fn, struct, etc)
  *  - scopes
- *
  */
 export interface WeslAST {
-  /** source text for this module */
+  /** Source text for this module. */
   srcModule: SrcModule;
-
-  /** root module element */
+  /** Root module element. */
   moduleElem: ModuleElem;
-
-  /** root scope for this module */
+  /** Root scope for this module. */
   rootScope: Scope;
-
-  /** imports found in this module */
+  /** Imports found in this module. */
   imports: ImportStatement[];
-
-  /** module level const_assert statements */
+  /** Module level const_assert statements. */
   moduleAsserts?: ConstAssertElem[];
 }
 
-/** an extended version of the AST */
+/** Extended AST with cached flattened imports. */
 export interface BindingAST extends WeslAST {
-  /* a flattened version of the import statements constructed on demand from import trees, and cached here */
+  /** Flattened import statements (cached on demand). */
   _flatImports?: FlatImport[];
 }
 
-/** stable and unstable state used during parsing */
-export interface WeslParseState
-  extends AppState<WeslParseContext, StableState> {
+/** Stable and unstable state used during parsing. */
+export interface WeslParseState {
   context: WeslParseContext;
   stable: StableState;
 }
 
-/** stable values used or accumulated during parsing */
+/** Stable values used or accumulated during parsing. */
 export type StableState = WeslAST;
 
-/** unstable values used during parse collection */
+/** Unstable values used during parse collection. */
 export interface WeslParseContext {
   scope: Scope; // current scope (points somewhere in rootScope)
   openElems: OpenElem[]; // elems that are collecting their contents
 }
 
-/**
- * An error when parsing WESL fails. Designed to be human-readable.
- */
+/** Human-readable error when parsing WESL fails. */
 export class WeslParseError extends Error {
   span: Span;
   src: SrcModule;
@@ -84,69 +71,18 @@ export class WeslParseError extends Error {
     let message = `${opts.src.debugFilePath}:${lineNum}:${linePos}`;
     message += ` error: ${opts.cause.message}\n`;
     message += errorHighlight(source, opts.cause.span).join("\n");
-    super(message, {
-      cause: opts.cause,
-    });
+    super(message, { cause: opts.cause });
     this.span = opts.cause.span;
     this.src = opts.src;
   }
 }
 
-/** Parse a WESL file. Throws on error. */
+/** Parse a WESL file. */
 export function parseSrcModule(srcModule: SrcModule): WeslAST {
-  const stream = new WeslStream(srcModule.src);
-
-  const appState = blankWeslParseState(srcModule);
-
-  const init: ParserInit = { stream, appState };
-  try {
-    const parseResult = weslRoot.parse(init);
-    if (parseResult === null) {
-      throw new Error("parseWESL failed");
-    }
-  } catch (e) {
-    if (e instanceof ParseError) {
-      const [lineNumber, lineColumn] = offsetToLineNumber(
-        e.span[0],
-        srcModule.src,
-      );
-      const error = new WeslParseError({ cause: e, src: srcModule });
-      throwClickableError({
-        url: srcModule.debugFilePath,
-        text: srcModule.src,
-        error,
-        lineNumber,
-        lineColumn,
-        length: e.span[1] - e.span[0],
-      });
-    } else {
-      throw e;
-    }
-  }
-
-  return appState.stable as WeslAST;
-}
-
-export function blankWeslParseState(srcModule: SrcModule): WeslParseState {
-  const rootScope = emptyScope(null);
-  const moduleElem = null as any; // we'll fill this in later
-  return {
-    context: { scope: rootScope, openElems: [] },
-    stable: { srcModule, imports: [], rootScope, moduleElem },
-  };
-}
-
-export function syntheticWeslParseState(): WeslParseState {
-  const srcModule: SrcModule = {
-    modulePath: "package::test",
-    debugFilePath: "./test.wesl",
-    src: "",
-  };
-
-  return blankWeslParseState(srcModule);
+  return parseWesl(srcModule);
 }
 
-/** @return a flattened form of the import tree for convenience in binding idents. */
+/** @return flattened form of import tree for binding idents. */
 export function flatImports(
   ast: BindingAST,
   conditions?: Conditions,

package/src/RawEmit.ts

@@ -56,11 +56,8 @@ export function typeListToString(params: TypeTemplateParameter[]): string {
 
 export function typeParamToString(param?: TypeTemplateParameter): string {
   if (param === undefined) return "?";
-  if (typeof param === "string") return param;
-
-  if (param.kind === "expression") return contentsToString(param);
   if (param.kind === "type") return typeRefToString(param);
-  assertUnreachable(param);
+  return expressionToString(param);
 }
 
 export function typeRefToString(t?: TypeRefElem): string {

package/src/Reflection.ts

@@ -1,4 +1,3 @@
-import { matchOneOf } from "mini-parse";
 import type {
   BindingStructElem,
   NameElem,
@@ -11,6 +10,7 @@ import type {
 import { assertThat } from "./Assertions.ts";
 import type { TransformedAST, WeslJsPlugin } from "./Linker.ts";
 import { identElemLog } from "./LinkerUtil.ts";
+import { matchOneOf } from "./parse/stream/RegexHelpers.ts";
 import type { RefIdent } from "./Scope.ts";
 import {
   multisampledTextureTypes,

package/src/Scope.ts

@@ -38,6 +38,9 @@ export interface RefIdent extends IdentBase {
   refersTo?: Ident; // import or decl ident in scope to which this ident refers. undefined before binding
   std?: true; // true if this is a standard wgsl identifier (like sin, or u32)
 
+  /** True for identifiers in @if/@elif conditions. Binding skips these (for now). */
+  conditionRef?: true;
+
   // LATER consider tracking the current ast in BindIdents so that this field is unnecessary
   ast: WeslAST; // AST from module that contains this ident (to find imports during decl binding)
 

package/src/Span.ts

@@ -0,0 +1,2 @@
+/** A range, from start (inclusive) to end (exclusive). */
+export type Span = readonly [number, number];

package/src/SrcMap.ts

@@ -0,0 +1,208 @@
+/** A source map file, and a path for debug purposes. */
+export interface SrcWithPath {
+  /** User friendly path */
+  path?: string;
+  text: string;
+}
+
+export interface SrcMapEntry {
+  src: SrcWithPath;
+  srcStart: number;
+  srcEnd: number;
+  destStart: number;
+  destEnd: number;
+}
+
+export interface SrcPosition {
+  src: SrcWithPath;
+  position: number;
+}
+
+/** map text ranges in multiple src texts to a single dest text */
+export class SrcMap {
+  entries: SrcMapEntry[];
+  dest: SrcWithPath;
+
+  constructor(dest: SrcWithPath, entries: SrcMapEntry[] = []) {
+    this.dest = dest;
+    this.entries = entries;
+  }
+
+  /** add a new mapping from src to dest ranges (must be non-overlapping in destination) */
+  addEntries(newEntries: SrcMapEntry[]): void {
+    this.entries.push(...newEntries);
+  }
+
+  /** given positions in the dest string, return corresponding positions in the src strings */
+  mapPositions(...positions: number[]): SrcPosition[] {
+    return positions.map(p => this.destToSrc(p));
+  }
+
+  /** internally compress adjacent entries where possible */
+  compact(): void {
+    if (!this.entries.length) return;
+    let prev = this.entries[0];
+    const newEntries: SrcMapEntry[] = [prev];
+
+    for (let i = 1; i < this.entries.length; i++) {
+      const e = this.entries[i];
+      if (
+        e.src.path === prev.src.path &&
+        e.src.text === prev.src.text &&
+        prev.destEnd === e.destStart &&
+        prev.srcEnd === e.srcStart
+      ) {
+        // combine adjacent range entries into one
+        prev.destEnd = e.destEnd;
+        prev.srcEnd = e.srcEnd;
+      } else {
+        newEntries.push(e);
+        prev = e;
+      }
+    }
+    this.entries = newEntries;
+  }
+
+  /** sort in destination order */
+  sort(): void {
+    this.entries.sort((a, b) => a.destStart - b.destStart);
+  }
+
+  /**
+   * This SrcMap's destination is a src for the other srcmap,
+   * so combine the two and return the result.
+   */
+  merge(other: SrcMap): SrcMap {
+    if (other === this) return this;
+
+    const mappedEntries = other.entries.filter(
+      e => e.src.path === this.dest.path && e.src.text === this.dest.text,
+    );
+    if (mappedEntries.length === 0) {
+      console.log("other source map does not link to this one");
+      return other;
+    }
+    sortSrc(mappedEntries);
+    const newEntries = mappedEntries.map(e => {
+      const { src, position: srcStart } = this.destToSrc(e.srcStart);
+      const { src: endSrc, position: srcEnd } = this.destToSrc(e.srcEnd);
+      if (endSrc !== src) throw new Error("NYI, need to split");
+      const newEntry: SrcMapEntry = {
+        src,
+        srcStart,
+        srcEnd,
+        destStart: e.destStart,
+        destEnd: e.destEnd,
+      };
+      return newEntry;
+    });
+
+    const otherSources = other.entries.filter(
+      e => e.src.path !== this.dest.path || e.src.text !== this.dest.text,
+    );
+
+    const newMap = new SrcMap(other.dest, [...otherSources, ...newEntries]);
+    newMap.sort();
+    return newMap;
+  }
+
+  /** @return the source position corresponding to a provided destination position */
+  destToSrc(destPos: number): SrcPosition {
+    const entry = this.entries.find(
+      e => e.destStart <= destPos && e.destEnd >= destPos,
+    );
+    if (!entry) {
+      return { src: this.dest, position: destPos };
+    }
+    return {
+      src: entry.src,
+      position: entry.srcStart + destPos - entry.destStart,
+    };
+  }
+}
+
+/** sort entries in place by src start position */
+function sortSrc(entries: SrcMapEntry[]): void {
+  entries.sort((a, b) => a.srcStart - b.srcStart);
+}
+
+/** Incrementally append to a string, tracking source references */
+export class SrcMapBuilder {
+  #fragments: string[] = [];
+  #destLength = 0;
+  #entries: SrcMapEntry[] = [];
+  source: SrcWithPath;
+
+  constructor(source: SrcWithPath) {
+    this.source = source;
+  }
+
+  /** append a string fragment to the destination string */
+  add(fragment: string, srcStart: number, srcEnd: number): void {
+    const destStart = this.#destLength;
+    this.#destLength += fragment.length;
+    const destEnd = this.#destLength;
+
+    this.#fragments.push(fragment);
+    this.#entries.push({
+      src: this.source,
+      srcStart,
+      srcEnd,
+      destStart,
+      destEnd,
+    });
+  }
+
+  /**
+   * Append a fragment to the destination string,
+   * mapping source to the previous,
+   * and guessing that the source fragment is just as long as the dest fragment.
+   */
+  appendNext(fragment: string): void {
+    const lastEnd = this.#entries.at(-1)?.destEnd ?? 0;
+    this.add(fragment, lastEnd, lastEnd + fragment.length);
+  }
+
+  addSynthetic(
+    fragment: string,
+    syntheticSource: string,
+    srcStart: number,
+    srcEnd: number,
+  ): void {
+    const destStart = this.#destLength;
+    this.#destLength += fragment.length;
+    const destEnd = this.#destLength;
+
+    this.#fragments.push(fragment);
+    this.#entries.push({
+      src: { text: syntheticSource },
+      srcStart,
+      srcEnd,
+      destStart,
+      destEnd,
+    });
+  }
+
+  /** append a synthetic newline, mapped to previous source location */
+  addNl(): void {
+    const lastEntry = this.#entries.at(-1) ?? { srcStart: 0, srcEnd: 0 };
+    const { srcStart, srcEnd } = lastEntry;
+    this.add("\n", srcStart, srcEnd);
+  }
+
+  /** copy a string fragment from the src to the destination string */
+  addCopy(srcStart: number, srcEnd: number): void {
+    const fragment = this.source.text.slice(srcStart, srcEnd);
+    this.add(fragment, srcStart, srcEnd);
+  }
+
+  /** return a SrcMap */
+  static build(builders: SrcMapBuilder[]): SrcMap {
+    const map = new SrcMap(
+      { text: builders.map(b => b.#fragments.join("")).join("") },
+      builders.flatMap(b => b.#entries),
+    );
+    map.compact();
+    return map;
+  }
+}

package/src/Stream.ts

@@ -0,0 +1,30 @@
+import type { Span } from "./Span.ts";
+
+/**
+ * Interface for a tokenizer. Returns a "next token", and can be reset to
+ * previously saved positions (checkpoints).
+ */
+export interface Stream<T extends Token> {
+  /** Returns the current position */
+  checkpoint(): number;
+  /** Restores a position */
+  reset(position: number): void;
+  /**
+   * Returns the next token, or `null` if the end of the stream has been reached.
+   * Always leaves `checkpoint` right after the token.
+   */
+  nextToken(): T | null;
+  /** src text */
+  src: string;
+}
+
+/** A text token */
+export interface Token {
+  kind: string;
+  text: string;
+  span: Span;
+}
+
+export interface TypedToken<Kind extends string> extends Token {
+  kind: Kind;
+}

package/src/TransformBindingStructs.ts

@@ -1,4 +1,3 @@
-import { tracing } from "mini-parse";
 import type {
   AbstractElem,
   AttributeElem,
@@ -13,6 +12,7 @@ import type {
 } from "./AbstractElems.ts";
 import type { TransformedAST, WeslJsPlugin } from "./Linker.ts";
 import { visitAst } from "./LinkerUtil.ts";
+import { debug } from "./Logging.ts";
 import { findDecl } from "./LowerAndEmit.ts";
 import { minimallyMangledName } from "./Mangler.ts";
 import {
@@ -307,7 +307,7 @@ export function transformBindingReference(
   const refName = memberRef.member.name;
   const structMember = struct.members.find(m => m.name.name === refName)!;
   if (!structMember || !structMember.mangledVarName) {
-    if (tracing) console.log(`missing mangledVarName for ${refName}`);
+    if (debug) console.log(`missing mangledVarName for ${refName}`);
     return { kind: "synthetic", text: refName };
   }
   const { extraComponents } = memberRef;

package/src/Util.ts

@@ -1,4 +1,4 @@
-import type { Span } from "mini-parse";
+import type { Span } from "./Span.ts";
 
 export function multiKeySet<A, B, V>(
   m: Map<A, Map<B, V>>,