wesl 0.6.0-pre2

package/src/Slicer.ts

@@ -0,0 +1,127 @@
+import { SrcMap, SrcMapEntry } from "mini-parse";
+import { last, scan } from "./Util.js";
+
+/** specify a start,end portion of a string to be replaced */
+export interface SliceReplace {
+  start: number;
+  end: number;
+  replacement: string;
+}
+
+interface SlicingProgress {
+  srcPos: number;
+  destPos: number;
+  results: string[];
+  entries: SrcMapEntry[];
+}
+
+/**
+ * Rewrite a string by replacing segments them with provided texts.
+ *
+ * example:
+ * src:
+ *  aaabbbbbc
+ *     ^    ^
+ *     St   End Repl='XXX'
+ *
+ * returns a srcMap with the new text and mappings from the original text to the new text
+ *  aaaXXXc
+ */
+export function sliceReplace(
+  src: string,
+  slices: SliceReplace[],
+  start = 0,
+  end = src.length,
+): SrcMap {
+  const sorted = [...slices].sort((a, b) => a.start - b.start);
+  const initProgress = { srcPos: start, destPos: 0, results: [], entries: [] };
+  const slicePogress = scan(sorted, oneSlice, initProgress);
+  const lastProgress = finalProgress2(slicePogress);
+
+  const { results, entries } = lastProgress;
+  const text = results.join("");
+  const srcMap = new SrcMap(text, entries);
+  return srcMap;
+
+  /** visit one slice, return progress */
+  function oneSlice(
+    slice: SliceReplace,
+    progress: SlicingProgress,
+  ): SlicingProgress {
+    // dlog({ slice });
+    // update text with copy and replacement
+    const copyText = src.slice(progress.srcPos, slice.start);
+    const copied = replaceOne(copyText, slice.start, progress);
+    const replaced = replaceOne(slice.replacement, slice.end, copied);
+
+    return replaced;
+  }
+
+  /** add provided text to the result, advance src position, and add a srcMap entry
+   * @return the accumulated progress */
+  function replaceOne(
+    replacement: string,
+    newSrcPos: number,
+    progress: SlicingProgress,
+  ): SlicingProgress {
+    const { destPos, entries } = progress;
+    const newDestPos = destPos + replacement.length;
+
+    // new srcMap entry if there is a replacement text (otherwise there's nothing to map dest to src)
+    let newEntries = entries;
+    if (replacement) {
+      const { srcPos } = progress;
+      newEntries = entries.concat({
+        src,
+        srcStart: srcPos,
+        srcEnd: newSrcPos,
+        destStart: destPos,
+        destEnd: newDestPos,
+      });
+    }
+
+    // update results text and progress
+    const { results } = progress;
+    const newResults = replacement ? results.concat(replacement) : results;
+    return {
+      srcPos: newSrcPos,
+      destPos: newDestPos,
+      results: newResults,
+      entries: newEntries,
+    };
+  }
+
+  /**
+   * If there's any trailing text uncovered by the slices before the end,
+   * add a result and srcMap entry
+   *
+   * @return the accumulated progress
+   */
+  function finalProgress2(progress: SlicingProgress[]): SlicingProgress {
+    const lastProgress = last(progress) ?? initProgress;
+    const { srcPos } = lastProgress;
+    return replaceOne(src.slice(srcPos, end), end, lastProgress);
+  }
+}
+
+const tokenRegex = /\b(\w+)\b/gi;
+/** find strings in a text -
+ * found strings must be 'tokens', surrounded by spaces or punctuation
+ *
+ * @return SliceReplace elements
+ */
+export function sliceWords(
+  text: string,
+  replace: Record<string, string>,
+): SliceReplace[] {
+  const tokens = [...text.matchAll(tokenRegex)];
+  const find = Object.keys(replace);
+  const matches = tokens.filter(m => find.includes(m[0]));
+  const slices = matches.map(m => {
+    const start = m.index;
+    const end = start + m[0].length;
+    const replacement = replace[m[0]];
+    return { start, end, replacement };
+  });
+  return slices;
+}

package/src/StandardTypes.ts

@@ -0,0 +1,66 @@
+export const stdFns = `bitcast all any select arrayLength 
+  abs acos acosh asin asinh atan atanh atan2 ceil clamp cos cosh 
+  countLeadingZeros countOneBits countTrailingZeros cross 
+  degrees determinant distance dot dot4UI8Packed dot4I8Packed 
+  exp exp2 extractBits faceForward firstLeadingBit firstTrailingBit 
+  floor fma fract frexp inserBits inverseSqrt ldexp length log log2
+  max min mix modf normalize pow quantizeToF16 radians reflect refract
+  reverseBits round saturate sign sin sinh smoothstep sqrt step tan tanh
+  transpose trunc
+  dpdx dpdxCoarse dpdxFine dpdy dpdyCoarse dpdyFine fwidth 
+  fwdithCoarse fwidthFine
+  textureDimensions textureGather textureGatherCompare textureLoad
+  textureNumLayers textureNumLevels textureNumSamples
+  textureSample textureSampleBias textureSampleCompare textureSampleCompareLevel
+  textureSampleGrad textureSampleLevel textureSampleBaseClampToEdge
+  textureStore
+  atomicLoad atomicStore atomicAdd atomicSub atomicMax atomicMin
+  atomicOr atomicXor atomicExchange atomicCompareExchangeWeak
+  pack4x8snorm pack4x8unorm pack4xI8 pack4xU8 pack4xI8Clamp pack4xU8Clamp
+  pack2x16snorm pack2x16unorm pack2x16float
+  unpack4x8snorm unpack4x8unorm unpack4xI8 unpack4xU8 
+  unpack2x16snorm unpack2x16unorm unpack2x16float
+  storageBarrier textureBarrier workgroupBarrier workgroupUniformLoad
+  `.split(/\s+/);
+
+export const stdTypes = `array atomic bool f16 f32 i32 
+  mat2x2 mat2x3 mat2x4 mat3x2 mat3x3 mat3x4 mat4x2 mat4x3 mat4x4
+  mat2x2f mat2x3f mat2x4f mat3x2f mat3x3f mat3x4f
+  mat4x2f mat4x3f mat4x4f
+  mat2x2h mat2x3h mat2x4h mat3x2h mat3x3h mat3x4h
+  mat4x2h mat4x3h mat4x4h
+  u32 vec2 vec3 vec4 ptr
+  vec2i vec3i vec4i vec2u vec3u vec4u
+  vec2f vec3f vec4f vec2h vec3h vec4h
+  texture_1d texture_2d texture_2d_array texture_3d 
+  texture_cube texture_cube_array
+  texture_multisampled_2d texture_depth_multisampled_2d
+  texture_external
+  texture_storage_1d texture_storage_2d texture_storage_2d_array
+  texture_storage_3d
+  texture_depth_2d texture_depth_2d_array texture_depth_cube
+  texture_depth_cube_array
+  sampler sampler_comparison
+  rgba8unorm rgba8snorm rgba8uint rgba8sint
+  rgba16uint rgba16sint rgba16float 
+  r32uint r32sint r32float rg32uint rg32sint rg32float
+  rgba32uint rgba32sint rgba32float
+  bgra8unorm 
+  function uniform
+  `.split(/\s+/); // LATER handle 'function' in template parser?
+
+/* Note the texel formats like rgba8unorm are here because they appear in type position
+ in <templates> for texture_storage_* types. 
+ (We could parse texture_storage types specially, but user code is unlikely to alias 
+  the texture format names with e.g. a 'struct rbga8unorm .)
+*/
+
+/** return true if the name is for a built in type (not a user struct) */
+export function stdType(name: string): boolean {
+  return stdTypes.includes(name);
+}
+
+/** return true if the name is for a built in fn (not a user function) */
+export function stdFn(name: string): boolean {
+  return stdFns.includes(name) || stdType(name);
+}

package/src/Util.ts

@@ -0,0 +1,112 @@
+export function multiKeySet<A, B, V>(
+  m: Map<A, Map<B, V>>,
+  a: A,
+  b: B,
+  v: V,
+): void {
+  const bMap = m.get(a) || new Map();
+  m.set(a, bMap);
+  bMap.set(b, v);
+}
+
+const tokenRegex = /\b(\w+)\b/gi;
+/** replace strings in a text according to a relacement map
+ * replaced strings must be 'tokens', surrounded by spaces or punctuation
+ */
+export function replaceWords(
+  text: string,
+  replace: Record<string, string>,
+): string {
+  return text.replaceAll(tokenRegex, s => (s in replace ? replace[s] : s));
+}
+
+/** return an array partitioned into possibly overlapping groups */
+export function grouped<T>(a: T[], size: number, stride = size): T[][] {
+  const groups = [];
+  for (let i = 0; i < a.length; i += stride) {
+    groups.push(a.slice(i, i + size));
+  }
+  return groups;
+}
+
+/** group an array into subarrays by a key function */
+export function groupBy<T, K>(a: T[], key: (t: T) => K): Map<K, T[]> {
+  const groups = new Map<K, T[]>();
+  for (const t of a) {
+    const k = key(t);
+    const group = groups.get(k) || [];
+    group.push(t);
+    groups.set(k, group);
+  }
+  return groups;
+}
+
+/** partition an array into two parts by a discriminator function */
+export function partition<T>(a: T[], partFn: (t: T) => boolean): [T[], T[]] {
+  const yesPart: T[] = [];
+  const noPart: T[] = [];
+  for (const t of a) {
+    if (partFn(t)) yesPart.push(t);
+    else noPart.push(t);
+  }
+  return [yesPart, noPart];
+}
+
+/** run an carrying function over every element in an array,
+ * i.e. an inclusive prefix scan */
+export function scan<T, U>(array: T[], fn: (a: T, b: U) => U, zero: U): U[] {
+  const result = [zero];
+
+  let current = zero;
+  for (let i = 0; i < array.length; i++) {
+    current = fn(array[i], current);
+    result.push(current);
+  }
+  return result;
+}
+
+/** return a new record by replacing values in 'a' with 'b' as a map.
+ * values in 'a' that are not in 'b' are unchanged.
+ * e.g. {a: "b", x: 9}, {b: 1} yields {a: 1, x: 9}
+ */
+export function mapForward(
+  a: Record<string, string>,
+  b: Record<string, any>,
+): Record<string, any> {
+  const combined = Object.entries(a).map(([key, value]) => {
+    const mappedValue = value in b ? b[value] : value;
+    return [key, mappedValue];
+  });
+  return Object.fromEntries(combined);
+}
+
+/** return the last element of an array or undefined */
+export function last<T>(a: T[]): T | undefined {
+  return a[a.length - 1];
+}
+
+/**
+ * Overlap two arrays, returning the tail of b if a is a prefix of b.
+ * Otherwise, return undefined.
+ */
+export function overlapTail<T>(a: T[], b: T[]): T[] | undefined {
+  let overlapSize = Math.min(a.length, b.length);
+
+  while (overlapSize > 0) {
+    const suffix = a.slice(-overlapSize);
+    const prefix = b.slice(0, overlapSize);
+    if (arrayEquals(suffix, prefix)) {
+      break;
+    } else {
+      overlapSize--;
+    }
+  }
+
+  if (overlapSize) {
+    return b.slice(overlapSize);
+  }
+}
+
+function arrayEquals(a: any[], b: any[]): boolean {
+  return a.length === b.length && a.every((val, index) => val === b[index]);
+}

package/src/WESLCollect.ts

@@ -0,0 +1,336 @@
+import { CollectContext, CollectPair, tracing } from "mini-parse";
+import {
+  AbstractElem,
+  AliasElem,
+  ConstElem,
+  DeclarationElem,
+  DeclIdentElem,
+  ElemWithContents,
+  FnElem,
+  GlobalVarElem,
+  ImportElem,
+  ModuleElem,
+  NameElem,
+  OverrideElem,
+  ParamElem,
+  RefIdentElem,
+  StructElem,
+  StructMemberElem,
+  TextElem,
+  VarElem,
+} from "./AbstractElems.ts";
+import {
+  ImportTree,
+  PathSegment,
+  SegmentList,
+  SimpleSegment,
+} from "./ImportTree.ts";
+import {
+  StableState,
+  WeslAST,
+  WeslParseContext,
+  WeslParseState,
+} from "./ParseWESL.ts";
+import { DeclIdent, emptyBodyScope, RefIdent, Scope } from "./Scope.ts";
+
+/** add an elem to the .contents array of the currently containing element */
+function addToOpenElem(cc: CollectContext, elem: AbstractElem): void {
+  const weslContext: WeslParseContext = cc.app.context;
+  const { openElems } = weslContext;
+  if (openElems && openElems.length) {
+    const open = openElems[openElems.length - 1];
+    open.contents.push(elem);
+  }
+}
+
+/** create reference Ident and add to context */
+export function refIdent(cc: CollectContext): RefIdentElem {
+  const { src, start, end } = cc;
+  const app = cc.app as WeslParseState;
+  const { scope } = app.context;
+  const { srcModule } = app.stable;
+  const originalName = src.slice(start, end);
+
+  const kind = "ref";
+  const ident: RefIdent = { kind, originalName, ast: cc.app.stable, scope };
+  const identElem: RefIdentElem = { kind, start, end, srcModule, ident };
+  ident.refIdentElem = identElem;
+
+  saveIdent(cc, identElem);
+  return identElem;
+}
+
+/** create declaration Ident and add to context */
+export function declIdentElem(cc: CollectContext): DeclIdentElem {
+  const { src, start, end } = cc;
+  const app = cc.app as WeslParseState;
+  const { srcModule } = app.stable;
+  const originalName = src.slice(start, end);
+
+  const kind = "decl";
+  const declElem = null as any; // we'll set declElem later
+  const ident: DeclIdent = { kind, originalName, scope: null as any, declElem }; // we'll set declElem later
+  const identElem: DeclIdentElem = { kind, start, end, srcModule, ident };
+
+  saveIdent(cc, identElem);
+  return identElem;
+}
+
+let identId = 0;
+/** add Ident to current open scope, add IdentElem to current open element */
+function saveIdent(
+  cc: CollectContext,
+  identElem: RefIdentElem | DeclIdentElem,
+) {
+  const { ident } = identElem;
+  ident.id = identId++;
+  const weslContext: WeslParseContext = cc.app.context;
+  weslContext.scope.idents.push(ident);
+  addToOpenElem(cc, identElem);
+}
+
+/** start a new child Scope */
+function startScope(cc: CollectContext) {
+  const { scope } = cc.app.context as WeslParseContext;
+  const newScope = emptyBodyScope(scope);
+  scope.children.push(newScope);
+  cc.app.context.scope = newScope;
+  // srcLog(cc.src, cc.start, "startScope", newScope.id);
+}
+
+/* close current Scope and set current scope to parent */
+function completeScope(cc: CollectContext): Scope {
+  const weslContext = cc.app.context as WeslParseContext;
+  const completedScope = weslContext.scope;
+  // srcLog(cc.src, cc.start, "completeScope", completedScope.id);
+  // console.log(scopeIdentTree(completedScope));
+  const { parent } = completedScope;
+  if (parent) {
+    weslContext.scope = parent;
+  } else if (tracing) {
+    const { idents, kind } = completedScope;
+    console.log("ERR: completeScope, no parent scope", { kind, idents });
+  }
+  return completedScope;
+}
+
+// prettier-ignore
+export type OpenElem<T extends AbstractElem = AbstractElem> = 
+  Pick< T, "kind" > & { contents: AbstractElem[] };
+
+// prettier-ignore
+export type PartElem<T extends AbstractElem = AbstractElem> = 
+  Pick< T, "kind" | "start" | "end" > & { contents: AbstractElem[] };
+
+type VarLikeElem =
+  | GlobalVarElem
+  | VarElem
+  | ConstElem
+  | OverrideElem
+  | AliasElem;
+
+export function collectVarLike<E extends VarLikeElem>(
+  kind: E["kind"],
+): CollectPair<E> {
+  return collectElem(kind, (cc: CollectContext, openElem: PartElem<E>) => {
+    // dlog({ tags: [...Object.keys(cc.tags)] });
+    const name = cc.tags.declIdent?.[0] as DeclIdentElem;
+    const typeRef = cc.tags.typeRef?.[0] as RefIdentElem;
+    const decl_scope = cc.tags.decl_scope?.[0] as Scope;
+    const partElem = { ...openElem, name, typeRef };
+    const varElem = withTextCover(partElem, cc) as E;
+    (name.ident as DeclIdent).declElem = varElem as DeclarationElem;
+    name.ident.scope = decl_scope;
+    return varElem;
+  });
+}
+
+export function collectFn(): CollectPair<FnElem> {
+  return collectElem("fn", (cc: CollectContext, openElem: PartElem<FnElem>) => {
+    const name = cc.tags.fnName?.[0] as DeclIdentElem;
+    const body_scope = cc.tags.body_scope?.[0] as Scope;
+    const params: ParamElem[] = cc.tags.fnParam?.flat(3) ?? [];
+    const returnType: RefIdentElem | undefined = cc.tags.returnType?.flat(3)[0];
+    const partElem: FnElem = { ...openElem, name, params, returnType };
+    const fnElem = withTextCover(partElem, cc);
+    (name.ident as DeclIdent).declElem = fnElem;
+    name.ident.scope = body_scope;
+
+    return fnElem;
+  });
+}
+
+export function collectFnParam(): CollectPair<ParamElem> {
+  return collectElem(
+    "param",
+    (cc: CollectContext, openElem: PartElem<ParamElem>) => {
+      const name = cc.tags.paramName?.[0]! as DeclIdentElem;
+      const typeRef = cc.tags.typeRef?.[0]! as RefIdentElem;
+      const elem: ParamElem = { ...openElem, name, typeRef };
+      const paramElem = withTextCover(elem, cc);
+      name.ident.declElem = paramElem;
+
+      // name.ident.scope = pseudoScope(typeRef); // TODO set scope for param?
+      return paramElem;
+    },
+  );
+}
+
+export function collectStruct(): CollectPair<StructElem> {
+  return collectElem(
+    "struct",
+    (cc: CollectContext, openElem: PartElem<StructElem>) => {
+      const name = cc.tags.typeName?.[0] as DeclIdentElem;
+      const members = cc.tags.members as StructMemberElem[];
+      const structElem = { ...openElem, name, members };
+      const elem = withTextCover(structElem, cc);
+      (name.ident as DeclIdent).declElem = elem as DeclarationElem;
+      name.ident.scope = cc.tags.struct_scope?.[0] as Scope;
+
+      return elem;
+    },
+  );
+}
+
+export function collectStructMember(): CollectPair<StructMemberElem> {
+  return collectElem(
+    "member",
+    (cc: CollectContext, openElem: PartElem<StructMemberElem>) => {
+      const name = cc.tags.nameElem?.[0]!;
+      const typeRef = cc.tags.typeRef?.[0];
+      const partElem = { ...openElem, name, typeRef };
+      return withTextCover(partElem, cc);
+    },
+  );
+}
+
+export function collectNameElem(cc: CollectContext): NameElem {
+  const { start, end, src, app } = cc;
+  const { srcModule } = app.stable as WeslAST;
+  const name = src.slice(start, end);
+  const elem: NameElem = { kind: "name", srcModule, start, end, name };
+  addToOpenElem(cc, elem);
+  return elem;
+}
+
+// prettier-ignore
+export function collectModule(): 
+    CollectPair<ModuleElem > {
+  return collectElem(
+    "module",
+    (cc: CollectContext, openElem: PartElem<ModuleElem>) => {
+      const ccComplete = { ...cc, start: 0, end: cc.src.length }; // force module to cover entire source despite ws skipping
+      const moduleElem: ModuleElem = withTextCover(openElem, ccComplete);
+      const weslState: StableState = cc.app.stable;
+      weslState.moduleElem = moduleElem;
+      return moduleElem;
+    },
+  );
+}
+
+export function importList(cc: CollectContext): SegmentList {
+  const list = cc.tags.list as PathSegment[];
+  return new SegmentList(list);
+}
+
+export function importSegment(cc: CollectContext): SimpleSegment {
+  const segOrig = cc.tags.segment?.[0] as string;
+  const seg = segOrig === "." ? "package" : segOrig; // TODO convert legacy syntax for now
+  return new SimpleSegment(seg, cc.tags.as?.[0]);
+}
+
+export function importTree(cc: CollectContext): ImportTree {
+  const path = cc.tags.p?.flat() as PathSegment[]; // LATER fix typing
+  return new ImportTree(path);
+}
+
+export function importElem(): CollectPair<ImportElem> {
+  return collectElem(
+    "import",
+    (cc: CollectContext, openElem: PartElem<ImportElem>) => {
+      const path = cc.tags.p as PathSegment[]; // LATER ts typing
+      const imports = new ImportTree(path);
+      const partialElem: ImportElem = { ...openElem, imports };
+      const importElem = withTextCover(partialElem, cc);
+      (cc.app.stable as StableState).imports.push(imports);
+      return importElem;
+    },
+  );
+}
+
+/** collect a scope start starts before and ends after a parser */
+export function scopeCollect(): CollectPair<void> {
+  return {
+    before: startScope,
+    after: completeScope,
+  };
+}
+
+export function collectSimpleElem<V extends AbstractElem & ElemWithContents>(
+  kind: V["kind"],
+): CollectPair<V> {
+  return collectElem(kind, (cc, part) => withTextCover(part, cc) as V);
+}
+
+function collectElem<V extends AbstractElem>(
+  kind: V["kind"],
+  fn: (cc: CollectContext, partialElem: PartElem<V>) => V,
+): CollectPair<V> {
+  return {
+    before: (cc: CollectContext) => {
+      const partialElem = { kind, contents: [] };
+      const weslContext: WeslParseContext = cc.app.context;
+      weslContext.openElems.push(partialElem);
+    },
+    after: (cc: CollectContext) => {
+      // TODO refine start?
+      const weslContext: WeslParseContext = cc.app.context;
+      const partialElem = weslContext.openElems.pop()!;
+      console.assert(partialElem && partialElem.kind === kind);
+      const elem = fn(cc, { ...partialElem, start: cc.start, end: cc.end });
+      addToOpenElem(cc, elem);
+      return elem;
+    },
+  };
+}
+
+/**
+ * @return a copy of the element with contents extended
+ * to include TextElems to cover the entire range.
+ */
+function withTextCover<T extends ElemWithContents>(
+  elem: T,
+  cc: CollectContext,
+): T {
+  const contents = coverWithText(cc, elem.contents);
+  return { ...elem, contents };
+}
+
+/** cover the entire source range with Elems by creating TextElems to
+ * cover any parts of the source that are not covered by other elems
+ * @returns the existing elems combined with any new TextElems, in src order */
+function coverWithText(
+  cc: CollectContext,
+  existing: AbstractElem[],
+): AbstractElem[] {
+  let { start: pos } = cc;
+  const { end, app } = cc;
+  const ast: WeslAST = app.stable;
+  const sorted = existing.sort((a, b) => a.start - b.start);
+
+  const elems = sorted.flatMap(elem => {
+    const result = pos < elem.start ? [makeTextElem(elem.start), elem] : [elem];
+    pos = elem.end;
+    return result;
+  });
+
+  if (pos < end) {
+    elems.push(makeTextElem(end));
+  }
+
+  return elems;
+
+  function makeTextElem(end: number): TextElem {
+    return { kind: "text", start: pos, end, srcModule: ast.srcModule };
+  }
+}