wesl 0.7.26 → 0.7.28

package/src/parse/AttachComments.ts

@@ -0,0 +1,289 @@
+import type {
+  AbstractElem,
+  AbstractElemBase,
+  CommentElem,
+  ModuleElem,
+  SyntheticElem,
+} from "../AbstractElems.ts";
+import { childElems } from "../LinkerUtil.ts";
+import type { SrcModule } from "../Scope.ts";
+import type { ParsingContext } from "./ParsingContext.ts";
+import type { CommentTrivia } from "./WeslStream.ts";
+
+/** An AST node with a source position (i.e. not a synthetic elem). Every such
+ *  node can carry comments via the {@link AbstractElemBase} fields. */
+type Positioned = Exclude<AbstractElem, SyntheticElem>;
+
+/** Positioned children per node, computed once per attach pass. Attached
+ *  comments are not structural fields, so entries never go stale. */
+type ChildCache = Map<Positioned, Positioned[]>;
+
+const tab = 0x09;
+const lineFeed = 0x0a;
+const verticalTab = 0x0b;
+const formFeed = 0x0c;
+const carriageReturn = 0x0d;
+const space = 0x20;
+const nextLine = 0x85;
+const leftToRightMark = 0x200e;
+const rightToLeftMark = 0x200f;
+const lineSeparator = 0x2028;
+const paragraphSeparator = 0x2029;
+
+/**
+ * Attach every recorded comment to a node in the parsed tree.
+ *
+ * Each comment run (a contiguous group of comments between two real tokens) is
+ * anchored to the deepest node that contains it in a gap between children, then
+ * distributed to the surrounding children:
+ * - a comment that begins its own line leads the next child (`commentsBefore`);
+ * - an inline comment trails the previous child (`commentsAfter`), unless it
+ *   hugs the next child -- only blank space and comments, no separator token,
+ *   between them -- in which case it leads the next child instead;
+ * - comments before a closing token with no following child trail the last child,
+ *   or land in an empty block's `innerComments`.
+ *
+ * Descending into expressions (via {@link childElems}) means interior comments
+ * like the one in `foo(1, /* x *\/ 2)` are preserved on the `2`, not dropped.
+ */
+export function attachComments(
+  ctx: ParsingContext,
+  moduleElem: ModuleElem,
+): void {
+  const { srcModule } = ctx.state.stable;
+  const cache: ChildCache = new Map();
+  for (const run of ctx.stream.commentRuns()) {
+    const anchor = deepestContaining(
+      moduleElem,
+      run[0].start,
+      runEnd(run),
+      cache,
+    );
+    distribute(anchor, run, srcModule, cache);
+  }
+}
+
+/** The deepest node whose span contains the whole [start, end) range, found by
+ *  descending into the child that brackets it. Comments live in gaps between
+ *  tokens, so the result is the node holding the gap, with the run between two
+ *  of its children. */
+function deepestContaining(
+  root: Positioned,
+  start: number,
+  end: number,
+  cache: ChildCache,
+): Positioned {
+  let node = root;
+  while (true) {
+    const child = positionedChildren(node, cache).find(
+      c => c.start <= start && end <= c.end,
+    );
+    if (!child) return node;
+    node = child;
+  }
+}
+
+function runEnd(run: CommentTrivia[]): number {
+  return run[run.length - 1].end;
+}
+
+/** Split a comment run between the previous child (trailing) and the next child
+ *  (leading) of its anchor, or onto the last child / inner comments when it
+ *  dangles before a closing token. */
+function distribute(
+  anchor: Positioned,
+  run: CommentTrivia[],
+  srcModule: SrcModule,
+  cache: ChildCache,
+): void {
+  const children = positionedChildren(anchor, cache);
+  const start = run[0].start;
+  const end = runEnd(run);
+
+  // the AbstractElem nodes bracketing the comment run.
+  //   prev ends at/before the comments
+  //   next starts at/after the comments
+  //   prev, next may be missing:
+  //     prev when the run leads the anchor's first child
+  //     next when the run dangles after the last child
+  //     both when the anchor has no children (e.g. an empty block)
+  const prev = children.findLast(c => c.end <= start);
+  const next = children.find(c => c.start >= end);
+
+  if (!next) {
+    // run dangles after the last child, before the anchor's closing token
+    if (prev) addComments(prev, "commentsAfter", run, srcModule);
+    // empty block: nothing to attach to, so the comments live inside it
+    else if (anchor.kind === "block")
+      anchor.innerComments = run.map(t => makeComment(t, srcModule));
+    // empty non-block container: keep the comments rather than drop them
+    else addComments(anchor, "commentsBefore", run, srcModule);
+    return;
+  }
+
+  // one run in a gap can hold comments belonging to both sides -- some trailing
+  // prev (on its line), some leading next -- so split it at the boundary:
+  // [0, split) trail prev, [split, end) lead next.
+  const split = splitPoint(prev, next, run, srcModule.src);
+  if (prev && split > 0)
+    addComments(prev, "commentsAfter", run.slice(0, split), srcModule);
+  if (split < run.length)
+    addComments(next, "commentsBefore", run.slice(split), srcModule);
+}
+
+function positionedChildren(node: Positioned, cache: ChildCache): Positioned[] {
+  let children = cache.get(node);
+  if (children === undefined) {
+    children = positioned(childElems(node));
+    cache.set(node, children);
+  }
+  return children;
+}
+
+/** Append converted comments to an element's leading or trailing list. */
+function addComments(
+  elem: AbstractElemBase,
+  field: "commentsBefore" | "commentsAfter",
+  trivia: CommentTrivia[],
+  srcModule: SrcModule,
+): void {
+  const comments = trivia.map(t => makeComment(t, srcModule));
+  const existing = elem[field];
+  elem[field] = existing ? [...existing, ...comments] : comments;
+}
+
+function makeComment(trivia: CommentTrivia, srcModule: SrcModule): CommentElem {
+  const { style, start, end } = trivia;
+  const comment: CommentElem = {
+    kind: "comment",
+    style,
+    start,
+    end,
+    srcModule,
+  };
+  // a fully blank line above the comment is preserved in the output
+  if (lineBreaksBefore(srcModule.src, start) >= 2) comment.blankBefore = true;
+  return comment;
+}
+
+/**
+ * Index into `run` where it flips from trailing `prev` to leading `next`:
+ * caller attaches run[0, split) to `prev` and run[split, end) to `next`.
+ *
+ * The aim is to keep each comment with the code it describes, so it stays
+ * meaningful after the tree is reordered or reformatted. That follows how people
+ * write comments: a comment on its own line documents what comes after it, while
+ * a comment sharing a line with code documents that code. Hence:
+ * - with no previous child the whole run leads (split 0);
+ * - otherwise the split is the first comment that begins its own line;
+ * - failing that (an all-inline run), it is the start of the suffix that hugs
+ *   `next` on the same line, so the comment in `foo(1, /* x *\/ 2)` documents,
+ *   and lands on, the `2`.
+ */
+function splitPoint(
+  prev: Positioned | undefined,
+  next: Positioned,
+  run: CommentTrivia[],
+  src: string,
+): number {
+  if (!prev) return 0;
+  const ownLine = firstOwnLine(run, src);
+  return ownLine >= 0 ? ownLine : hugsNextStart(run, next.start, src);
+}
+
+/** Source-positioned children, sorted in source order. Synthetic elems (no
+ *  source position) cannot anchor comments and are dropped. Children usually
+ *  arrive already in source order, so the sort is skipped when possible.
+ *  Filtering and order-checking share one pass: this runs per node, so it's
+ *  kept deliberately lean. */
+function positioned(elems: readonly AbstractElem[]): Positioned[] {
+  const result: Positioned[] = [];
+  let sorted = true; // starts in source order until proven otherwise
+  let lastStart = -1;
+  for (const e of elems) {
+    if (e.kind === "synthetic") continue;
+    if (e.start < lastStart) sorted = false;
+    lastStart = e.start;
+    result.push(e);
+  }
+  if (!sorted) result.sort((a, b) => a.start - b.start);
+  return result;
+}
+
+// Blankspace classification per https://www.w3.org/TR/WGSL/#blankspace-and-line-breaks
+// (the tokenizer matches the same set via the blankspaces/lineBreak regexes).
+
+/** Count line breaks in the whitespace run immediately before `pos`, capped at 2:
+ *  callers only need none / one line break / a blank line. Scans backward over
+ *  blankspace, stopping at the first non-blankspace char, so cost is the gap
+ *  length, not the source length. `\r\n` counts as one break. */
+function lineBreaksBefore(src: string, pos: number): number {
+  let count = 0;
+  for (let i = pos - 1; i >= 0; i--) {
+    const c = src.charCodeAt(i);
+    if (isLineBreak(c)) {
+      if (c === lineFeed && src.charCodeAt(i - 1) === carriageReturn) i--; // \r\n
+    } else if (isInlineSpace(c)) {
+      continue; // still inside the run
+    } else {
+      break; // run ended at a non-blankspace char
+    }
+    if (++count >= 2) return 2;
+  }
+  return count;
+}
+
+/** Index of the first comment that begins its own line (after a line break),
+ *  or -1 if none do. */
+function firstOwnLine(run: CommentTrivia[], src: string): number {
+  return run.findIndex(t => lineBreaksBefore(src, t.start) >= 1);
+}
+
+/** Start index of the trailing suffix of `run` that is joined to `next` by
+ *  same-line whitespace only (no line break, no separator token between them).
+ *  Returns run.length when nothing hugs `next`. */
+function hugsNextStart(
+  run: CommentTrivia[],
+  nextStart: number,
+  src: string,
+): number {
+  let suffixStart = run.length; // nothing hugs `next` until proven otherwise
+  let rightStart = nextStart; // start of the neighbor just right of this comment
+  for (let i = run.length - 1; i >= 0; i--) {
+    const comment = run[i];
+    // stop once more than blank space sits between this comment and its neighbor
+    if (!sameLineGap(src, comment.end, rightStart)) break;
+    suffixStart = i;
+    rightStart = comment.start;
+  }
+  return suffixStart;
+}
+
+/** A WGSL line break code point. `\r\n` is two of these; callers coalesce it. */
+function isLineBreak(c: number): boolean {
+  return (
+    c === lineFeed ||
+    c === carriageReturn ||
+    c === verticalTab ||
+    c === formFeed ||
+    c === nextLine ||
+    c === lineSeparator ||
+    c === paragraphSeparator
+  );
+}
+
+/** WGSL blankspace that stays on the same line (not a line break). */
+function isInlineSpace(c: number): boolean {
+  return (
+    c === space || c === tab || c === leftToRightMark || c === rightToLeftMark
+  );
+}
+
+/** True when [from, to) is inline blankspace only (no line break): the two ends
+ *  sit on the same line with nothing but same-line spaces between. */
+function sameLineGap(src: string, from: number, to: number): boolean {
+  for (let i = from; i < to; i++) {
+    if (!isInlineSpace(src.charCodeAt(i))) return false;
+  }
+  return true;
+}

package/src/parse/ExpressionUtil.ts

@@ -21,7 +21,8 @@ export function makeLiteral(token: WeslToken<"keyword" | "number">): Literal {
 }
 
 export function makeUnaryOperator(token: WeslToken<"symbol">): UnaryOperator {
-  return { value: token.text as UnaryOperator["value"], span: token.span };
+  const [start, end] = token.span;
+  return { value: token.text as UnaryOperator["value"], start, end };
 }
 
 export function makeBinaryOperator(token: {
@@ -36,12 +37,11 @@ export function makeUnaryExpression(
   operator: UnaryOperator,
   expr: ExpressionElem,
 ): UnaryExpression {
-  const [start] = operator.span;
   return {
     kind: "unary-expression",
     operator,
     expression: expr,
-    start,
+    start: operator.start,
     end: expr.end,
   };
 }

package/src/parse/Keywords.ts

@@ -1,4 +1,4 @@
-// Use https://github.com/wgsl-tooling-wg/wgsl-spec to check this list in the future
+// Use https://github.com/webgpu-tools/wgsl-spec to check this list in the future
 // I recommend checking whether a new list and the current list are equal
 
 /** https://www.w3.org/TR/WGSL/#keyword-summary */

package/src/parse/ParseAttribute.ts

@@ -6,7 +6,6 @@ import type {
   DiagnosticRule,
   ElifAttribute,
   ElseAttribute,
-  ExpressionElem,
   IfAttribute,
   InterpolateAttribute,
   NameElem,
@@ -15,7 +14,6 @@ import type {
   UnknownExpressionElem,
 } from "../AbstractElems.ts";
 import { ParseError } from "../ParseError.ts";
-import { beginElem, finishContents } from "./ContentsHelpers.ts";
 import { parseExpression } from "./ParseExpression.ts";
 import {
   expect,
@@ -23,6 +21,7 @@ import {
   makeNameElem,
   parseCommaList,
   parseMany,
+  throwParseError,
 } from "./ParseUtil.ts";
 import type { ParsingContext } from "./ParsingContext.ts";
 
@@ -47,6 +46,27 @@ export function parseElifAttribute(ctx: ParsingContext): ElifAttribute | null {
   return parseConditionalAttribute(ctx, "elif", makeElifAttribute);
 }
 
+/** Parse WESL conditional attributes (@if, @elif, @else) */
+export function parseWeslConditional(
+  ctx: ParsingContext,
+): AttributeElem | null {
+  const { stream } = ctx;
+  const peeked = stream.peek();
+  if (peeked?.text !== "@") return null;
+  const startPos = peeked.span[0]; // Use token position, not stream checkpoint
+
+  const ifAttr = parseIfAttribute(ctx);
+  if (ifAttr) return attributeElem(ifAttr, startPos, stream.checkpoint());
+
+  const elifAttr = parseElifAttribute(ctx);
+  if (elifAttr) return attributeElem(elifAttr, startPos, stream.checkpoint());
+
+  const elseAttr = parseElseAttribute(ctx);
+  if (elseAttr) return attributeElem(elseAttr, startPos, stream.checkpoint());
+
+  return null;
+}
+
 /**
  * Grammar: attribute :
  *   '@' ident_pattern_token argument_expression_list ?
@@ -57,18 +77,9 @@ export function parseElifAttribute(ctx: ParsingContext): ElifAttribute | null {
  * WESL extensions: @if, @elif, @else
  */
 function parseAttribute(ctx: ParsingContext): AttributeElem | null {
-  const { stream } = ctx;
-  const startPos = stream.checkpoint();
-  if (!stream.matchText("@")) return null;
-  stream.reset(startPos);
-
-  const weslAttr = parseWeslConditional(ctx);
-  if (weslAttr) return weslAttr;
+  if (ctx.stream.peek()?.text !== "@") return null;
 
-  const stdAttr = parseStandardAttribute(ctx);
-  if (stdAttr) return stdAttr;
-
-  return null;
+  return parseWeslConditional(ctx) ?? parseStandardAttribute(ctx);
 }
 
 /** Parse `@if(expr)` or `@elif(expr)` conditional attributes. */
@@ -82,16 +93,21 @@ function parseConditionalAttribute<T>(
   if (!stream.matchSequence("@", keyword)) return null;
 
   expect(stream, "(", `@${keyword}`);
+  // Past `@keyword(` we're committed: a missing/invalid condition is a hard
+  // parse error, not a backtrack (the stream is already advanced past `(`).
   const expr = parseExpression(ctx, true);
-  if (!expr) return null;
+  if (!expr) throwParseError(stream, `Expected expression after @${keyword}(`);
 
   stream.matchText(",");
   expect(stream, ")", `@${keyword} expression`);
 
-  const translateTimeExpr = makeTranslateTimeExpressionElem({
-    value: expr,
-    span: [startPos, stream.checkpoint()],
-  });
+  // TODO remove translate-time once we drop v1
+  const translateTimeExpr: TranslateTimeExpressionElem = {
+    kind: "translate-time-expression",
+    expression: expr,
+    start: startPos,
+    end: stream.checkpoint(),
+  };
   return makeAttr(translateTimeExpr);
 }
 
@@ -107,25 +123,12 @@ function makeElifAttribute(param: TranslateTimeExpressionElem): ElifAttribute {
   return { kind: "@elif", param } as const;
 }
 
-/** Parse WESL conditional attributes (@if, @elif, @else) */
-export function parseWeslConditional(
-  ctx: ParsingContext,
-): AttributeElem | null {
-  const { stream } = ctx;
-  const peeked = stream.peek();
-  if (peeked?.text !== "@") return null;
-  const startPos = peeked.span[0]; // Use token position, not stream checkpoint
-
-  const ifAttr = parseIfAttribute(ctx);
-  if (ifAttr) return attributeElem(ifAttr, startPos, stream.checkpoint());
-
-  const elifAttr = parseElifAttribute(ctx);
-  if (elifAttr) return attributeElem(elifAttr, startPos, stream.checkpoint());
-
-  const elseAttr = parseElseAttribute(ctx);
-  if (elseAttr) return attributeElem(elseAttr, startPos, stream.checkpoint());
-
-  return null;
+function attributeElem(
+  attribute: Attribute,
+  start: number,
+  end: number,
+): AttributeElem {
+  return { kind: "attribute", attribute, start, end };
 }
 
 /** Parse a standard attribute (not @if/@elif/@else) */
@@ -137,10 +140,7 @@ function parseStandardAttribute(ctx: ParsingContext): AttributeElem | null {
   const startPos = atToken.span[0]; // Use actual @ position, not before whitespace
 
   const nameToken = stream.peek();
-  if (
-    !nameToken ||
-    (nameToken.kind !== "word" && nameToken.kind !== "keyword")
-  ) {
+  if (nameToken?.kind !== "word" && nameToken?.kind !== "keyword") {
     stream.reset(resetPos);
     return null;
   }
@@ -171,26 +171,6 @@ function parseStandardAttribute(ctx: ParsingContext): AttributeElem | null {
   return attributeElem(stdAttr, startPos, stream.checkpoint());
 }
 
-// TODO remove translate-time once we drop v1
-function makeTranslateTimeExpressionElem(args: {
-  value: ExpressionElem;
-  span: [number, number];
-}): TranslateTimeExpressionElem {
-  return {
-    kind: "translate-time-expression",
-    expression: args.value,
-    span: args.span,
-  };
-}
-
-function attributeElem(
-  attribute: Attribute,
-  start: number,
-  end: number,
-): AttributeElem {
-  return { kind: "attribute", attribute, start, end, contents: [] };
-}
-
 function parseBuiltinAttribute(
   ctx: ParsingContext,
   startPos: number,
@@ -224,11 +204,6 @@ function parseInterpolateAttribute(
   return attributeElem(interpolateAttr, startPos, stream.checkpoint());
 }
 
-function parseNameElem(ctx: ParsingContext): NameElem {
-  const nameToken = expectWord(ctx.stream, "Expected identifier");
-  return makeNameElem(nameToken);
-}
-
 /** @diagnostic(severity, rule) or @diagnostic(severity, namespace.rule) */
 function parseDiagnosticAttribute(
   ctx: ParsingContext,
@@ -263,12 +238,15 @@ function parseAttributeParams(ctx: ParsingContext): UnknownExpressionElem[] {
   return parseCommaList(ctx, parseAttrParam);
 }
 
+function parseNameElem(ctx: ParsingContext): NameElem {
+  const nameToken = expectWord(ctx.stream, "Expected identifier");
+  return makeNameElem(nameToken);
+}
+
 function parseAttrParam(ctx: ParsingContext): UnknownExpressionElem {
   const { stream } = ctx;
   const start = stream.checkpoint();
-  beginElem(ctx, "expression");
-  parseExpression(ctx);
-  const end = stream.checkpoint();
-  const contents = finishContents(ctx, start, end);
-  return { kind: "expression", start, end, contents };
+  const expression = parseExpression(ctx);
+  if (!expression) throwParseError(stream, "Expected attribute parameter");
+  return { kind: "expression", expression, start, end: stream.checkpoint() };
 }

package/src/parse/ParseCall.ts

@@ -24,11 +24,10 @@ export function parseCallSuffix(
   if (current.kind !== "ref" && current.kind !== "type") return null;
 
   const { stream } = ctx;
-  // TypeRefElem has templates already; RefIdentElem needs separate parsing
+  // Constructor calls (e.g. vec4<f32>(...)) already have their template args
+  // on the TypeRefElem; only function calls need template args parsed here.
   const templateArgs =
-    current.kind === "type"
-      ? (current.templateParams ?? null)
-      : parseCallTemplateArgs(ctx);
+    current.kind === "ref" ? parseCallTemplateArgs(ctx) : null;
 
   if (!stream.matchText("(")) return null;