yuku-parser 0.5.17 → 0.5.18

package/README.md

@@ -74,22 +74,20 @@ const { line, column } = result.locOf(result.program.body[0].start);
 
 Lines are 1-based and columns are 0-based, matching ESTree's `loc` convention. The lookup is an O(log n) binary search: tens of nanoseconds per call, safe to invoke per node during a walk.
 
-For bulk work — building your own line/column index, integrating with another source-map or diagnostics library, or skipping the per-call binary search — read `result.lineStarts` directly. It's a sorted array of UTF-16 offsets where each line begins: `lineStarts[i]` is the start of line `i + 1`. Decoded lazily on first access and cached.
-
 ## Walking the AST
 
-The AST is standard ESTree, so any ESTree-compatible walker works. For example, with [zimmerframe](https://github.com/sveltejs/zimmerframe):
+[`yuku-ast`](https://www.npmjs.com/package/yuku-ast) is the companion toolkit built for this AST: a typed walker, node builders (`b`), type guards (`is`), and identifier validators.
 
 ```ts
-import { parse, type Node } from "yuku-parser";
-import { walk } from "zimmerframe";
+import { parse } from "yuku-parser";
+import { walk } from "yuku-ast";
 
 const { program } = parse(`
   const message = "hello";
   console.log(message);
 `);
 
-walk(program as Node, null, {
+walk(program, {
   Identifier(node) {
     console.log(node.name);
   },
@@ -99,6 +97,8 @@ walk(program as Node, null, {
 });
 ```
 
+The AST is also standard ESTree, so any ESTree-compatible walker (e.g. [zimmerframe](https://github.com/sveltejs/zimmerframe)) works just as well.
+
 ## Options
 
 All options are optional.
@@ -121,7 +121,7 @@ const result = parse(source, {
 | `preserveParens`             | `true`, `false`                           | `true`     | Keep `ParenthesizedExpression` nodes in the AST. When false, parentheses are stripped and only the inner expression is kept. |
 | `allowReturnOutsideFunction` | `true`, `false`                           | `false`    | Allow `return` statements outside of functions, at the top level.                                                            |
 | `semanticErrors`             | `true`, `false`                           | `false`    | Run semantic analysis and report semantic errors alongside syntax errors.                                                    |
-| `attachComments`             | `true`, `false`                           | `false`    | Collect comments and attach them to host AST nodes. See [Comments](#comments).                                               |
+| `attachComments`             | `true`, `false`                           | `false`    | Also attach each comment to its host AST node. The flat `result.comments` list is always present. See [Comments](#comments). |
 
 ## Result
 
@@ -130,6 +130,7 @@ const result = parse(source, {
 ```ts
 interface ParseResult {
   program: Program;
+  comments: Comment[]; // every comment in source order
   diagnostics: Diagnostic[];
   lineStarts: number[];
   locOf(offset: number): { line: number; column: number };
@@ -163,63 +164,58 @@ This incurs a very small performance overhead. If your build pipeline already ha
 
 ## Comments
 
-Comments are attached to the AST node they sit next to. Enable collection with `attachComments`, then read them off any node:
+Every comment is always in `result.comments`, a flat list in source order with each comment's source span:
 
 ```js
-const { program } = parse(
-  `// header\nfunction foo() {} // trailing`,
-  { attachComments: true },
-);
+const { comments } = parse(`// a line comment\nconst x = 1; /* a block comment */`);
 
-const fn = program.body[0];
-for (const c of fn.comments ?? []) {
-  console.log(c.position, c.type, c.value);
+for (const c of comments) {
+  console.log(c.type, JSON.stringify(c.value), c.start, c.end);
 }
-// before Line " header"
-// after  Line " trailing"
+// Line " a line comment" 0 17
+// Block " a block comment " 31 52
 ```
 
-Each comment is:
+Each entry is:
 
 ```ts
 interface Comment {
   type: "Line" | "Block";
-  position: "before" | "after" | "inside";
-  sameLine: boolean;
   value: string; // body without delimiters
+  start: number; // byte offset, delimiter included
+  end: number;   // byte offset, delimiter included
 }
 ```
 
-`position` tells you where the comment sits relative to its host:
-
-- `"before"`: leading the host node.
-- `"after"`: trailing the host node.
-- `"inside"`: interior to an otherwise empty host, like `function f() { /* hi */ }`.
-
-`sameLine` is `true` when the comment shares a source line with the host's adjacent edge (host's start for `before`, host's end for `after`). For `inside` it is always `false`.
-
-When `attachComments` is disabled (the default), comments are skipped like whitespace and no `comments` field appears on any node.
-
-### Why comments are attached to nodes
-
-Comments are part of the source. Any tool that reads or rewrites code needs to know which comments belong with which node, and that link has to survive AST transforms. Yuku puts a single `comments` array on each node, with a `position` field on each comment (`"before"`, `"after"`, `"inside"`). It is the simplest shape that holds up under transforms and stays cheap to decode. Every other approach trades away one of those properties.
+The span (`start`/`end`) covers the whole comment, delimiters included, so `source.slice(c.start, c.end)` returns the raw text.
 
-**Flat offset-indexed array (ESTree default, Oxc).** A separate `comments: [...]` array on the parse result, sibling to the program, each entry pointing at a source range. Fine for read-only analysis. It falls apart the moment you transform the tree.
+### Attaching comments to nodes
 
-- *Transforms desync the array.* Insert, delete, or move a node and every offset downstream of the edit is wrong. The comment that used to sit between `a` and `b` may now sit inside something unrelated, or hang in empty space between two nodes. Re-syncing means walking the entire array and rewriting offsets on every edit, and even then the meaning of "the comment between these two things" cannot really be preserved by offsets alone.
-- *Codegen cannot trust the offsets.* A printer takes whatever AST it is handed. With offset-based comments, a transformed AST will either print comments in the wrong places or drop them entirely. With attached comments, when you move a node its comments come with it. The printer reads them off `node.comments` and emits them next to the node, no reconciliation needed.
-- *Lookup is awkward.* "Give me the comments belonging to this node" with a flat array is a binary search at best and a linear scan at worst. `node.comments` is O(1) and always exact.
+Set `attachComments: true` to also hang each comment on the AST node it sits next to, read off `node.comments`. This is what a codegen pass needs, since attached comments move with their node through transforms.
 
-**Parser returns a flat list, you attach on the JS side (Acorn).** Acorn's `onComment` option pushes every comment into an array you provide, and that is all it does. To get them onto AST nodes you have to run a separate JS pass like `escodegen.attachComments`, which walks the tree, scans the comment list alongside it, and tags each comment as leading or trailing on the nearest node. That work is unavoidable if you want to transform the tree, and doing it in JavaScript is slow. Yuku does the same attachment in Zig as part of the parse. On the 7.8 MB `typescript.js` (30k+ comments), turning `attachComments` on adds only around 4ms.
+```js
+const { program } = parse(`// header\nfunction foo() {} // trailing`, { attachComments: true });
 
-**Three separate fields per node (Babel).** `leadingComments`, `trailingComments`, and `innerComments`. Same idea as Yuku, but split across three arrays. Two costs come with the split.
+const fn = program.body[0];
+for (const c of fn.comments ?? []) {
+  console.log(c.position, c.type, c.value);
+}
+// before Line " header"
+// after  Line " trailing"
+```
 
-- *Decode work.* Three fields per node means three slots to check, three potential array allocations, and roughly 3x the work on the JS side for the same information.
-- *Duplication.* A comment that sits between two nodes is the `trailingComments` of one and the `leadingComments` of the other. Transforms have to track both copies and keep them in sync.
+Each attached comment is:
 
-A single `comments` array with a `position` field carries the same meaning, with one entry per comment and one place to look.
+```ts
+interface AttachedComment {
+  type: "Line" | "Block";
+  position: "before" | "after" | "inside";
+  sameLine: boolean;
+  value: string; // body without delimiters
+}
+```
 
-Even with `attachComments` enabled, Yuku still parses the same file roughly 4x faster than Babel does without comments attached at all. When the flag is off (the default), comments are skipped like whitespace and there is no attachment work at all.
+`position` is where the comment sits relative to its host: `"before"` (leading), `"after"` (trailing), or `"inside"` (interior to an otherwise empty host like `function f() { /* hi */ }`). `sameLine` is `true` when the comment shares a source line with the host's adjacent edge.
 
 ## License
 

package/decode.js

@@ -68,22 +68,24 @@ function decode(buffer, source) {
         extraCount = _u32[1],
         spLen = _u32[2];
   const commentCount = _u32[4],
-        lineStartsCount = _u32[5],
-        diagCount = _u32[6],
-        progIdx = _u32[7];
-  const _flags = _u32[8];
+        lineStartsCount = _u32[6],
+        diagCount = _u32[7],
+        progIdx = _u32[8];
+  const attachedCommentCount = _u32[5];
+  const _flags = _u32[9];
   const _isTs = !!(_flags & 1);
-  const _attachComments = !!(_flags & 2);
-  const _firstNa = _u32[9];
-  const _nodesOff = 40;
+  const _attached = !!(_flags & 2);
+  const _firstNa = _u32[10];
+  const _nodesOff = 44;
   const eOff = _nodesOff + nodeCount * 48;
   const _extraBase = eOff >> 2;
   const _spOff = eOff + extraCount * 4;
   // sections after the variable-length string pool may be at any
   // byte alignment, so reads against them use `dv`, not `_u32`.
   const dv = new DataView(buffer);
-  const _coOff = _spOff + spLen;
-  const _cOff = _attachComments ? _coOff + (nodeCount + 1) * 4 : _coOff;
+  const _aoOff = _spOff + spLen;
+  const _acOff = _attached ? _aoOff + (nodeCount + 1) * 4 : _aoOff;
+  const _cOff = _acOff + attachedCommentCount * 12;
   function _poolDecode(s, e) {
     const a = _spOff + s - _srcLen, b = _spOff + e - _srcLen;
     let hasEd = false;
@@ -155,10 +157,10 @@ function decode(buffer, source) {
     if (rest !== NULL) p.push(node(rest));
     return p;
   }
-  function _commentsOf(a, e) {
+  function _attachedCommentsOf(a, e) {
     const out = new Array(e - a);
     for (let j = a; j < e; j++) {
-      const o = _cOff + j * 12;
+      const o = _acOff + j * 12;
       const cf = _u8[o + 0];
       const vs = dv.getUint32(o + 4, true),
             ve = dv.getUint32(o + 8, true);
@@ -176,9 +178,9 @@ function decode(buffer, source) {
     // skip unwrap cases whose inner node already carries its own
     // comments (e.g. formal_parameter returning its pattern)
     if (r && r.type !== undefined && r.comments === undefined) {
-      const off = _coOff + i * 4;
+      const off = _aoOff + i * 4;
       const a = dv.getUint32(off, true), e = dv.getUint32(off + 4, true);
-      if (a !== e) r.comments = _commentsOf(a, e);
+      if (a !== e) r.comments = _attachedCommentsOf(a, e);
     }
     return r;
   }
@@ -609,9 +611,27 @@ function decode(buffer, source) {
     case 170: return { type: "JSXSpreadChild", start, end, expression: f1 !== NULL ? node(f1) : null };
     }
   }
-  const node = _attachComments ? nodeWithComments : _decode;
-  const lsOff = _cOff + commentCount * 12;
+  const node = _attached ? nodeWithComments : _decode;
+  const lsOff = _cOff + commentCount * 20;
   const dOff = lsOff + lineStartsCount * 4;
+  function _decodeComments() {
+    const out = new Array(commentCount);
+    for (let j = 0; j < commentCount; j++) {
+      const o = _cOff + j * 20;
+      const cf = _u8[o + 0];
+      const vs = dv.getUint32(o + 4, true),
+            ve = dv.getUint32(o + 8, true);
+      const ss = dv.getUint32(o + 12, true),
+            se = dv.getUint32(o + 16, true);
+      out[j] = {
+        type: (cf & 1) ? "Block" : "Line",
+        value: str(vs, ve),
+        start: _p(ss),
+        end: _p(se),
+      };
+    }
+    return out;
+  }
   function _decodeLineStarts() {
     const out = new Array(lineStartsCount);
     if (_firstNa >= _srcLen) {
@@ -659,7 +679,7 @@ function decode(buffer, source) {
     }
     return out;
   }
-  let _program, _lineStarts, _diagnostics;
+  let _program, _lineStarts, _diagnostics, _comments;
   function _getLineStarts() {
     if (_lineStarts === undefined) _lineStarts = _decodeLineStarts();
     return _lineStarts;
@@ -668,6 +688,11 @@ function decode(buffer, source) {
     get program() {
       return _program !== undefined ? _program : (_program = node(progIdx));
     },
+    get comments() {
+      return _comments !== undefined
+        ? _comments
+        : (_comments = _decodeComments());
+    },
     get diagnostics() {
       return _diagnostics !== undefined
         ? _diagnostics

package/index.d.ts

@@ -41,18 +41,33 @@ interface ParseOptions {
    */
   semanticErrors?: boolean;
   /**
-   * Collect comments and attach them to host AST nodes via
-   * {@link BaseNode.comments}.
+   * Also attach each comment to the AST node it sits next to, via
+   * {@link BaseNode.comments}. The flat {@link ParseResult.comments} list is
+   * always present regardless.
    * @default false
    */
   attachComments?: boolean;
 }
 
-/** Whether a {@link Comment} came from a line or block source comment. */
+/** Whether a comment came from a line or block source comment. */
 type CommentType = "Line" | "Block";
 
 /**
- * Position of a {@link Comment} relative to its host node.
+ * A source comment in the flat {@link ParseResult.comments} list, carrying its
+ * source span.
+ */
+interface Comment {
+  type: CommentType;
+  /** Comment text without the delimiters. */
+  value: string;
+  /** Byte offset of the comment start (delimiter included). */
+  start: number;
+  /** Byte offset of the comment end (delimiter included). */
+  end: number;
+}
+
+/**
+ * Position of an {@link AttachedComment} relative to its host node.
  *
  * - `before`: leading the host.
  * - `after`: trailing the host.
@@ -61,13 +76,13 @@ type CommentType = "Line" | "Block";
 type CommentPosition = "before" | "after" | "inside";
 
 /**
- * A source code comment attached to a single host AST node.
+ * A comment attached to a single host AST node, via {@link BaseNode.comments}.
  *
  * `sameLine` is true when the comment shares a source line with the
  * host's adjacent edge (host's start for `before`, host's end for
  * `after`). For `inside` it is always `false`.
  */
-interface Comment {
+interface AttachedComment {
   type: CommentType;
   position: CommentPosition;
   sameLine: boolean;
@@ -119,6 +134,8 @@ interface SourceLocation {
 interface ParseResult {
   /** Root ESTree/TypeScript-ESTree AST node. */
   program: Program;
+  /** Every comment in source order, each with its source span. */
+  comments: Comment[];
   /** Syntax diagnostics, and semantic diagnostics when {@link ParseOptions.semanticErrors} is enabled. */
   diagnostics: Diagnostic[];
   /**
@@ -165,9 +182,9 @@ interface BaseNode {
   end: number;
   /**
    * Comments attached to this node in source order. Present only when
-   * {@link ParseOptions.attachComments} is enabled.
+   * {@link ParseOptions.attachComments} is true.
    */
-  comments?: Comment[];
+  comments?: AttachedComment[];
 }
 
 type Span = BaseNode;
@@ -605,7 +622,7 @@ interface Directive extends BaseNode {
 
 interface BlockStatement extends BaseNode {
   type: "BlockStatement";
-  body: Statement[];
+  body: (Statement | Directive)[];
 }
 
 interface IfStatement extends BaseNode {
@@ -1495,7 +1512,7 @@ interface TSModuleDeclaration extends BaseNode {
 
 interface TSModuleBlock extends BaseNode {
   type: "TSModuleBlock";
-  body: Statement[];
+  body: ProgramStatement[];
 }
 
 interface TSParameterProperty extends BaseNode {
@@ -1609,9 +1626,17 @@ interface Program extends BaseNode {
   type: "Program";
   sourceType: ModuleKind;
   hashbang: Hashbang | null;
-  body: Array<Statement | ModuleDeclaration | Directive>;
+  body: ProgramStatement[];
 }
 
+/**
+ * An element of `Program.body`. Unlike {@link Statement}, this also includes
+ * {@link ModuleDeclaration} (`import`/`export`) and {@link Directive}, which
+ * are only valid at the top level of a program, never in nested statement
+ * positions such as a block, loop, or `if` body.
+ */
+type ProgramStatement = Statement | ModuleDeclaration | Directive;
+
 type Declaration =
   | FunctionDeclaration
   | ClassDeclaration
@@ -1761,6 +1786,7 @@ export type {
   ParseOptions,
   ParseResult,
   Comment,
+  AttachedComment,
   CommentType,
   CommentPosition,
   Diagnostic,
@@ -1773,6 +1799,7 @@ export type {
   BaseNode,
   Span,
   Program,
+  ProgramStatement,
   Statement,
   Expression,
   Declaration,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yuku-parser",
-  "version": "0.5.17",
+  "version": "0.5.18",
   "description": "High-performance JavaScript/TypeScript parser",
   "license": "MIT",
   "repository": {
@@ -17,17 +17,17 @@
     "decode.js"
   ],
   "optionalDependencies": {
-    "@yuku-parser/binding-linux-x64-gnu": "0.5.17",
-    "@yuku-parser/binding-linux-arm64-gnu": "0.5.17",
-    "@yuku-parser/binding-linux-arm-gnu": "0.5.17",
-    "@yuku-parser/binding-linux-x64-musl": "0.5.17",
-    "@yuku-parser/binding-linux-arm64-musl": "0.5.17",
-    "@yuku-parser/binding-linux-arm-musl": "0.5.17",
-    "@yuku-parser/binding-darwin-x64": "0.5.17",
-    "@yuku-parser/binding-darwin-arm64": "0.5.17",
-    "@yuku-parser/binding-win32-x64": "0.5.17",
-    "@yuku-parser/binding-win32-arm64": "0.5.17",
-    "@yuku-parser/binding-freebsd-x64": "0.5.17"
+    "@yuku-parser/binding-linux-x64-gnu": "0.5.18",
+    "@yuku-parser/binding-linux-arm64-gnu": "0.5.18",
+    "@yuku-parser/binding-linux-arm-gnu": "0.5.18",
+    "@yuku-parser/binding-linux-x64-musl": "0.5.18",
+    "@yuku-parser/binding-linux-arm64-musl": "0.5.18",
+    "@yuku-parser/binding-linux-arm-musl": "0.5.18",
+    "@yuku-parser/binding-darwin-x64": "0.5.18",
+    "@yuku-parser/binding-darwin-arm64": "0.5.18",
+    "@yuku-parser/binding-win32-x64": "0.5.18",
+    "@yuku-parser/binding-win32-arm64": "0.5.18",
+    "@yuku-parser/binding-freebsd-x64": "0.5.18"
   },
   "keywords": [
     "acorn",