sqllens 1.9.1 → 1.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -158,15 +158,29 @@ export declare class SqlDocument {
158
158
  * NOT replace the cache entry: the first occurrence keeps its stable cross-edit identity (the
159
159
  * common case), and only intra-doc duplicates — rare — pay a re-parse per build. */
160
160
  private buildCell;
161
- /** Build the ONE cell for a TEMPLATED document: span [0, text.length) the templated build path
162
- * bypasses `splitStatements` entirely (one cell, whole text), and its products come from a single
163
- * `engine.parse(text, dialect, { provider })` call rather than the plain per-dialect `parse()`.
164
- * `r.tokens`/`r.diagnostics` are ALREADY document coordinates (the cell always starts at 0), so —
165
- * unlike `buildCell` nothing is shifted. Mirrors `buildCell`'s CachedCell/StatementCell shapes
166
- * so every downstream consumer (analyze(), cellAt(), nodeAt()…) sees the same structure whether
167
- * the document is plain or templated. Cached in the SAME cross-edit `_cellCache` as plain cells,
168
- * under a prefixed key so a templated cell can never collide with a plain one for the same text. */
161
+ /** Build the WHOLE-TEXT cell of a TEMPLATED document: span [0, text.length), its products from a
162
+ * single `engine.parse(text, dialect, { provider })` call rather than the plain per-dialect
163
+ * `parse()`. It is the document's one cell when the placeholder splits into one statement (every
164
+ * single-statement model), and the carrier of the engine result (`templated`) that a multi-cell
165
+ * document's slices are built from. `r.tokens`/`r.diagnostics` are ALREADY document coordinates
166
+ * (the cell always starts at 0), so, unlike `buildCell`, nothing is shifted. Mirrors
167
+ * `buildCell`'s CachedCell/StatementCell shapes so every downstream consumer (analyze(), cellAt(),
168
+ * nodeAt()…) sees the same structure whether the document is plain or templated. Cached in the SAME
169
+ * cross-edit `_cellCache` as plain cells, under a prefixed key so a templated cell can never
170
+ * collide with a plain one for the same text. */
169
171
  private buildTemplatedCell;
172
+ /** Build one statement cell of a MULTI-cell templated document for `span` (a `splitStatements`
173
+ * span over the engine's placeholder). The cell is the plain parse of its placeholder slice,
174
+ * cell-relative like a plain cell, with the whole-document tags correlated onto it by
175
+ * `engine.parseCell` (provider-resolved source names + `template` markers, so the fill never
176
+ * reaches scope/qualify). Cached across edits like a plain cell: the key is the placeholder
177
+ * slice (what parsed) plus the raw slice (which tags sit inside it), the engine + provider
178
+ * version (what the tags resolve to) and `setsKey` (the `{% set %}`/`{% macro %}`/`{% for %}`
179
+ * tags anywhere in the text that steer a bare `{{ t }}` binding). The document-level products
180
+ * are projected onto the cell by span rather than re-derived: `tokens` is the unified SQL+jinja
181
+ * stream sliced, `diagnostics` the scrubbed set filtered (the last cell absorbs end-of-text),
182
+ * `errors` that count. `handedOut` dedupes intra-document duplicates exactly as `buildCell` does. */
183
+ private buildTemplatedSlice;
170
184
  /** Build a document for `text` in `dialect`. Total: never throws, even on broken / mid-edit input.
171
185
  * Starts a FRESH cell cache — cross-edit reuse comes from withText(), not create(). Pass
172
186
  * `templating` to parse through an injected TemplateEngine (jinja-SQL etc.) instead of the plain
@@ -295,8 +309,7 @@ export declare class SqlDocument {
295
309
  * subset, see `scopeOutputColumns`'s pipe doc comment. Falls through to this document's own
296
310
  * (single-arm) answer when there are no variants; there is no pre-existing single-doc
297
311
  * equivalent to delegate to, unlike unionSymbols/unionDiagnostics, so the no-variant case is
298
- * just the one-arm instance of the same algorithm. A MULTI-STATEMENT document (no variants: the
299
- * templated door always forces exactly one cell, so the two "multi" shapes never overlap) merges
312
+ * just the one-arm instance of the same algorithm. A MULTI-STATEMENT document (or arm) merges
300
313
  * every statement CELL's own CTEs instead, each shifted from cell-relative to DOCUMENT coordinates
301
314
  * (the same shift `analyze()` already applies to symbols/diagnostics for a multi-cell document);
302
315
  * the compound facade itself carries no CTEs, so cells are the real per-statement source. Memoized
@@ -315,13 +328,16 @@ export declare class SqlDocument {
315
328
  span: Span;
316
329
  }[];
317
330
  private buildUnionOutputColumns;
318
- /** One "arm" `unionCtes`/`unionOutputColumns` aggregate over, unified across the two shapes that
319
- * can each independently make a document "multi" (never both at once: the templated door always
320
- * builds exactly one statement cell, see its own comment above): a templated document's real
321
- * variants (each a full arm SqlDocument, already in DOCUMENT coordinates, zero shift), or, for a
322
- * plain multi-statement document, each statement CELL (cell-relative scopes, shifted to document
323
- * coordinates by the cell's start, mirroring `buildAnalysis`'s per-cell shift). A single-cell,
324
- * non-templated document is the trivial one-arm case of the same shape (zero shift, this
325
- * document's own scopes/qualification). */
331
+ /** The "arms" `unionCtes`/`unionOutputColumns` aggregate over, unified across the two shapes that
332
+ * make a document "multi", VARIANTS FIRST (ruled 2026-09-13): a templated document's real
333
+ * variants are the unit, each arm SqlDocument (already in DOCUMENT coordinates, since a
334
+ * realization is length-preserving) contributing its own statement cells through `ownArms`; a
335
+ * document without variants contributes its own cells the same way. A region can span a cell
336
+ * boundary, so cells-first (arms inside each cell) is not an option. */
326
337
  private armsData;
338
+ /** This document's own arms, variants ignored: each statement CELL of a multi-cell document
339
+ * (cell-relative scopes, shifted to document coordinates by the cell's start, mirroring
340
+ * `buildAnalysis`'s per-cell shift), or the single-cell document itself (zero shift, its own
341
+ * scopes/qualification). */
342
+ private ownArms;
327
343
  }
@@ -155,12 +155,35 @@ export class SqlDocument {
155
155
  this._provider = opts.provider;
156
156
  let cells;
157
157
  let backing;
158
+ let templated;
158
159
  if (opts.templating) {
159
- // The templated door: ONE cell spanning the whole text, bypassing splitStatements —
160
- // its products come from the engine, not the plain per-dialect parse (see buildTemplatedCell).
161
- const built = this.buildTemplatedCell(text, opts.templating, opts.provider);
162
- cells = [built.cell];
163
- backing = [built.cached];
160
+ // The templated door: the engine runs ONCE over the whole text (buildTemplatedCell), then the
161
+ // placeholder it saw is split into statement cells exactly like a plain document's text. The
162
+ // fill is length- and newline-preserving, so the spans are document coordinates already, and
163
+ // each cell is the plain parse of its placeholder slice with the whole-document tags
164
+ // correlated onto it by the engine (buildTemplatedSlice). One span (every single-statement
165
+ // model, or a tiling failure) keeps the whole-text cell itself: byte-identical products,
166
+ // same cache entry as before cells existed on this door.
167
+ const whole = this.buildTemplatedCell(text, opts.templating, opts.provider);
168
+ const r = whole.cached.templated;
169
+ const spans = opts.templating.parseCell ? splitStatements(r.placeholder, dialect) : [whole.cell.span];
170
+ if (spans.length === 1) {
171
+ cells = [whole.cell];
172
+ backing = [whole.cached];
173
+ templated = r;
174
+ }
175
+ else {
176
+ const setsKey = setResolutionKey(r, text);
177
+ const handedOut = new Set();
178
+ cells = [];
179
+ backing = [];
180
+ for (const span of spans) {
181
+ const built = this.buildTemplatedSlice(span, r, opts.templating, opts.provider, setsKey, handedOut);
182
+ cells.push(built.cell);
183
+ backing.push(built.cached);
184
+ }
185
+ templated = withCellCorrelation(r, cells, backing);
186
+ }
164
187
  }
165
188
  else {
166
189
  // Split into per-statement cells and parse each independently, reusing unchanged cells from
@@ -178,7 +201,7 @@ export class SqlDocument {
178
201
  }
179
202
  this.statements = Object.freeze(cells);
180
203
  this._cells = backing;
181
- this.templated = opts.templating ? backing[0].templated : undefined;
204
+ this.templated = templated;
182
205
  // Whole-document facade. tokens/diagnostics/errors are the cheap concat/sum across cells.
183
206
  this.tokens = cells.flatMap((c) => c.tokens);
184
207
  this.diagnostics = cells.flatMap((c) => c.diagnostics);
@@ -254,14 +277,16 @@ export class SqlDocument {
254
277
  });
255
278
  return { cell, cached };
256
279
  }
257
- /** Build the ONE cell for a TEMPLATED document: span [0, text.length) the templated build path
258
- * bypasses `splitStatements` entirely (one cell, whole text), and its products come from a single
259
- * `engine.parse(text, dialect, { provider })` call rather than the plain per-dialect `parse()`.
260
- * `r.tokens`/`r.diagnostics` are ALREADY document coordinates (the cell always starts at 0), so —
261
- * unlike `buildCell` nothing is shifted. Mirrors `buildCell`'s CachedCell/StatementCell shapes
262
- * so every downstream consumer (analyze(), cellAt(), nodeAt()…) sees the same structure whether
263
- * the document is plain or templated. Cached in the SAME cross-edit `_cellCache` as plain cells,
264
- * under a prefixed key so a templated cell can never collide with a plain one for the same text. */
280
+ /** Build the WHOLE-TEXT cell of a TEMPLATED document: span [0, text.length), its products from a
281
+ * single `engine.parse(text, dialect, { provider })` call rather than the plain per-dialect
282
+ * `parse()`. It is the document's one cell when the placeholder splits into one statement (every
283
+ * single-statement model), and the carrier of the engine result (`templated`) that a multi-cell
284
+ * document's slices are built from. `r.tokens`/`r.diagnostics` are ALREADY document coordinates
285
+ * (the cell always starts at 0), so, unlike `buildCell`, nothing is shifted. Mirrors
286
+ * `buildCell`'s CachedCell/StatementCell shapes so every downstream consumer (analyze(), cellAt(),
287
+ * nodeAt()…) sees the same structure whether the document is plain or templated. Cached in the SAME
288
+ * cross-edit `_cellCache` as plain cells, under a prefixed key so a templated cell can never
289
+ * collide with a plain one for the same text. */
265
290
  buildTemplatedCell(text, engine, provider) {
266
291
  const span = { start: 0, end: text.length };
267
292
  // Collision-proofed against a plain cell's `dialect + " " + text` key by the "templated "
@@ -301,6 +326,67 @@ export class SqlDocument {
301
326
  });
302
327
  return { cell, cached };
303
328
  }
329
+ /** Build one statement cell of a MULTI-cell templated document for `span` (a `splitStatements`
330
+ * span over the engine's placeholder). The cell is the plain parse of its placeholder slice,
331
+ * cell-relative like a plain cell, with the whole-document tags correlated onto it by
332
+ * `engine.parseCell` (provider-resolved source names + `template` markers, so the fill never
333
+ * reaches scope/qualify). Cached across edits like a plain cell: the key is the placeholder
334
+ * slice (what parsed) plus the raw slice (which tags sit inside it), the engine + provider
335
+ * version (what the tags resolve to) and `setsKey` (the `{% set %}`/`{% macro %}`/`{% for %}`
336
+ * tags anywhere in the text that steer a bare `{{ t }}` binding). The document-level products
337
+ * are projected onto the cell by span rather than re-derived: `tokens` is the unified SQL+jinja
338
+ * stream sliced, `diagnostics` the scrubbed set filtered (the last cell absorbs end-of-text),
339
+ * `errors` that count. `handedOut` dedupes intra-document duplicates exactly as `buildCell` does. */
340
+ buildTemplatedSlice(span, r, engine, provider, setsKey, handedOut) {
341
+ const rawText = this.text.slice(span.start, span.end);
342
+ const placeholderText = r.placeholder.slice(span.start, span.end);
343
+ const providerVersion = provider?.version ?? 0;
344
+ // `setsKey` is length-prefixed and the two slices are equal-length (the fill is
345
+ // length-preserving), so the plain concatenation is unambiguous without a separator byte.
346
+ const key = `templated-cell ${engine.name}@${providerVersion} ${this.dialect} ${setsKey.length}:${setsKey}${placeholderText}${rawText}`;
347
+ let cached = this._cellCache.get(key);
348
+ if (cached !== undefined && handedOut.has(cached))
349
+ cached = undefined; // intra-doc duplicate
350
+ if (cached === undefined) {
351
+ const c = engine.parseCell(r, span, this.text, this.dialect, { provider });
352
+ cached = {
353
+ text: rawText,
354
+ category: c.sql.ast.statement ?? "other",
355
+ ast: c.sql.ast,
356
+ cst: c.sql.cst,
357
+ // Resolve scopes from the already-lowered (marker-carrying) ast: never re-parse.
358
+ scopes: toScopes(c.sql.ast, { dialect: this.dialect }),
359
+ tokens: c.sql.tokens,
360
+ errors: c.sql.errors,
361
+ diagnostics: c.sql.diagnostics,
362
+ analysis: new WeakMap(),
363
+ correlation: {
364
+ tagStartOf: new Map(c.links.map((l) => [l.node, l.tagStart])),
365
+ primaryAt: new Map(c.links.filter((l) => l.primary).map((l) => [l.tagStart, l.node])),
366
+ },
367
+ };
368
+ // Cache only the FIRST product for a key (see buildCell: duplicates stay uncached).
369
+ if (this._cellCache.get(key) === undefined)
370
+ this._cellCache.set(key, cached);
371
+ }
372
+ handedOut.add(cached);
373
+ const last = span.end === this.text.length;
374
+ const inCell = (offset) => offset >= span.start && (offset < span.end || last);
375
+ const tokens = r.tokens.filter((t) => inCell(t.start));
376
+ const diagnostics = r.diagnostics.filter((d) => inCell(d.offset ?? this.lines.offsetAt(d.line - 1, d.column)));
377
+ const cell = Object.freeze({
378
+ span,
379
+ text: cached.text,
380
+ category: cached.category,
381
+ ast: cached.ast,
382
+ cst: cached.cst,
383
+ scopes: cached.scopes,
384
+ tokens,
385
+ errors: diagnostics.length,
386
+ diagnostics,
387
+ });
388
+ return { cell, cached };
389
+ }
304
390
  /** Build a document for `text` in `dialect`. Total: never throws, even on broken / mid-edit input.
305
391
  * Starts a FRESH cell cache — cross-edit reuse comes from withText(), not create(). Pass
306
392
  * `templating` to parse through an injected TemplateEngine (jinja-SQL etc.) instead of the plain
@@ -716,8 +802,7 @@ export class SqlDocument {
716
802
  * subset, see `scopeOutputColumns`'s pipe doc comment. Falls through to this document's own
717
803
  * (single-arm) answer when there are no variants; there is no pre-existing single-doc
718
804
  * equivalent to delegate to, unlike unionSymbols/unionDiagnostics, so the no-variant case is
719
- * just the one-arm instance of the same algorithm. A MULTI-STATEMENT document (no variants: the
720
- * templated door always forces exactly one cell, so the two "multi" shapes never overlap) merges
805
+ * just the one-arm instance of the same algorithm. A MULTI-STATEMENT document (or arm) merges
721
806
  * every statement CELL's own CTEs instead, each shifted from cell-relative to DOCUMENT coordinates
722
807
  * (the same shift `analyze()` already applies to symbols/diagnostics for a multi-cell document);
723
808
  * the compound facade itself carries no CTEs, so cells are the real per-statement source. Memoized
@@ -783,27 +868,23 @@ export class SqlDocument {
783
868
  }
784
869
  return order.map((name) => ({ name, span: byName.get(name) }));
785
870
  }
786
- /** One "arm" `unionCtes`/`unionOutputColumns` aggregate over, unified across the two shapes that
787
- * can each independently make a document "multi" (never both at once: the templated door always
788
- * builds exactly one statement cell, see its own comment above): a templated document's real
789
- * variants (each a full arm SqlDocument, already in DOCUMENT coordinates, zero shift), or, for a
790
- * plain multi-statement document, each statement CELL (cell-relative scopes, shifted to document
791
- * coordinates by the cell's start, mirroring `buildAnalysis`'s per-cell shift). A single-cell,
792
- * non-templated document is the trivial one-arm case of the same shape (zero shift, this
793
- * document's own scopes/qualification). */
871
+ /** The "arms" `unionCtes`/`unionOutputColumns` aggregate over, unified across the two shapes that
872
+ * make a document "multi", VARIANTS FIRST (ruled 2026-09-13): a templated document's real
873
+ * variants are the unit, each arm SqlDocument (already in DOCUMENT coordinates, since a
874
+ * realization is length-preserving) contributing its own statement cells through `ownArms`; a
875
+ * document without variants contributes its own cells the same way. A region can span a cell
876
+ * boundary, so cells-first (arms inside each cell) is not an option. */
794
877
  armsData(s) {
878
+ if (this.variants.length > 0)
879
+ return this.variants.flatMap((v) => v.doc().ownArms(s));
880
+ return this.ownArms(s);
881
+ }
882
+ /** This document's own arms, variants ignored: each statement CELL of a multi-cell document
883
+ * (cell-relative scopes, shifted to document coordinates by the cell's start, mirroring
884
+ * `buildAnalysis`'s per-cell shift), or the single-cell document itself (zero shift, its own
885
+ * scopes/qualification). */
886
+ ownArms(s) {
795
887
  const ZERO = { line: 0, col: 0, offset: 0 };
796
- if (this.variants.length > 0) {
797
- return this.variants.map((v) => {
798
- const doc = v.doc();
799
- return {
800
- scopeRoot: doc.scopes.root,
801
- qualification: doc.analyze(s).qualification,
802
- dialect: doc.dialect,
803
- base: ZERO,
804
- };
805
- });
806
- }
807
888
  if (this.statements.length > 1) {
808
889
  return this.statements.map((cell, i) => {
809
890
  const p = this.lines.positionAt(cell.span.start);
@@ -825,6 +906,52 @@ export class SqlDocument {
825
906
  ];
826
907
  }
827
908
  }
909
+ /** The cache-key component of a templated cell's tag correlation that the cell's own text does not
910
+ * determine: the raw text of every `{% set %}` / `{% macro %}` / `{% for %}` tag in the document. A
911
+ * bare `{{ t }}` source binds through a literal `{% set t = ref(...) %}` declared ANYWHERE, and an
912
+ * inline macro or a `for` target anywhere disables that binding (apply-tags' `resolveSets`), so a
913
+ * change to any of them must miss every cell. */
914
+ function setResolutionKey(r, text) {
915
+ const parts = [];
916
+ for (const t of r.tags) {
917
+ if (t.kind !== "control")
918
+ continue;
919
+ if (t.keyword === "set" || t.keyword === "macro" || t.keyword === "for")
920
+ parts.push(text.slice(t.tagSpan.start, t.tagSpan.end));
921
+ }
922
+ // A control tag's own text never contains a tag boundary, so a newline join is unambiguous.
923
+ return parts.join("\n");
924
+ }
925
+ /** The `templated` facade of a MULTI-cell templated document: the whole-text engine result with
926
+ * `tagOf`/`nodeOf` answering from the CELLS' own correlations (the per-statement IR consumers reach
927
+ * through `statements`/`cellAt`/`nodeAt`), never from the whole-text parse, whose nodes a multi-cell
928
+ * document does not expose (its `ast` is the compound facade). A cell's join names its tag by
929
+ * cell-relative start, so the answer is always one of THIS parse's TagNodes, whichever parse the
930
+ * cached cell was built under. Everything else is the engine's. */
931
+ function withCellCorrelation(r, cells, backing) {
932
+ const tagAt = new Map(r.tags.map((t) => [t.tagSpan.start, t]));
933
+ return {
934
+ ...r,
935
+ tagOf: (node) => {
936
+ for (let i = 0; i < backing.length; i++) {
937
+ const rel = backing[i].correlation?.tagStartOf.get(node);
938
+ if (rel !== undefined)
939
+ return tagAt.get(cells[i].span.start + rel);
940
+ }
941
+ return undefined;
942
+ },
943
+ nodeOf: (tag) => {
944
+ for (let i = 0; i < backing.length; i++) {
945
+ const span = cells[i].span;
946
+ if (tag.tagSpan.start < span.start || tag.tagSpan.start >= span.end)
947
+ continue;
948
+ return backing[i].correlation?.primaryAt.get(tag.tagSpan.start - span.start);
949
+ }
950
+ return undefined;
951
+ },
952
+ diagnosticsOf: (tag) => r.diagnosticsOf(tag),
953
+ };
954
+ }
828
955
  /** Dedup `items` by a string key, keeping the FIRST occurrence of each key — arm/document order, so
829
956
  * the "first live arm wins" representative-data rule falls out of plain array order. */
830
957
  function dedupBy(items, keyOf) {
@@ -2,7 +2,8 @@ import type { Dialect } from "../dialect.js";
2
2
  export interface StatementCellSpan {
3
3
  /** doc offset, inclusive — cell text includes leading trivia. */
4
4
  start: number;
5
- /** doc offset, exclusive — includes the trailing separator (`;` / GO line). */
5
+ /** doc offset, exclusive — includes the trailing separator (`;` / GO line); the document's last
6
+ * cell also includes whatever trivia follows its separator, up to `text.length`. */
6
7
  end: number;
7
8
  }
8
9
  /**
@@ -26,7 +26,10 @@ const NON_OPENER_END_SUFFIXES = new Set(["IF", "WHILE", "FOR", "LOOP", "REPEAT"]
26
26
  function wholeDoc(text) {
27
27
  return [{ start: 0, end: text.length }];
28
28
  }
29
- /** Every offset in `text` where a top-level separator ends (exclusive), in ascending order. */
29
+ /** Every offset in `text` where a top-level separator ends (exclusive), in ascending order. A
30
+ * separator that no channel-0 token follows (only whitespace, comments, the final newline) is not
31
+ * a split end: the trivia after it belongs to the cell it terminates, so a single terminated
32
+ * statement is one cell rather than a statement plus a token-less tail cell. */
30
33
  function findSplitEnds(text, tokens, dialect) {
31
34
  const channel0 = tokens.filter((t) => t.channel === 0);
32
35
  const ends = [];
@@ -80,6 +83,11 @@ function findSplitEnds(text, tokens, dialect) {
80
83
  }
81
84
  }
82
85
  }
86
+ // Tokens are in source order, so the last channel-0 token decides whether anything real follows
87
+ // the last separator.
88
+ const lastReal = channel0[channel0.length - 1];
89
+ if (ends.length > 0 && (lastReal === undefined || lastReal.start < ends[ends.length - 1]))
90
+ ends.pop();
83
91
  return ends;
84
92
  }
85
93
  /** Turn ascending split-end offsets into contiguous cells tiling `[0, text.length)`. A doc
@@ -1,6 +1,13 @@
1
1
  import type { QueryExpr } from "../ir/ir.js";
2
2
  import type { TemplateCall, TemplateProvider } from "../qualify/template-provider.js";
3
3
  import type { MacroCall, TagNode } from "./tag-ast.js";
4
+ /** A statement cell's start in the document: 0-based line, 0-based column, char offset (the shape
5
+ * `LineIndex.positionAt` answers). Zero = the whole-text parse. */
6
+ export interface CellBase {
7
+ line: number;
8
+ column: number;
9
+ offset: number;
10
+ }
4
11
  /** The rebuilt AST plus the tag↔node correlations collected while building it. */
5
12
  export interface TagCorrelation {
6
13
  ast: QueryExpr;
@@ -8,6 +15,12 @@ export interface TagCorrelation {
8
15
  byNode: WeakMap<object, TagNode>;
9
16
  /** TagNode → the IR node it became (undefined-by-absence for tags with no IR presence). */
10
17
  byTag: Map<TagNode, object>;
18
+ /** Every attached (node, tag) pair in attach order: the enumerable form of `byNode`, for a
19
+ * caller that must re-key the join to another parse's TagNodes (a cached statement cell). */
20
+ links: {
21
+ node: object;
22
+ tag: TagNode;
23
+ }[];
11
24
  }
12
25
  /**
13
26
  * Rewrite templated FROM/JOIN sources in `ast` to carry their provider-resolved name (when the
@@ -15,8 +28,13 @@ export interface TagCorrelation {
15
28
  * containment. Returns the SAME `ast` reference when nothing correlates (structural sharing);
16
29
  * returns a re-frozen rebuilt tree otherwise. Total, never throws; the correlation maps are empty
17
30
  * (not absent) on the no-op and error paths.
31
+ *
32
+ * `base` is the cell's start position when `ast` is ONE STATEMENT CELL's parse of the placeholder
33
+ * (`parseTemplatedCell`): the node offsets are cell-relative while `tags` and `text` are the whole
34
+ * document's, so containment compares `offset + base.offset`, and every span this transform writes
35
+ * into the IR is rebased to cell coordinates. Zero (the default) is the whole-text parse.
18
36
  */
19
- export declare function applyTemplateTags(ast: QueryExpr, tags: TagNode[], text: string, provider: TemplateProvider): TagCorrelation;
37
+ export declare function applyTemplateTags(ast: QueryExpr, tags: TagNode[], text: string, provider: TemplateProvider, base?: CellBase): TagCorrelation;
20
38
  /**
21
39
  * The provider key of a tag-AST MacroCall — name + package + literal args, with kwargs
22
40
  * carried separately (the channel-agreed TemplateCall contract: quote-stripped, escapes
@@ -30,6 +30,7 @@ import { debugRethrow } from "../debug.js";
30
30
  import { freezeIR } from "../ir/freeze.js";
31
31
  import { qualifiedNameOf, synthesizedQualifiedName } from "../ir/qualified-name.js";
32
32
  import { resolveBehavior } from "../dialect-behavior/registry.js";
33
+ const ZERO_BASE = { line: 0, column: 0, offset: 0 };
33
34
  /** Record a freshly built node's correlation to the tag it came from, then return it unchanged
34
35
  * (a passthrough so call sites stay expression-shaped). A scalar-slot tag lowers to BOTH a
35
36
  * column Expr (`kind: "column"`) and a parallel ColumnRef record (`kind: "columnref"`, same
@@ -38,6 +39,7 @@ import { resolveBehavior } from "../dialect-behavior/registry.js";
38
39
  * of which one this walk visits first. */
39
40
  function attach(ctx, node, tag) {
40
41
  ctx.byNode.set(node, tag);
42
+ ctx.links.push({ node, tag });
41
43
  const existing = ctx.byTag.get(tag);
42
44
  if (existing?.kind !== "column")
43
45
  ctx.byTag.set(tag, node);
@@ -49,25 +51,33 @@ function attach(ctx, node, tag) {
49
51
  * containment. Returns the SAME `ast` reference when nothing correlates (structural sharing);
50
52
  * returns a re-frozen rebuilt tree otherwise. Total, never throws; the correlation maps are empty
51
53
  * (not absent) on the no-op and error paths.
54
+ *
55
+ * `base` is the cell's start position when `ast` is ONE STATEMENT CELL's parse of the placeholder
56
+ * (`parseTemplatedCell`): the node offsets are cell-relative while `tags` and `text` are the whole
57
+ * document's, so containment compares `offset + base.offset`, and every span this transform writes
58
+ * into the IR is rebased to cell coordinates. Zero (the default) is the whole-text parse.
52
59
  */
53
- export function applyTemplateTags(ast, tags, text, provider) {
60
+ export function applyTemplateTags(ast, tags, text, provider, base = ZERO_BASE) {
54
61
  const byNode = new WeakMap();
55
62
  const byTag = new Map();
63
+ const links = [];
56
64
  try {
57
65
  // config is a no-output tag (whitespace-filled), so it can never yield a table
58
66
  // source and stays out of the correlation set even though ExprTag admits it.
59
67
  // An incomplete/mid-typing call (`{{ ref('cu`) is NOT a resolved source, so skip it.
60
68
  const relTags = tags.filter((t) => (t.kind === "call" && !t.incomplete) || t.kind === "other");
61
69
  if (relTags.length === 0)
62
- return { ast, byNode, byTag };
70
+ return { ast, byNode, byTag, links };
63
71
  const nameConfig = ast.dialect !== undefined ? resolveBehavior(ast.dialect).nameConfig : undefined;
64
72
  const ctx = {
65
73
  relTags,
66
74
  sets: resolveSets(tags, text, provider),
67
75
  text,
68
76
  provider,
77
+ base,
69
78
  byNode,
70
79
  byTag,
80
+ links,
71
81
  ...(nameConfig ? { nameConfig } : {}),
72
82
  };
73
83
  const next = transformQuery(ast, ctx);
@@ -75,11 +85,11 @@ export function applyTemplateTags(ast, tags, text, provider) {
75
85
  // fill gets a `template` marker (span + provider key), so inference resolves it
76
86
  // through the provider and qualify never checks the placeholder as a real column.
77
87
  const marked = markTemplateExprs(next, ctx);
78
- return { ast: marked === ast ? ast : freezeIR(marked), byNode, byTag };
88
+ return { ast: marked === ast ? ast : freezeIR(marked), byNode, byTag, links };
79
89
  }
80
90
  catch (e) {
81
91
  debugRethrow(e);
82
- return { ast, byNode: new WeakMap(), byTag: new Map() };
92
+ return { ast, byNode: new WeakMap(), byTag: new Map(), links: [] };
83
93
  }
84
94
  }
85
95
  /** The TemplateExprInfo for a scalar-slot tag: its span + (when an identity is extractable)
@@ -89,14 +99,14 @@ function exprInfoOf(tag, ctx) {
89
99
  // A call tag (ref/source/var/env_var/a macro) carries its provider key straight off the call,
90
100
  // callOf reads name + literal args from the source, uniform across every callee.
91
101
  if (tag.kind === "call")
92
- return { span: tag.tagSpan, call: callOf(tag, ctx.text) };
102
+ return { span: cellSpan(tag.tagSpan, ctx), call: callOf(tag, ctx.text) };
93
103
  // A non-call `other` tag: a bare `{{ t }}` resolving through a single-call `{% set t = … %}`
94
104
  // carries that RHS call; anything else is opaque.
95
105
  const ident = bareIdentOf(tag, ctx.text);
96
106
  const resolved = ident !== undefined ? ctx.sets.get(ident) : undefined;
97
107
  if (resolved)
98
- return { span: tag.tagSpan, call: resolved.call };
99
- return { span: tag.tagSpan };
108
+ return { span: cellSpan(tag.tagSpan, ctx), call: resolved.call };
109
+ return { span: cellSpan(tag.tagSpan, ctx) };
100
110
  }
101
111
  /** `{{ var('x') }}` / `{{ env_var('Y', …) }}` — the name + first literal arg, lexically. */
102
112
  const VALUE_CALL_TAG = /^\{\{-?\s*(var|env_var)\s*\(\s*(['"])([^'"\\]*)\2\s*(,[\s\S]*?)?\)\s*(?:\|[\s\S]*)?-?\}\}$/;
@@ -129,7 +139,7 @@ function markTemplateExprs(node, ctx) {
129
139
  if ((isColumnExpr || isColumnRef) && rec.template === undefined) {
130
140
  const start = rec.cst?.start?.start;
131
141
  if (start !== undefined) {
132
- const tag = containingTag(ctx.relTags, start);
142
+ const tag = containingTag(ctx.relTags, start + ctx.base.offset);
133
143
  if (tag)
134
144
  return attach(ctx, { ...rec, template: exprInfoOf(tag, ctx) }, tag);
135
145
  }
@@ -280,6 +290,22 @@ function containingTag(tags, offset) {
280
290
  }
281
291
  return undefined;
282
292
  }
293
+ /** A document-coordinate tag span rebased to the cell's coordinates (the inverse of
294
+ * src/document/shift.ts's `shiftPartSpan`): a span on the cell's first line also loses the cell's
295
+ * start column. The whole-text parse (zero base) keeps the span object itself. */
296
+ function cellSpan(p, ctx) {
297
+ const b = ctx.base;
298
+ if (b.offset === 0 && b.line === 0 && b.column === 0)
299
+ return p;
300
+ return {
301
+ start: p.start - b.offset,
302
+ end: p.end - b.offset,
303
+ line: p.line - b.line,
304
+ column: p.line === b.line + 1 ? p.column - b.column : p.column,
305
+ endLine: p.endLine - b.line,
306
+ endColumn: p.endLine === b.line + 1 ? p.endColumn - b.column : p.endColumn,
307
+ };
308
+ }
283
309
  function transformQuery(q, ctx) {
284
310
  const ctes = mapShared(q.ctes, (c) => transformCte(c, ctx));
285
311
  const body = transformBody(q.body, ctx);
@@ -401,7 +427,7 @@ function transformTableSource(src, ctx) {
401
427
  const startTok = src.cst?.start;
402
428
  if (!startTok)
403
429
  return src;
404
- const tag = containingTag(ctx.relTags, startTok.start);
430
+ const tag = containingTag(ctx.relTags, startTok.start + ctx.base.offset);
405
431
  if (!tag)
406
432
  return src;
407
433
  // A placeholder-fill alias sits INSIDE the tag span: a multi-line tag fills one
@@ -414,7 +440,7 @@ function transformTableSource(src, ctx) {
414
440
  // fill limitation, out of apply-tags' reach); making it `undefined` here is honest,
415
441
  // where `jjj…` was a fabrication.
416
442
  const aliasTok = src.aliasCst?.start;
417
- const base = aliasTok != null && inSpan(aliasTok.start, tag.tagSpan) ? withoutAlias(src) : src;
443
+ const base = aliasTok != null && inSpan(aliasTok.start + ctx.base.offset, tag.tagSpan) ? withoutAlias(src) : src;
418
444
  // An unresolved source's name is the RAW TAG TEXT — the bytes the user actually wrote. The
419
445
  // placeholder fill is scaffolding this library invented so the grammar parses; letting it
420
446
  // escape as a relation name (scope sources, lineage dependencies, go-to-def) is fabrication
@@ -432,7 +458,8 @@ function transformTableSource(src, ctx) {
432
458
  : qualifiedNameOf(parts, ctx.nameConfig);
433
459
  return { ...b, relation };
434
460
  };
435
- // NOTE: `template.span` intentionally aliases `tag.tagSpan` BY REFERENCE. freezeIR
461
+ // NOTE: on the whole-text parse `template.span` intentionally aliases `tag.tagSpan` BY
462
+ // REFERENCE (a statement cell's parse gets a rebased copy, see `cellSpan`). freezeIR
436
463
  // therefore also freezes the TagNode.tagSpan object returned in `.tags`, benign
437
464
  // since spans are read-only. Every call marker carries its `call`, the provider key
438
465
  // the semantic layer resolves the relation and its columns through (relation-columns.ts).
@@ -444,7 +471,7 @@ function transformTableSource(src, ctx) {
444
471
  // Either way the `call` keeps it consultable, so an unresolved call is not a dead end: a
445
472
  // provider added later resolves it. ref vs source is not stored here, it is call.name.
446
473
  const named = rel ? renamed(base, [...rel.nameParts], true) : renamed(base, rawTagName, false);
447
- const template = { kind: "call", span: tag.tagSpan, call };
474
+ const template = { kind: "call", span: cellSpan(tag.tagSpan, ctx), call };
448
475
  return attach(ctx, { ...named, template }, tag);
449
476
  }
450
477
  // Non-call expression tag (var / env_var / other) in a FROM slot. A bare `{{ t }}` resolving
@@ -455,9 +482,14 @@ function transformTableSource(src, ctx) {
455
482
  const resolved = ident !== undefined ? ctx.sets.get(ident) : undefined;
456
483
  if (resolved) {
457
484
  const named = resolved.name ? renamed(base, [...resolved.name], true) : renamed(base, rawTagName, false);
458
- const template = { kind: "call", span: tag.tagSpan, indirect: true, call: resolved.call };
485
+ const template = {
486
+ kind: "call",
487
+ span: cellSpan(tag.tagSpan, ctx),
488
+ indirect: true,
489
+ call: resolved.call,
490
+ };
459
491
  return attach(ctx, { ...named, template }, tag);
460
492
  }
461
- const template = { kind: "expr", span: tag.tagSpan, opaque: true };
493
+ const template = { kind: "expr", span: cellSpan(tag.tagSpan, ctx), opaque: true };
462
494
  return attach(ctx, { ...renamed(base, rawTagName, false), template }, tag);
463
495
  }
@@ -1,4 +1,4 @@
1
- import { parseTemplated } from "./parse.js";
1
+ import { parseTemplated, parseTemplatedCell } from "./parse.js";
2
2
  import { templateVariants } from "./variants.js";
3
3
  /** The minijinja template engine (the Rust engine dbt Fusion uses — the grammar
4
4
  * oracle for what we accept). The shipped, and only, TemplateEngine. */
@@ -7,5 +7,6 @@ export function minijinja() {
7
7
  name: "minijinja",
8
8
  parse: (text, dialect, opts) => parseTemplated(text, dialect, opts),
9
9
  variants: (text, dialect) => templateVariants(text, dialect),
10
+ parseCell: (whole, span, text, dialect, opts) => parseTemplatedCell(whole, span, text, dialect, opts),
10
11
  };
11
12
  }
@@ -1,6 +1,6 @@
1
1
  import type { Dialect } from "../dialect.js";
2
2
  import type { Token } from "../token/token.js";
3
- import type { MacroShape, TemplatedParseOptions, TemplatedParseResult } from "../template/engine.js";
3
+ import type { MacroShape, TemplatedCellResult, TemplatedParseOptions, TemplatedParseResult } from "../template/engine.js";
4
4
  import type { ExpansionShape } from "../qualify/template-provider.js";
5
5
  import type { TemplateCall } from "../ir/ir.js";
6
6
  export type { TagNode, MacroCall } from "./tag-ast.js";
@@ -28,3 +28,18 @@ export declare function parseTemplated(text: string, dialect: Dialect, opts?: Te
28
28
  * of parseTemplated. Total — never throws.
29
29
  */
30
30
  export declare function tokenizeTemplated(text: string, dialect: Dialect, opts?: TemplatedParseOptions): Token[];
31
+ /**
32
+ * One statement cell of a templated document (`TemplateEngine.parseCell`): the plain per-dialect
33
+ * parse of the placeholder slice `[span.start, span.end)`, the same batch-of-one path a plain
34
+ * document's cells take, so IR / CST / tokens / diagnostics are CELL-relative, with `whole`'s tags
35
+ * correlated onto it by DOCUMENT offset (a node's cell offset plus the cell's start). The cell's
36
+ * sources then carry their provider-resolved names and `template` markers exactly as the
37
+ * whole-text parse's do, never the fill; a marker's `span` is rebased to cell coordinates like
38
+ * every other span in the cell IR, and the tag↔node join is returned as cell-relative `links`
39
+ * so a cached cell re-keys it to whatever parse it is later reused under.
40
+ * Total: `applyTemplateTags` leaves the plain parse in place on any internal surprise.
41
+ */
42
+ export declare function parseTemplatedCell(whole: TemplatedParseResult, span: {
43
+ start: number;
44
+ end: number;
45
+ }, text: string, dialect: Dialect, opts?: TemplatedParseOptions): TemplatedCellResult;
@@ -651,3 +651,40 @@ export function parseTemplated(text, dialect, opts) {
651
651
  export function tokenizeTemplated(text, dialect, opts) {
652
652
  return parseTemplated(text, dialect, opts).tokens;
653
653
  }
654
+ /**
655
+ * One statement cell of a templated document (`TemplateEngine.parseCell`): the plain per-dialect
656
+ * parse of the placeholder slice `[span.start, span.end)`, the same batch-of-one path a plain
657
+ * document's cells take, so IR / CST / tokens / diagnostics are CELL-relative, with `whole`'s tags
658
+ * correlated onto it by DOCUMENT offset (a node's cell offset plus the cell's start). The cell's
659
+ * sources then carry their provider-resolved names and `template` markers exactly as the
660
+ * whole-text parse's do, never the fill; a marker's `span` is rebased to cell coordinates like
661
+ * every other span in the cell IR, and the tag↔node join is returned as cell-relative `links`
662
+ * so a cached cell re-keys it to whatever parse it is later reused under.
663
+ * Total: `applyTemplateTags` leaves the plain parse in place on any internal surprise.
664
+ */
665
+ export function parseTemplatedCell(whole, span, text, dialect, opts) {
666
+ const sql = parse(whole.placeholder.slice(span.start, span.end), dialect);
667
+ const provider = opts?.provider ?? OPEN_PROVIDER;
668
+ const correlation = applyTemplateTags(sql.ast, whole.tags, text, provider, cellBaseOf(text, span.start));
669
+ return {
670
+ sql: { ...sql, ast: correlation.ast },
671
+ links: correlation.links.map(({ node, tag }) => ({
672
+ node,
673
+ tagStart: tag.tagSpan.start - span.start,
674
+ primary: correlation.byTag.get(tag) === node,
675
+ })),
676
+ };
677
+ }
678
+ /** The cell start's 0-based line / column / char offset in `text` (`\n` is the line break, the
679
+ * convention every span in the pipeline follows; a `\r` is an ordinary column). */
680
+ function cellBaseOf(text, offset) {
681
+ let line = 0;
682
+ let lineStart = 0;
683
+ for (let i = 0; i < offset; i++) {
684
+ if (text.charCodeAt(i) === 10) {
685
+ line++;
686
+ lineStart = i + 1;
687
+ }
688
+ }
689
+ return { line, column: offset - lineStart, offset };
690
+ }
@@ -88,6 +88,29 @@ export interface TemplatedParseResult {
88
88
  * scrubber widened to it. Empty array when none. */
89
89
  diagnosticsOf(tag: TagNode): SyntaxDiagnostic[];
90
90
  }
91
+ /** One template-marked node of a statement cell's IR and the tag it came from, the tag named by
92
+ * its CELL-relative start offset rather than by object: a cell is cached across edits and reused
93
+ * under a later parse whose TagNodes are fresh objects, so the join is re-keyed to that parse's
94
+ * tag at the same offset. `primary` marks the tag's one answer (a scalar-slot tag marks both a
95
+ * column expression and its column-ref record; the expression is primary). */
96
+ export interface TemplatedCellLink {
97
+ node: object;
98
+ /** `tag.tagSpan.start - cell.span.start`. */
99
+ tagStart: number;
100
+ primary: boolean;
101
+ }
102
+ /** One statement CELL of a templated document (`TemplateEngine.parseCell`): the plain per-dialect
103
+ * parse of the cell's placeholder slice, in CELL-relative coordinates like a plain document's
104
+ * cells, with the whole-document tags correlated onto it (provider-resolved source names,
105
+ * `template` markers, and the two-spine join). */
106
+ export interface TemplatedCellResult {
107
+ /** The cell's SQL parse over its placeholder slice (ast / cst / tokens / errors / diagnostics),
108
+ * every span cell-relative. A `template` marker's span is cell-relative too. */
109
+ sql: ParseResultIR;
110
+ /** The tag↔node join over THIS cell's IR, position- and identity-independent (see
111
+ * `TemplatedCellLink`). Empty when nothing correlates. */
112
+ links: TemplatedCellLink[];
113
+ }
91
114
  /** A template engine: the syntax front end for one templating language over
92
115
  * SQL. `parse` must satisfy the engine contract the conformance suite
93
116
  * checks — tokens tile the source byte-for-byte, every span in original
@@ -99,4 +122,13 @@ export interface TemplateEngine {
99
122
  parse(text: string, dialect: Dialect, opts?: TemplatedParseOptions): TemplatedParseResult;
100
123
  /** Optional: coherent per-branch variant enumeration, for engines with control-flow arms. */
101
124
  variants?(text: string, dialect: Dialect): TemplateVariant[];
125
+ /** Optional: the products of ONE statement cell of a templated document, the plain parse of
126
+ * `whole.placeholder`'s slice `[span.start, span.end)` (cell-relative) with `whole`'s tags
127
+ * correlated onto it. `whole` is this engine's own `parse` result for the full `text`, `span`
128
+ * a `splitStatements` span over that placeholder. An engine without it keeps the templated
129
+ * `SqlDocument` door at one whole-text cell. */
130
+ parseCell?(whole: TemplatedParseResult, span: {
131
+ start: number;
132
+ end: number;
133
+ }, text: string, dialect: Dialect, opts?: TemplatedParseOptions): TemplatedCellResult;
102
134
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sqllens",
3
- "version": "1.9.1",
3
+ "version": "1.10.1",
4
4
  "description": "A TypeScript SQL parser and static analyzer: parse, resolve names, infer types, and trace column lineage across many SQL dialects (Databricks, T-SQL, Snowflake, BigQuery, Redshift, PostgreSQL, DuckDB, Trino, SQLite, MySQL).",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",