sqllens 1.10.1 → 1.11.0

package/README.md

@@ -367,7 +367,11 @@ that run on incomplete, mid-edit text. They never need a clean parse:
 - `SqlDocument` is a persistent, immutable, position-addressable per-file model.
   It runs `parse → resolveScopes` once (plus lazy `analyze(schema)`), caches the
   result, and answers `tokenAt` / `nodeAt`. An edit yields a new document; an
-  O(log n) `LineIndex` maps positions to offsets.
+  O(log n) `LineIndex` maps positions to offsets. Its `statements` are the
+  per-statement cells; each cell's `span` carries the separator token that ends
+  it (`;`, or the `GO` word), so the statement's own text ends where that starts.
+- `statementSpans(text, dialect, { templating? })`: the same cell spans without
+  the per-cell parses, for code lenses and run-at-cursor ranges on every change.
 - `completeAt(doc, offset, schema?)`: scope-aware completion (keywords, columns,
   tables, functions) from an ATN (Augmented Transition Network, the grammar's
   state-machine form) candidate walk over the grammar, our own, with no third-party

package/dist/api.d.ts

@@ -116,6 +116,7 @@ export type { Token, TokenRole } from "./token/token.js";
 export { SqlDocument, type DocumentAnalysis, type StatementCell, type DocumentVariant, type UnionCte, } from "./document/document.js";
 export { LineIndex } from "./document/line-index.js";
 export type { StatementCellSpan } from "./document/split.js";
+export { statementSpans } from "./document/spans.js";
 export { complete, completeAt, type Completion, type CompletionResult, type ReplaceRange, type CompleteOptions, type CandidateIdentity, type CandidateDecoration, type DecorateCandidate, } from "./completion/complete.js";
 export { jinjaSlotAt, type JinjaSlot } from "./completion/jinja-slot.js";
 export { signatureAt, type SignatureHelpInfo, type SignatureLabel } from "./signature/signature.js";

package/dist/api.js

@@ -202,6 +202,9 @@ export { tokenize } from "./token/tokenize.js";
 // these at call time, never at module-eval time.
 export { SqlDocument, } from "./document/document.js";
 export { LineIndex } from "./document/line-index.js";
+// The cell spans alone (plain or templated), for a consumer that needs statement ranges without
+// the per-cell parses; identical to the spans the document's cells carry.
+export { statementSpans } from "./document/spans.js";
 // Scope-aware completion over a SqlDocument — the broken-input editor feature (keywords + schema
 // tables/columns + function names at the caret). Total: never throws.
 export { complete, completeAt, } from "./completion/complete.js";

package/dist/document/document.js

@@ -168,7 +168,8 @@ export class SqlDocument {
             const r = whole.cached.templated;
             const spans = opts.templating.parseCell ? splitStatements(r.placeholder, dialect) : [whole.cell.span];
             if (spans.length === 1) {
-                cells = [whole.cell];
+                // The whole-text cell, under the split's own span so it carries its separator.
+                cells = [Object.freeze({ ...whole.cell, span: spans[0] })];
                 backing = [whole.cached];
                 templated = r;
             }

package/dist/document/spans.d.ts

@@ -0,0 +1,14 @@
+import type { Dialect } from "../dialect.js";
+import type { TemplateProvider } from "../qualify/template-provider.js";
+import type { TemplateEngine } from "../template/engine.js";
+import { type StatementCellSpan } from "./split.js";
+/**
+ * The statement cell spans `SqlDocument.create(text, dialect, opts)` would produce, without the
+ * per-cell parses. Plain text splits directly; with `templating` the engine's placeholder is
+ * split (its `placeholder` hook when it has one, else its full `parse`), and an engine without
+ * `parseCell` answers one whole-text span, exactly as the document door does. Total: never throws.
+ */
+export declare function statementSpans(text: string, dialect: Dialect, opts?: {
+    templating?: TemplateEngine;
+    provider?: TemplateProvider;
+}): StatementCellSpan[];

package/dist/document/spans.js

@@ -0,0 +1,34 @@
+// ---------------------------------------------------------------------------
+// statementSpans() — the statement cell spans of a text without building the
+// document. The SqlDocument constructor derives its cells from exactly this
+// split (the plain text, or the engine's placeholder for a templated one), so
+// the spans answered here are the spans `doc.statements[i].span` would carry;
+// a consumer that only needs "where do the statements start and end" (code
+// lenses, run-at-cursor ranges) pays the lex and the split, not the parses.
+// ---------------------------------------------------------------------------
+import { debugRethrow } from "../debug.js";
+import { splitStatements } from "./split.js";
+/**
+ * The statement cell spans `SqlDocument.create(text, dialect, opts)` would produce, without the
+ * per-cell parses. Plain text splits directly; with `templating` the engine's placeholder is
+ * split (its `placeholder` hook when it has one, else its full `parse`), and an engine without
+ * `parseCell` answers one whole-text span, exactly as the document door does. Total: never throws.
+ */
+export function statementSpans(text, dialect, opts = {}) {
+    const engine = opts.templating;
+    if (!engine)
+        return splitStatements(text, dialect);
+    if (!engine.parseCell)
+        return [{ start: 0, end: text.length }];
+    let placeholder;
+    try {
+        placeholder = engine.placeholder
+            ? engine.placeholder(text, dialect, { provider: opts.provider })
+            : engine.parse(text, dialect, { provider: opts.provider }).placeholder;
+    }
+    catch (e) {
+        debugRethrow(e);
+        placeholder = text;
+    }
+    return splitStatements(placeholder, dialect);
+}

package/dist/document/split.d.ts

@@ -5,13 +5,23 @@ export interface StatementCellSpan {
     /** doc offset, exclusive — includes the trailing separator (`;` / GO line); the document's last
      *  cell also includes whatever trivia follows its separator, up to `text.length`. */
     end: number;
+    /** The separator token that terminates this cell, as doc offsets (`start` inclusive, `end`
+     *  exclusive): the `;`, or the `GO` word itself (not its line). Absent when the cell has no
+     *  separator (an unterminated final statement, a separator-free document, the whole-document
+     *  fallback). The statement's own text ends where this starts, before any trailing trivia. */
+    separator?: {
+        start: number;
+        end: number;
+    };
 }
 /**
  * Split `text` into top-level statement cells using `tokenize(text, dialect)`.
  * Total: never throws. Splits at channel-0 `;` at compound depth 0 (BEGIN/CASE
  * increment, END decrements, floor 0; a T-SQL `BEGIN TRAN`/`TRANSACTION`/
  * `DISTRIBUTED` does not open a depth level) plus, for T-SQL, a `GO` batch
- * separator alone on its line. Returns the whole doc as one cell when
+ * separator alone on its line; text ending inside an open level splits at every
+ * separator from the unclosed opener on (see `findSplitEnds`). Each cell carries
+ * the separator token that ends it. Returns the whole doc as one cell when
  * splitting is unsafe (the tiling invariant fails) or pointless (no separators).
  */
 export declare function splitStatements(text: string, dialect: Dialect): StatementCellSpan[];

package/dist/document/split.js

@@ -26,14 +26,43 @@ const NON_OPENER_END_SUFFIXES = new Set(["IF", "WHILE", "FOR", "LOOP", "REPEAT"]
 function wholeDoc(text) {
     return [{ start: 0, end: text.length }];
 }
-/** Every offset in `text` where a top-level separator ends (exclusive), in ascending order. A
- *  separator that no channel-0 token follows (only whitespace, comments, the final newline) is not
- *  a split end: the trivia after it belongs to the cell it terminates, so a single terminated
- *  statement is one cell rather than a statement plus a token-less tail cell. */
+/** The split point a `;` or a T-SQL `GO` at `channel0[i]` makes, ignoring depth, or undefined when
+ *  the token is neither. A GO batch separator must sit alone on its line among channel-0 tokens
+ *  (otherwise it is an identifier/alias use of the word); its cell end is the end of that line. */
+function separatorAt(text, channel0, i, dialect) {
+    const t = channel0[i];
+    const separator = { start: t.start, end: t.stop + 1 };
+    if (t.text === ";")
+        return { end: t.stop + 1, separator };
+    if (dialect === "tsql" && t.text.toUpperCase() === "GO") {
+        const prev = channel0[i - 1];
+        const next = channel0[i + 1];
+        const alone = (!prev || prev.line !== t.line) && (!next || next.line !== t.line);
+        if (!alone)
+            return undefined;
+        const nl = text.indexOf("\n", t.stop + 1);
+        return { end: nl === -1 ? text.length : nl + 1, separator };
+    }
+    return undefined;
+}
+/** Every top-level split point in `text`, in ascending order, plus `tail`: the separator of a
+ *  terminated final statement that nothing real follows (only whitespace, comments, the final
+ *  newline). That separator is not a split end, the trivia after it belongs to the cell it
+ *  terminates, so a single terminated statement is one cell rather than a statement plus a
+ *  token-less tail cell; `tail` lets that cell still carry its separator.
+ *
+ *  Depth: `BEGIN`/`CASE` open a level, `END` closes one, and only a separator at depth 0 splits.
+ *  When the text ends INSIDE an open level (an unclosed CASE mid-typing, jinja arms that leave
+ *  the placeholder with more openers than closers), the walk cannot know where that block ends,
+ *  and one cell silently spanning several statements is the worse failure (a "run statement at
+ *  cursor" would run them all): the split points before the outermost unclosed opener stand, and
+ *  from that opener on every separator splits regardless of depth. Balanced text is unaffected. */
 function findSplitEnds(text, tokens, dialect) {
     const channel0 = tokens.filter((t) => t.channel === 0);
-    const ends = [];
+    let ends = [];
     let depth = 0;
+    /** The outermost currently-open level: where it opened and how many split points preceded it. */
+    let opener;
     for (let i = 0; i < channel0.length; i++) {
         const t = channel0[i];
         const upper = t.text.toUpperCase();
@@ -41,10 +70,15 @@ function findSplitEnds(text, tokens, dialect) {
             // `BEGIN TRAN`/`TRANSACTION`/`DISTRIBUTED` (T-SQL) starts a transaction, not a
             // scripting compound — it has no matching END, so it must not open a depth level.
             const next = channel0[i + 1];
-            if (!next || !TRAN_WORDS.has(next.text.toUpperCase()))
+            if (!next || !TRAN_WORDS.has(next.text.toUpperCase())) {
+                if (depth === 0)
+                    opener = { index: i, endsBefore: ends.length };
                 depth++;
+            }
         }
         else if (upper === "CASE") {
+            if (depth === 0)
+                opener = { index: i, endsBefore: ends.length };
             depth++;
         }
         else if (upper === "END") {
@@ -66,42 +100,46 @@ function findSplitEnds(text, tokens, dialect) {
             else {
                 depth = Math.max(0, depth - 1);
             }
-        }
-        else if (t.text === ";") {
             if (depth === 0)
-                ends.push(t.stop + 1);
+                opener = undefined;
         }
-        else if (dialect === "tsql" && upper === "GO" && depth === 0) {
-            // A GO batch separator must sit alone on its line among channel-0 tokens —
-            // otherwise it's an identifier/alias use of the word `GO`, not a separator.
-            const prev = channel0[i - 1];
-            const next = channel0[i + 1];
-            const alone = (!prev || prev.line !== t.line) && (!next || next.line !== t.line);
-            if (alone) {
-                const nl = text.indexOf("\n", t.stop + 1);
-                ends.push(nl === -1 ? text.length : nl + 1);
-            }
+        else if (depth === 0) {
+            const sep = separatorAt(text, channel0, i, dialect);
+            if (sep)
+                ends.push(sep);
+        }
+    }
+    if (depth > 0 && opener) {
+        // Unclosed level: keep the split points before it, then split at every separator from it on.
+        ends = ends.slice(0, opener.endsBefore);
+        for (let i = opener.index; i < channel0.length; i++) {
+            const sep = separatorAt(text, channel0, i, dialect);
+            if (sep)
+                ends.push(sep);
         }
     }
     // Tokens are in source order, so the last channel-0 token decides whether anything real follows
     // the last separator.
     const lastReal = channel0[channel0.length - 1];
-    if (ends.length > 0 && (lastReal === undefined || lastReal.start < ends[ends.length - 1]))
+    const last = ends[ends.length - 1];
+    if (last && (lastReal === undefined || lastReal.start < last.end)) {
         ends.pop();
-    return ends;
+        return { ends, tail: last.separator };
+    }
+    return { ends };
 }
-/** Turn ascending split-end offsets into contiguous cells tiling `[0, text.length)`. A doc
- *  with no separators is one cell; trailing text after the last separator is its own cell —
- *  but when the last separator already reaches `length` there is no trailing cell to add. */
-function buildCells(splitEnds, length) {
+/** Turn ascending split points into contiguous cells tiling `[0, text.length)`. A doc with no
+ *  split points is one cell; text after the last split point is the final cell, carrying `tail`
+ *  (its own separator, when it is a terminated statement whose trailing trivia was folded in). */
+function buildCells(splitEnds, tail, length) {
     const spans = [];
     let start = 0;
-    for (const end of splitEnds) {
-        spans.push({ start, end });
-        start = end;
+    for (const e of splitEnds) {
+        spans.push({ start, end: e.end, separator: e.separator });
+        start = e.end;
     }
     if (start < length || spans.length === 0)
-        spans.push({ start, end: length });
+        spans.push(tail ? { start, end: length, separator: tail } : { start, end: length });
     return spans;
 }
 /** The tiling invariant: contiguous, starts at 0, ends at `length`, in order. */
@@ -123,14 +161,16 @@ function tiles(spans, length) {
  * Total: never throws. Splits at channel-0 `;` at compound depth 0 (BEGIN/CASE
  * increment, END decrements, floor 0; a T-SQL `BEGIN TRAN`/`TRANSACTION`/
  * `DISTRIBUTED` does not open a depth level) plus, for T-SQL, a `GO` batch
- * separator alone on its line. Returns the whole doc as one cell when
+ * separator alone on its line; text ending inside an open level splits at every
+ * separator from the unclosed opener on (see `findSplitEnds`). Each cell carries
+ * the separator token that ends it. Returns the whole doc as one cell when
  * splitting is unsafe (the tiling invariant fails) or pointless (no separators).
  */
 export function splitStatements(text, dialect) {
     try {
         const tokens = tokenize(text, dialect);
-        const splitEnds = findSplitEnds(text, tokens, dialect);
-        const spans = buildCells(splitEnds, text.length);
+        const { ends, tail } = findSplitEnds(text, tokens, dialect);
+        const spans = buildCells(ends, tail, text.length);
         return tiles(spans, text.length) ? spans : wholeDoc(text);
     }
     catch (e) {

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export { parse, analyze, toAst, toScopes, qualify, lineage, deriveSymbols, TypeInfo, type Nullability, Lineage, originsOfExpr, lineageAt, lineageOf, type LineageHop, type ViaStep, tokenize, SqlDocument, LineIndex, complete, completeAt, type Completion, type CompletionResult, type ReplaceRange, type CompleteOptions, type CandidateIdentity, type CandidateDecoration, type DecorateCandidate, jinjaSlotAt, type JinjaSlot, signatureAt, SIGNATURES, lookupSignature, hasSignature, renderSignature, FN_DOCS, lookupFnDoc, referencesAt, type Occurrence, type Occurrences, frameAt, type Frame, clausesOf, type ClauseInfo, type ClauseKind, setOpArmsOf, type SetOpArm, type SetOpArms, dialectSymbols, type DialectSymbols, dialectVocabulary, type DialectVocabulary, reservedKeywords, type KeywordEntry, type KeywordReservation, CallbackSchema, type SchemaProvider, type TableResolver, DERIVED_DIALECTS, resolveDialect, foldIdentifier, displayName, type IdentKind, type SignatureHelpInfo, type SignatureLabel, type FnSignature, type ParamSig, type RenderSignatureOptions, type FnDoc, type Dialect, type DialectOpts, type ParseResultIR, type Analysis, type Token, type TokenRole, type DocumentAnalysis, type StatementCell, type StatementCellSpan, type DocumentVariant, type UnionCte, } from "./api.js";
+export { parse, analyze, toAst, toScopes, qualify, lineage, deriveSymbols, TypeInfo, type Nullability, Lineage, originsOfExpr, lineageAt, lineageOf, type LineageHop, type ViaStep, tokenize, SqlDocument, LineIndex, complete, completeAt, type Completion, type CompletionResult, type ReplaceRange, type CompleteOptions, type CandidateIdentity, type CandidateDecoration, type DecorateCandidate, jinjaSlotAt, type JinjaSlot, signatureAt, SIGNATURES, lookupSignature, hasSignature, renderSignature, FN_DOCS, lookupFnDoc, referencesAt, type Occurrence, type Occurrences, frameAt, type Frame, clausesOf, type ClauseInfo, type ClauseKind, setOpArmsOf, type SetOpArm, type SetOpArms, dialectSymbols, type DialectSymbols, dialectVocabulary, type DialectVocabulary, reservedKeywords, type KeywordEntry, type KeywordReservation, CallbackSchema, type SchemaProvider, type TableResolver, DERIVED_DIALECTS, resolveDialect, foldIdentifier, displayName, type IdentKind, type SignatureHelpInfo, type SignatureLabel, type FnSignature, type ParamSig, type RenderSignatureOptions, type FnDoc, type Dialect, type DialectOpts, type ParseResultIR, type Analysis, type Token, type TokenRole, type DocumentAnalysis, type StatementCell, type StatementCellSpan, statementSpans, type DocumentVariant, type UnionCte, } from "./api.js";
 export { SqlSession, type SessionOptions } from "./session.js";
 export { parseDatabricks } from "./databricks/parse.js";
 export { parseTSql } from "./tsql/parse.js";

package/dist/index.js

@@ -10,7 +10,7 @@
 //   - Building blocks: the per-dialect parse*/lower and the raw shared passes, for callers who
 //                 want a specific tier or the raw CST escape hatch.
 // --- Uniform entry, lift helpers, typed wrappers, composable passes (src/api.ts) ---
-export { parse, analyze, toAst, toScopes, qualify, lineage, deriveSymbols, TypeInfo, Lineage, originsOfExpr, lineageAt, lineageOf, tokenize, SqlDocument, LineIndex, complete, completeAt, jinjaSlotAt, signatureAt, SIGNATURES, lookupSignature, hasSignature, renderSignature, FN_DOCS, lookupFnDoc, referencesAt, frameAt, clausesOf, setOpArmsOf, dialectSymbols, dialectVocabulary, reservedKeywords, CallbackSchema, DERIVED_DIALECTS, resolveDialect, foldIdentifier, displayName, } from "./api.js";
+export { parse, analyze, toAst, toScopes, qualify, lineage, deriveSymbols, TypeInfo, Lineage, originsOfExpr, lineageAt, lineageOf, tokenize, SqlDocument, LineIndex, complete, completeAt, jinjaSlotAt, signatureAt, SIGNATURES, lookupSignature, hasSignature, renderSignature, FN_DOCS, lookupFnDoc, referencesAt, frameAt, clausesOf, setOpArmsOf, dialectSymbols, dialectVocabulary, reservedKeywords, CallbackSchema, DERIVED_DIALECTS, resolveDialect, foldIdentifier, displayName, statementSpans, } from "./api.js";
 // SqlSession — the verb-shaped facade over one SqlDocument (offset in, answer out; pure delegation).
 export { SqlSession } from "./session.js";
 // --- Per-dialect building blocks: parse* (CST + errors) and lower* (CST → IR), kept as one

package/dist/minijinja/engine.js

@@ -1,4 +1,4 @@
-import { parseTemplated, parseTemplatedCell } from "./parse.js";
+import { parseTemplated, parseTemplatedCell, placeholderOf } from "./parse.js";
 import { templateVariants } from "./variants.js";
 /** The minijinja template engine (the Rust engine dbt Fusion uses — the grammar
  *  oracle for what we accept). The shipped, and only, TemplateEngine. */
@@ -7,6 +7,7 @@ export function minijinja() {
         name: "minijinja",
         parse: (text, dialect, opts) => parseTemplated(text, dialect, opts),
         variants: (text, dialect) => templateVariants(text, dialect),
+        placeholder: (text, _dialect, opts) => placeholderOf(text, opts),
         parseCell: (whole, span, text, dialect, opts) => parseTemplatedCell(whole, span, text, dialect, opts),
     };
 }

package/dist/minijinja/parse.d.ts

@@ -43,3 +43,7 @@ export declare function parseTemplatedCell(whole: TemplatedParseResult, span: {
     start: number;
     end: number;
 }, text: string, dialect: Dialect, opts?: TemplatedParseOptions): TemplatedCellResult;
+/** The placeholder `parseTemplated` would parse for `text` (`TemplateEngine.placeholder`): the
+ *  segmenter alone, no SQL parse. Total: the text itself when segmentation gives up, the same
+ *  floor `parseTemplated`'s degrade path reports as its placeholder. */
+export declare function placeholderOf(text: string, opts?: TemplatedParseOptions): string;

package/dist/minijinja/parse.js

@@ -675,6 +675,18 @@ export function parseTemplatedCell(whole, span, text, dialect, opts) {
         })),
     };
 }
+/** The placeholder `parseTemplated` would parse for `text` (`TemplateEngine.placeholder`): the
+ *  segmenter alone, no SQL parse. Total: the text itself when segmentation gives up, the same
+ *  floor `parseTemplated`'s degrade path reports as its placeholder. */
+export function placeholderOf(text, opts) {
+    try {
+        return segment(text, opts?.provider ?? OPEN_PROVIDER).placeholder;
+    }
+    catch (e) {
+        debugRethrow(e);
+        return text;
+    }
+}
 /** The cell start's 0-based line / column / char offset in `text` (`\n` is the line break, the
  *  convention every span in the pipeline follows; a `\r` is an ordinary column). */
 function cellBaseOf(text, offset) {

package/dist/template/engine.d.ts

@@ -122,6 +122,10 @@ export interface TemplateEngine {
     parse(text: string, dialect: Dialect, opts?: TemplatedParseOptions): TemplatedParseResult;
     /** Optional: coherent per-branch variant enumeration, for engines with control-flow arms. */
     variants?(text: string, dialect: Dialect): TemplateVariant[];
+    /** Optional: the placeholder-filled SQL text `parse` would see for `text` (see
+     *  `TemplatedParseResult.placeholder`), without running the SQL parse. Total: on any internal
+     *  surprise answer `text` itself. Backs `statementSpans`, which only needs the split. */
+    placeholder?(text: string, dialect: Dialect, opts?: TemplatedParseOptions): string;
     /** Optional: the products of ONE statement cell of a templated document, the plain parse of
      *  `whole.placeholder`'s slice `[span.start, span.end)` (cell-relative) with `whole`'s tags
      *  correlated onto it. `whole` is this engine's own `parse` result for the full `text`, `span`

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "sqllens",
-	"version": "1.10.1",
+	"version": "1.11.0",
 	"description": "A TypeScript SQL parser and static analyzer: parse, resolve names, infer types, and trace column lineage across many SQL dialects (Databricks, T-SQL, Snowflake, BigQuery, Redshift, PostgreSQL, DuckDB, Trino, SQLite, MySQL).",
 	"main": "./dist/index.js",
 	"types": "./dist/index.d.ts",