sqllens 1.8.0 → 1.9.0

package/dist/bigquery/parse.d.ts

@@ -9,3 +9,7 @@ export type { ParseResult } from "../parse-result.js";
  * only when SLL fails — same result LL alone would give, just faster on valid input.
  */
 export declare function parseBigQuery(sql: string): ParseResult;
+/** The fragment entries (src/fragment.ts): the grammar's own rules for a statement batch
+ *  (`root`), an expression, a FROM-slot source, a CTE list (no WITH) and a select list;
+ *  the driver anchors each at EOF. For templated macro bodies. */
+export declare const fragmentGrammar: import("../fragment-grammar.js").FragmentGrammar;

package/dist/bigquery/parse.js

@@ -1,6 +1,7 @@
-import { BailErrorStrategy, CharStream, CommonTokenStream, PredictionMode } from "antlr4ng";
+import { BailErrorStrategy, CharStream, CommonTokenStream, PredictionMode, Token as AntlrToken, } from "antlr4ng";
 import { GoogleSQLLexer } from "../generated/bigquery/GoogleSQLLexer.js";
 import { GoogleSQLParser } from "../generated/bigquery/GoogleSQLParser.js";
+import { defineFragmentGrammar, separatedList } from "../fragment-grammar.js";
 import { dotPathTokenSource } from "./dot-path.js";
 import { postParseDiagnostics } from "./post-validate.js";
 import { makeErrorCollector } from "../parse-diagnostics.js";
@@ -67,3 +68,29 @@ export function parseBigQuery(sql) {
         return withTokens({ tree, errors: diagnostics.length, diagnostics, sllFallback: true });
     }
 }
+/** The fragment entries (src/fragment.ts): the grammar's own rules for a statement batch
+ *  (`root`), an expression, a FROM-slot source, a CTE list (no WITH) and a select list;
+ *  the driver anchors each at EOF. For templated macro bodies. */
+export const fragmentGrammar = defineFragmentGrammar({
+    // The same token pipeline as parseBigQuery: the DOT_IDENTIFIER path rewrite and the
+    // literal-escape validation ride the lex; the post-parse tree checks ride each parse.
+    lex: (text) => {
+        const lexer = new GoogleSQLLexer(CharStream.fromString(text));
+        lexer.removeErrorListeners();
+        const { source, escapeDiagnostics } = dotPathTokenSource(text, lexer);
+        const stream = new CommonTokenStream(source);
+        stream.fill();
+        return { tokens: stream.getTokens().filter((t) => t.type !== AntlrToken.EOF), diagnostics: escapeDiagnostics };
+    },
+    newLexer: (input) => new GoogleSQLLexer(input),
+    newParser: (tokens) => new GoogleSQLParser(tokens),
+    postParse: postParseDiagnostics,
+    separator: GoogleSQLLexer.COMMA_SYMBOL,
+    entries: {
+        statement: (p) => p.root(),
+        expression: (p) => p.expression(),
+        tableSource: (p) => p.from_clause_contents(),
+        cteList: separatedList((p) => p.with_clause_entry(), GoogleSQLLexer.COMMA_SYMBOL),
+        selectList: (p) => p.select_list(),
+    },
+});

package/dist/databricks/parse.d.ts

@@ -9,3 +9,7 @@ export type { ParseResult } from "../parse-result.js";
  * result LL alone would produce, so correctness is unchanged — just faster.
  */
 export declare function parseDatabricks(sql: string): ParseResult;
+/** The fragment entries (src/fragment.ts): the grammar's own rules for a statement batch
+ *  (`multiStatement`), an expression, a FROM-slot source, a CTE list (no WITH) and a select list;
+ *  the driver anchors each at EOF. For templated macro bodies. */
+export declare const fragmentGrammar: import("../fragment-grammar.js").FragmentGrammar;

package/dist/databricks/parse.js

@@ -1,6 +1,7 @@
 import { BailErrorStrategy, CharStream, CommonTokenStream, PredictionMode, } from "antlr4ng";
 import { DatabricksLexer } from "../generated/databricks/DatabricksLexer.js";
 import { DatabricksParser } from "../generated/databricks/DatabricksParser.js";
+import { defineFragmentGrammar, separatedList } from "../fragment-grammar.js";
 import { makeErrorCollector } from "../parse-diagnostics.js";
 import { CONSUMED_AS_RULES, deriveConsumedAs } from "../token/consumed-as.js";
 import { mapTokens } from "../token/map.js";
@@ -72,3 +73,18 @@ function attachErrorCounter(lexer, parser, listener) {
     parser.removeErrorListeners();
     parser.addErrorListener(listener);
 }
+/** The fragment entries (src/fragment.ts): the grammar's own rules for a statement batch
+ *  (`multiStatement`), an expression, a FROM-slot source, a CTE list (no WITH) and a select list;
+ *  the driver anchors each at EOF. For templated macro bodies. */
+export const fragmentGrammar = defineFragmentGrammar({
+    newLexer: (input) => new DatabricksLexer(input),
+    newParser: (tokens) => new DatabricksParser(tokens),
+    separator: DatabricksLexer.COMMA,
+    entries: {
+        statement: (p) => p.multiStatement(),
+        expression: (p) => p.expression(),
+        tableSource: (p) => p.relation(),
+        cteList: separatedList((p) => p.namedQuery(), DatabricksLexer.COMMA),
+        selectList: (p) => p.namedExpressionSeq(),
+    },
+});

package/dist/duckdb/parse.d.ts

@@ -8,3 +8,7 @@ export type { ParseResult } from "../parse-result.js";
  * only when SLL fails — same result LL alone would give, just faster on valid input.
  */
 export declare function parseDuckdb(sql: string): ParseResult;
+/** The fragment entries (src/fragment.ts): the grammar's own rules for a statement batch
+ *  (`root`), an expression, a FROM-slot source, a CTE list (no WITH) and a select list;
+ *  the driver anchors each at EOF. For templated macro bodies. */
+export declare const fragmentGrammar: import("../fragment-grammar.js").FragmentGrammar;

package/dist/duckdb/parse.js

@@ -1,6 +1,7 @@
 import { BailErrorStrategy, CharStream, CommonTokenStream, PredictionMode, } from "antlr4ng";
 import { DuckdbLexer } from "../generated/duckdb/DuckdbLexer.js";
 import { DuckdbParser } from "../generated/duckdb/DuckdbParser.js";
+import { defineFragmentGrammar } from "../fragment-grammar.js";
 import { makeErrorCollector } from "../parse-diagnostics.js";
 import { CONSUMED_AS_RULES, deriveConsumedAs } from "../token/consumed-as.js";
 import { mapTokens } from "../token/map.js";
@@ -63,3 +64,18 @@ function attachErrorCounter(lexer, parser, listener) {
     parser.removeErrorListeners();
     parser.addErrorListener(listener);
 }
+/** The fragment entries (src/fragment.ts): the grammar's own rules for a statement batch
+ *  (`root`), an expression, a FROM-slot source, a CTE list (no WITH) and a select list;
+ *  the driver anchors each at EOF. For templated macro bodies. */
+export const fragmentGrammar = defineFragmentGrammar({
+    newLexer: (input) => new DuckdbLexer(input),
+    newParser: (tokens) => new DuckdbParser(tokens),
+    separator: DuckdbLexer.COMMA,
+    entries: {
+        statement: (p) => p.root(),
+        expression: (p) => p.a_expr(),
+        tableSource: (p) => p.table_ref(),
+        cteList: (p) => p.cte_list(),
+        selectList: (p) => p.target_list(),
+    },
+});

package/dist/fragment-grammar.d.ts

@@ -0,0 +1,51 @@
+import { CharStream, CommonTokenStream, type Lexer, type Parser, type ParserRuleContext, Token as AntlrToken } from "antlr4ng";
+import { type SyntaxDiagnostic } from "./parse-diagnostics.js";
+/** What a fragment parses as. `statement` is the dialect's ordinary full-file entry. */
+export type FragmentKind = "statement" | "expression" | "tableSource" | "cteList" | "selectList";
+/** Every kind, in the order a macro body is tried: the most common reading first. */
+export declare const FRAGMENT_KINDS: readonly FragmentKind[];
+export interface FragmentLex {
+    /** Text-native tokens, trivia included, EOF excluded. */
+    tokens: AntlrToken[];
+    /** Token-derived diagnostics the dialect's statement entry would also report (bigquery's
+     *  literal-escape validation); positioned in the text's coordinates. */
+    diagnostics: SyntaxDiagnostic[];
+}
+export interface FragmentParse {
+    tree: ParserRuleContext;
+    /** Empty when the slice parsed clean as this kind (the rule is EOF-anchored). */
+    diagnostics: SyntaxDiagnostic[];
+}
+export interface FragmentGrammar {
+    /** Lex a whole text exactly as the dialect's statement entry does (same token rewrites). */
+    lex(text: string): FragmentLex;
+    /**
+     * Parse an already-lexed slice as `kind`. `bail` = SLL + bail strategy: returns undefined on
+     * the first syntax error (fast clean/not-clean probe); otherwise full LL with recovery and
+     * positioned diagnostics.
+     */
+    parse(slice: readonly AntlrToken[], kind: FragmentKind, bail: boolean): FragmentParse | undefined;
+}
+/** A fragment entry: drives one or more of the grammar's OWN rules over the parser. The grammar
+ *  is untouched (no `x EOF` wrapper rules: an extra caller widens a rule's SLL follow set and
+ *  flips prediction on unrelated input); the EOF anchor is checked by the driver instead. */
+export type FragmentEntry<P extends Parser> = (parser: P) => ParserRuleContext;
+export interface FragmentGrammarSpec<P extends Parser> {
+    /** Lex a whole text; defaults to the plain lexer with no diagnostics. */
+    lex?: (text: string) => FragmentLex;
+    newLexer: (input: CharStream) => Lexer;
+    newParser: (tokens: CommonTokenStream) => P;
+    entries: {
+        readonly [K in FragmentKind]: FragmentEntry<P>;
+    };
+    /** The list separator token type (COMMA). A list body (`cteList`/`selectList`) may end with
+     *  one: a macro's CTE list often ends `),` because the caller appends more CTEs. */
+    separator: number;
+    /** Tree-walking checks the dialect's statement entry runs after the parse (bigquery). */
+    postParse?: (tree: ParserRuleContext) => SyntaxDiagnostic[];
+}
+/** `item (sep item)*` driven from outside the grammar, for dialects without a list rule
+ *  (a CTE list without its WITH, a select list). The tree is the FIRST item's. */
+export declare function separatedList<P extends Parser>(item: FragmentEntry<P>, separator: number): FragmentEntry<P>;
+/** Bind a dialect's lexer, parser and fragment entry rules into a `FragmentGrammar`. */
+export declare function defineFragmentGrammar<P extends Parser>(spec: FragmentGrammarSpec<P>): FragmentGrammar;

package/dist/fragment-grammar.js

@@ -0,0 +1,103 @@
+// ---------------------------------------------------------------------------
+// Fragment grammar: the per-dialect entry set for parsing an SQL FRAGMENT — a
+// templated macro body that is not a statement (an expression, a FROM-slot
+// source, a CTE list pasted after WITH, a select list). Each dialect's parse.ts
+// binds its lexer, parser and the five EOF-anchored `*_fragment` rules through
+// `defineFragmentGrammar`; src/fragment.ts holds the registry and the driver.
+//
+// The parse runs over an already-lexed token slice (antlr4ng's ListTokenSource),
+// never over a re-lexed substring, so every token, span and diagnostic stays in
+// the coordinates of the text the tokens were lexed from.
+// ---------------------------------------------------------------------------
+import { BailErrorStrategy, CharStream, CommonTokenStream, ListTokenSource, PredictionMode, Token as AntlrToken, } from "antlr4ng";
+import { makeErrorCollector } from "./parse-diagnostics.js";
+/** Every kind, in the order a macro body is tried: the most common reading first. */
+export const FRAGMENT_KINDS = [
+    "statement",
+    "expression",
+    "tableSource",
+    "cteList",
+    "selectList",
+];
+const LIST_KINDS = new Set(["cteList", "selectList"]);
+/** `item (sep item)*` driven from outside the grammar, for dialects without a list rule
+ *  (a CTE list without its WITH, a select list). The tree is the FIRST item's. */
+export function separatedList(item, separator) {
+    return (parser) => {
+        const first = item(parser);
+        // A separator followed by EOF is a trailing one (`run` consumes it), not another item.
+        while (parser.inputStream.LA(1) === separator && parser.inputStream.LA(2) !== AntlrToken.EOF) {
+            parser.inputStream.consume(); // between rules there is no context to attach the separator to
+            item(parser);
+        }
+        return first;
+    };
+}
+/** The EOF anchor: the entry returned but input remains — the same diagnostic shape ANTLR
+ *  emits for an EOF-anchored rule, positioned at the leftover token. */
+function trailingInput(parser) {
+    const tok = parser.inputStream.LT(1);
+    if (!tok || tok.type === AntlrToken.EOF)
+        return undefined;
+    const text = tok.text ?? "";
+    return {
+        message: `extraneous input '${text}' expecting <EOF>`,
+        line: tok.line,
+        column: tok.column,
+        offset: tok.start,
+        length: text.length || 1,
+    };
+}
+/** Bind a dialect's lexer, parser and fragment entry rules into a `FragmentGrammar`. */
+export function defineFragmentGrammar(spec) {
+    const { newLexer, newParser, entries, separator, postParse } = spec;
+    /** The entry, then a trailing separator on a list body is consumed rather than left over. */
+    const run = (parser, kind) => {
+        const tree = entries[kind](parser);
+        const input = parser.inputStream;
+        if (LIST_KINDS.has(kind) && input.LA(1) === separator && input.LA(2) === AntlrToken.EOF)
+            input.consume();
+        return tree;
+    };
+    const lex = spec.lex ??
+        ((text) => {
+            const lexer = newLexer(CharStream.fromString(text));
+            lexer.removeErrorListeners();
+            return { tokens: lexer.getAllTokens(), diagnostics: [] };
+        });
+    return {
+        lex,
+        parse(slice, kind, bail) {
+            const tokens = new CommonTokenStream(new ListTokenSource([...slice]));
+            const parser = newParser(tokens);
+            const collector = makeErrorCollector();
+            parser.removeErrorListeners();
+            parser.addErrorListener(collector.listener);
+            const sim = parser.interpreter;
+            if (bail) {
+                parser.errorHandler = new BailErrorStrategy();
+                sim.predictionMode = PredictionMode.SLL;
+                let tree;
+                try {
+                    tree = run(parser, kind);
+                }
+                catch {
+                    return undefined;
+                }
+                // A grammar action can report through the listener without throwing (bigquery's
+                // join-balance check); that is not a clean parse either. Nor is leftover input.
+                if (collector.diagnostics.length > 0 || trailingInput(parser))
+                    return undefined;
+                const post = postParse?.(tree) ?? [];
+                return post.length === 0 ? { tree, diagnostics: [] } : undefined;
+            }
+            sim.predictionMode = PredictionMode.LL;
+            const tree = run(parser, kind);
+            const trailing = trailingInput(parser);
+            return {
+                tree,
+                diagnostics: [...collector.diagnostics, ...(trailing ? [trailing] : []), ...(postParse?.(tree) ?? [])],
+            };
+        },
+    };
+}

package/dist/fragment.d.ts

@@ -0,0 +1,32 @@
+import type { ParserRuleContext } from "antlr4ng";
+import type { Dialect } from "./dialect.js";
+import { type FragmentKind } from "./fragment-grammar.js";
+import type { SyntaxDiagnostic } from "./parse-diagnostics.js";
+export type { FragmentKind } from "./fragment-grammar.js";
+/** A half-open [start, end) offset range of the lexed text (a `PartSpan` fits). */
+export interface FragmentRange {
+    start: number;
+    end: number;
+}
+export interface FragmentResult {
+    /** The kind the ranges parsed as; when `clean` is false, the kind that got furthest. */
+    kind: FragmentKind;
+    /** True when the ranges parsed with zero syntax errors as `kind`. */
+    clean: boolean;
+    tree: ParserRuleContext;
+    /** Positioned in the text's coordinates. Empty when clean. */
+    diagnostics: SyntaxDiagnostic[];
+}
+export interface FragmentSession {
+    /**
+     * Parse the tokens inside `ranges` (a token counts when it starts inside one) as a fragment.
+     * Returns undefined when the ranges hold no default-channel token (nothing to parse; no
+     * verdict). Never throws: every attempt runs a recovering parser over a fixed kind list.
+     */
+    parse(ranges: readonly FragmentRange[], kinds?: readonly FragmentKind[]): FragmentResult | undefined;
+    /** The first kind the ranges parse clean as, or undefined (no token, or no clean reading).
+     *  Probes only (SLL + bail); never a diagnostic. */
+    verdict(ranges: readonly FragmentRange[], kinds?: readonly FragmentKind[]): FragmentKind | undefined;
+}
+/** Lex `text` once with `dialect`'s statement-entry token pipeline; parse ranges of it after. */
+export declare function openFragments(text: string, dialect: Dialect): FragmentSession;

package/dist/fragment.js

@@ -0,0 +1,93 @@
+// ---------------------------------------------------------------------------
+// Fragment parsing: parse RANGES of a text as an SQL fragment, trying each
+// FragmentKind in order until one parses clean. The consumer is the minijinja
+// front end (a `{% macro %}` body is whatever gets pasted at the call site: a
+// statement, an expression, a FROM-slot source, a CTE list, a select list); the
+// statement entry is right for a model file and wrong for most macro bodies.
+//
+// The text is lexed ONCE (`openFragments`) and every parse cuts its token slice
+// from that lex, so the tokens (and every diagnostic they position) stay in the
+// text's own coordinates: no remap. A fast SLL+bail probe per kind finds the
+// clean reading; when none is clean the kinds are re-run with recovery and the
+// reading that got furthest before its first error supplies the diagnostics —
+// a heuristic for the broken-input case only (the reading closest to what was
+// meant), never a claim: `clean` is false and the region carries no verdict.
+// ---------------------------------------------------------------------------
+import { FRAGMENT_KINDS } from "./fragment-grammar.js";
+import { fragmentGrammar as bigquery } from "./bigquery/parse.js";
+import { fragmentGrammar as databricks } from "./databricks/parse.js";
+import { fragmentGrammar as duckdb } from "./duckdb/parse.js";
+import { fragmentGrammar as mysql } from "./mysql/parse.js";
+import { fragmentGrammar as postgres } from "./postgres/parse.js";
+import { fragmentGrammar as redshift } from "./redshift/parse.js";
+import { fragmentGrammar as snowflake } from "./snowflake/parse.js";
+import { fragmentGrammar as sqlite } from "./sqlite/parse.js";
+import { fragmentGrammar as trino } from "./trino/parse.js";
+import { fragmentGrammar as tsql } from "./tsql/parse.js";
+const FRAGMENT_GRAMMARS = {
+    databricks,
+    tsql,
+    snowflake,
+    bigquery,
+    redshift,
+    postgres,
+    duckdb,
+    trino,
+    sqlite,
+    mysql,
+};
+/** Lex `text` once with `dialect`'s statement-entry token pipeline; parse ranges of it after. */
+export function openFragments(text, dialect) {
+    const grammar = FRAGMENT_GRAMMARS[dialect];
+    const lexed = grammar.lex(text);
+    const inRanges = (offset, ranges) => ranges.some((r) => offset >= r.start && offset < r.end);
+    const sliceOf = (ranges) => {
+        const slice = lexed.tokens.filter((t) => inRanges(t.start, ranges));
+        return slice.some((t) => t.channel === 0) ? slice : undefined;
+    };
+    // Token-derived diagnostics of a slice (bigquery literal escapes) hold under every reading.
+    const lexDiagsOf = (ranges) => lexed.diagnostics.filter((d) => d.offset !== undefined && inRanges(d.offset, ranges));
+    const probe = (slice, kinds) => {
+        for (const kind of kinds) {
+            const clean = grammar.parse(slice, kind, true);
+            if (clean)
+                return { kind, clean: true, tree: clean.tree, diagnostics: [] };
+        }
+        return undefined;
+    };
+    return {
+        verdict(ranges, kinds = FRAGMENT_KINDS) {
+            const slice = sliceOf(ranges);
+            if (!slice || lexDiagsOf(ranges).length > 0)
+                return undefined;
+            return probe(slice, kinds)?.kind;
+        },
+        parse(ranges, kinds = FRAGMENT_KINDS) {
+            const slice = sliceOf(ranges);
+            if (!slice)
+                return undefined;
+            const lexDiags = lexDiagsOf(ranges);
+            if (lexDiags.length === 0) {
+                const clean = probe(slice, kinds);
+                if (clean)
+                    return clean;
+            }
+            let best;
+            let bestAt = -1;
+            for (const kind of kinds) {
+                const attempt = grammar.parse(slice, kind, false);
+                if (!attempt)
+                    continue;
+                const diagnostics = [...lexDiags, ...attempt.diagnostics];
+                if (diagnostics.length === 0)
+                    return { kind, clean: true, tree: attempt.tree, diagnostics };
+                const at = attempt.diagnostics[0]?.offset ?? Number.MAX_SAFE_INTEGER;
+                if (at > bestAt) {
+                    bestAt = at;
+                    best = { kind, clean: false, tree: attempt.tree, diagnostics };
+                }
+            }
+            return best;
+        },
+    };
+}

package/dist/minijinja/index.d.ts

@@ -1,5 +1,6 @@
 export { minijinja } from "./engine.js";
-export { parseTemplated, tokenizeTemplated } from "./parse.js";
+export { parseTemplated, tokenizeTemplated, shapesForCall } from "./parse.js";
+export type { MacroShape } from "../template/engine.js";
 export type { TemplatedParseResult, TemplatedParseOptions } from "../template/engine.js";
 export type { TagNode, MacroCall } from "./parse.js";
 export { templateRegions, templateSymbols } from "./regions.js";

package/dist/minijinja/index.js

@@ -3,6 +3,6 @@
 // canonically declared in src/template/engine.ts and re-exported both there
 // and here.
 export { minijinja } from "./engine.js";
-export { parseTemplated, tokenizeTemplated } from "./parse.js";
+export { parseTemplated, tokenizeTemplated, shapesForCall } from "./parse.js";
 export { templateRegions, templateSymbols } from "./regions.js";
 export { templateVariants } from "./variants.js";

package/dist/minijinja/parse.d.ts

@@ -1,10 +1,20 @@
 import type { Dialect } from "../dialect.js";
 import type { Token } from "../token/token.js";
-import type { TemplatedParseOptions, TemplatedParseResult } from "../template/engine.js";
+import type { MacroShape, TemplatedParseOptions, TemplatedParseResult } from "../template/engine.js";
+import type { ExpansionShape } from "../qualify/template-provider.js";
+import type { TemplateCall } from "../ir/ir.js";
 export type { TagNode, MacroCall } from "./tag-ast.js";
 export type { TemplateRegion, TemplateArm, TemplateSymbol } from "./regions.js";
 export { templateRegions, templateSymbols } from "./regions.js";
-export type { TemplatedParseOptions, TemplatedParseResult } from "../template/engine.js";
+export type { MacroShape, TemplatedParseOptions, TemplatedParseResult } from "../template/engine.js";
+/**
+ * The shapes a specific CALL of a macro takes: `macro.shapes`, with the keyword-parameter hole
+ * (if any) resolved from the call's own literal argument (positional or keyword) or the
+ * parameter's default. A call whose keyword is not a literal, or names a word that opens no
+ * known clause, resolves to nothing for that hole (never-wrong). Pure: definition text + call
+ * text, no project knowledge; a host's `shapeOf(call)` is `shapesForCall(index.get(call.name), call)`.
+ */
+export declare function shapesForCall(macro: MacroShape, call: TemplateCall): ExpansionShape[];
 /**
  * Parse raw jinja-SQL: one whole-document jinja lex (segment()), the untouched
  * per-dialect SQL parse over the resulting placeholder, a per-tag jinja parse over