sqllens 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/package.json +91 -91
package/README.md
CHANGED
|
@@ -1,255 +1,255 @@
|
|
|
1
|
-
# sqllens
|
|
2
|
-
|
|
3
|
-
A TypeScript SQL parser and static analyzer. It parses SQL into a tree, lowers it
|
|
4
|
-
to a dialect-neutral IR, and runs a semantic layer over that IR: name resolution
|
|
5
|
-
(scope), schema-fed qualification, type inference, and column lineage. Give it a
|
|
6
|
-
query and it tells you the query's sources, its output columns, their types, and
|
|
7
|
-
where each column comes from. The parsers are generated TypeScript on the
|
|
8
|
-
[antlr4ng](https://github.com/mike-lischke/antlr4ng) runtime.
|
|
9
|
-
|
|
10
|
-
The front end is error-tolerant and token-first, so the same library powers
|
|
11
|
-
editor tooling — an LSP and a SQL debugger — over incomplete, mid-edit text. See
|
|
12
|
-
[Editor / language tooling](#editor--language-tooling).
|
|
13
|
-
|
|
14
|
-
## Dialects
|
|
15
|
-
|
|
16
|
-
| Dialect | Parse + lower | Semantic layer | Notes |
|
|
17
|
-
|---|---|---|---|
|
|
18
|
-
| Databricks (Spark SQL) | yes | yes | grammar forked from apache/spark |
|
|
19
|
-
| T-SQL | yes | yes | grammar forked from grammars-v4 `sql/tsql` |
|
|
20
|
-
| Snowflake | yes | yes | grammar forked from grammars-v4 `sql/snowflake` |
|
|
21
|
-
| BigQuery (GoogleSQL) | yes | yes | grammar forked from `bytebase/parser` `googlesql/`; gated against ZetaSQL's `.test` corpus |
|
|
22
|
-
| Redshift | yes | yes | grammar forked from Bytebase's Postgres-derived Redshift grammar (BSD-3) |
|
|
23
|
-
| PostgreSQL | yes | yes | grammar forked from `bytebase/parser` `postgresql/` (BSD-3, PG18 keywords) |
|
|
24
|
-
| DuckDB | yes | yes | grammar forked from this repo's own postgres pair (no open ANTLR grammar exists) |
|
|
25
|
-
| Trino | yes | yes | grammar is the first-party trinodb `SqlBase.g4` (release 482), mechanically split; covers dbt-trino + dbt-athena |
|
|
26
|
-
|
|
27
|
-
The semantic layer is dialect-agnostic: it operates on the shared IR and runs
|
|
28
|
-
unchanged on every dialect. Only the parse and lower stages are dialect-specific.
|
|
29
|
-
|
|
30
|
-
## The pipeline
|
|
31
|
-
|
|
32
|
-
```
|
|
33
|
-
parse → lower → resolveScopes → qualify → infer / lineage / symbols
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
- **parse** — text → concrete syntax tree (CST), with a syntax-error count.
|
|
37
|
-
- **lower** — CST → a dialect-neutral IR (`QueryExpr` / `SelectExpr` / `Expr` …);
|
|
38
|
-
also reports the statement kind (query / dml / ddl / …).
|
|
39
|
-
- **resolveScopes** — a schema-free symbol table: visible sources, CTE
|
|
40
|
-
resolution, output columns.
|
|
41
|
-
- **qualify** — with a schema: `*` expansion, unknown-table/column diagnostics,
|
|
42
|
-
column types.
|
|
43
|
-
- **infer / lineage / symbols** — type inference, base-table lineage per output
|
|
44
|
-
column, and a kind×modifier symbol model.
|
|
45
|
-
|
|
46
|
-
## Status
|
|
47
|
-
|
|
48
|
-
Pre-release, and not yet published to npm. The library is consumed as TypeScript
|
|
49
|
-
(no build emit yet — packaging is a later step). The public API (`src/index.ts`)
|
|
50
|
-
is uniform across all eight dialects: `parse` and `analyze` take the dialect as a
|
|
51
|
-
parameter, and every per-dialect `parse*` / `lower` plus the shared passes stay
|
|
52
|
-
exported as lower-level building blocks. The editor-facing surface — `tokenize`,
|
|
53
|
-
`SqlDocument`, `complete`, `signatureAt` — lives on the same barrel.
|
|
54
|
-
|
|
55
|
-
## Usage
|
|
56
|
-
|
|
57
|
-
`dialect` is `"databricks" | "tsql" | "snowflake" | "bigquery" | "redshift" | "postgres" | "duckdb" | "trino"`.
|
|
58
|
-
|
|
59
|
-
Those eight grammars serve more dbt adapters than that, because several adapters
|
|
60
|
-
are SQL front ends over an engine already covered. `adapterDialect` resolves a
|
|
61
|
-
profiles.yml `type:` value (or a dialect name) to the dialect that parses its
|
|
62
|
-
SQL — so consumers don't re-derive the family knowledge:
|
|
63
|
-
|
|
64
|
-
```ts
|
|
65
|
-
import { adapterDialect, ADAPTER_DIALECTS } from "sqllens";
|
|
66
|
-
|
|
67
|
-
adapterDialect("athena"); // "trino" — Athena engine v3 executes on Trino
|
|
68
|
-
adapterDialect("glue"); // "databricks" — AWS Glue runs Spark; Databricks SQL = Spark SQL
|
|
69
|
-
adapterDialect("fabric"); // "tsql" — same for "synapse" and "sqlserver"
|
|
70
|
-
adapterDialect("presto"); // "trino" — the pre-rename Trino adapter
|
|
71
|
-
adapterDialect("oracle"); // undefined — not served; never a guess
|
|
72
|
-
```
|
|
73
|
-
|
|
74
|
-
The map is exact by contract: only adapters whose SQL surface the corpus gates
|
|
75
|
-
genuinely represent are listed. The LSP's `.sqllens.json` accepts adapter types
|
|
76
|
-
through the same map, so `{ "dialect": "athena" }` works in rules and `default`.
|
|
77
|
-
|
|
78
|
-
The surface is
|
|
79
|
-
**layered** — each tier is a terminal value you can stop at — and **composable**:
|
|
80
|
-
every semantic method takes the closest upstream result (so passing it does no
|
|
81
|
-
rework) or a raw string / IR via an idempotent lift helper.
|
|
82
|
-
|
|
83
|
-
```ts
|
|
84
|
-
import { parse, analyze, Schema } from "sqllens";
|
|
85
|
-
|
|
86
|
-
// Tier 1 — just the IR. No semantic layer pulled in.
|
|
87
|
-
const { ast, errors, cst } = parse("SELECT a, b FROM t WHERE a > 1", "tsql");
|
|
88
|
-
// ast = dialect-neutral IR (frozen — no pass mutates it); cst = raw antlr tree (escape hatch)
|
|
89
|
-
// ast.statement -> "query" | "dml" | "ddl" | …
|
|
90
|
-
|
|
91
|
-
// Whole pipeline in one call.
|
|
92
|
-
const schema = new Schema({ t: { a: "int", b: "string" } });
|
|
93
|
-
const a = analyze("SELECT a, b FROM t", "tsql", { schema });
|
|
94
|
-
a.scopes; // name resolution (ScopeTree)
|
|
95
|
-
a.diagnostics; // unknown-table/column diagnostics
|
|
96
|
-
a.qualification.columnsOf(a.scopes.root); // * expansion
|
|
97
|
-
a.types.typeOf(expr, scope); // per-expression types
|
|
98
|
-
a.lineage.originsOf("a"); // base-table origins of an output column
|
|
99
|
-
a.symbols; // kind × modifier symbol model
|
|
100
|
-
```
|
|
101
|
-
|
|
102
|
-
Compose tier by tier — pass any upstream result (or a string) to any later pass,
|
|
103
|
-
and only the missing steps run. No exported signature takes or returns a raw
|
|
104
|
-
`Map`/`Set`/`Record`:
|
|
105
|
-
|
|
106
|
-
```ts
|
|
107
|
-
import { parse, qualify, lineage, deriveSymbols, toScopes, Schema } from "sqllens";
|
|
108
|
-
|
|
109
|
-
const { ast } = parse(sql, "snowflake");
|
|
110
|
-
const scopes = toScopes(ast, { dialect: "snowflake" }); // idempotent lift; identity if already a ScopeTree
|
|
111
|
-
qualify(scopes, schema); // reuses scopes — never re-parses or re-resolves
|
|
112
|
-
lineage(scopes, schema); // safe to call on the same scopes, in any order
|
|
113
|
-
deriveSymbols(scopes); // independent results, no cross-contamination
|
|
114
|
-
```
|
|
115
|
-
|
|
116
|
-
The per-dialect entries (`parseDatabricks` / `parseTSql` / `parseSnowflake` /
|
|
117
|
-
`parseBigQuery` / `parseRedshift` / `parsePostgres` / `parseDuckdb` / `parseTrino`,
|
|
118
|
-
each `lower`, and the raw `resolveScopes` / `inferType`) remain exported for
|
|
119
|
-
callers that want a single stage.
|
|
120
|
-
|
|
121
|
-
## Editor / language tooling
|
|
122
|
-
|
|
123
|
-
The front end is error-tolerant and token-first, so it serves editor features
|
|
124
|
-
that run on incomplete, mid-edit text — they never need a clean parse:
|
|
125
|
-
|
|
126
|
-
- **`tokenize(sql, dialect)`** and **`parse(...).tokens`** give a first-class token
|
|
127
|
-
stream: every token with its exact span, role, and channel. Always available,
|
|
128
|
-
even when the parse has errors.
|
|
129
|
-
- **`lower()` never throws** on broken or partial input — you get a flagged
|
|
130
|
-
`query` IR back, so every downstream pass stays total.
|
|
131
|
-
- **`SqlDocument`** is a persistent, immutable, position-addressable per-file
|
|
132
|
-
model. It runs `parse → resolveScopes` once (plus lazy `analyze(schema)`),
|
|
133
|
-
caches the result, and answers `tokenAt` / `nodeAt`. An edit yields a new
|
|
134
|
-
document; an O(log n) `LineIndex` maps positions ↔ offsets.
|
|
135
|
-
- **`complete(doc, offset, schema?)`** — scope-aware completion (keywords,
|
|
136
|
-
columns, tables, functions) from an ATN candidate walk over the grammar (our
|
|
137
|
-
own, no third-party dependency).
|
|
138
|
-
- **`signatureAt(doc, offset)`** — parameter hints from a curated per-dialect
|
|
139
|
-
function-signature table; the long tail degrades to name + active-argument.
|
|
140
|
-
- **`referencesAt(scopes, offset, schema?)`** — every occurrence (plus the
|
|
141
|
-
declaration) of the symbol under the cursor; backs find-references, document
|
|
142
|
-
highlight, and code-lens reference counts.
|
|
143
|
-
|
|
144
|
-
```ts
|
|
145
|
-
import { SqlDocument, Schema } from "sqllens";
|
|
146
|
-
|
|
147
|
-
const doc = SqlDocument.create("SELECT amount FROM sales", "databricks");
|
|
148
|
-
doc.tokens; // first-class token stream (spans + roles)
|
|
149
|
-
doc.tokenAt(7); // token under an offset
|
|
150
|
-
const next = doc.withText("SELECT amount, id FROM sales", 2); // immutable edit → new doc
|
|
151
|
-
```
|
|
152
|
-
|
|
153
|
-
## Language server
|
|
154
|
-
|
|
155
|
-
An LSP (Language Server Protocol) server built on the library, in `src/lsp/`. It
|
|
156
|
-
holds one `SqlDocument` per open file (rebuilt on edit) and reaches the library
|
|
157
|
-
only through the public API surface above — it adds no analysis of its own, only
|
|
158
|
-
protocol translation.
|
|
159
|
-
|
|
160
|
-
LSP is a large protocol — roughly thirty request types across document-sync,
|
|
161
|
-
language, and workspace features — so "supports LSP" is not one bit but a long
|
|
162
|
-
checklist. A SQL server needs a subset, but more of it maps to SQL than it first
|
|
163
|
-
looks — a CTE / view / model is the SQL analog of a definition, and the
|
|
164
|
-
dependency graph between them is a call hierarchy. A few features genuinely don't
|
|
165
|
-
apply (type hierarchy, document color, monikers); a few are deliberately deferred
|
|
166
|
-
(formatting, project-wide navigation). The coverage, feature by feature:
|
|
167
|
-
|
|
168
|
-
**Language features**
|
|
169
|
-
|
|
170
|
-
| Feature | Status |
|
|
171
|
-
| --- | --- |
|
|
172
|
-
| Completion (+ resolve) | ✅ |
|
|
173
|
-
| Hover | ✅ |
|
|
174
|
-
| Hover — nullability | ✅ (` — not null` / ` — nullable` suffix when provable) |
|
|
175
|
-
| Signature help | ✅ |
|
|
176
|
-
| Go to definition | ✅ |
|
|
177
|
-
| Find references | ✅ |
|
|
178
|
-
| Document highlight | ✅ |
|
|
179
|
-
| Document symbols | ✅ |
|
|
180
|
-
| Folding range | ✅ |
|
|
181
|
-
| Selection range | ✅ |
|
|
182
|
-
| Semantic tokens (full / range / delta) | ✅ all three |
|
|
183
|
-
| Inlay hints | ✅ (no resolve) |
|
|
184
|
-
| Code lens | ✅ (no resolve) |
|
|
185
|
-
| Go to declaration | ◻️ not yet |
|
|
186
|
-
| Go to type definition | ◻️ not yet |
|
|
187
|
-
| Go to implementation | ◻️ not yet — name → its defining query (view / model); needs the project model |
|
|
188
|
-
| Call hierarchy | ◻️ not yet — the CTE / dbt-model dependency graph |
|
|
189
|
-
| Document link | ◻️ not yet |
|
|
190
|
-
| Linked editing range | ◻️ not yet — live alias / name sync-edit |
|
|
191
|
-
| Code action (quick fixes) | ◻️ next phase |
|
|
192
|
-
| Rename (+ prepare) | ◻️ next phase |
|
|
193
|
-
| Formatting / range / on-type | ◻️ deferred (external formatter) |
|
|
194
|
-
| Inline values | ◻️ debugger surface |
|
|
195
|
-
| Type hierarchy | — n/a — SQL has no type-inheritance relation |
|
|
196
|
-
| Document color | — n/a — no color literals |
|
|
197
|
-
| Moniker | — n/a — LSIF / cross-repo indexing concern |
|
|
198
|
-
|
|
199
|
-
**Diagnostics & document sync**
|
|
200
|
-
|
|
201
|
-
| Feature | Status |
|
|
202
|
-
| --- | --- |
|
|
203
|
-
| Diagnostics — push (`publishDiagnostics`) | ✅ |
|
|
204
|
-
| Diagnostics — call signature (arity / argument type) | ✅ (curated tables; never-wrong, per-dialect coercion) |
|
|
205
|
-
| Diagnostics — pull (document) | ✅ |
|
|
206
|
-
| Diagnostics — pull (workspace) | ◻️ not yet |
|
|
207
|
-
| Text sync — open / change / close | ✅ (full-document) |
|
|
208
|
-
| Incremental sync | ◻️ full-document only (fine at SQL file sizes) |
|
|
209
|
-
| Save notifications (`didSave` / `willSave`) | ◻️ not yet |
|
|
210
|
-
| Notebook document sync | ◻️ not yet |
|
|
211
|
-
|
|
212
|
-
**Workspace features**
|
|
213
|
-
|
|
214
|
-
| Feature | Status |
|
|
215
|
-
| --- | --- |
|
|
216
|
-
| Workspace symbols | ◻️ needs a project / multi-file model |
|
|
217
|
-
| Execute command | ◻️ not yet |
|
|
218
|
-
| Configuration / watched-files | ◻️ not yet (protocol config; file-based `.sqllens.json` config exists) |
|
|
219
|
-
| File operations (create / rename / delete) | ◻️ not yet |
|
|
220
|
-
|
|
221
|
-
Legend: ✅ implemented · ◻️ not yet / deferred · — not applicable to SQL. The
|
|
222
|
-
deferred items are tracked work: rename and
|
|
223
|
-
code actions are the next LSP phase, workspace symbols need the project model,
|
|
224
|
-
and formatting is expected to wrap an existing external formatter.
|
|
225
|
-
|
|
226
|
-
## Generating the parsers
|
|
227
|
-
|
|
228
|
-
`src/generated/` is a build product and is gitignored. After a fresh clone, or
|
|
229
|
-
after editing any `.g4`, generate the parsers (the lexer must generate before the
|
|
230
|
-
parser, which the driver handles):
|
|
231
|
-
|
|
232
|
-
```bash
|
|
233
|
-
npm run gen -- databricks # | tsql | snowflake | bigquery | redshift | postgres | duckdb | trino
|
|
234
|
-
npm run typecheck
|
|
235
|
-
npm test
|
|
236
|
-
```
|
|
237
|
-
|
|
238
|
-
## Architecture
|
|
239
|
-
|
|
240
|
-
One folder per dialect; no shared "core" grammar and no grammar inheritance. Each
|
|
241
|
-
dialect is a standalone pair of split `.g4` files (a lexer grammar + a parser
|
|
242
|
-
grammar), forked from its best starting point and edited in place. Everything
|
|
243
|
-
downstream of `lower` is shared and dialect-neutral.
|
|
244
|
-
|
|
245
|
-
## Contributing
|
|
246
|
-
|
|
247
|
-
See [CONTRIBUTING.md](CONTRIBUTING.md). In short: the conformance corpora are the
|
|
248
|
-
gate — a grammar change that regresses a corpus is not done — and grammar work is
|
|
249
|
-
test-driven against those corpora.
|
|
250
|
-
|
|
251
|
-
## License
|
|
252
|
-
|
|
253
|
-
MIT — see [LICENSE](LICENSE). The forked grammars under `grammars/` keep their
|
|
254
|
-
upstream licenses (Apache-2.0 for Databricks; MIT for T-SQL and Snowflake; BSD-3
|
|
255
|
-
for BigQuery and Redshift); see [THIRD-PARTY-NOTICES.md](THIRD-PARTY-NOTICES.md).
|
|
1
|
+
# sqllens
|
|
2
|
+
|
|
3
|
+
A TypeScript SQL parser and static analyzer. It parses SQL into a tree, lowers it
|
|
4
|
+
to a dialect-neutral IR, and runs a semantic layer over that IR: name resolution
|
|
5
|
+
(scope), schema-fed qualification, type inference, and column lineage. Give it a
|
|
6
|
+
query and it tells you the query's sources, its output columns, their types, and
|
|
7
|
+
where each column comes from. The parsers are generated TypeScript on the
|
|
8
|
+
[antlr4ng](https://github.com/mike-lischke/antlr4ng) runtime.
|
|
9
|
+
|
|
10
|
+
The front end is error-tolerant and token-first, so the same library powers
|
|
11
|
+
editor tooling — an LSP and a SQL debugger — over incomplete, mid-edit text. See
|
|
12
|
+
[Editor / language tooling](#editor--language-tooling).
|
|
13
|
+
|
|
14
|
+
## Dialects
|
|
15
|
+
|
|
16
|
+
| Dialect | Parse + lower | Semantic layer | Notes |
|
|
17
|
+
|---|---|---|---|
|
|
18
|
+
| Databricks (Spark SQL) | yes | yes | grammar forked from apache/spark |
|
|
19
|
+
| T-SQL | yes | yes | grammar forked from grammars-v4 `sql/tsql` |
|
|
20
|
+
| Snowflake | yes | yes | grammar forked from grammars-v4 `sql/snowflake` |
|
|
21
|
+
| BigQuery (GoogleSQL) | yes | yes | grammar forked from `bytebase/parser` `googlesql/`; gated against ZetaSQL's `.test` corpus |
|
|
22
|
+
| Redshift | yes | yes | grammar forked from Bytebase's Postgres-derived Redshift grammar (BSD-3) |
|
|
23
|
+
| PostgreSQL | yes | yes | grammar forked from `bytebase/parser` `postgresql/` (BSD-3, PG18 keywords) |
|
|
24
|
+
| DuckDB | yes | yes | grammar forked from this repo's own postgres pair (no open ANTLR grammar exists) |
|
|
25
|
+
| Trino | yes | yes | grammar is the first-party trinodb `SqlBase.g4` (release 482), mechanically split; covers dbt-trino + dbt-athena |
|
|
26
|
+
|
|
27
|
+
The semantic layer is dialect-agnostic: it operates on the shared IR and runs
|
|
28
|
+
unchanged on every dialect. Only the parse and lower stages are dialect-specific.
|
|
29
|
+
|
|
30
|
+
## The pipeline
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
parse → lower → resolveScopes → qualify → infer / lineage / symbols
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
- **parse** — text → concrete syntax tree (CST), with a syntax-error count.
|
|
37
|
+
- **lower** — CST → a dialect-neutral IR (`QueryExpr` / `SelectExpr` / `Expr` …);
|
|
38
|
+
also reports the statement kind (query / dml / ddl / …).
|
|
39
|
+
- **resolveScopes** — a schema-free symbol table: visible sources, CTE
|
|
40
|
+
resolution, output columns.
|
|
41
|
+
- **qualify** — with a schema: `*` expansion, unknown-table/column diagnostics,
|
|
42
|
+
column types.
|
|
43
|
+
- **infer / lineage / symbols** — type inference, base-table lineage per output
|
|
44
|
+
column, and a kind×modifier symbol model.
|
|
45
|
+
|
|
46
|
+
## Status
|
|
47
|
+
|
|
48
|
+
Pre-release, and not yet published to npm. The library is consumed as TypeScript
|
|
49
|
+
(no build emit yet — packaging is a later step). The public API (`src/index.ts`)
|
|
50
|
+
is uniform across all eight dialects: `parse` and `analyze` take the dialect as a
|
|
51
|
+
parameter, and every per-dialect `parse*` / `lower` plus the shared passes stay
|
|
52
|
+
exported as lower-level building blocks. The editor-facing surface — `tokenize`,
|
|
53
|
+
`SqlDocument`, `complete`, `signatureAt` — lives on the same barrel.
|
|
54
|
+
|
|
55
|
+
## Usage
|
|
56
|
+
|
|
57
|
+
`dialect` is `"databricks" | "tsql" | "snowflake" | "bigquery" | "redshift" | "postgres" | "duckdb" | "trino"`.
|
|
58
|
+
|
|
59
|
+
Those eight grammars serve more dbt adapters than that, because several adapters
|
|
60
|
+
are SQL front ends over an engine already covered. `adapterDialect` resolves a
|
|
61
|
+
profiles.yml `type:` value (or a dialect name) to the dialect that parses its
|
|
62
|
+
SQL — so consumers don't re-derive the family knowledge:
|
|
63
|
+
|
|
64
|
+
```ts
|
|
65
|
+
import { adapterDialect, ADAPTER_DIALECTS } from "sqllens";
|
|
66
|
+
|
|
67
|
+
adapterDialect("athena"); // "trino" — Athena engine v3 executes on Trino
|
|
68
|
+
adapterDialect("glue"); // "databricks" — AWS Glue runs Spark; Databricks SQL = Spark SQL
|
|
69
|
+
adapterDialect("fabric"); // "tsql" — same for "synapse" and "sqlserver"
|
|
70
|
+
adapterDialect("presto"); // "trino" — the pre-rename Trino adapter
|
|
71
|
+
adapterDialect("oracle"); // undefined — not served; never a guess
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
The map is exact by contract: only adapters whose SQL surface the corpus gates
|
|
75
|
+
genuinely represent are listed. The LSP's `.sqllens.json` accepts adapter types
|
|
76
|
+
through the same map, so `{ "dialect": "athena" }` works in rules and `default`.
|
|
77
|
+
|
|
78
|
+
The surface is
|
|
79
|
+
**layered** — each tier is a terminal value you can stop at — and **composable**:
|
|
80
|
+
every semantic method takes the closest upstream result (so passing it does no
|
|
81
|
+
rework) or a raw string / IR via an idempotent lift helper.
|
|
82
|
+
|
|
83
|
+
```ts
|
|
84
|
+
import { parse, analyze, Schema } from "sqllens";
|
|
85
|
+
|
|
86
|
+
// Tier 1 — just the IR. No semantic layer pulled in.
|
|
87
|
+
const { ast, errors, cst } = parse("SELECT a, b FROM t WHERE a > 1", "tsql");
|
|
88
|
+
// ast = dialect-neutral IR (frozen — no pass mutates it); cst = raw antlr tree (escape hatch)
|
|
89
|
+
// ast.statement -> "query" | "dml" | "ddl" | …
|
|
90
|
+
|
|
91
|
+
// Whole pipeline in one call.
|
|
92
|
+
const schema = new Schema({ t: { a: "int", b: "string" } });
|
|
93
|
+
const a = analyze("SELECT a, b FROM t", "tsql", { schema });
|
|
94
|
+
a.scopes; // name resolution (ScopeTree)
|
|
95
|
+
a.diagnostics; // unknown-table/column diagnostics
|
|
96
|
+
a.qualification.columnsOf(a.scopes.root); // * expansion
|
|
97
|
+
a.types.typeOf(expr, scope); // per-expression types
|
|
98
|
+
a.lineage.originsOf("a"); // base-table origins of an output column
|
|
99
|
+
a.symbols; // kind × modifier symbol model
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Compose tier by tier — pass any upstream result (or a string) to any later pass,
|
|
103
|
+
and only the missing steps run. No exported signature takes or returns a raw
|
|
104
|
+
`Map`/`Set`/`Record`:
|
|
105
|
+
|
|
106
|
+
```ts
|
|
107
|
+
import { parse, qualify, lineage, deriveSymbols, toScopes, Schema } from "sqllens";
|
|
108
|
+
|
|
109
|
+
const { ast } = parse(sql, "snowflake");
|
|
110
|
+
const scopes = toScopes(ast, { dialect: "snowflake" }); // idempotent lift; identity if already a ScopeTree
|
|
111
|
+
qualify(scopes, schema); // reuses scopes — never re-parses or re-resolves
|
|
112
|
+
lineage(scopes, schema); // safe to call on the same scopes, in any order
|
|
113
|
+
deriveSymbols(scopes); // independent results, no cross-contamination
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The per-dialect entries (`parseDatabricks` / `parseTSql` / `parseSnowflake` /
|
|
117
|
+
`parseBigQuery` / `parseRedshift` / `parsePostgres` / `parseDuckdb` / `parseTrino`,
|
|
118
|
+
each `lower`, and the raw `resolveScopes` / `inferType`) remain exported for
|
|
119
|
+
callers that want a single stage.
|
|
120
|
+
|
|
121
|
+
## Editor / language tooling
|
|
122
|
+
|
|
123
|
+
The front end is error-tolerant and token-first, so it serves editor features
|
|
124
|
+
that run on incomplete, mid-edit text — they never need a clean parse:
|
|
125
|
+
|
|
126
|
+
- **`tokenize(sql, dialect)`** and **`parse(...).tokens`** give a first-class token
|
|
127
|
+
stream: every token with its exact span, role, and channel. Always available,
|
|
128
|
+
even when the parse has errors.
|
|
129
|
+
- **`lower()` never throws** on broken or partial input — you get a flagged
|
|
130
|
+
`query` IR back, so every downstream pass stays total.
|
|
131
|
+
- **`SqlDocument`** is a persistent, immutable, position-addressable per-file
|
|
132
|
+
model. It runs `parse → resolveScopes` once (plus lazy `analyze(schema)`),
|
|
133
|
+
caches the result, and answers `tokenAt` / `nodeAt`. An edit yields a new
|
|
134
|
+
document; an O(log n) `LineIndex` maps positions ↔ offsets.
|
|
135
|
+
- **`complete(doc, offset, schema?)`** — scope-aware completion (keywords,
|
|
136
|
+
columns, tables, functions) from an ATN candidate walk over the grammar (our
|
|
137
|
+
own, no third-party dependency).
|
|
138
|
+
- **`signatureAt(doc, offset)`** — parameter hints from a curated per-dialect
|
|
139
|
+
function-signature table; the long tail degrades to name + active-argument.
|
|
140
|
+
- **`referencesAt(scopes, offset, schema?)`** — every occurrence (plus the
|
|
141
|
+
declaration) of the symbol under the cursor; backs find-references, document
|
|
142
|
+
highlight, and code-lens reference counts.
|
|
143
|
+
|
|
144
|
+
```ts
|
|
145
|
+
import { SqlDocument, Schema } from "sqllens";
|
|
146
|
+
|
|
147
|
+
const doc = SqlDocument.create("SELECT amount FROM sales", "databricks");
|
|
148
|
+
doc.tokens; // first-class token stream (spans + roles)
|
|
149
|
+
doc.tokenAt(7); // token under an offset
|
|
150
|
+
const next = doc.withText("SELECT amount, id FROM sales", 2); // immutable edit → new doc
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Language server
|
|
154
|
+
|
|
155
|
+
An LSP (Language Server Protocol) server built on the library, in `src/lsp/`. It
|
|
156
|
+
holds one `SqlDocument` per open file (rebuilt on edit) and reaches the library
|
|
157
|
+
only through the public API surface above — it adds no analysis of its own, only
|
|
158
|
+
protocol translation.
|
|
159
|
+
|
|
160
|
+
LSP is a large protocol — roughly thirty request types across document-sync,
|
|
161
|
+
language, and workspace features — so "supports LSP" is not one bit but a long
|
|
162
|
+
checklist. A SQL server needs a subset, but more of it maps to SQL than it first
|
|
163
|
+
looks — a CTE / view / model is the SQL analog of a definition, and the
|
|
164
|
+
dependency graph between them is a call hierarchy. A few features genuinely don't
|
|
165
|
+
apply (type hierarchy, document color, monikers); a few are deliberately deferred
|
|
166
|
+
(formatting, project-wide navigation). The coverage, feature by feature:
|
|
167
|
+
|
|
168
|
+
**Language features**
|
|
169
|
+
|
|
170
|
+
| Feature | Status |
|
|
171
|
+
| --- | --- |
|
|
172
|
+
| Completion (+ resolve) | ✅ |
|
|
173
|
+
| Hover | ✅ |
|
|
174
|
+
| Hover — nullability | ✅ (` — not null` / ` — nullable` suffix when provable) |
|
|
175
|
+
| Signature help | ✅ |
|
|
176
|
+
| Go to definition | ✅ |
|
|
177
|
+
| Find references | ✅ |
|
|
178
|
+
| Document highlight | ✅ |
|
|
179
|
+
| Document symbols | ✅ |
|
|
180
|
+
| Folding range | ✅ |
|
|
181
|
+
| Selection range | ✅ |
|
|
182
|
+
| Semantic tokens (full / range / delta) | ✅ all three |
|
|
183
|
+
| Inlay hints | ✅ (no resolve) |
|
|
184
|
+
| Code lens | ✅ (no resolve) |
|
|
185
|
+
| Go to declaration | ◻️ not yet |
|
|
186
|
+
| Go to type definition | ◻️ not yet |
|
|
187
|
+
| Go to implementation | ◻️ not yet — name → its defining query (view / model); needs the project model |
|
|
188
|
+
| Call hierarchy | ◻️ not yet — the CTE / dbt-model dependency graph |
|
|
189
|
+
| Document link | ◻️ not yet |
|
|
190
|
+
| Linked editing range | ◻️ not yet — live alias / name sync-edit |
|
|
191
|
+
| Code action (quick fixes) | ◻️ next phase |
|
|
192
|
+
| Rename (+ prepare) | ◻️ next phase |
|
|
193
|
+
| Formatting / range / on-type | ◻️ deferred (external formatter) |
|
|
194
|
+
| Inline values | ◻️ debugger surface |
|
|
195
|
+
| Type hierarchy | — n/a — SQL has no type-inheritance relation |
|
|
196
|
+
| Document color | — n/a — no color literals |
|
|
197
|
+
| Moniker | — n/a — LSIF / cross-repo indexing concern |
|
|
198
|
+
|
|
199
|
+
**Diagnostics & document sync**
|
|
200
|
+
|
|
201
|
+
| Feature | Status |
|
|
202
|
+
| --- | --- |
|
|
203
|
+
| Diagnostics — push (`publishDiagnostics`) | ✅ |
|
|
204
|
+
| Diagnostics — call signature (arity / argument type) | ✅ (curated tables; never-wrong, per-dialect coercion) |
|
|
205
|
+
| Diagnostics — pull (document) | ✅ |
|
|
206
|
+
| Diagnostics — pull (workspace) | ◻️ not yet |
|
|
207
|
+
| Text sync — open / change / close | ✅ (full-document) |
|
|
208
|
+
| Incremental sync | ◻️ full-document only (fine at SQL file sizes) |
|
|
209
|
+
| Save notifications (`didSave` / `willSave`) | ◻️ not yet |
|
|
210
|
+
| Notebook document sync | ◻️ not yet |
|
|
211
|
+
|
|
212
|
+
**Workspace features**
|
|
213
|
+
|
|
214
|
+
| Feature | Status |
|
|
215
|
+
| --- | --- |
|
|
216
|
+
| Workspace symbols | ◻️ needs a project / multi-file model |
|
|
217
|
+
| Execute command | ◻️ not yet |
|
|
218
|
+
| Configuration / watched-files | ◻️ not yet (protocol config; file-based `.sqllens.json` config exists) |
|
|
219
|
+
| File operations (create / rename / delete) | ◻️ not yet |
|
|
220
|
+
|
|
221
|
+
Legend: ✅ implemented · ◻️ not yet / deferred · — not applicable to SQL. The
|
|
222
|
+
deferred items are tracked work: rename and
|
|
223
|
+
code actions are the next LSP phase, workspace symbols need the project model,
|
|
224
|
+
and formatting is expected to wrap an existing external formatter.
|
|
225
|
+
|
|
226
|
+
## Generating the parsers
|
|
227
|
+
|
|
228
|
+
`src/generated/` is a build product and is gitignored. After a fresh clone, or
|
|
229
|
+
after editing any `.g4`, generate the parsers (the lexer must generate before the
|
|
230
|
+
parser, which the driver handles):
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
npm run gen -- databricks # | tsql | snowflake | bigquery | redshift | postgres | duckdb | trino
|
|
234
|
+
npm run typecheck
|
|
235
|
+
npm test
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
## Architecture
|
|
239
|
+
|
|
240
|
+
One folder per dialect; no shared "core" grammar and no grammar inheritance. Each
|
|
241
|
+
dialect is a standalone pair of split `.g4` files (a lexer grammar + a parser
|
|
242
|
+
grammar), forked from its best starting point and edited in place. Everything
|
|
243
|
+
downstream of `lower` is shared and dialect-neutral.
|
|
244
|
+
|
|
245
|
+
## Contributing
|
|
246
|
+
|
|
247
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). In short: the conformance corpora are the
|
|
248
|
+
gate — a grammar change that regresses a corpus is not done — and grammar work is
|
|
249
|
+
test-driven against those corpora.
|
|
250
|
+
|
|
251
|
+
## License
|
|
252
|
+
|
|
253
|
+
MIT — see [LICENSE](LICENSE). The forked grammars under `grammars/` keep their
|
|
254
|
+
upstream licenses (Apache-2.0 for Databricks; MIT for T-SQL and Snowflake; BSD-3
|
|
255
|
+
for BigQuery and Redshift); see [THIRD-PARTY-NOTICES.md](THIRD-PARTY-NOTICES.md).
|
package/package.json
CHANGED
|
@@ -1,91 +1,91 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "sqllens",
|
|
3
|
-
"version": "0.1.
|
|
4
|
-
"description": "A TypeScript SQL parser and static analyzer: parse, resolve names, infer types, and trace column lineage across eight SQL dialects (Databricks, T-SQL, Snowflake, BigQuery, Redshift, PostgreSQL, DuckDB, Trino).",
|
|
5
|
-
"main": "./dist/index.js",
|
|
6
|
-
"types": "./dist/index.d.ts",
|
|
7
|
-
"exports": {
|
|
8
|
-
".": {
|
|
9
|
-
"types": "./dist/index.d.ts",
|
|
10
|
-
"default": "./dist/index.js"
|
|
11
|
-
}
|
|
12
|
-
},
|
|
13
|
-
"files": [
|
|
14
|
-
"dist",
|
|
15
|
-
"LICENSE",
|
|
16
|
-
"THIRD-PARTY-NOTICES.md"
|
|
17
|
-
],
|
|
18
|
-
"engines": {
|
|
19
|
-
"node": ">=20.11"
|
|
20
|
-
},
|
|
21
|
-
"directories": {
|
|
22
|
-
"doc": "docs",
|
|
23
|
-
"test": "tests"
|
|
24
|
-
},
|
|
25
|
-
"keywords": [
|
|
26
|
-
"sql",
|
|
27
|
-
"parser",
|
|
28
|
-
"sql-parser",
|
|
29
|
-
"analyzer",
|
|
30
|
-
"ast",
|
|
31
|
-
"lineage",
|
|
32
|
-
"type-inference",
|
|
33
|
-
"name-resolution",
|
|
34
|
-
"lsp",
|
|
35
|
-
"databricks",
|
|
36
|
-
"tsql",
|
|
37
|
-
"snowflake",
|
|
38
|
-
"bigquery",
|
|
39
|
-
"redshift",
|
|
40
|
-
"postgres",
|
|
41
|
-
"duckdb",
|
|
42
|
-
"trino",
|
|
43
|
-
"antlr"
|
|
44
|
-
],
|
|
45
|
-
"author": "Niclas Olofsson",
|
|
46
|
-
"repository": {
|
|
47
|
-
"type": "git",
|
|
48
|
-
"url": "git+https://github.com/NiclasOlofsson/sqllens.git"
|
|
49
|
-
},
|
|
50
|
-
"bugs": {
|
|
51
|
-
"url": "https://github.com/NiclasOlofsson/sqllens/issues"
|
|
52
|
-
},
|
|
53
|
-
"homepage": "https://github.com/NiclasOlofsson/sqllens#readme",
|
|
54
|
-
"scripts": {
|
|
55
|
-
"gen": "node tools/gen.mjs",
|
|
56
|
-
"gen:all": "npm run gen -- databricks && npm run gen -- tsql && npm run gen -- snowflake && npm run gen -- bigquery && npm run gen -- redshift && npm run gen -- postgres && npm run gen -- duckdb && npm run gen -- trino && npm run gen -- minijinja",
|
|
57
|
-
"build": "npm run gen:all && tsc -p tsconfig.build.json",
|
|
58
|
-
"prepublishOnly": "npm run build",
|
|
59
|
-
"lsp": "node --import tsx src/lsp/main.ts",
|
|
60
|
-
"typecheck": "tsgo -p tsconfig.json",
|
|
61
|
-
"test": "vitest run",
|
|
62
|
-
"test:corpus": "vitest run --config vitest.corpus.config.ts",
|
|
63
|
-
"test:all": "npm test && npm run test:corpus",
|
|
64
|
-
"format": "prettier --write .",
|
|
65
|
-
"format:check": "prettier --check ."
|
|
66
|
-
},
|
|
67
|
-
"license": "MIT",
|
|
68
|
-
"type": "module",
|
|
69
|
-
"devDependencies": {
|
|
70
|
-
"@semantic-release/changelog": "^6.0.3",
|
|
71
|
-
"@semantic-release/git": "^10.0.1",
|
|
72
|
-
"@semantic-release/github": "^12.0.9",
|
|
73
|
-
"@semantic-release/npm": "^13.1.5",
|
|
74
|
-
"@types/node": "^25.9.2",
|
|
75
|
-
"@typescript/native-preview": "^7.0.0-dev.20260605.1",
|
|
76
|
-
"antlr-ng": "^1.0.10",
|
|
77
|
-
"minimatch": "^10.2.5",
|
|
78
|
-
"prettier": "^3.8.3",
|
|
79
|
-
"semantic-release": "^25.0.5",
|
|
80
|
-
"tsx": "^4.22.4",
|
|
81
|
-
"typescript": "^6.0.3",
|
|
82
|
-
"vitest": "^4.1.8",
|
|
83
|
-
"vscode-languageserver": "^10.0.1",
|
|
84
|
-
"vscode-languageserver-protocol": "^3.18.1",
|
|
85
|
-
"vscode-languageserver-textdocument": "^1.0.12",
|
|
86
|
-
"vscode-languageserver-types": "^3.18.0"
|
|
87
|
-
},
|
|
88
|
-
"dependencies": {
|
|
89
|
-
"antlr4ng": "^3.0.16"
|
|
90
|
-
}
|
|
91
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "sqllens",
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "A TypeScript SQL parser and static analyzer: parse, resolve names, infer types, and trace column lineage across eight SQL dialects (Databricks, T-SQL, Snowflake, BigQuery, Redshift, PostgreSQL, DuckDB, Trino).",
|
|
5
|
+
"main": "./dist/index.js",
|
|
6
|
+
"types": "./dist/index.d.ts",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"default": "./dist/index.js"
|
|
11
|
+
}
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"dist",
|
|
15
|
+
"LICENSE",
|
|
16
|
+
"THIRD-PARTY-NOTICES.md"
|
|
17
|
+
],
|
|
18
|
+
"engines": {
|
|
19
|
+
"node": ">=20.11"
|
|
20
|
+
},
|
|
21
|
+
"directories": {
|
|
22
|
+
"doc": "docs",
|
|
23
|
+
"test": "tests"
|
|
24
|
+
},
|
|
25
|
+
"keywords": [
|
|
26
|
+
"sql",
|
|
27
|
+
"parser",
|
|
28
|
+
"sql-parser",
|
|
29
|
+
"analyzer",
|
|
30
|
+
"ast",
|
|
31
|
+
"lineage",
|
|
32
|
+
"type-inference",
|
|
33
|
+
"name-resolution",
|
|
34
|
+
"lsp",
|
|
35
|
+
"databricks",
|
|
36
|
+
"tsql",
|
|
37
|
+
"snowflake",
|
|
38
|
+
"bigquery",
|
|
39
|
+
"redshift",
|
|
40
|
+
"postgres",
|
|
41
|
+
"duckdb",
|
|
42
|
+
"trino",
|
|
43
|
+
"antlr"
|
|
44
|
+
],
|
|
45
|
+
"author": "Niclas Olofsson",
|
|
46
|
+
"repository": {
|
|
47
|
+
"type": "git",
|
|
48
|
+
"url": "git+https://github.com/NiclasOlofsson/sqllens.git"
|
|
49
|
+
},
|
|
50
|
+
"bugs": {
|
|
51
|
+
"url": "https://github.com/NiclasOlofsson/sqllens/issues"
|
|
52
|
+
},
|
|
53
|
+
"homepage": "https://github.com/NiclasOlofsson/sqllens#readme",
|
|
54
|
+
"scripts": {
|
|
55
|
+
"gen": "node tools/gen.mjs",
|
|
56
|
+
"gen:all": "npm run gen -- databricks && npm run gen -- tsql && npm run gen -- snowflake && npm run gen -- bigquery && npm run gen -- redshift && npm run gen -- postgres && npm run gen -- duckdb && npm run gen -- trino && npm run gen -- minijinja",
|
|
57
|
+
"build": "npm run gen:all && tsc -p tsconfig.build.json",
|
|
58
|
+
"prepublishOnly": "npm run build",
|
|
59
|
+
"lsp": "node --import tsx src/lsp/main.ts",
|
|
60
|
+
"typecheck": "tsgo -p tsconfig.json",
|
|
61
|
+
"test": "vitest run",
|
|
62
|
+
"test:corpus": "vitest run --config vitest.corpus.config.ts",
|
|
63
|
+
"test:all": "npm test && npm run test:corpus",
|
|
64
|
+
"format": "prettier --write .",
|
|
65
|
+
"format:check": "prettier --check ."
|
|
66
|
+
},
|
|
67
|
+
"license": "MIT",
|
|
68
|
+
"type": "module",
|
|
69
|
+
"devDependencies": {
|
|
70
|
+
"@semantic-release/changelog": "^6.0.3",
|
|
71
|
+
"@semantic-release/git": "^10.0.1",
|
|
72
|
+
"@semantic-release/github": "^12.0.9",
|
|
73
|
+
"@semantic-release/npm": "^13.1.5",
|
|
74
|
+
"@types/node": "^25.9.2",
|
|
75
|
+
"@typescript/native-preview": "^7.0.0-dev.20260605.1",
|
|
76
|
+
"antlr-ng": "^1.0.10",
|
|
77
|
+
"minimatch": "^10.2.5",
|
|
78
|
+
"prettier": "^3.8.3",
|
|
79
|
+
"semantic-release": "^25.0.5",
|
|
80
|
+
"tsx": "^4.22.4",
|
|
81
|
+
"typescript": "^6.0.3",
|
|
82
|
+
"vitest": "^4.1.8",
|
|
83
|
+
"vscode-languageserver": "^10.0.1",
|
|
84
|
+
"vscode-languageserver-protocol": "^3.18.1",
|
|
85
|
+
"vscode-languageserver-textdocument": "^1.0.12",
|
|
86
|
+
"vscode-languageserver-types": "^3.18.0"
|
|
87
|
+
},
|
|
88
|
+
"dependencies": {
|
|
89
|
+
"antlr4ng": "^3.0.16"
|
|
90
|
+
}
|
|
91
|
+
}
|