dbt-cst 0.1.0__tar.gz

dbt_cst-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+/target
+.venv/
+__pycache__/
+*.so
+.pytest_cache/
+/dist

dbt_cst-0.1.0/.python-version

@@ -0,0 +1 @@
+3.11

dbt_cst-0.1.0/Cargo.lock

@@ -0,0 +1,236 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "dbt-cst"
+version = "0.1.0"
+dependencies = [
+ "nom",
+ "parking_lot",
+ "pyo3",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "inventory"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.190"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
+
+[[package]]
+name = "lock_api"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "nom"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.29.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655"
+dependencies = [
+ "inventory",
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.29.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.29.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.29.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "smallvec"
+version = "1.16.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"

dbt_cst-0.1.0/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "dbt-cst"
+version = "0.1.0"
+edition = "2024"
+description = "Lossless concrete syntax tree parser for dbt models (Jinja templated SQL) with Python bindings"
+readme = "README.md"
+
+[lib]
+name = "dbt_cst"
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = []
+# Enabled by maturin when building the Python extension module.
+python = ["dep:pyo3"]
+
+[dependencies]
+nom = "8.0.0"
+parking_lot = "0.12.5"
+pyo3 = { version = "0.29.3", features = ["abi3-py39", "extension-module", "multiple-pymethods"], optional = true }

dbt_cst-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+# MIT License
+
+Copyright (c) 2026 Albert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dbt_cst-0.1.0/PKG-INFO

@@ -0,0 +1,165 @@
+Metadata-Version: 2.4
+Name: dbt-cst
+Version: 0.1.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+License-File: LICENSE
+Summary: Lossless dbt model parser: read, modify and rewrite Jinja templated SQL with its formatting intact
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/livinlefevreloca/dbt-cst
+
+# dbt-cst
+
+A lossless dbt model parser written in Rust with Python bindings. It parses a model (Jinja
+templated SQL) into a concrete syntax tree (CST) that keeps every comment, blank line,
+indentation and Jinja tag, so a file can be read, modified and written back out with its
+formatting intact:
+
+```python
+from dbt_cst import parse
+
+doc = parse(source)
+assert str(doc) == source  # byte for byte, for every model in albert-dbt-analytics-transforms
+```
+
+The tree groups the SQL into queries, clauses, select columns and CTEs, and the Jinja into
+blocks, without interpreting either: a column is the text between two commas, a WHERE clause
+is the text between its keyword and the next clause keyword. That is enough to add and remove
+columns in a model someone has customized by hand, which rewriting the file from a template
+cannot do. The Python package is an abi3 wheel built against Python 3.9, so one wheel works on
+Python 3.9 and 3.11.
+
+## Python usage
+
+```python
+from dbt_cst import parse, DbtSyntaxError
+
+doc = parse(path.read_text())
+
+    doc.config["materialized"]                  # "view"
+    doc.config["materialized"] = "table"        # quoted like the call's other strings
+    doc.config["tags"] = ["pii"]                # added in the call's layout: one per line, trailing comma
+    doc.config.set_raw("schema", 'var("schema")')
+    del doc.config["alias"]
+
+    # Example: Modifying multiple config values
+    doc.config["materialized"] = "table"
+    doc.config["schema"] = "analytics"
+    doc.config["tags"] = ["pii", "finance"]
+    doc.config["enabled"] = False
+
+    # Example: Using raw expressions for dynamic values
+    doc.config.set_raw("alias", 'var("model_alias")')
+    doc.config.set_raw("schema", 'var("schema")')
+
+    # Example: Removing config values
+    if "description" in doc.config:
+        del doc.config["description"]
+
+    doc.sources                                 # [("trading", "apex_apexentry")]
+    doc.refs                                    # [("users",), ("package", "model")]
+
+query = doc.query                           # the model's own SELECT, after any CTEs
+select = query.select
+select.names                                # ["account_id", "activity_id", "percent"]
+"account_id" in select                      # True, compared ignoring case
+
+select.add("user_id")                       # appended in the select's comma style
+select.add("amount / 100 AS dollars", index=select.index("account_id") + 1)
+select.remove("activity_id")                # with its comma and the comments above it
+select.find("percent").text = '"percent" AS pct'
+
+renamed = query.cte("renamed").query.select # CTE bodies are queries too
+for query in doc.queries():                 # every query, subqueries included
+    ...
+
+path.write_text(str(doc))
+```
+
+### API
+
+`parse(source) -> Document` raises `DbtSyntaxError` (a `ValueError`) with `line`, `column` and
+`offset` attributes. Only a Jinja tag that is never closed is an error. SQL that does not fit
+the expected shape still parses: it ends up as unstructured tokens and still round trips.
+
+| Object     | Members                                                                                         |
+| ---------- | ----------------------------------------------------------------------------------------------- |
+| `Document` | `config`, `query`, `queries()`, `sources`, `refs`, `str()`                                      |
+| `Config`   | `c[key]`, `get(key, default)`, `c[key] = value`, `del c[key]`, `raw(key)`, `set_raw(key, src)`, `keys()`, `in`, `len`, `iter` |
+| `Query`    | `select` (the first), `selects` (one per UNION branch), `ctes`, `cte(name)`, `clauses`            |
+| `Cte`      | `name`, `query`                                                                                  |
+| `Clause`   | `name` (`FROM`, `GROUP BY`, `UNION ALL`, ...), `str()`                                           |
+| `Select`   | `columns`, `names`, `find(name)`, `index(name)`, `add(text, index=None)`, `remove(name_or_column)`, `in`, `len`, `iter` |
+| `Column`   | `text` (settable), `expr`, `alias`, `name`, `is_templated`, `select`, `detach()`                   |
+
+`Config` values are Jinja literals: strings, numbers, booleans, `None`, lists and dicts.
+Reading an argument whose value is an expression such as `var("x")` raises `ValueError`; use
+`raw()` for its source text.
+
+`Column.name` is the alias, or the last part of a plain column reference (`t.id` gives `id`),
+unquoted. It is `None` for an unaliased expression or a bare Jinja tag such as
+`{{ dbt_utils.star(...) }}`.
+
+Handles are live: a `Column` from `select.columns` stays attached to the document, and two
+handles to the same node compare equal.
+
+## How it parses
+
+1. **Lexing** (`src/lexer.rs`) splits the file into tokens covering every byte. Jinja tags end
+   where Jinja says they do, skipping string literals and nested brackets, so
+   `{{ {'a': 1} }}` and `{{ "}}" }}` are one tag each. A tag inside a SQL string or comment is
+   absorbed into that token. `{% raw %}` sections are one token.
+2. **Grouping** (`src/parser.rs`) pairs `(`/`)`, `[`/`]` and Jinja block tags with one stack.
+   Any `{% name %}` whose `{% endname %}` appears in the file is opened tentatively, so tags
+   from Jinja, dbt (`materialization`, `test`, `snapshot`) and packages all work without a
+   list of names. A tag that never closes, like a one line `{% set %}`, or one whose end would
+   cross a parenthesis, is put back as a plain token. `elif` and `else` split blocks into
+   branches.
+3. **Structure**: a top level `SELECT` or `WITH` starts a query, which is split into clauses at
+   top level clause keywords. SELECT items are split at top level commas, and WITH clauses into
+   CTEs. "Top level" means outside any bracket or Jinja block, so commas in function calls and
+   keywords in subqueries never split anything. A Jinja block that supplies its own commas,
+   like `{% for c in cols %}{{ c }},{% endfor %}`, separates the columns around it.
+
+The lexer is written with nom, like looker-cst's parser. The one hand written scan is the end
+of a `{{ }}` or `{% %}` tag, which tracks bracket depth and skips string literals the way
+Jinja's own lexer does.
+
+## Rust usage
+
+```rust
+let doc = dbt_cst::parse(&source)?;
+let query = doc.query().unwrap();
+let select = dbt_cst::query::clauses_named(&query.read(), "SELECT").remove(0);
+dbt_cst::edit::insert_item(&mut select.write(), usize::MAX, "user_id")?;
+assert!(doc.to_string().contains("user_id"));
+```
+
+`cargo run --example dump -- path/to/model.sql` prints the tree.
+
+## Development
+
+```sh
+cargo test                           # Rust tests, cloning the corpus into target/corpus on first run
+uv run pytest                        # builds the extension with maturin, then runs the Python tests
+DBT_EXTRA_REPOS=../albert-dbt-analytics-transforms cargo test --release --test corpus
+```
+
+The corpus is the 14 public dbt projects listed in `tests/corpus_repos.txt` (dbt-utils,
+dbt-expectations, automate-dv, the Fivetran packages and others, about 1,700 `.sql` files).
+Both test suites shallow clone them into `target/corpus` on first use, or into `DBT_CORPUS`
+when set. A failed clone skips those files unless `REQUIRE_DBT_CORPUS` is set, as in CI.
+`DBT_EXTRA_REPOS` adds more checkouts, separated by `:`.
+
+- `tests/corpus.rs` round trips every file, adds and removes a column in every SELECT,
+  removes each column in turn, and adds and removes a config argument.
+- `tests/python/test_corpus.py` does the same read, modify and write cycle through the Python
+  API, one test per file.
+- `tests/fixtures/` holds awkward inputs (CRLF, unicode, unterminated quotes, unbalanced
+  brackets, unpaired Jinja blocks) that both suites round trip, and `edge_cases.sql`, a
+  customized model whose structure both suites check.
+- `tests/python/test_albert_models.py` checks every model in a sibling
+  `albert-dbt-analytics-transforms` checkout (or `ALBERT_DBT_REPO`) against the field list
+  DBTGenerator reads today.
+

dbt_cst-0.1.0/README.md

@@ -0,0 +1,153 @@
+# dbt-cst
+
+A lossless dbt model parser written in Rust with Python bindings. It parses a model (Jinja
+templated SQL) into a concrete syntax tree (CST) that keeps every comment, blank line,
+indentation and Jinja tag, so a file can be read, modified and written back out with its
+formatting intact:
+
+```python
+from dbt_cst import parse
+
+doc = parse(source)
+assert str(doc) == source  # byte for byte, for every model in albert-dbt-analytics-transforms
+```
+
+The tree groups the SQL into queries, clauses, select columns and CTEs, and the Jinja into
+blocks, without interpreting either: a column is the text between two commas, a WHERE clause
+is the text between its keyword and the next clause keyword. That is enough to add and remove
+columns in a model someone has customized by hand, which rewriting the file from a template
+cannot do. The Python package is an abi3 wheel built against Python 3.9, so one wheel works on
+Python 3.9 and 3.11.
+
+## Python usage
+
+```python
+from dbt_cst import parse, DbtSyntaxError
+
+doc = parse(path.read_text())
+
+    doc.config["materialized"]                  # "view"
+    doc.config["materialized"] = "table"        # quoted like the call's other strings
+    doc.config["tags"] = ["pii"]                # added in the call's layout: one per line, trailing comma
+    doc.config.set_raw("schema", 'var("schema")')
+    del doc.config["alias"]
+
+    # Example: Modifying multiple config values
+    doc.config["materialized"] = "table"
+    doc.config["schema"] = "analytics"
+    doc.config["tags"] = ["pii", "finance"]
+    doc.config["enabled"] = False
+
+    # Example: Using raw expressions for dynamic values
+    doc.config.set_raw("alias", 'var("model_alias")')
+    doc.config.set_raw("schema", 'var("schema")')
+
+    # Example: Removing config values
+    if "description" in doc.config:
+        del doc.config["description"]
+
+    doc.sources                                 # [("trading", "apex_apexentry")]
+    doc.refs                                    # [("users",), ("package", "model")]
+
+query = doc.query                           # the model's own SELECT, after any CTEs
+select = query.select
+select.names                                # ["account_id", "activity_id", "percent"]
+"account_id" in select                      # True, compared ignoring case
+
+select.add("user_id")                       # appended in the select's comma style
+select.add("amount / 100 AS dollars", index=select.index("account_id") + 1)
+select.remove("activity_id")                # with its comma and the comments above it
+select.find("percent").text = '"percent" AS pct'
+
+renamed = query.cte("renamed").query.select # CTE bodies are queries too
+for query in doc.queries():                 # every query, subqueries included
+    ...
+
+path.write_text(str(doc))
+```
+
+### API
+
+`parse(source) -> Document` raises `DbtSyntaxError` (a `ValueError`) with `line`, `column` and
+`offset` attributes. Only a Jinja tag that is never closed is an error. SQL that does not fit
+the expected shape still parses: it ends up as unstructured tokens and still round trips.
+
+| Object     | Members                                                                                         |
+| ---------- | ----------------------------------------------------------------------------------------------- |
+| `Document` | `config`, `query`, `queries()`, `sources`, `refs`, `str()`                                      |
+| `Config`   | `c[key]`, `get(key, default)`, `c[key] = value`, `del c[key]`, `raw(key)`, `set_raw(key, src)`, `keys()`, `in`, `len`, `iter` |
+| `Query`    | `select` (the first), `selects` (one per UNION branch), `ctes`, `cte(name)`, `clauses`            |
+| `Cte`      | `name`, `query`                                                                                  |
+| `Clause`   | `name` (`FROM`, `GROUP BY`, `UNION ALL`, ...), `str()`                                           |
+| `Select`   | `columns`, `names`, `find(name)`, `index(name)`, `add(text, index=None)`, `remove(name_or_column)`, `in`, `len`, `iter` |
+| `Column`   | `text` (settable), `expr`, `alias`, `name`, `is_templated`, `select`, `detach()`                   |
+
+`Config` values are Jinja literals: strings, numbers, booleans, `None`, lists and dicts.
+Reading an argument whose value is an expression such as `var("x")` raises `ValueError`; use
+`raw()` for its source text.
+
+`Column.name` is the alias, or the last part of a plain column reference (`t.id` gives `id`),
+unquoted. It is `None` for an unaliased expression or a bare Jinja tag such as
+`{{ dbt_utils.star(...) }}`.
+
+Handles are live: a `Column` from `select.columns` stays attached to the document, and two
+handles to the same node compare equal.
+
+## How it parses
+
+1. **Lexing** (`src/lexer.rs`) splits the file into tokens covering every byte. Jinja tags end
+   where Jinja says they do, skipping string literals and nested brackets, so
+   `{{ {'a': 1} }}` and `{{ "}}" }}` are one tag each. A tag inside a SQL string or comment is
+   absorbed into that token. `{% raw %}` sections are one token.
+2. **Grouping** (`src/parser.rs`) pairs `(`/`)`, `[`/`]` and Jinja block tags with one stack.
+   Any `{% name %}` whose `{% endname %}` appears in the file is opened tentatively, so tags
+   from Jinja, dbt (`materialization`, `test`, `snapshot`) and packages all work without a
+   list of names. A tag that never closes, like a one line `{% set %}`, or one whose end would
+   cross a parenthesis, is put back as a plain token. `elif` and `else` split blocks into
+   branches.
+3. **Structure**: a top level `SELECT` or `WITH` starts a query, which is split into clauses at
+   top level clause keywords. SELECT items are split at top level commas, and WITH clauses into
+   CTEs. "Top level" means outside any bracket or Jinja block, so commas in function calls and
+   keywords in subqueries never split anything. A Jinja block that supplies its own commas,
+   like `{% for c in cols %}{{ c }},{% endfor %}`, separates the columns around it.
+
+The lexer is written with nom, like looker-cst's parser. The one hand written scan is the end
+of a `{{ }}` or `{% %}` tag, which tracks bracket depth and skips string literals the way
+Jinja's own lexer does.
+
+## Rust usage
+
+```rust
+let doc = dbt_cst::parse(&source)?;
+let query = doc.query().unwrap();
+let select = dbt_cst::query::clauses_named(&query.read(), "SELECT").remove(0);
+dbt_cst::edit::insert_item(&mut select.write(), usize::MAX, "user_id")?;
+assert!(doc.to_string().contains("user_id"));
+```
+
+`cargo run --example dump -- path/to/model.sql` prints the tree.
+
+## Development
+
+```sh
+cargo test                           # Rust tests, cloning the corpus into target/corpus on first run
+uv run pytest                        # builds the extension with maturin, then runs the Python tests
+DBT_EXTRA_REPOS=../albert-dbt-analytics-transforms cargo test --release --test corpus
+```
+
+The corpus is the 14 public dbt projects listed in `tests/corpus_repos.txt` (dbt-utils,
+dbt-expectations, automate-dv, the Fivetran packages and others, about 1,700 `.sql` files).
+Both test suites shallow clone them into `target/corpus` on first use, or into `DBT_CORPUS`
+when set. A failed clone skips those files unless `REQUIRE_DBT_CORPUS` is set, as in CI.
+`DBT_EXTRA_REPOS` adds more checkouts, separated by `:`.
+
+- `tests/corpus.rs` round trips every file, adds and removes a column in every SELECT,
+  removes each column in turn, and adds and removes a config argument.
+- `tests/python/test_corpus.py` does the same read, modify and write cycle through the Python
+  API, one test per file.
+- `tests/fixtures/` holds awkward inputs (CRLF, unicode, unterminated quotes, unbalanced
+  brackets, unpaired Jinja blocks) that both suites round trip, and `edge_cases.sql`, a
+  customized model whose structure both suites check.
+- `tests/python/test_albert_models.py` checks every model in a sibling
+  `albert-dbt-analytics-transforms` checkout (or `ALBERT_DBT_REPO`) against the field list
+  DBTGenerator reads today.

dbt_cst-0.1.0/examples/dump.rs

@@ -0,0 +1,22 @@
+//! Prints the tree of a dbt file, one node or token per line.
+//!
+//!     cargo run --example dump -- path/to/model.sql
+
+use dbt_cst::{Child, Node, parse};
+
+fn main() {
+    let path = std::env::args().nth(1).expect("usage: dump <file.sql>");
+    let source = std::fs::read_to_string(&path).expect("readable file");
+    let doc = parse(&source).unwrap_or_else(|e| panic!("{path}: {e}"));
+    print_node(&doc.root.read(), 0);
+}
+
+fn print_node(node: &Node, depth: usize) {
+    println!("{}{:?}", "  ".repeat(depth), node.kind);
+    for child in &node.children {
+        match child {
+            Child::Token(token) => println!("{}{:?} {:?}", "  ".repeat(depth + 1), token.kind, token.text),
+            Child::Node(node) => print_node(&node.read(), depth + 1),
+        }
+    }
+}

dbt_cst-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["maturin>=1.9,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "dbt-cst"
+version = "0.1.0"
+description = "Lossless dbt model parser: read, modify and rewrite Jinja templated SQL with its formatting intact"
+requires-python = ">=3.9"
+readme = "README.md"
+license = { file = "LICENSE" }
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+]
+
+[project.urls]
+Repository = "https://github.com/livinlefevreloca/dbt-cst"
+
+[dependency-groups]
+dev = ["pytest>=8"]
+
+[tool.maturin]
+python-source = "python"
+module-name = "dbt_cst._native"
+features = ["python"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests/python"]
+
+[tool.uv]
+# Rebuild the extension when the Rust sources change.
+cache-keys = [{ file = "pyproject.toml" }, { file = "Cargo.toml" }, { file = "src/**/*.rs" }]