ripple-sql 0.1.18__tar.gz → 0.1.19__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/CHANGELOG.md +20 -1
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/PKG-INFO +34 -2
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/README.md +33 -1
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/pyproject.toml +4 -1
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/cache.py +18 -2
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/declared.py +2 -1
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/graph.py +10 -2
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/loaders/dbt.py +39 -189
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/loaders/dbt_config.py +30 -26
- ripple_sql-0.1.19/src/ripple/loaders/dbt_info_schema.py +134 -0
- ripple_sql-0.1.19/src/ripple/loaders/dbt_profile.py +430 -0
- ripple_sql-0.1.19/src/ripple/loaders/dbt_profile_render.py +160 -0
- ripple_sql-0.1.19/src/ripple/loaders/env_vars.py +101 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/loaders/sidecar.py +2 -1
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/loaders/types.py +8 -10
- ripple_sql-0.1.19/src/ripple/loaders/yaml_text.py +49 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/lookml.py +2 -1
- ripple_sql-0.1.19/src/ripple/mcp_budget.py +390 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/mcp_handlers.py +67 -24
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/mcp_server.py +35 -121
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/mcp_tools.py +48 -9
- ripple_sql-0.1.19/src/ripple/mcp_wait.py +131 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/project.py +7 -0
- ripple_sql-0.1.19/src/ripple/refresher.py +69 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/schema_fetch.py +128 -180
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/semantic.py +3 -4
- ripple_sql-0.1.19/src/ripple/service_build.py +229 -0
- ripple_sql-0.1.19/src/ripple/service_usage.py +81 -0
- ripple_sql-0.1.19/src/ripple/service_warehouse.py +447 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/sourcefiles.py +4 -3
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/ingest.js +2 -3
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/usage/cli.py +2 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/usage/collect.py +22 -33
- ripple_sql-0.1.19/src/ripple/usage/discover.py +125 -0
- ripple_sql-0.1.19/src/ripple/warehouse_choice.py +193 -0
- ripple_sql-0.1.19/src/ripple/warehouse_config/__init__.py +24 -0
- ripple_sql-0.1.19/src/ripple/warehouse_config/locate.py +164 -0
- ripple_sql-0.1.19/src/ripple/warehouse_config/process.py +103 -0
- ripple_sql-0.1.19/src/ripple/warehouse_config/snowflake.py +373 -0
- ripple_sql-0.1.19/src/ripple/warehouse_config/snowflake_rules.py +182 -0
- ripple_sql-0.1.19/src/ripple/warehouse_config/toml_read.py +324 -0
- ripple_sql-0.1.19/src/ripple/warehouse_next.py +151 -0
- ripple_sql-0.1.18/src/ripple/service_warehouse.py +0 -223
- ripple_sql-0.1.18/src/ripple/usage/discover.py +0 -194
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/.gitignore +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/LICENSE +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/__init__.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/answer.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/answer_page.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/__init__.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/binding.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/header_sense.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/naming.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/readers.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/report.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/check_file/text.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/ci.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/ci_signature.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/cli.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/cli_files.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/dbt_catalog.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/digest.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/doctor.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/__init__.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/budget.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/column_lineage.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/column_ref.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/cte_tracing.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/dependencies.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/dialect.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/dispatch.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/extraction.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/generators.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/jinja.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/json_sources.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/macro_source.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/pipeline.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/preprocess.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/safe_gen.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/schema_qualification.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/scope.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/select_sources.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/sql_script.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/statement.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/tech_debt.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/tsql_catalog.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/tsql_scalar_vars.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/tsql_tvf.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/tsql_xml.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/types.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/unused_deps.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/engine/validation.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/home.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/loaders/__init__.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/loaders/identity.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/loaders/sqldir.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/names.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/notify/__init__.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/notify/config.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/notify/formats.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/notify/safe.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/notify/send.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/py.typed +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/render.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/render_shims.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/review_reasons.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/schema_drop.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/schemas.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/server.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/star_resolution.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/answer.css +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/answer.html +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/answer_twin.js +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/check.js +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/explore.js +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/find.js +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/focus.js +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/fonts/OFL-atkinson-hyperlegible-next.txt +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/fonts/OFL-jetbrains-mono.txt +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/fonts/atkinson-hyperlegible-next-latin.woff2 +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/static/home.js +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/usage/__init__.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/usage/ingest.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/usage/report.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/warehouse_columns.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/warehouse_memory.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/watch.py +0 -0
- {ripple_sql-0.1.18 → ripple_sql-0.1.19}/src/ripple/watch_store.py +0 -0
|
@@ -6,6 +6,24 @@ Notable changes to Ripple. The format follows
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.1.19] - 2026-10-08
|
|
10
|
+
|
|
11
|
+
Found by a Snowflake trial on Oct 8 2026: an analytics engineer on GitLab's 1,908-model dbt project, driving `ripple mcp` the way Claude Code or Cursor would, across twelve real setups.
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
- Ripple finds Snowflake connections wherever `snow` keeps them. It asks `snow connection list` first (it never connects) and otherwise follows snow's own rules: `SNOWFLAKE_HOME`, `~/.snowflake` only when it exists, then the platform folder (`~/Library/Application Support/snowflake` on a Mac, `~/.config/snowflake` or `XDG_CONFIG_HOME` on Linux, `%LOCALAPPDATA%\snowflake` on Windows). On a fresh Mac it used to see no connection at all. `connections.toml` replaces `config.toml`'s connections, as in snow; the default comes from `SNOWFLAKE_DEFAULT_CONNECTION_NAME`, `config.toml`, or a connection named `default`. A file snow refuses (readable by others, broken TOML) is named with the fix, for example `chmod 600 "<path>"`. Checked against real snow 3.29 on 24 setups.
|
|
15
|
+
- Ripple knows a dbt project's warehouse from `profiles.yml` wherever dbt finds it: `DBT_PROFILES_DIR`, the current folder when it is inside the repo, `~/.dbt`, then folders like `profile/` in the repo. The profile and target follow dbt (`DBT_PROFILE`, `DBT_TARGET`, the `target:` key, a target named `default`, dbt Fusion's list targets). Only the adapter type, the profile and target names and the file path are kept; a password on a broken YAML line never shows up anywhere. `RIPPLE_DBT_HOME=off` skips the home folder. It used to read only a `profiles.yml` beside `dbt_project.yml`, so most Snowflake users got "warehouse unknown".
|
|
16
|
+
- MCP results fit what clients accept. One budget for every tool: about 8,000 characters by default, `detail="full"` or `limit` for up to 24,000, `offset` for the next page, and a `cut` record naming what was left out and the exact call for the rest. Counts and the verdict come first, `next` last. `ingest_schema` returned 957K characters on GitLab (the whole schemas file, twice); `breaks` 126K.
|
|
17
|
+
- No MCP call blocks while Ripple builds or fetches. A first call on a large project answers within 2 s with "Ripple is reading the project, 712 of 1,908 models · ask again in about 30 s", and progress notifications go out when the client asks for them. `fetch_schema`, `collect_usage` and `ingest_schema` start in the background and report through `lineage_summary` and `usage_summary`. Claude Desktop, Codex and Cursor's CLI stop a call at 60 s; the first call used to take 72 to 97 s.
|
|
18
|
+
- Every MCP call stays fast after a column fetch. A status check re-read all of a project's dbt YAML several times per call (1,206 parses on GitLab's 201 sources.yml files) and fought the rebuild for Python's lock, so a call right after a fetch took 50 to 67 s. Each YAML file is now parsed once until it changes (with libyaml when installed), and the warehouse state is refreshed in the background and served with its age. On GitLab every call answers within 3 s. "Ask again in about N s" now follows the last real build time, clamped to 5 to 120 s (it once said 1,150 s).
|
|
19
|
+
- `next` leads with what blocks the warehouse read and the exact fix: a connection file to chmod, which connection to pick, `DBT_TARGET` or a source database variable to add to the server's env block. When sign-in can prompt (a password with an MFA push, Okta, a browser), it tells the agent to ask the user before fetching.
|
|
20
|
+
- `env_var()` in a repo's dbt files can no longer pull a secret into Ripple. Every read goes through one gate: a credential-looking name (`AWS_SECRET_ACCESS_KEY`, `GITHUB_TOKEN`, ...) is refused, a value used as a table name must look like one, and messages name the variable, never its value. A sources.yml naming `AWS_SECRET_ACCESS_KEY` put its value into the SQL sent to `snow`.
|
|
21
|
+
- On Windows, a warehouse CLI no longer sees a terminal on stdin (the NUL device reports one), so it cannot wait on a prompt. CLIs are found only in absolute PATH folders, never the repo, and in the usual install folders when an app started from the Dock or Start menu has a short PATH (for example `/opt/homebrew/bin`), with a note saying so.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
- Automatic column reads run only for sign-ins that never prompt: key pair (`SNOWFLAKE_JWT`), OAuth client credentials, programmatic access tokens, workload identity. A password connection now waits for the user, because it can send an MFA push.
|
|
25
|
+
- ClickHouse, SQL Server, Fabric, Synapse, Athena and other adapters parse in their own dialect instead of Snowflake with a warning (one adapter map).
|
|
26
|
+
|
|
9
27
|
## [0.1.18] - 2026-10-07
|
|
10
28
|
|
|
11
29
|
### Added
|
|
@@ -354,7 +372,8 @@ First public release.
|
|
|
354
372
|
### Removed
|
|
355
373
|
- The dark whole-graph canvas that `ripple serve` used to open (`static/index.html`). Every surface now draws one answer.
|
|
356
374
|
|
|
357
|
-
[Unreleased]: https://github.com/bteh/ripple/compare/v0.1.
|
|
375
|
+
[Unreleased]: https://github.com/bteh/ripple/compare/v0.1.19...HEAD
|
|
376
|
+
[0.1.19]: https://github.com/bteh/ripple/compare/v0.1.18...v0.1.19
|
|
358
377
|
[0.1.18]: https://github.com/bteh/ripple/compare/v0.1.17...v0.1.18
|
|
359
378
|
[0.1.17]: https://github.com/bteh/ripple/compare/v0.1.16...v0.1.17
|
|
360
379
|
[0.1.16]: https://github.com/bteh/ripple/releases/tag/v0.1.16
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ripple-sql
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.19
|
|
4
4
|
Summary: Offline column-level SQL lineage. See what breaks before you merge.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bteh/ripple
|
|
6
6
|
Project-URL: Repository, https://github.com/bteh/ripple
|
|
@@ -137,6 +137,34 @@ default_tools_approval_mode = "approve"
|
|
|
137
137
|
Then ask: *"what breaks if I change `orders.amount` to cents?"* If the client
|
|
138
138
|
lists ripple but the tools do nothing, `ripple doctor` tells you why.
|
|
139
139
|
|
|
140
|
+
On a big project a full answer can run to hundreds of thousands of
|
|
141
|
+
characters, and clients cut or spill results long before that (Cline keeps
|
|
142
|
+
8,000 characters, Codex cuts the middle at about 10,000 tokens, Claude Code
|
|
143
|
+
writes anything over 50,000 to a file). So every tool result stays under
|
|
144
|
+
8,000 characters by default. The counts, the verdict and `next` are never
|
|
145
|
+
cut; long lists are, nearest hops or worst blockers first, and the result's
|
|
146
|
+
`cut` field says how many it showed out of how many, with the exact call for
|
|
147
|
+
the rest. `breaks`, `trace`, `check_file`, `model_columns`,
|
|
148
|
+
`unresolved_tables`, `usage_summary` and `lineage_summary` take three extra
|
|
149
|
+
arguments: `offset` (skip that many rows), `limit` (rows per page) and
|
|
150
|
+
`detail` (`summary`, the default, or `full`). Asked for more, a result still
|
|
151
|
+
stops at 24,000 characters. `page=true` on `breaks` and `trace` is the HTML
|
|
152
|
+
answer page for a person, not paging.
|
|
153
|
+
|
|
154
|
+
No tool call waits more than about 2 seconds. Claude Desktop, Codex and the
|
|
155
|
+
Cursor CLI give up on a call after 60 seconds, and reading a 2,000-model
|
|
156
|
+
project the first time takes longer than that. So the server starts reading
|
|
157
|
+
the project the moment it starts, and a call that arrives before it is done
|
|
158
|
+
answers at once with `building: true`, how many models are read, and when to
|
|
159
|
+
ask again ("Ripple is reading the project, 712 of 1,908 models · ask again in
|
|
160
|
+
about 30 s"). The disk cache makes a restart instant. `fetch_schema` and
|
|
161
|
+
`ingest_schema` work the same way: a slow warehouse or rebuild answers
|
|
162
|
+
"started" and the result lands in `lineage_summary` (`warehouse.fetching`,
|
|
163
|
+
`warehouse.rebuilding`, then `warehouse.last_fetch` with what was saved and
|
|
164
|
+
how many more models are fully analyzed). A client that sends a
|
|
165
|
+
`progressToken` also gets progress notifications while a call waits.
|
|
166
|
+
`RIPPLE_MCP_WAIT_S` changes the wait.
|
|
167
|
+
|
|
140
168
|
## Is it actually right?
|
|
141
169
|
|
|
142
170
|
Most SQL lineage tools are confidently wrong, so Ripple ships a benchmark you can rerun instead of a claim you have to trust. Real public dbt projects, hand-verified column lineage, every tool scored by the same function. The corpus and the harness are both in this repo.
|
|
@@ -241,6 +269,10 @@ Your SQL probably reads tables defined outside the repo, and without their colum
|
|
|
241
269
|
|
|
242
270
|
Ripple never stores, prompts for, or sends a credential, and never opens a connection of its own: it runs the CLI you already signed in to, with the connection you marked default (or the only one). It starts on its own only when signing in needs no browser; otherwise the page shows one button, "Fetch from Snowflake · prod". Several connections and no default: the page asks once and remembers the name in your cache, not in the repo. `RIPPLE_WAREHOUSE=off` turns it off.
|
|
243
271
|
|
|
272
|
+
For a dbt project, Ripple learns the warehouse from `profiles.yml` the way dbt finds it: `DBT_PROFILES_DIR`, then your working directory, the project folder, `~/.dbt`, and folders like `profile/` in the repo. It picks the target from `DBT_TARGET` or the profile's `target:`, and reads only each target's `type`, never a password or token. An MCP server started by a desktop app does not see your shell's exports, so put `DBT_TARGET` or `DBT_PROFILES_DIR` in the server's `env` block when you need them, and any variable your sources read with `env_var()` (GitLab's sources name their database with `SNOWFLAKE_LOAD_DATABASE`, so its block carries `"SNOWFLAKE_LOAD_DATABASE": "RAW"`). Restart the server after changing it. `RIPPLE_DBT_HOME=off` stops it reading `~/.dbt`, and `--dialect` overrides all of it.
|
|
273
|
+
|
|
274
|
+
What Ripple reports about your profile, in the CLI, the answer page and to an MCP client: the adapter type, the profile and target names, the other targets of that profile with their adapter types (so it can say "set DBT_TARGET=prod"), and the path of the one file it used (or, when it found none, the places it looked). It never reports other profile files on your machine. A `{{ env_var('X') }}` in `type`, `target` or `profile` is read only for `DBT_*` names and names ending in `_TARGET`, `_PROFILE`, `_TYPE`, `_ENV` and the like, never for a name that looks like a credential (`TOKEN`, `KEY`, `SECRET`, `PASSWORD`). A value from the environment is kept only when it names a known adapter, a target of that profile, or a profile a file has; otherwise Ripple names the variable and not its value.
|
|
275
|
+
|
|
244
276
|
The columns land in `.ripple/schemas.json` with where they came from and when, which you can commit so the whole team gets the coverage. From a terminal:
|
|
245
277
|
|
|
246
278
|
```bash
|
|
@@ -249,7 +281,7 @@ ripple unresolved # which tables still block coverage, worst fir
|
|
|
249
281
|
ripple ingest-schema cols.csv # no CLI? an information_schema.columns export works too
|
|
250
282
|
```
|
|
251
283
|
|
|
252
|
-
An agent connected over MCP is told all of this when it connects: it calls `fetch_schema
|
|
284
|
+
An agent connected over MCP is told all of this when it connects: it calls `fetch_schema` (on its own only when the sign-in cannot prompt; otherwise it asks you first), or, when Ripple cannot read the warehouse and the agent can, queries `information_schema.columns` itself and hands the names to `ingest_schema`.
|
|
253
285
|
|
|
254
286
|
## A file arrives from another team
|
|
255
287
|
|
|
@@ -96,6 +96,34 @@ default_tools_approval_mode = "approve"
|
|
|
96
96
|
Then ask: *"what breaks if I change `orders.amount` to cents?"* If the client
|
|
97
97
|
lists ripple but the tools do nothing, `ripple doctor` tells you why.
|
|
98
98
|
|
|
99
|
+
On a big project a full answer can run to hundreds of thousands of
|
|
100
|
+
characters, and clients cut or spill results long before that (Cline keeps
|
|
101
|
+
8,000 characters, Codex cuts the middle at about 10,000 tokens, Claude Code
|
|
102
|
+
writes anything over 50,000 to a file). So every tool result stays under
|
|
103
|
+
8,000 characters by default. The counts, the verdict and `next` are never
|
|
104
|
+
cut; long lists are, nearest hops or worst blockers first, and the result's
|
|
105
|
+
`cut` field says how many it showed out of how many, with the exact call for
|
|
106
|
+
the rest. `breaks`, `trace`, `check_file`, `model_columns`,
|
|
107
|
+
`unresolved_tables`, `usage_summary` and `lineage_summary` take three extra
|
|
108
|
+
arguments: `offset` (skip that many rows), `limit` (rows per page) and
|
|
109
|
+
`detail` (`summary`, the default, or `full`). Asked for more, a result still
|
|
110
|
+
stops at 24,000 characters. `page=true` on `breaks` and `trace` is the HTML
|
|
111
|
+
answer page for a person, not paging.
|
|
112
|
+
|
|
113
|
+
No tool call waits more than about 2 seconds. Claude Desktop, Codex and the
|
|
114
|
+
Cursor CLI give up on a call after 60 seconds, and reading a 2,000-model
|
|
115
|
+
project the first time takes longer than that. So the server starts reading
|
|
116
|
+
the project the moment it starts, and a call that arrives before it is done
|
|
117
|
+
answers at once with `building: true`, how many models are read, and when to
|
|
118
|
+
ask again ("Ripple is reading the project, 712 of 1,908 models · ask again in
|
|
119
|
+
about 30 s"). The disk cache makes a restart instant. `fetch_schema` and
|
|
120
|
+
`ingest_schema` work the same way: a slow warehouse or rebuild answers
|
|
121
|
+
"started" and the result lands in `lineage_summary` (`warehouse.fetching`,
|
|
122
|
+
`warehouse.rebuilding`, then `warehouse.last_fetch` with what was saved and
|
|
123
|
+
how many more models are fully analyzed). A client that sends a
|
|
124
|
+
`progressToken` also gets progress notifications while a call waits.
|
|
125
|
+
`RIPPLE_MCP_WAIT_S` changes the wait.
|
|
126
|
+
|
|
99
127
|
## Is it actually right?
|
|
100
128
|
|
|
101
129
|
Most SQL lineage tools are confidently wrong, so Ripple ships a benchmark you can rerun instead of a claim you have to trust. Real public dbt projects, hand-verified column lineage, every tool scored by the same function. The corpus and the harness are both in this repo.
|
|
@@ -200,6 +228,10 @@ Your SQL probably reads tables defined outside the repo, and without their colum
|
|
|
200
228
|
|
|
201
229
|
Ripple never stores, prompts for, or sends a credential, and never opens a connection of its own: it runs the CLI you already signed in to, with the connection you marked default (or the only one). It starts on its own only when signing in needs no browser; otherwise the page shows one button, "Fetch from Snowflake · prod". Several connections and no default: the page asks once and remembers the name in your cache, not in the repo. `RIPPLE_WAREHOUSE=off` turns it off.
|
|
202
230
|
|
|
231
|
+
For a dbt project, Ripple learns the warehouse from `profiles.yml` the way dbt finds it: `DBT_PROFILES_DIR`, then your working directory, the project folder, `~/.dbt`, and folders like `profile/` in the repo. It picks the target from `DBT_TARGET` or the profile's `target:`, and reads only each target's `type`, never a password or token. An MCP server started by a desktop app does not see your shell's exports, so put `DBT_TARGET` or `DBT_PROFILES_DIR` in the server's `env` block when you need them, and any variable your sources read with `env_var()` (GitLab's sources name their database with `SNOWFLAKE_LOAD_DATABASE`, so its block carries `"SNOWFLAKE_LOAD_DATABASE": "RAW"`). Restart the server after changing it. `RIPPLE_DBT_HOME=off` stops it reading `~/.dbt`, and `--dialect` overrides all of it.
|
|
232
|
+
|
|
233
|
+
What Ripple reports about your profile, in the CLI, the answer page and to an MCP client: the adapter type, the profile and target names, the other targets of that profile with their adapter types (so it can say "set DBT_TARGET=prod"), and the path of the one file it used (or, when it found none, the places it looked). It never reports other profile files on your machine. A `{{ env_var('X') }}` in `type`, `target` or `profile` is read only for `DBT_*` names and names ending in `_TARGET`, `_PROFILE`, `_TYPE`, `_ENV` and the like, never for a name that looks like a credential (`TOKEN`, `KEY`, `SECRET`, `PASSWORD`). A value from the environment is kept only when it names a known adapter, a target of that profile, or a profile a file has; otherwise Ripple names the variable and not its value.
|
|
234
|
+
|
|
203
235
|
The columns land in `.ripple/schemas.json` with where they came from and when, which you can commit so the whole team gets the coverage. From a terminal:
|
|
204
236
|
|
|
205
237
|
```bash
|
|
@@ -208,7 +240,7 @@ ripple unresolved # which tables still block coverage, worst fir
|
|
|
208
240
|
ripple ingest-schema cols.csv # no CLI? an information_schema.columns export works too
|
|
209
241
|
```
|
|
210
242
|
|
|
211
|
-
An agent connected over MCP is told all of this when it connects: it calls `fetch_schema
|
|
243
|
+
An agent connected over MCP is told all of this when it connects: it calls `fetch_schema` (on its own only when the sign-in cannot prompt; otherwise it asks you first), or, when Ripple cannot read the warehouse and the agent can, queries `information_schema.columns` itself and hands the names to `ingest_schema`.
|
|
212
244
|
|
|
213
245
|
## A file arrives from another team
|
|
214
246
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ripple-sql"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.19"
|
|
4
4
|
description = "Offline column-level SQL lineage. See what breaks before you merge."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -70,6 +70,9 @@ testpaths = ["tests"]
|
|
|
70
70
|
# a hung test fails with its stack instead of running until the CI job limit
|
|
71
71
|
timeout = 180
|
|
72
72
|
timeout_method = "thread"
|
|
73
|
+
markers = [
|
|
74
|
+
"asks_snow: lets the test run `snow connection list` (a fake on PATH, or the real one)",
|
|
75
|
+
]
|
|
73
76
|
|
|
74
77
|
[tool.ruff]
|
|
75
78
|
line-length = 100
|
|
@@ -16,7 +16,7 @@ import os
|
|
|
16
16
|
import pickle
|
|
17
17
|
from pathlib import Path
|
|
18
18
|
|
|
19
|
-
from ripple.sourcefiles import stamp
|
|
19
|
+
from ripple.sourcefiles import source_files, stamp
|
|
20
20
|
|
|
21
21
|
logger = logging.getLogger(__name__)
|
|
22
22
|
|
|
@@ -44,10 +44,26 @@ def _fingerprint(root: Path, dialect: str | None) -> str:
|
|
|
44
44
|
digest.update(f"{src.name}|{src.stat().st_mtime_ns}".encode())
|
|
45
45
|
except OSError:
|
|
46
46
|
pass
|
|
47
|
-
digest.update(
|
|
47
|
+
digest.update(inputs_stamp(root, dialect).encode())
|
|
48
48
|
return digest.hexdigest()
|
|
49
49
|
|
|
50
50
|
|
|
51
|
+
def inputs_stamp(root: Path, dialect: str | None, skip: Path | None = None) -> str:
|
|
52
|
+
"""What a graph was built from right now. The disk cache and the live
|
|
53
|
+
server both use it, so a change either notices, the other does too.
|
|
54
|
+
skip leaves one file out, to ask whether anything else changed."""
|
|
55
|
+
files = [f for f in source_files(root) if f != skip]
|
|
56
|
+
out = stamp(root, files)
|
|
57
|
+
if dialect is None:
|
|
58
|
+
# the dialect may come from ~/.dbt or DBT_TARGET, which no repo
|
|
59
|
+
# file stamps; --dialect users skip this so their key is unchanged
|
|
60
|
+
from ripple.loaders.dbt_profile import selection_key
|
|
61
|
+
|
|
62
|
+
keys = [selection_key(f.parent) for f in files if f.name == "dbt_project.yml"]
|
|
63
|
+
out += "|" + "|".join(keys)
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
|
|
51
67
|
def load(root: Path, dialect: str | None):
|
|
52
68
|
path = _cache_dir() / f"graph-{_fingerprint(root, dialect)}.pickle"
|
|
53
69
|
if not path.exists():
|
|
@@ -11,6 +11,7 @@ import copy
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
|
|
13
13
|
from ripple.check_file.readers import CannotCheck
|
|
14
|
+
from ripple.loaders.yaml_text import safe_load
|
|
14
15
|
|
|
15
16
|
# roots tuple -> (signature, declarations), one entry per roots tuple
|
|
16
17
|
_CACHE: dict[tuple, tuple[tuple, list]] = {}
|
|
@@ -89,7 +90,7 @@ def _read(roots: list[Path]):
|
|
|
89
90
|
own = root / "ripple.yml"
|
|
90
91
|
if own.is_file():
|
|
91
92
|
try:
|
|
92
|
-
parsed =
|
|
93
|
+
parsed = safe_load(own.read_text(encoding="utf-8", errors="replace")) or {}
|
|
93
94
|
except yaml.YAMLError as e:
|
|
94
95
|
raise BrokenRippleYml(
|
|
95
96
|
f"ripple.yml is not valid yaml ({str(e).splitlines()[0]})"
|
|
@@ -1319,11 +1319,13 @@ class LineageGraph:
|
|
|
1319
1319
|
|
|
1320
1320
|
# ---- export ----
|
|
1321
1321
|
|
|
1322
|
-
def
|
|
1322
|
+
def failure_reasons(self) -> dict[str, str]:
|
|
1323
|
+
"""Every model that failed, fell back or timed out, with its error.
|
|
1324
|
+
stats() keeps the first 20 for the CLI; MCP pages the whole list."""
|
|
1323
1325
|
by_status = defaultdict(list)
|
|
1324
1326
|
for report in self.reports.values():
|
|
1325
1327
|
by_status[report.status].append(report.name)
|
|
1326
|
-
|
|
1328
|
+
return {
|
|
1327
1329
|
name: self.reports[name].error
|
|
1328
1330
|
for name in [
|
|
1329
1331
|
*by_status.get("failed", []),
|
|
@@ -1331,6 +1333,12 @@ class LineageGraph:
|
|
|
1331
1333
|
*by_status.get("timed_out", []),
|
|
1332
1334
|
]
|
|
1333
1335
|
}
|
|
1336
|
+
|
|
1337
|
+
def stats(self) -> dict:
|
|
1338
|
+
by_status = defaultdict(list)
|
|
1339
|
+
for report in self.reports.values():
|
|
1340
|
+
by_status[report.status].append(report.name)
|
|
1341
|
+
failed = self.failure_reasons()
|
|
1334
1342
|
return {
|
|
1335
1343
|
"mode": self.project.mode,
|
|
1336
1344
|
"dialect": self.project.dialect,
|
|
@@ -19,6 +19,13 @@ from ripple.loaders.dbt_config import (
|
|
|
19
19
|
_project_vars,
|
|
20
20
|
_seed_dirs,
|
|
21
21
|
)
|
|
22
|
+
from ripple.loaders.dbt_info_schema import ( # noqa: F401 tests and the cache import these from here
|
|
23
|
+
INFO_SCHEMA,
|
|
24
|
+
_manifest_from_info_schema,
|
|
25
|
+
_source_parts,
|
|
26
|
+
info_schema_files,
|
|
27
|
+
)
|
|
28
|
+
from ripple.loaders.dbt_profile import resolve_profile
|
|
22
29
|
from ripple.loaders.types import (
|
|
23
30
|
_REF_RE,
|
|
24
31
|
_SOURCE_RE,
|
|
@@ -34,57 +41,6 @@ from ripple.loaders.types import (
|
|
|
34
41
|
logger = logging.getLogger(__name__)
|
|
35
42
|
|
|
36
43
|
|
|
37
|
-
def _profile_adapter(root: Path) -> str | None:
|
|
38
|
-
"""The adapter dbt is configured to use, from profiles.yml beside the project.
|
|
39
|
-
|
|
40
|
-
dbt names the profile in dbt_project.yml and defines it in profiles.yml. That
|
|
41
|
-
is the project telling us its warehouse, and it was never read: every raw dbt
|
|
42
|
-
project was called snowflake regardless, so ClickHouse/dbt-clickhouse
|
|
43
|
-
reported snowflake without a word.
|
|
44
|
-
|
|
45
|
-
Only the project-local profiles.yml is read. ~/.dbt/profiles.yml belongs to
|
|
46
|
-
whoever is running the command, not to the repo, and reading a developer's
|
|
47
|
-
home directory to analyze a checkout would be a surprise.
|
|
48
|
-
"""
|
|
49
|
-
path = root / "profiles.yml"
|
|
50
|
-
if not path.is_file():
|
|
51
|
-
return None
|
|
52
|
-
try:
|
|
53
|
-
import yaml
|
|
54
|
-
|
|
55
|
-
doc = yaml.safe_load(path.read_text(errors="replace", encoding="utf-8")) or {}
|
|
56
|
-
except Exception:
|
|
57
|
-
return None
|
|
58
|
-
if not isinstance(doc, dict):
|
|
59
|
-
return None
|
|
60
|
-
wanted = _project_profile_name(root)
|
|
61
|
-
for name, profile in doc.items():
|
|
62
|
-
if not isinstance(profile, dict) or (wanted and name != wanted):
|
|
63
|
-
continue
|
|
64
|
-
outputs = profile.get("outputs")
|
|
65
|
-
if not isinstance(outputs, dict) or not outputs:
|
|
66
|
-
continue
|
|
67
|
-
target = profile.get("target")
|
|
68
|
-
chosen = outputs.get(target) if target in outputs else next(iter(outputs.values()))
|
|
69
|
-
if isinstance(chosen, dict) and chosen.get("type"):
|
|
70
|
-
return str(chosen["type"]).lower()
|
|
71
|
-
return None
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def _project_profile_name(root: Path) -> str | None:
|
|
75
|
-
path = root / "dbt_project.yml"
|
|
76
|
-
if not path.is_file():
|
|
77
|
-
return None
|
|
78
|
-
try:
|
|
79
|
-
import yaml
|
|
80
|
-
|
|
81
|
-
doc = yaml.safe_load(path.read_text(errors="replace", encoding="utf-8")) or {}
|
|
82
|
-
except Exception:
|
|
83
|
-
return None
|
|
84
|
-
profile = doc.get("profile") if isinstance(doc, dict) else None
|
|
85
|
-
return str(profile) if profile else None
|
|
86
|
-
|
|
87
|
-
|
|
88
44
|
def resolve_dbt_dialect(
|
|
89
45
|
root: Path, dialect: str | None, sample: str = ""
|
|
90
46
|
) -> tuple[str, str | None]:
|
|
@@ -95,17 +51,30 @@ def resolve_dbt_dialect(
|
|
|
95
51
|
does not say it guessed is the failure this exists to prevent: a wrong one
|
|
96
52
|
costs whole models, and the reader has no way to know that is why.
|
|
97
53
|
"""
|
|
54
|
+
resolved, note, _ = _resolve_with_source(root, dialect, sample)
|
|
55
|
+
return resolved, note
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _resolve_with_source(
|
|
59
|
+
root: Path, dialect: str | None, sample: str = ""
|
|
60
|
+
) -> tuple[str, str | None, dict]:
|
|
61
|
+
"""resolve_dbt_dialect plus where the answer came from (dbt_profile's
|
|
62
|
+
ProfileResolution as a dict), which the warehouse fetch reports."""
|
|
98
63
|
if dialect:
|
|
99
|
-
return dialect, None
|
|
100
|
-
|
|
64
|
+
return dialect, None, {"kind": "flag", "platform": dialect}
|
|
65
|
+
found = resolve_profile(root)
|
|
66
|
+
source = found.as_dict()
|
|
67
|
+
adapter = found.platform
|
|
101
68
|
if adapter and adapter in ADAPTER_TO_DIALECT:
|
|
102
|
-
return ADAPTER_TO_DIALECT[adapter], None
|
|
69
|
+
return ADAPTER_TO_DIALECT[adapter], None, source
|
|
103
70
|
if adapter:
|
|
104
71
|
return (
|
|
105
72
|
DEFAULT_DIALECT,
|
|
106
73
|
f"dbt profile declares '{adapter}', which Ripple cannot parse; "
|
|
107
74
|
f"guessed {DEFAULT_DIALECT}. Pass --dialect to override.",
|
|
75
|
+
source,
|
|
108
76
|
)
|
|
77
|
+
why = f" ({found.reason})" if found.kind == "unusable" and found.reason else ""
|
|
109
78
|
if sample:
|
|
110
79
|
from ripple.engine.sql_script import sniff_dialect
|
|
111
80
|
|
|
@@ -113,13 +82,15 @@ def resolve_dbt_dialect(
|
|
|
113
82
|
if sniffed:
|
|
114
83
|
return (
|
|
115
84
|
sniffed,
|
|
116
|
-
f"No dbt profile or manifest declares a warehouse; the SQL reads as "
|
|
85
|
+
f"No dbt profile or manifest declares a warehouse{why}; the SQL reads as "
|
|
117
86
|
f"{sniffed}. Pass --dialect to override.",
|
|
87
|
+
source,
|
|
118
88
|
)
|
|
119
89
|
return (
|
|
120
90
|
DEFAULT_DIALECT,
|
|
121
|
-
f"No dbt profile or manifest declares a warehouse; guessed "
|
|
91
|
+
f"No dbt profile or manifest declares a warehouse{why}; guessed "
|
|
122
92
|
f"{DEFAULT_DIALECT}. Pass --dialect to override.",
|
|
93
|
+
source,
|
|
123
94
|
)
|
|
124
95
|
|
|
125
96
|
|
|
@@ -162,135 +133,6 @@ def _load_one_dbt(project_dir: Path, dialect: str | None) -> Project:
|
|
|
162
133
|
return project
|
|
163
134
|
|
|
164
135
|
|
|
165
|
-
INFO_SCHEMA = Path("target") / "info_schema" / "v1"
|
|
166
|
-
# dbt 2's Parquet tables that hold SQL nodes, and the resource type each row is
|
|
167
|
-
NODE_TABLES = {"models": "model", "seeds": "seed", "snapshots": "snapshot"}
|
|
168
|
-
SQL_PARENTS = ("model.", "source.", "seed.", "snapshot.")
|
|
169
|
-
SQL_CHILDREN = ("model.", "snapshot.", "seed.")
|
|
170
|
-
# the columns the reader uses, so a large project's raw and compiled SQL is
|
|
171
|
-
# the only heavy thing read, once, and nothing else is materialised
|
|
172
|
-
NODE_COLUMNS = (
|
|
173
|
-
"unique_id",
|
|
174
|
-
"name",
|
|
175
|
-
"resource_type",
|
|
176
|
-
"package_name",
|
|
177
|
-
"original_file_path",
|
|
178
|
-
"alias",
|
|
179
|
-
"raw_code",
|
|
180
|
-
"compiled_code",
|
|
181
|
-
"compiled_path",
|
|
182
|
-
"database_name",
|
|
183
|
-
"schema_name",
|
|
184
|
-
"relation_name",
|
|
185
|
-
"identifier",
|
|
186
|
-
"enabled",
|
|
187
|
-
)
|
|
188
|
-
EDGE_COLUMNS = ("parent_unique_id", "child_unique_id")
|
|
189
|
-
PROJECT_COLUMNS = ("adapter_type", "dbt_version")
|
|
190
|
-
INFO_SCHEMA_FILES = tuple(f"dbt.{t}.parquet" for t in (*NODE_TABLES, "sources", "edges", "project"))
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
def info_schema_files(root: Path) -> list[Path]:
|
|
194
|
-
"""The Parquet files the reader consumes, for whoever fingerprints the
|
|
195
|
-
project: a change to them must invalidate a cached graph."""
|
|
196
|
-
folder = root / INFO_SCHEMA
|
|
197
|
-
return [folder / name for name in INFO_SCHEMA_FILES if (folder / name).is_file()]
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
def _source_parts(uid: str, package: str | None, name: str | None) -> tuple[str, str]:
|
|
201
|
-
"""(source_name, table_name) from a source unique_id, which is
|
|
202
|
-
source.<package>.<source_name>.<table>. A source or table name may hold
|
|
203
|
-
a dot, so the table is taken from the row's own name and the source is
|
|
204
|
-
what sits between the package and it, never a fixed position."""
|
|
205
|
-
parts = uid.split(".")
|
|
206
|
-
if len(parts) < 4 or parts[0] != "source":
|
|
207
|
-
return "", ""
|
|
208
|
-
prefix = f"source.{package}." if package else ""
|
|
209
|
-
rest = uid[len(prefix) :] if prefix and uid.startswith(prefix) else ".".join(parts[2:])
|
|
210
|
-
if name and rest.endswith(f".{name}") and len(rest) > len(name) + 1:
|
|
211
|
-
return rest[: -len(name) - 1], name
|
|
212
|
-
return parts[-2], name or parts[-1]
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
def _manifest_from_info_schema(root: Path) -> tuple[dict | None, str | None]:
|
|
216
|
-
"""dbt 2's Parquet information schema (target/info_schema/v1/, written by
|
|
217
|
-
--generate-info-schema) folded into the manifest shape the manifest
|
|
218
|
-
reader takes, so one reader serves both artifacts. A team that turns
|
|
219
|
-
manifest.json off with --no-write-json still gets compiled lineage.
|
|
220
|
-
(manifest, note): the note says why the folder could not be used."""
|
|
221
|
-
folder = root / INFO_SCHEMA
|
|
222
|
-
node_files = [folder / f"dbt.{t}.parquet" for t in (*NODE_TABLES, "sources")]
|
|
223
|
-
if not any(f.is_file() for f in node_files):
|
|
224
|
-
return None, None
|
|
225
|
-
try:
|
|
226
|
-
import pyarrow.parquet as pq
|
|
227
|
-
except ImportError:
|
|
228
|
-
return None, (
|
|
229
|
-
"target/info_schema found (dbt 2's Parquet artifacts) but pyarrow is not installed; "
|
|
230
|
-
"pip install pyarrow to read it. Reading raw model SQL instead."
|
|
231
|
-
)
|
|
232
|
-
|
|
233
|
-
def rows(name: str, wanted: tuple[str, ...]) -> list[dict]:
|
|
234
|
-
path = folder / f"dbt.{name}.parquet"
|
|
235
|
-
if not path.is_file():
|
|
236
|
-
return []
|
|
237
|
-
have = set(pq.read_schema(path).names)
|
|
238
|
-
return pq.read_table(path, columns=[c for c in wanted if c in have]).to_pylist()
|
|
239
|
-
|
|
240
|
-
try:
|
|
241
|
-
project = (rows("project", PROJECT_COLUMNS) or [{}])[0]
|
|
242
|
-
parents: dict[str, list[str]] = {}
|
|
243
|
-
for edge in rows("edges", EDGE_COLUMNS):
|
|
244
|
-
parent, child = edge.get("parent_unique_id") or "", edge.get("child_unique_id") or ""
|
|
245
|
-
if child.startswith(SQL_CHILDREN) and parent.startswith(SQL_PARENTS):
|
|
246
|
-
parents.setdefault(child, []).append(parent)
|
|
247
|
-
nodes: dict[str, dict] = {}
|
|
248
|
-
for table, kind in NODE_TABLES.items():
|
|
249
|
-
for row in rows(table, NODE_COLUMNS):
|
|
250
|
-
uid = row.get("unique_id") or ""
|
|
251
|
-
if not uid or str(row.get("enabled", "True")).lower() == "false":
|
|
252
|
-
continue
|
|
253
|
-
nodes[uid] = {
|
|
254
|
-
"resource_type": row.get("resource_type") or kind,
|
|
255
|
-
"name": row.get("name") or uid,
|
|
256
|
-
"alias": row.get("alias"),
|
|
257
|
-
"compiled_code": row.get("compiled_code") or "",
|
|
258
|
-
"compiled_path": row.get("compiled_path") or "",
|
|
259
|
-
"raw_code": row.get("raw_code") or "",
|
|
260
|
-
"database": row.get("database_name"),
|
|
261
|
-
"schema": row.get("schema_name"),
|
|
262
|
-
"relation_name": row.get("relation_name"),
|
|
263
|
-
"original_file_path": row.get("original_file_path") or "",
|
|
264
|
-
"package_name": row.get("package_name"),
|
|
265
|
-
"depends_on": {"nodes": parents.get(uid, [])},
|
|
266
|
-
}
|
|
267
|
-
sources: dict[str, dict] = {}
|
|
268
|
-
for row in rows("sources", NODE_COLUMNS):
|
|
269
|
-
uid = row.get("unique_id") or ""
|
|
270
|
-
source_name, table_name = _source_parts(uid, row.get("package_name"), row.get("name"))
|
|
271
|
-
if not source_name:
|
|
272
|
-
continue
|
|
273
|
-
sources[uid] = {
|
|
274
|
-
"source_name": source_name,
|
|
275
|
-
"name": table_name,
|
|
276
|
-
"database": row.get("database_name"),
|
|
277
|
-
"schema": row.get("schema_name"),
|
|
278
|
-
"identifier": row.get("identifier"),
|
|
279
|
-
"relation_name": row.get("relation_name"),
|
|
280
|
-
}
|
|
281
|
-
except (
|
|
282
|
-
Exception
|
|
283
|
-
) as e: # pyarrow raises its own hierarchy; a broken artifact is a note, not a crash
|
|
284
|
-
return None, f"target/info_schema could not be read ({e}); reading raw model SQL instead."
|
|
285
|
-
metadata = {
|
|
286
|
-
"adapter_type": project.get("adapter_type"),
|
|
287
|
-
"dbt_version": project.get("dbt_version"),
|
|
288
|
-
"dbt_schema_version": "https://schemas.getdbt.com/dbt/info_schema/v1",
|
|
289
|
-
"artifact": "info_schema",
|
|
290
|
-
}
|
|
291
|
-
return {"metadata": metadata, "nodes": nodes, "sources": sources}, None
|
|
292
|
-
|
|
293
|
-
|
|
294
136
|
def _find_manifest(root: Path) -> dict | None:
|
|
295
137
|
candidate = root / "target" / "manifest.json"
|
|
296
138
|
if not candidate.exists():
|
|
@@ -364,12 +206,15 @@ def _compiled_on_disk(root: Path, node: dict, inputs_mtime: float) -> str:
|
|
|
364
206
|
def _load_from_manifest(root: Path, manifest: dict, dialect: str | None) -> Project:
|
|
365
207
|
metadata = manifest.get("metadata", {})
|
|
366
208
|
adapter = (metadata.get("adapter_type") or "").lower()
|
|
367
|
-
resolved_dialect = dialect or ADAPTER_TO_DIALECT.get(adapter,
|
|
209
|
+
resolved_dialect = dialect or ADAPTER_TO_DIALECT.get(adapter, DEFAULT_DIALECT)
|
|
368
210
|
warnings: list[str] = []
|
|
369
|
-
|
|
211
|
+
assumed = not dialect and adapter not in ADAPTER_TO_DIALECT
|
|
212
|
+
if assumed:
|
|
370
213
|
warnings.append(
|
|
371
|
-
f"Unknown adapter '{adapter}', assuming
|
|
214
|
+
f"Unknown adapter '{adapter}', assuming {DEFAULT_DIALECT}. Pass --dialect to override."
|
|
372
215
|
)
|
|
216
|
+
source = {"kind": "flag", "platform": dialect} if dialect else {"kind": "manifest"}
|
|
217
|
+
source.setdefault("platform", adapter or None)
|
|
373
218
|
|
|
374
219
|
models: list[Model] = []
|
|
375
220
|
inputs_mtime: float | None = None # the macro walk runs once, and only when a row needs it
|
|
@@ -436,6 +281,8 @@ def _load_from_manifest(root: Path, manifest: dict, dialect: str | None) -> Proj
|
|
|
436
281
|
models=models,
|
|
437
282
|
sources=sources,
|
|
438
283
|
warnings=warnings,
|
|
284
|
+
dialect_assumed=assumed,
|
|
285
|
+
dialect_source=source,
|
|
439
286
|
jinja_vars=_project_vars(root),
|
|
440
287
|
)
|
|
441
288
|
|
|
@@ -453,9 +300,11 @@ def _load_dbt_raw(root: Path, dialect: str | None) -> Project:
|
|
|
453
300
|
sample = "\n".join(p.read_text(errors="replace", encoding="utf-8")[:4000] for _, p in pairs)[
|
|
454
301
|
:200000
|
|
455
302
|
]
|
|
456
|
-
resolved_dialect, dialect_note =
|
|
303
|
+
resolved_dialect, dialect_note, dialect_source = _resolve_with_source(root, dialect, sample)
|
|
457
304
|
if dialect_note:
|
|
458
305
|
warnings.append(dialect_note)
|
|
306
|
+
if dialect_source.get("note"):
|
|
307
|
+
warnings.append(dialect_source["note"])
|
|
459
308
|
project_vars = _project_vars(root)
|
|
460
309
|
enabled_rules = _enabled_overrides(root, resolved_dialect, project_vars)
|
|
461
310
|
models: list[Model] = []
|
|
@@ -541,6 +390,7 @@ def _load_dbt_raw(root: Path, dialect: str | None) -> Project:
|
|
|
541
390
|
dialect=resolved_dialect,
|
|
542
391
|
mode="dbt-raw",
|
|
543
392
|
dialect_assumed=dialect_note is not None,
|
|
393
|
+
dialect_source=dialect_source,
|
|
544
394
|
models=models,
|
|
545
395
|
sources=sources,
|
|
546
396
|
warnings=warnings,
|