parrant 0.17.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parrant-0.17.1/LICENSE +10 -0
- parrant-0.17.1/PKG-INFO +276 -0
- parrant-0.17.1/README.md +252 -0
- parrant-0.17.1/parrant/artifacts/adapter_mapping.py +98 -0
- parrant-0.17.1/parrant/artifacts/catalog.py +91 -0
- parrant-0.17.1/parrant/artifacts/exceptions.py +15 -0
- parrant-0.17.1/parrant/artifacts/manifest.py +445 -0
- parrant-0.17.1/parrant/artifacts/registry.py +718 -0
- parrant-0.17.1/parrant/cli/main.py +1116 -0
- parrant-0.17.1/parrant/lineage/backtest.py +518 -0
- parrant-0.17.1/parrant/lineage/changeset.py +766 -0
- parrant-0.17.1/parrant/lineage/ci.py +269 -0
- parrant-0.17.1/parrant/lineage/display/__init__.py +5 -0
- parrant-0.17.1/parrant/lineage/display/backtest.py +115 -0
- parrant-0.17.1/parrant/lineage/display/base.py +50 -0
- parrant-0.17.1/parrant/lineage/display/dot.py +135 -0
- parrant-0.17.1/parrant/lineage/display/html/__init__.py +0 -0
- parrant-0.17.1/parrant/lineage/display/html/explore.py +1256 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/base.css +198 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/components.css +332 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/explore.css +837 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/graph.css +676 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/impact.css +2307 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/layout.css +87 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/model-details.css +337 -0
- parrant-0.17.1/parrant/lineage/display/html/static/css/policy.css +402 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/app.js +25 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/config.js +62 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/dataProcessor.js +641 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/explore.js +679 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/graph.js +1101 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/impact.js +1009 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/interactions.js +481 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/model-details.js +174 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/policy.js +247 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/renderer.js +1870 -0
- parrant-0.17.1/parrant/lineage/display/html/static/js/utils.js +398 -0
- parrant-0.17.1/parrant/lineage/display/html/static/vendor/d3.v7.min.js +2 -0
- parrant-0.17.1/parrant/lineage/display/html/templates/graph.html +255 -0
- parrant-0.17.1/parrant/lineage/display/json.py +97 -0
- parrant-0.17.1/parrant/lineage/display/markdown.py +660 -0
- parrant-0.17.1/parrant/lineage/display/text.py +73 -0
- parrant-0.17.1/parrant/lineage/policy.py +1046 -0
- parrant-0.17.1/parrant/lineage/policy_init.py +405 -0
- parrant-0.17.1/parrant/lineage/provider.py +198 -0
- parrant-0.17.1/parrant/lineage/semantic_diff.py +154 -0
- parrant-0.17.1/parrant/lineage/service.py +986 -0
- parrant-0.17.1/parrant/lineage/sqlglot_provider.py +87 -0
- parrant-0.17.1/parrant/lineage/verdict.py +366 -0
- parrant-0.17.1/parrant/metabase/__init__.py +17 -0
- parrant-0.17.1/parrant/metabase/artifact.py +68 -0
- parrant-0.17.1/parrant/metabase/cli.py +130 -0
- parrant-0.17.1/parrant/metabase/client.py +212 -0
- parrant-0.17.1/parrant/metabase/extract.py +194 -0
- parrant-0.17.1/parrant/metabase/join.py +77 -0
- parrant-0.17.1/parrant/metabase/reach.py +247 -0
- parrant-0.17.1/parrant/metabase/resolvers.py +463 -0
- parrant-0.17.1/parrant/metabase/warehouse_meta.py +158 -0
- parrant-0.17.1/parrant/models/__init__.py +0 -0
- parrant-0.17.1/parrant/models/schema.py +846 -0
- parrant-0.17.1/parrant/parser/__init__.py +3 -0
- parrant-0.17.1/parrant/parser/sql_parser.py +1005 -0
- parrant-0.17.1/parrant/parser/sql_parser_utils.py +125 -0
- parrant-0.17.1/pyproject.toml +92 -0
parrant-0.17.1/LICENSE
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright 2025 Fszta
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
10
|
+
|
parrant-0.17.1/PKG-INFO
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: parrant
|
|
3
|
+
Version: 0.17.1
|
|
4
|
+
Summary: Parrant — the offline pre-merge column-impact gate for dbt: parry the breaking changes, warrant the safe ones
|
|
5
|
+
License: MIT
|
|
6
|
+
Author: Fszta
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Requires-Dist: click (>=8.1.0,<9.0.0)
|
|
15
|
+
Requires-Dist: fastapi (>=0.116.0,<1.0.0)
|
|
16
|
+
Requires-Dist: graphviz (>=0.20.3,<0.21.0)
|
|
17
|
+
Requires-Dist: pydantic (>=2.10.6,<3.0.0)
|
|
18
|
+
Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
|
|
19
|
+
Requires-Dist: requests (>=2.32.0,<3.0.0)
|
|
20
|
+
Requires-Dist: sqlglot (>=26.8.0,<27.0.0)
|
|
21
|
+
Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
<p align="center">
|
|
25
|
+
<img src="assets/parrant-logo.svg" alt="Parrant" width="96" height="96">
|
|
26
|
+
</p>
|
|
27
|
+
|
|
28
|
+
<h1 align="center">Parrant</h1>
|
|
29
|
+
|
|
30
|
+
<p align="center">
|
|
31
|
+
<strong>Parry the breaking changes, warrant the safe ones.</strong><br>
|
|
32
|
+
Column-level lineage and change-impact analysis for dbt — answer
|
|
33
|
+
<em>"what breaks if I change this column?"</em> in seconds, without ever running your warehouse.
|
|
34
|
+
</p>
|
|
35
|
+
|
|
36
|
+
> **Formerly `dbt-col-lineage`.** Same tool, new name. `pip install parrant` (the old
|
|
37
|
+
> `dbt-col-lineage` package and the `dbt-col-lineage` command still work for now).
|
|
38
|
+
|
|
39
|
+
[](https://github.com/Fszta/parrant/actions/workflows/test.yml)
|
|
40
|
+
[](https://pypi.org/project/parrant/)
|
|
41
|
+
[](https://pypi.org/project/parrant/)
|
|
42
|
+
[](https://pypi.org/project/parrant/)
|
|
43
|
+
[](LICENSE)
|
|
44
|
+
|
|
45
|
+
📖 **[Documentation](https://fszta.github.io/parrant/)** · 🚀 [Live Demo](https://dbt-column-lineage.onrender.com) · 🐛 [Report Bug](https://github.com/Fszta/parrant/issues) · 💡 [Request Feature](https://github.com/Fszta/parrant/issues)
|
|
46
|
+
|
|
47
|
+
<!-- TODO(rebrand): rename the Render demo service so the URL becomes parrant.onrender.com (infra step, not code) -->
|
|
48
|
+

|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## "I already have lineage in dbt docs."
|
|
53
|
+
|
|
54
|
+
You have *model*-level lineage. When you rename, retype, or drop a single **column**,
|
|
55
|
+
the dbt DAG can't tell you which downstream columns, transformations, or dashboards
|
|
56
|
+
actually break — it only knows model A feeds model B. That's the gap this closes.
|
|
57
|
+
|
|
58
|
+
| | **dbt docs / DAG** | **Parrant** |
|
|
59
|
+
|---|---|---|
|
|
60
|
+
| Lineage granularity | Model → model | **Column → column** |
|
|
61
|
+
| *"What breaks if I change `orders.amount`?"* | Guess from the model graph | **Exact affected columns, models & exposures** |
|
|
62
|
+
| Pass-through vs. real logic | ❌ | ✅ flags columns whose SQL actually recomputes the value |
|
|
63
|
+
| Needs a warehouse connection | `dbt docs serve` | **No — reads artifacts, runs anywhere** |
|
|
64
|
+
| Blast-radius check in CI | ❌ | ✅ sticky PR comment + severity gate |
|
|
65
|
+
| Machine-readable for AI agents | ❌ | ✅ one JSON document, built for automation |
|
|
66
|
+
|
|
67
|
+
It reads only your dbt artifacts (`manifest.json` + `catalog.json`) and parses the
|
|
68
|
+
compiled SQL statically with [sqlglot](https://github.com/tobymao/sqlglot).
|
|
69
|
+
**It never connects to your warehouse and never runs dbt models.**
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Comment the blast radius on every dbt PR
|
|
74
|
+
|
|
75
|
+
Point it at CI and it posts a **sticky comment on the pull request** showing exactly
|
|
76
|
+
what a change breaks — the downstream models, columns, and business-facing **exposures**
|
|
77
|
+
(dashboards, apps) it touches — so every reviewer sees the impact before they approve.
|
|
78
|
+
Optionally **fail the check** when a change is too risky.
|
|
79
|
+
|
|
80
|
+

|
|
81
|
+
|
|
82
|
+
Add it as a GitHub Action:
|
|
83
|
+
|
|
84
|
+
```yaml
|
|
85
|
+
# .github/workflows/impact.yml
|
|
86
|
+
permissions:
|
|
87
|
+
pull-requests: write # so the check can post its comment
|
|
88
|
+
|
|
89
|
+
# ...build base- and head-branch dbt artifacts, then:
|
|
90
|
+
- uses: Fszta/parrant@v0
|
|
91
|
+
with:
|
|
92
|
+
manifest: artifacts/head/manifest.json
|
|
93
|
+
catalog: artifacts/head/catalog.json
|
|
94
|
+
base-manifest: artifacts/base/manifest.json
|
|
95
|
+
base-catalog: artifacts/base/catalog.json
|
|
96
|
+
fail-on: none # start non-blocking; flip to exposures|critical once trusted
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
**How the loop works:** a PR opens → CI builds dbt artifacts for the base and PR
|
|
100
|
+
branches → the action diffs them, traces every affected column, model, and exposure,
|
|
101
|
+
and posts **one** sticky comment (found-and-updated via a hidden marker, so re-runs
|
|
102
|
+
edit the same comment instead of spamming the thread).
|
|
103
|
+
|
|
104
|
+
**The severity gate (`fail-on`)** decides when an impactful change should block the PR:
|
|
105
|
+
|
|
106
|
+
| `fail-on` | Blocks the PR when… |
|
|
107
|
+
|-------------|---------------------|
|
|
108
|
+
| `none` | never — comment only (**default**, the safe on-ramp) |
|
|
109
|
+
| `tests` | a change **provably breaks a dbt test** (removes/renames a column a `not_null`/`unique`/`relationships` test still targets) — the objective, false-positive-free level to block on (needs `base-manifest`) |
|
|
110
|
+
| `exposures` | a change reaches a business-facing **exposure** (dashboard / app) |
|
|
111
|
+
| `critical` | a downstream column **recomputes derived logic** (not just a pass-through) |
|
|
112
|
+
| `any` | any downstream column is affected at all |
|
|
113
|
+
|
|
114
|
+
The action also emits step outputs for your own gating/reporting: `affected_models`,
|
|
115
|
+
`affected_columns`, `affected_exposures`, `provable_breaks`, `verdict` (`safe`/`review`/`block`),
|
|
116
|
+
`tripped_level`, and `overrides_applied`.
|
|
117
|
+
|
|
118
|
+
**Escape hatch, not off-switch.** When the gate flags a change the author knows is fine, an
|
|
119
|
+
in-code **override pragma** in the head model acknowledges *that one change* with a mandatory
|
|
120
|
+
reason — `-- lineage:allow-change reason="…"`, or `-- lineage:allow-break reason="…"` for the
|
|
121
|
+
one thing that can lower a provable-break block. It lives in the PR's SQL (diffable, reviewed,
|
|
122
|
+
logged), so the first false positive tunes the gate instead of disarming it repo-wide. Every
|
|
123
|
+
honored override is surfaced in the PR comment and counted in `overrides_applied`; run
|
|
124
|
+
`parrant impact --no-overrides` (or set the action's `no-overrides: true`) to see the raw
|
|
125
|
+
gate. See the [override guide](https://github.com/Fszta/parrant/blob/main/docs/decision-engine/policy-gate.md#overriding-a-verdict-the-in-code-escape-hatch).
|
|
126
|
+
|
|
127
|
+
Pin `@v0` for updates within the current major (like `actions/checkout@v4`), or an
|
|
128
|
+
exact release — `@v0.17.0` — for reproducible builds. The action installs the CLI
|
|
129
|
+
bundled at whichever ref you pin, so the tool always matches the tag. A complete
|
|
130
|
+
runnable workflow lives at
|
|
131
|
+
[`docs/examples/impact-pr-check.yml`](docs/examples/impact-pr-check.yml).
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Quick start (local)
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
pip install parrant
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Generate your dbt artifacts once — this is the only step that touches dbt, and it
|
|
142
|
+
still never connects to your warehouse:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
dbt compile # produces target/manifest.json
|
|
146
|
+
dbt docs generate # produces target/catalog.json (column metadata)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Then explore your column lineage in the browser — no flags needed, it reads `target/`
|
|
150
|
+
by default:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
parrant --explore
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Open `http://127.0.0.1:8000`, pick a column, and click **Analyze Impact** to see the
|
|
157
|
+
columns that need review, the pass-through columns, and the affected models and
|
|
158
|
+
exposures. **[Try the live demo →](https://dbt-column-lineage.onrender.com)** — no
|
|
159
|
+
install required.
|
|
160
|
+
|
|
161
|
+

|
|
162
|
+
|
|
163
|
+
> Works even when `manifest.json` has no embedded `compiled_code` (e.g. from
|
|
164
|
+
> `dbt parse`), as long as `target/compiled/**` exists — it falls back to the
|
|
165
|
+
> compiled SQL on disk.
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## Machine-readable output (built for agents & automation)
|
|
170
|
+
|
|
171
|
+
Emit any column's lineage and downstream impact as a **single JSON document** — a
|
|
172
|
+
stable contract you can pipe into an LLM tool call, a CI script, or your own tooling:
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
parrant --select stg_accounts.account_id+ --format json \
|
|
176
|
+
--manifest target/manifest.json --catalog target/catalog.json
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
Selector grammar (works for `text`, `json`, and `dot` output):
|
|
180
|
+
|
|
181
|
+
| Selector | Meaning |
|
|
182
|
+
|---|---|
|
|
183
|
+
| `+model.col` | upstream only (where the value comes from) |
|
|
184
|
+
| `model.col+` | downstream only (what it feeds) |
|
|
185
|
+
| `model.col` | both directions |
|
|
186
|
+
|
|
187
|
+
The JSON splits `upstream`/`downstream` into `models`, `sources`, `direct_refs`, and
|
|
188
|
+
`exposures`, plus an `impact` block summarising the affected models, columns, and
|
|
189
|
+
exposures. Use `--format dot` for Graphviz.
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## Run the impact report locally
|
|
194
|
+
|
|
195
|
+
The `impact` command derives the *set* of changed columns for a branch and reports one
|
|
196
|
+
consolidated blast radius, ranked by severity:
|
|
197
|
+
**`removed` > `type_changed` > `logic_changed` > `added`**.
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
# Reliable two-manifest diff (base branch vs. current)
|
|
201
|
+
parrant impact \
|
|
202
|
+
--manifest target/manifest.json --catalog target/catalog.json \
|
|
203
|
+
--base-manifest base/manifest.json --base-catalog base/catalog.json
|
|
204
|
+
|
|
205
|
+
# Git-diff fallback when only one manifest is available
|
|
206
|
+
parrant impact --git-base main
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
It defaults to a human-readable Markdown summary (exposures first, then a blast-radius
|
|
210
|
+
table); add `--format json` for the machine-readable report. Add `--ci` to post the
|
|
211
|
+
sticky PR comment and apply the `--fail-on` gate.
|
|
212
|
+
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## Beyond the blast radius: a decision engine
|
|
216
|
+
|
|
217
|
+
The impact report is the foundation; on top of it the tool now turns a PR into a **decision**,
|
|
218
|
+
on the principle *"diff cheaply, rebuild selectively."* All of this is additive — skip the flags
|
|
219
|
+
and the tool behaves exactly as before.
|
|
220
|
+
|
|
221
|
+
- **Semantic categorization** — every changed column is labelled breaking vs **provably
|
|
222
|
+
cosmetic**, so a refactor that doesn't change any value doesn't get flagged.
|
|
223
|
+
- **A metadata-agnostic policy gate** — you author rules (`predicate → block/warn/build/test/notify`)
|
|
224
|
+
over *any* dbt `meta`, the change kind, the semantic signal, and the lineage reach. The tool
|
|
225
|
+
ships the engine; you ship the rules. `critical` / `pii` are example configs, never built-ins.
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
parrant impact --base-manifest base/manifest.json --base-catalog base/catalog.json \
|
|
229
|
+
--policy policy.yml --fail-on policy
|
|
230
|
+
```
|
|
231
|
+
- **Scaffold a starter policy, don't start from a blank file** — `parrant policy init`
|
|
232
|
+
reads your manifest + catalog and writes a heavily-commented, safe-by-construction
|
|
233
|
+
`parrant.policy.yml` keyed only to signals the scan confirmed exist: it enables the
|
|
234
|
+
provable-break block when column-targeted tests are found and an exposure guard when exposures
|
|
235
|
+
are found, and offers every dbt-`meta` key you actually use as a commented template prefixed with
|
|
236
|
+
its real coverage. The result runs green on day one — no rage-block.
|
|
237
|
+
|
|
238
|
+
```bash
|
|
239
|
+
parrant policy init --manifest target/manifest.json --catalog target/catalog.json
|
|
240
|
+
```
|
|
241
|
+
- **Backtest a policy before you arm it** — `parrant policy test` replays a candidate
|
|
242
|
+
policy over your recent git history (`--last 30`) or a saved changeset corpus and reports, per
|
|
243
|
+
rule, what the gate *would* have ruled — including how many firings were driven by a fail-safe
|
|
244
|
+
UNKNOWN rather than a proven match, and which rules never fired at all. Offline and deterministic.
|
|
245
|
+
|
|
246
|
+
```bash
|
|
247
|
+
parrant policy test --policy policy.yml --last 30
|
|
248
|
+
```
|
|
249
|
+
- **Cross-boundary (Metabase) impact** — a separate credentialed `metabase-extract` step snapshots
|
|
250
|
+
Metabase into `metabase_lineage.json`; the offline gate then answers *"will this column change
|
|
251
|
+
break that dashboard?"* by folding dashboards into the same reach the policy engine scans.
|
|
252
|
+
|
|
253
|
+
Full guides: **[Decision Engine docs](https://fszta.github.io/parrant/decision-engine/overview/)**.
|
|
254
|
+
|
|
255
|
+
## Compatibility
|
|
256
|
+
|
|
257
|
+
Works with **any [sqlglot](https://github.com/tobymao/sqlglot) dialect** via `--adapter`
|
|
258
|
+
(auto-detected from your manifest by default). Verified against **Snowflake**,
|
|
259
|
+
**DuckDB**, **SQLite**, and **MS SQL Server / TSQL**; on BigQuery, Redshift, Postgres,
|
|
260
|
+
etc., pass `--adapter <dialect>` if auto-detection needs a nudge.
|
|
261
|
+
|
|
262
|
+
## Limitations
|
|
263
|
+
|
|
264
|
+
- Python models are not supported.
|
|
265
|
+
- Some SQL functions/syntax can't be parsed and cause the affected model to be skipped.
|
|
266
|
+
|
|
267
|
+
## Documentation
|
|
268
|
+
|
|
269
|
+
Full CLI reference — every flag (`--scope-git`, `--github-token`/`--repo`/`--pr-number`,
|
|
270
|
+
the complete `impact` surface), output formats, and CI recipes — lives at
|
|
271
|
+
**[fszta.github.io/parrant](https://fszta.github.io/parrant/)**.
|
|
272
|
+
|
|
273
|
+
## License
|
|
274
|
+
|
|
275
|
+
MIT
|
|
276
|
+
|
parrant-0.17.1/README.md
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="assets/parrant-logo.svg" alt="Parrant" width="96" height="96">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">Parrant</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<strong>Parry the breaking changes, warrant the safe ones.</strong><br>
|
|
9
|
+
Column-level lineage and change-impact analysis for dbt — answer
|
|
10
|
+
<em>"what breaks if I change this column?"</em> in seconds, without ever running your warehouse.
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
> **Formerly `dbt-col-lineage`.** Same tool, new name. `pip install parrant` (the old
|
|
14
|
+
> `dbt-col-lineage` package and the `dbt-col-lineage` command still work for now).
|
|
15
|
+
|
|
16
|
+
[](https://github.com/Fszta/parrant/actions/workflows/test.yml)
|
|
17
|
+
[](https://pypi.org/project/parrant/)
|
|
18
|
+
[](https://pypi.org/project/parrant/)
|
|
19
|
+
[](https://pypi.org/project/parrant/)
|
|
20
|
+
[](LICENSE)
|
|
21
|
+
|
|
22
|
+
📖 **[Documentation](https://fszta.github.io/parrant/)** · 🚀 [Live Demo](https://dbt-column-lineage.onrender.com) · 🐛 [Report Bug](https://github.com/Fszta/parrant/issues) · 💡 [Request Feature](https://github.com/Fszta/parrant/issues)
|
|
23
|
+
|
|
24
|
+
<!-- TODO(rebrand): rename the Render demo service so the URL becomes parrant.onrender.com (infra step, not code) -->
|
|
25
|
+

|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## "I already have lineage in dbt docs."
|
|
30
|
+
|
|
31
|
+
You have *model*-level lineage. When you rename, retype, or drop a single **column**,
|
|
32
|
+
the dbt DAG can't tell you which downstream columns, transformations, or dashboards
|
|
33
|
+
actually break — it only knows model A feeds model B. That's the gap this closes.
|
|
34
|
+
|
|
35
|
+
| | **dbt docs / DAG** | **Parrant** |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| Lineage granularity | Model → model | **Column → column** |
|
|
38
|
+
| *"What breaks if I change `orders.amount`?"* | Guess from the model graph | **Exact affected columns, models & exposures** |
|
|
39
|
+
| Pass-through vs. real logic | ❌ | ✅ flags columns whose SQL actually recomputes the value |
|
|
40
|
+
| Needs a warehouse connection | `dbt docs serve` | **No — reads artifacts, runs anywhere** |
|
|
41
|
+
| Blast-radius check in CI | ❌ | ✅ sticky PR comment + severity gate |
|
|
42
|
+
| Machine-readable for AI agents | ❌ | ✅ one JSON document, built for automation |
|
|
43
|
+
|
|
44
|
+
It reads only your dbt artifacts (`manifest.json` + `catalog.json`) and parses the
|
|
45
|
+
compiled SQL statically with [sqlglot](https://github.com/tobymao/sqlglot).
|
|
46
|
+
**It never connects to your warehouse and never runs dbt models.**
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## Comment the blast radius on every dbt PR
|
|
51
|
+
|
|
52
|
+
Point it at CI and it posts a **sticky comment on the pull request** showing exactly
|
|
53
|
+
what a change breaks — the downstream models, columns, and business-facing **exposures**
|
|
54
|
+
(dashboards, apps) it touches — so every reviewer sees the impact before they approve.
|
|
55
|
+
Optionally **fail the check** when a change is too risky.
|
|
56
|
+
|
|
57
|
+

|
|
58
|
+
|
|
59
|
+
Add it as a GitHub Action:
|
|
60
|
+
|
|
61
|
+
```yaml
|
|
62
|
+
# .github/workflows/impact.yml
|
|
63
|
+
permissions:
|
|
64
|
+
pull-requests: write # so the check can post its comment
|
|
65
|
+
|
|
66
|
+
# ...build base- and head-branch dbt artifacts, then:
|
|
67
|
+
- uses: Fszta/parrant@v0
|
|
68
|
+
with:
|
|
69
|
+
manifest: artifacts/head/manifest.json
|
|
70
|
+
catalog: artifacts/head/catalog.json
|
|
71
|
+
base-manifest: artifacts/base/manifest.json
|
|
72
|
+
base-catalog: artifacts/base/catalog.json
|
|
73
|
+
fail-on: none # start non-blocking; flip to exposures|critical once trusted
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
**How the loop works:** a PR opens → CI builds dbt artifacts for the base and PR
|
|
77
|
+
branches → the action diffs them, traces every affected column, model, and exposure,
|
|
78
|
+
and posts **one** sticky comment (found-and-updated via a hidden marker, so re-runs
|
|
79
|
+
edit the same comment instead of spamming the thread).
|
|
80
|
+
|
|
81
|
+
**The severity gate (`fail-on`)** decides when an impactful change should block the PR:
|
|
82
|
+
|
|
83
|
+
| `fail-on` | Blocks the PR when… |
|
|
84
|
+
|-------------|---------------------|
|
|
85
|
+
| `none` | never — comment only (**default**, the safe on-ramp) |
|
|
86
|
+
| `tests` | a change **provably breaks a dbt test** (removes/renames a column a `not_null`/`unique`/`relationships` test still targets) — the objective, false-positive-free level to block on (needs `base-manifest`) |
|
|
87
|
+
| `exposures` | a change reaches a business-facing **exposure** (dashboard / app) |
|
|
88
|
+
| `critical` | a downstream column **recomputes derived logic** (not just a pass-through) |
|
|
89
|
+
| `any` | any downstream column is affected at all |
|
|
90
|
+
|
|
91
|
+
The action also emits step outputs for your own gating/reporting: `affected_models`,
|
|
92
|
+
`affected_columns`, `affected_exposures`, `provable_breaks`, `verdict` (`safe`/`review`/`block`),
|
|
93
|
+
`tripped_level`, and `overrides_applied`.
|
|
94
|
+
|
|
95
|
+
**Escape hatch, not off-switch.** When the gate flags a change the author knows is fine, an
|
|
96
|
+
in-code **override pragma** in the head model acknowledges *that one change* with a mandatory
|
|
97
|
+
reason — `-- lineage:allow-change reason="…"`, or `-- lineage:allow-break reason="…"` for the
|
|
98
|
+
one thing that can lower a provable-break block. It lives in the PR's SQL (diffable, reviewed,
|
|
99
|
+
logged), so the first false positive tunes the gate instead of disarming it repo-wide. Every
|
|
100
|
+
honored override is surfaced in the PR comment and counted in `overrides_applied`; run
|
|
101
|
+
`parrant impact --no-overrides` (or set the action's `no-overrides: true`) to see the raw
|
|
102
|
+
gate. See the [override guide](https://github.com/Fszta/parrant/blob/main/docs/decision-engine/policy-gate.md#overriding-a-verdict-the-in-code-escape-hatch).
|
|
103
|
+
|
|
104
|
+
Pin `@v0` for updates within the current major (like `actions/checkout@v4`), or an
|
|
105
|
+
exact release — `@v0.17.0` — for reproducible builds. The action installs the CLI
|
|
106
|
+
bundled at whichever ref you pin, so the tool always matches the tag. A complete
|
|
107
|
+
runnable workflow lives at
|
|
108
|
+
[`docs/examples/impact-pr-check.yml`](docs/examples/impact-pr-check.yml).
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## Quick start (local)
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
pip install parrant
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Generate your dbt artifacts once — this is the only step that touches dbt, and it
|
|
119
|
+
still never connects to your warehouse:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
dbt compile # produces target/manifest.json
|
|
123
|
+
dbt docs generate # produces target/catalog.json (column metadata)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Then explore your column lineage in the browser — no flags needed, it reads `target/`
|
|
127
|
+
by default:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
parrant --explore
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Open `http://127.0.0.1:8000`, pick a column, and click **Analyze Impact** to see the
|
|
134
|
+
columns that need review, the pass-through columns, and the affected models and
|
|
135
|
+
exposures. **[Try the live demo →](https://dbt-column-lineage.onrender.com)** — no
|
|
136
|
+
install required.
|
|
137
|
+
|
|
138
|
+

|
|
139
|
+
|
|
140
|
+
> Works even when `manifest.json` has no embedded `compiled_code` (e.g. from
|
|
141
|
+
> `dbt parse`), as long as `target/compiled/**` exists — it falls back to the
|
|
142
|
+
> compiled SQL on disk.
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
## Machine-readable output (built for agents & automation)
|
|
147
|
+
|
|
148
|
+
Emit any column's lineage and downstream impact as a **single JSON document** — a
|
|
149
|
+
stable contract you can pipe into an LLM tool call, a CI script, or your own tooling:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
parrant --select stg_accounts.account_id+ --format json \
|
|
153
|
+
--manifest target/manifest.json --catalog target/catalog.json
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Selector grammar (works for `text`, `json`, and `dot` output):
|
|
157
|
+
|
|
158
|
+
| Selector | Meaning |
|
|
159
|
+
|---|---|
|
|
160
|
+
| `+model.col` | upstream only (where the value comes from) |
|
|
161
|
+
| `model.col+` | downstream only (what it feeds) |
|
|
162
|
+
| `model.col` | both directions |
|
|
163
|
+
|
|
164
|
+
The JSON splits `upstream`/`downstream` into `models`, `sources`, `direct_refs`, and
|
|
165
|
+
`exposures`, plus an `impact` block summarising the affected models, columns, and
|
|
166
|
+
exposures. Use `--format dot` for Graphviz.
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
## Run the impact report locally
|
|
171
|
+
|
|
172
|
+
The `impact` command derives the *set* of changed columns for a branch and reports one
|
|
173
|
+
consolidated blast radius, ranked by severity:
|
|
174
|
+
**`removed` > `type_changed` > `logic_changed` > `added`**.
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
# Reliable two-manifest diff (base branch vs. current)
|
|
178
|
+
parrant impact \
|
|
179
|
+
--manifest target/manifest.json --catalog target/catalog.json \
|
|
180
|
+
--base-manifest base/manifest.json --base-catalog base/catalog.json
|
|
181
|
+
|
|
182
|
+
# Git-diff fallback when only one manifest is available
|
|
183
|
+
parrant impact --git-base main
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
It defaults to a human-readable Markdown summary (exposures first, then a blast-radius
|
|
187
|
+
table); add `--format json` for the machine-readable report. Add `--ci` to post the
|
|
188
|
+
sticky PR comment and apply the `--fail-on` gate.
|
|
189
|
+
|
|
190
|
+
---
|
|
191
|
+
|
|
192
|
+
## Beyond the blast radius: a decision engine
|
|
193
|
+
|
|
194
|
+
The impact report is the foundation; on top of it the tool now turns a PR into a **decision**,
|
|
195
|
+
on the principle *"diff cheaply, rebuild selectively."* All of this is additive — skip the flags
|
|
196
|
+
and the tool behaves exactly as before.
|
|
197
|
+
|
|
198
|
+
- **Semantic categorization** — every changed column is labelled breaking vs **provably
|
|
199
|
+
cosmetic**, so a refactor that doesn't change any value doesn't get flagged.
|
|
200
|
+
- **A metadata-agnostic policy gate** — you author rules (`predicate → block/warn/build/test/notify`)
|
|
201
|
+
over *any* dbt `meta`, the change kind, the semantic signal, and the lineage reach. The tool
|
|
202
|
+
ships the engine; you ship the rules. `critical` / `pii` are example configs, never built-ins.
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
parrant impact --base-manifest base/manifest.json --base-catalog base/catalog.json \
|
|
206
|
+
--policy policy.yml --fail-on policy
|
|
207
|
+
```
|
|
208
|
+
- **Scaffold a starter policy, don't start from a blank file** — `parrant policy init`
|
|
209
|
+
reads your manifest + catalog and writes a heavily-commented, safe-by-construction
|
|
210
|
+
`parrant.policy.yml` keyed only to signals the scan confirmed exist: it enables the
|
|
211
|
+
provable-break block when column-targeted tests are found and an exposure guard when exposures
|
|
212
|
+
are found, and offers every dbt-`meta` key you actually use as a commented template prefixed with
|
|
213
|
+
its real coverage. The result runs green on day one — no rage-block.
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
parrant policy init --manifest target/manifest.json --catalog target/catalog.json
|
|
217
|
+
```
|
|
218
|
+
- **Backtest a policy before you arm it** — `parrant policy test` replays a candidate
|
|
219
|
+
policy over your recent git history (`--last 30`) or a saved changeset corpus and reports, per
|
|
220
|
+
rule, what the gate *would* have ruled — including how many firings were driven by a fail-safe
|
|
221
|
+
UNKNOWN rather than a proven match, and which rules never fired at all. Offline and deterministic.
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
parrant policy test --policy policy.yml --last 30
|
|
225
|
+
```
|
|
226
|
+
- **Cross-boundary (Metabase) impact** — a separate credentialed `metabase-extract` step snapshots
|
|
227
|
+
Metabase into `metabase_lineage.json`; the offline gate then answers *"will this column change
|
|
228
|
+
break that dashboard?"* by folding dashboards into the same reach the policy engine scans.
|
|
229
|
+
|
|
230
|
+
Full guides: **[Decision Engine docs](https://fszta.github.io/parrant/decision-engine/overview/)**.
|
|
231
|
+
|
|
232
|
+
## Compatibility
|
|
233
|
+
|
|
234
|
+
Works with **any [sqlglot](https://github.com/tobymao/sqlglot) dialect** via `--adapter`
|
|
235
|
+
(auto-detected from your manifest by default). Verified against **Snowflake**,
|
|
236
|
+
**DuckDB**, **SQLite**, and **MS SQL Server / TSQL**; on BigQuery, Redshift, Postgres,
|
|
237
|
+
etc., pass `--adapter <dialect>` if auto-detection needs a nudge.
|
|
238
|
+
|
|
239
|
+
## Limitations
|
|
240
|
+
|
|
241
|
+
- Python models are not supported.
|
|
242
|
+
- Some SQL functions/syntax can't be parsed and cause the affected model to be skipped.
|
|
243
|
+
|
|
244
|
+
## Documentation
|
|
245
|
+
|
|
246
|
+
Full CLI reference — every flag (`--scope-git`, `--github-token`/`--repo`/`--pr-number`,
|
|
247
|
+
the complete `impact` surface), output formats, and CI recipes — lives at
|
|
248
|
+
**[fszta.github.io/parrant](https://fszta.github.io/parrant/)**.
|
|
249
|
+
|
|
250
|
+
## License
|
|
251
|
+
|
|
252
|
+
MIT
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Centralized adapter/dialect normalization for parrant.
|
|
3
|
+
|
|
4
|
+
This module defines a single source of truth for mapping raw dbt adapter
|
|
5
|
+
names to sqlglot dialect names. For example, dbt may report the adapter
|
|
6
|
+
"sqlserver" while sqlglot expects the dialect name "tsql".
|
|
7
|
+
|
|
8
|
+
Most common warehouse adapters (snowflake, bigquery, redshift, databricks,
|
|
9
|
+
postgres, duckdb, ...) already share their name with the corresponding
|
|
10
|
+
sqlglot dialect, so they resolve correctly without an explicit entry. The
|
|
11
|
+
mapping below covers the adapters whose dbt name differs from the sqlglot
|
|
12
|
+
dialect name, plus a few identity entries that pin well-supported adapters
|
|
13
|
+
to their verified dialect.
|
|
14
|
+
|
|
15
|
+
Extend ADAPTER_TO_DIALECT as needed to support additional adapters.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
from typing import Dict, Optional, Set
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
# Mapping from dbt adapter name (metadata.adapter_type) to sqlglot dialect.
|
|
24
|
+
#
|
|
25
|
+
# Only entries verified against sqlglot's supported dialects belong here. When
|
|
26
|
+
# the dbt adapter name already equals a valid sqlglot dialect (snowflake,
|
|
27
|
+
# bigquery, redshift, databricks, postgres, duckdb, spark, trino, presto,
|
|
28
|
+
# athena, clickhouse, ...) an explicit identity entry is optional -- the
|
|
29
|
+
# fallthrough in normalize_adapter returns the lowercased name unchanged.
|
|
30
|
+
ADAPTER_TO_DIALECT: Dict[str, str] = {
|
|
31
|
+
# --- adapters whose dbt name differs from the sqlglot dialect name ---
|
|
32
|
+
# The T-SQL family: dbt reports sqlserver/synapse/fabric, sqlglot uses "tsql".
|
|
33
|
+
"sqlserver": "tsql",
|
|
34
|
+
"synapse": "tsql",
|
|
35
|
+
"fabric": "tsql",
|
|
36
|
+
# --- identity pins for common, verified adapters (documented support) ---
|
|
37
|
+
"snowflake": "snowflake",
|
|
38
|
+
"bigquery": "bigquery",
|
|
39
|
+
"redshift": "redshift",
|
|
40
|
+
"databricks": "databricks",
|
|
41
|
+
"spark": "spark",
|
|
42
|
+
"trino": "trino",
|
|
43
|
+
"presto": "presto",
|
|
44
|
+
"athena": "athena",
|
|
45
|
+
"postgres": "postgres",
|
|
46
|
+
"duckdb": "duckdb",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _known_sqlglot_dialects() -> Set[str]:
|
|
51
|
+
"""Return the set of dialect names sqlglot actually supports.
|
|
52
|
+
|
|
53
|
+
Derived from sqlglot at runtime so the check stays correct as sqlglot is
|
|
54
|
+
upgraded. Falls back to an empty set if the internal enum is unavailable,
|
|
55
|
+
in which case the unknown-dialect warning is simply skipped.
|
|
56
|
+
"""
|
|
57
|
+
try:
|
|
58
|
+
from sqlglot.dialects.dialect import Dialects
|
|
59
|
+
|
|
60
|
+
return {member.value for member in Dialects if member.value}
|
|
61
|
+
except Exception: # pragma: no cover - defensive, sqlglot API drift
|
|
62
|
+
return set()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
_KNOWN_DIALECTS: Set[str] = _known_sqlglot_dialects()
|
|
66
|
+
|
|
67
|
+
# Track adapters we have already warned about so the warning fires once per
|
|
68
|
+
# unresolved adapter instead of on every column parsed.
|
|
69
|
+
_warned_adapters: Set[str] = set()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def normalize_adapter(adapter_name: Optional[str]) -> Optional[str]:
|
|
73
|
+
"""Normalize a dbt adapter name to a sqlglot dialect name.
|
|
74
|
+
|
|
75
|
+
If adapter_name is None or empty, returns it unchanged.
|
|
76
|
+
If there is no mapping defined, returns the lowercased adapter_name.
|
|
77
|
+
|
|
78
|
+
When the resolved dialect is not a dialect sqlglot recognizes, a one-time
|
|
79
|
+
WARNING is emitted per adapter so the user gets a visible signal that SQL
|
|
80
|
+
parsing will fall back to sqlglot's default behavior instead of silently
|
|
81
|
+
degrading.
|
|
82
|
+
"""
|
|
83
|
+
if not adapter_name:
|
|
84
|
+
return adapter_name
|
|
85
|
+
lower = adapter_name.lower()
|
|
86
|
+
dialect = ADAPTER_TO_DIALECT.get(lower, lower)
|
|
87
|
+
|
|
88
|
+
if _KNOWN_DIALECTS and dialect not in _KNOWN_DIALECTS and lower not in _warned_adapters:
|
|
89
|
+
_warned_adapters.add(lower)
|
|
90
|
+
logger.warning(
|
|
91
|
+
"dbt adapter '%s' resolved to dialect '%s', which is not a known "
|
|
92
|
+
"sqlglot dialect; SQL parsing may be less accurate. Consider adding "
|
|
93
|
+
"a mapping in adapter_mapping.ADAPTER_TO_DIALECT.",
|
|
94
|
+
adapter_name,
|
|
95
|
+
dialect,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
return dialect
|