datawright-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datawright.py +3839 -0
- datawright_cli-0.1.0.dist-info/METADATA +129 -0
- datawright_cli-0.1.0.dist-info/RECORD +17 -0
- datawright_cli-0.1.0.dist-info/WHEEL +4 -0
- datawright_cli-0.1.0.dist-info/entry_points.txt +4 -0
- datawright_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- datawright_kit/assets/bi_export.py +98 -0
- datawright_kit/assets/config.yml +20 -0
- datawright_kit/assets/connectors/dagster.py +121 -0
- datawright_kit/assets/connectors/dbt_yaml.py +36 -0
- datawright_kit/assets/connectors/kafka_connect.py +34 -0
- datawright_kit/assets/connectors/lineage_yaml.py +59 -0
- datawright_kit/assets/connectors/lookml.py +151 -0
- datawright_kit/assets/connectors/metabase.py +140 -0
- datawright_kit/assets/connectors/mybatis.py +77 -0
- datawright_kit/assets/connectors/odcs.py +53 -0
- datawright_kit/assets/connectors/superset.py +171 -0
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: datawright-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lineage, impact reports and join paths compiled from your data repo's code, for AI agents and reviewers
|
|
5
|
+
Project-URL: Homepage, https://github.com/clementlemon02/datawright
|
|
6
|
+
Project-URL: Issues, https://github.com/clementlemon02/datawright/issues
|
|
7
|
+
Author: Clement
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agent-skills,ai-agents,airflow,data-engineering,dbt,lineage,mcp,sql
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Database
|
|
13
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Requires-Dist: pyyaml>=6
|
|
16
|
+
Requires-Dist: sqlglot>=30
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
<div align="center">
|
|
20
|
+
|
|
21
|
+
<img src="docs/logo.svg" alt="Datawright" width="360">
|
|
22
|
+
|
|
23
|
+
### Know what breaks before you merge.
|
|
24
|
+
|
|
25
|
+
`datawright` is a local command that reads your data repo's code and tells you, and your AI agent,<br>the lineage, join paths and column meanings. No warehouse access, no model tokens, one file.
|
|
26
|
+
|
|
27
|
+
[](https://github.com/clementlemon02/datawright/actions/workflows/ci.yml)
|
|
28
|
+
[](LICENSE)
|
|
29
|
+
[](https://www.python.org)
|
|
30
|
+
[](https://agentskills.io)
|
|
31
|
+
|
|
32
|
+
[Quick start](#quick-start) · [What you get](#what-you-get) · [Benchmarks](#how-well-does-it-work) · [FAQ](#faq) · [Reference](docs/reference.md) · [Roadmap](ROADMAP.md)
|
|
33
|
+
|
|
34
|
+
<img src="docs/demo.svg" alt="datawright impact, join and card output on the shop example" width="860">
|
|
35
|
+
|
|
36
|
+
</div>
|
|
37
|
+
|
|
38
|
+
## Why
|
|
39
|
+
|
|
40
|
+
Agents and reviewers guess. Datawright answers from your repo, with no warehouse access and no model tokens, in seconds.
|
|
41
|
+
|
|
42
|
+
| Without it | With it |
|
|
43
|
+
|---|---|
|
|
44
|
+
| "Which tables does this change break?" Nobody knows until a dashboard does. | A comment on every merge request: changed tables, the DAG tasks that run them, downstream tables and dashboards. |
|
|
45
|
+
| The agent joins two tables on whatever column names match. | `datawright join a b` writes the join your code already uses, with casts. |
|
|
46
|
+
| What does `status = 2` mean? Ask someone who left. | The card says `2 = approved`, learned from your own `CASE` expressions. |
|
|
47
|
+
| A rerun silently doubles yesterday's rows. | `datawright lint` flags appends with nothing clearing the slice first. |
|
|
48
|
+
|
|
49
|
+
## What you get
|
|
50
|
+
|
|
51
|
+
- **Impact on every merge request.** `datawright review` lists changed tables, the Airflow task that runs each, what's downstream (tables, Looker, Superset and Metabase dashboards), broken data contracts, and unsafe patterns.
|
|
52
|
+
- **Lineage down to the column.** Which columns feed which. Every table card names the file and line that writes and reads it, and links that were worked out rather than read plainly are marked `(inferred)`.
|
|
53
|
+
- **Join paths and meanings.** How tables join as your code joins them, what codes like `status = 2` mean, which key columns nothing uses any more.
|
|
54
|
+
- **Any agent can use it.** A command line, plus an MCP server (`datawright serve`) for clients with no shell.
|
|
55
|
+
- **11 agent skills on top** (set up, build, change, debug, test, review, optimize, on-call, tickets) that teach your agent to use it like a senior data engineer.
|
|
56
|
+
|
|
57
|
+
**Reads:** SQL · dbt · Airflow · Dagster · Spark · Flink · Kafka · Java (MyBatis) · Looker · Superset · Metabase · data contracts (ODCS). Anything else: a small connector.
|
|
58
|
+
**Agents:** Claude Code · Codex · Cursor · Gemini CLI · anything that reads [Agent Skills](https://agentskills.io).
|
|
59
|
+
|
|
60
|
+
## Quick start
|
|
61
|
+
|
|
62
|
+
You need [uv](https://docs.astral.sh/uv/) and git.
|
|
63
|
+
|
|
64
|
+
**1. Try the tool.** One minute, no agent needed. From your data repo's root:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv run https://raw.githubusercontent.com/clementlemon02/datawright/main/skills/data-context/scripts/datawright.py init
|
|
68
|
+
.data-context/dw card <a table you know>
|
|
69
|
+
.data-context/dw impact <that table>
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
On a feature branch, `.data-context/dw review --base main` prints the merge-request report. `.data-context/dw coverage` lists what it doesn't read in your repo yet.
|
|
73
|
+
|
|
74
|
+
**2. Give your agent the skills.** Claude Code:
|
|
75
|
+
```
|
|
76
|
+
/plugin marketplace add clementlemon02/datawright
|
|
77
|
+
/plugin install data-eng@datawright
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Codex, Cursor, Gemini CLI and others (swap `-a`):
|
|
81
|
+
```bash
|
|
82
|
+
npx skills add clementlemon02/datawright -a codex
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Then tell your agent **"set up this repo"** and ask: *"What breaks if I change `orders_daily`?"*, *"How do `sessions` and `regions` join?"*, *"Review my changes before I open a PR."*
|
|
86
|
+
|
|
87
|
+
No data repo handy? Try [examples/shop](examples/shop), a small warehouse with planted bugs.
|
|
88
|
+
|
|
89
|
+
## How well does it work?
|
|
90
|
+
|
|
91
|
+
| Test | Result |
|
|
92
|
+
|---|---|
|
|
93
|
+
| Table lineage vs dbt's own manifest, three public dbt projects | **100% precision, 99.5–100% recall** |
|
|
94
|
+
| Questions agents answered correctly on a hard synthetic warehouse (Sonnet) | **12 of 12 with Datawright, 7 of 12 without** |
|
|
95
|
+
| Same test, Haiku | 7–8 of 12 with it, 3 of 12 without |
|
|
96
|
+
| Index build, 220-file production Airflow repo | under 2 seconds |
|
|
97
|
+
|
|
98
|
+
Six public repos run in CI at pinned commits, so any change in what it finds is reviewed. The agent benchmark is small and home-made: [methods and limits](docs/semantics.md#results-so-far-the-join-path-benchmark).
|
|
99
|
+
|
|
100
|
+
## How is it different?
|
|
101
|
+
|
|
102
|
+
| | Data catalogs (DataHub, OpenMetadata) | dbt-centric review (Datafold, Recce, SQLMesh) | **Datawright** |
|
|
103
|
+
|---|---|---|---|
|
|
104
|
+
| Where it looks | Metadata ingested from your tools | Your dbt project and warehouse | Your repo's code, read directly |
|
|
105
|
+
| Beyond dbt (Airflow, Spark, Java, Kafka) | Through ingestion connectors | No | Yes |
|
|
106
|
+
| Comparing the actual data before and after | No | **Yes, row and value diffs** | Aggregates only |
|
|
107
|
+
| Impact comment on every merge request | Not their job | Yes | Yes |
|
|
108
|
+
| Built for an agent to query | Some, via MCP | Some | Yes: cards, `join`, MCP server |
|
|
109
|
+
| Setup | A service to run | A warehouse connection, often a paid plan | One file, no service, no credentials |
|
|
110
|
+
|
|
111
|
+
They coexist: Datawright imports from them and can [export to dbt, DataHub and OpenMetadata](docs/reference.md#everything-datawright-answers). If you only run dbt, you may already have most of the lineage; the merge-request report and join paths are what's new.
|
|
112
|
+
|
|
113
|
+
## FAQ
|
|
114
|
+
|
|
115
|
+
**Does it touch my warehouse or send data anywhere?** No. `datawright` reads code, makes no network calls and never writes. Optional read-only queries need a read-only user you set up.
|
|
116
|
+
|
|
117
|
+
**Which warehouse and SQL dialects?** Anything sqlglot parses, auto-detected, including Snowflake, BigQuery, Trino, StarRocks, Doris, Spark and DuckDB.
|
|
118
|
+
|
|
119
|
+
**What if it doesn't understand part of my repo?** `datawright coverage` lists what it skips. A connector, usually a few dozen lines, teaches it, and setup can write one for you.
|
|
120
|
+
|
|
121
|
+
**How do I know it's right?** Every answer points at the file and line it came from, `datawright coverage` lists what it can't read and how many links are inferred, and lineage is scored against dbt's own manifest in CI. It's a heuristic parser, so check surprising results and [report wrong lineage](docs/try-it.md).
|
|
122
|
+
|
|
123
|
+
**Is it production-ready?** It's a first release, tested by its author on public repos and one production platform. [Ten minutes and five answers](docs/try-it.md) from you would help the most.
|
|
124
|
+
|
|
125
|
+
## More
|
|
126
|
+
|
|
127
|
+
[Skills and every command](docs/reference.md) · [Architecture](ARCHITECTURE.md) · [Roadmap](ROADMAP.md) · [Landscape](docs/landscape.md) · [Contributing](ARCHITECTURE.md#contributing)
|
|
128
|
+
|
|
129
|
+
MIT license
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
datawright_kit/assets/bi_export.py,sha256=7XFWWE5yAR3ghyZYsopdqmdrXKgTzEa7Pi1C87JOFZg,4867
|
|
2
|
+
datawright_kit/assets/config.yml,sha256=hxpEVsdQTWgQmvZvDVbZuKiFx5BEPz8cKwtzx8qVNLA,2123
|
|
3
|
+
datawright_kit/assets/connectors/dagster.py,sha256=E90lqxb9aCnRM8e-YgYj2DCdZxzKdr72qsUWVPiN0EY,6008
|
|
4
|
+
datawright_kit/assets/connectors/dbt_yaml.py,sha256=g9v5wt14CEadOQrUgw0oFu-8tl6keAdvZ1kBaAGZ1pw,1626
|
|
5
|
+
datawright_kit/assets/connectors/kafka_connect.py,sha256=_7mY4jcw9eOqbVrSbii7GkRqf3K4x78QwJitt2YJzVw,2096
|
|
6
|
+
datawright_kit/assets/connectors/lineage_yaml.py,sha256=7mXh6Tg1MshQorVWIrnuA4QRRKCvhXw3tTQ96yZIeuQ,2375
|
|
7
|
+
datawright_kit/assets/connectors/lookml.py,sha256=D_kGphUP2ac91k3R1ENx06Lm3BLqpr36lZJQLWhD4DU,6713
|
|
8
|
+
datawright_kit/assets/connectors/metabase.py,sha256=JXb58jbNlyqJftIVzJUvtxij07EIjw2bXf8Cx2Xt6OY,6708
|
|
9
|
+
datawright_kit/assets/connectors/mybatis.py,sha256=LqwBjeTf7YKPgYnmqrgaXyktjT6Jd7JyXwK0Qb8jTSg,3696
|
|
10
|
+
datawright_kit/assets/connectors/odcs.py,sha256=Uk9LmpIgdafXEunFQmtq-N8Xucr_27tQTzP4pVSuDOM,2751
|
|
11
|
+
datawright_kit/assets/connectors/superset.py,sha256=kX-VsHKPIDWvutYSQNCGt1ODjhFL3b4XPmzORy_3F3w,7466
|
|
12
|
+
datawright.py,sha256=fn383B9PzWOwhh47vK9AlyCncUBgqjebU8-EHdyDdto,256594
|
|
13
|
+
datawright_cli-0.1.0.dist-info/METADATA,sha256=SjIPc0nkgp9oknY8R3UrQeGRg6aSzfmBr9ZObtirjao,7619
|
|
14
|
+
datawright_cli-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
15
|
+
datawright_cli-0.1.0.dist-info/entry_points.txt,sha256=nHa99ErHYbn3a9zekWgul3411x1uQyrhuO-azd2Awmo,101
|
|
16
|
+
datawright_cli-0.1.0.dist-info/licenses/LICENSE,sha256=IVtfmhVBqe9XKh809kclMKS4mRjCODwKpr6E55c9hyc,1064
|
|
17
|
+
datawright_cli-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Clement
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# /// script
|
|
2
|
+
# requires-python = ">=3.10"
|
|
3
|
+
# dependencies = []
|
|
4
|
+
# ///
|
|
5
|
+
"""Fetch BI definitions into .data-context/imports/ for datawright: read-only, run from the repo root.
|
|
6
|
+
|
|
7
|
+
uv run bi_export.py superset # SUPERSET_URL, plus SUPERSET_ACCESS_TOKEN or SUPERSET_USERNAME + SUPERSET_PASSWORD
|
|
8
|
+
uv run bi_export.py metabase # METABASE_URL, plus METABASE_API_KEY (Admin > Authentication > API keys) or METABASE_SESSION
|
|
9
|
+
|
|
10
|
+
Superset: every dashboard's export ZIP (datasets, charts, dashboards; never data or database passwords).
|
|
11
|
+
Metabase: databases' table and field names, questions, models and dashboards. Neither fetches rows.
|
|
12
|
+
Use an account that can see the dashboards but not change them. Schedule it (cron, an Airflow task) to keep datawright current.
|
|
13
|
+
"""
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import urllib.request
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
OUT = Path(".data-context/imports")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def env(name, required=True):
|
|
25
|
+
v = os.environ.get(name, "").strip()
|
|
26
|
+
if required and not v:
|
|
27
|
+
sys.exit(f"Set {name} in the environment (never in a file in the repo).")
|
|
28
|
+
return v
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def call(base, path, headers, data=None):
|
|
32
|
+
req = urllib.request.Request(base.rstrip("/") + path, headers={"Accept": "application/json", **headers},
|
|
33
|
+
data=json.dumps(data).encode() if data is not None else None)
|
|
34
|
+
if data is not None:
|
|
35
|
+
req.add_header("Content-Type", "application/json")
|
|
36
|
+
try:
|
|
37
|
+
with urllib.request.urlopen(req, timeout=120) as r:
|
|
38
|
+
body = r.read()
|
|
39
|
+
except urllib.error.HTTPError as e:
|
|
40
|
+
sys.exit(f"{path.split('?')[0]}: HTTP {e.code}") # never the response: it can echo credentials
|
|
41
|
+
return body if path.startswith("/api/v1/dashboard/export") else json.loads(body or b"null")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def superset():
|
|
45
|
+
base = env("SUPERSET_URL")
|
|
46
|
+
token = env("SUPERSET_ACCESS_TOKEN", required=False)
|
|
47
|
+
if not token:
|
|
48
|
+
login = {"username": env("SUPERSET_USERNAME"), "password": env("SUPERSET_PASSWORD"), "provider": "db", "refresh": False}
|
|
49
|
+
token = call(base, "/api/v1/security/login", {}, login)["access_token"]
|
|
50
|
+
auth = {"Authorization": f"Bearer {token}"}
|
|
51
|
+
ids, page = [], 0
|
|
52
|
+
while True:
|
|
53
|
+
q = urllib.parse.quote(f"(columns:!(id),page:{page},page_size:100)")
|
|
54
|
+
batch = [d["id"] for d in call(base, f"/api/v1/dashboard/?q={q}", auth).get("result") or []]
|
|
55
|
+
ids += batch
|
|
56
|
+
if len(batch) < 100:
|
|
57
|
+
break
|
|
58
|
+
page += 1
|
|
59
|
+
if not ids:
|
|
60
|
+
sys.exit("No dashboards visible to this account.")
|
|
61
|
+
OUT.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
zipped = call(base, "/api/v1/dashboard/export/?q=" + urllib.parse.quote(f"!({','.join(map(str, ids))})"), auth)
|
|
63
|
+
(OUT / "superset.zip").write_bytes(zipped)
|
|
64
|
+
print(f"wrote {OUT / 'superset.zip'}: {len(ids)} dashboards with their charts and datasets; datawright build unpacks it")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def metabase():
|
|
68
|
+
base = env("METABASE_URL")
|
|
69
|
+
key, session = env("METABASE_API_KEY", required=False), env("METABASE_SESSION", required=False)
|
|
70
|
+
if not (key or session):
|
|
71
|
+
sys.exit("Set METABASE_API_KEY (or METABASE_SESSION) in the environment.")
|
|
72
|
+
auth = {"x-api-key": key} if key else {"X-Metabase-Session": session}
|
|
73
|
+
dbs = call(base, "/api/database", auth)
|
|
74
|
+
dbs = dbs.get("data", dbs) if isinstance(dbs, dict) else dbs
|
|
75
|
+
tables = []
|
|
76
|
+
for db in dbs:
|
|
77
|
+
meta = call(base, f"/api/database/{db['id']}/metadata", auth)
|
|
78
|
+
tables += [{"id": t["id"], "db": db.get("name"), "schema": t.get("schema"), "name": t["name"],
|
|
79
|
+
"fields": [{"id": f["id"], "name": f["name"]} for f in t.get("fields") or []]} for t in meta.get("tables") or []]
|
|
80
|
+
keep = ("id", "name", "type", "description", "archived", "database_id", "dataset_query", "collection_id")
|
|
81
|
+
cards = [{k: c.get(k) for k in keep} | {"columns": [m.get("name") for m in c.get("result_metadata") or []]}
|
|
82
|
+
for c in call(base, "/api/card", auth) if not c.get("archived")]
|
|
83
|
+
dashboards = []
|
|
84
|
+
for d in call(base, "/api/dashboard", auth):
|
|
85
|
+
if d.get("archived"):
|
|
86
|
+
continue
|
|
87
|
+
full = call(base, f"/api/dashboard/{d['id']}", auth)
|
|
88
|
+
dashcards = full.get("dashcards") or full.get("ordered_cards") or []
|
|
89
|
+
dashboards.append({"id": d["id"], "name": d["name"], "description": d.get("description"),
|
|
90
|
+
"cards": sorted({dc["card_id"] for dc in dashcards if dc.get("card_id")})})
|
|
91
|
+
out = OUT / "metabase"
|
|
92
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
93
|
+
(out / "metabase.json").write_text(json.dumps({"tables": tables, "cards": cards, "dashboards": dashboards}, indent=1))
|
|
94
|
+
print(f"wrote {out / 'metabase.json'}: {len(tables)} tables, {len(cards)} questions and models, {len(dashboards)} dashboards")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
{"superset": superset, "metabase": metabase}.get(sys.argv[1] if len(sys.argv) > 1 else "", lambda: sys.exit(__doc__))()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# datawright settings. All optional.
|
|
2
|
+
dialect: # empty = auto-detect on each build; or pin the engine: snowflake, bigquery, databricks, spark, hive, postgres, redshift, trino, duckdb, mysql, starrocks, clickhouse, ...
|
|
3
|
+
exclude: [] # glob patterns to skip, e.g. ["legacy/**", "scratch/*.sql"]
|
|
4
|
+
repos: [] # other repos to index with this one, e.g. ["../flink-jobs", "../warehouse-ddl"]: when DAGs here run jobs whose code lives there
|
|
5
|
+
price_per_mtok: 3 # your model's input price, USD per million tokens (used by `datawright savings`)
|
|
6
|
+
seconds_per_file: 4 # how long one file read takes your agent, in seconds (used by `datawright savings`)
|
|
7
|
+
track: true # false stops logging lookups to .data-context/usage.jsonl
|
|
8
|
+
query_command: # CLI for `datawright query` on a read-only, non-production connection. {sql} becomes the query, $VARS expand. Examples:
|
|
9
|
+
# psql "$DATABASE_URL" --csv -c {sql}
|
|
10
|
+
# snow sql --format CSV -q {sql}
|
|
11
|
+
# bq query --use_legacy_sql=false --format=csv --maximum_bytes_billed=1000000000 {sql} (LIMIT doesn't cut BigQuery's bytes billed)
|
|
12
|
+
# duckdb -readonly -csv dev.duckdb {sql}
|
|
13
|
+
# mysql --batch -e {sql} (MySQL, StarRocks; credentials in ~/.my.cnf)
|
|
14
|
+
# trino --output-format CSV_HEADER --execute {sql}
|
|
15
|
+
# clickhouse client --format CSVWithNames --query {sql}
|
|
16
|
+
env: {} # values of deploy-time variables set outside these repos (Helm, CI secrets), e.g. {KAFKA_TOPIC_ORDERS: shop.orders}:
|
|
17
|
+
# jobs that read a topic or table name from the environment then join the rest of the lineage. Names and values only, never secrets
|
|
18
|
+
layers: [] # your warehouse layers, earliest first, if not a common convention: e.g. [ods, dwd, dws, ads]; `lint` flags lineage running backwards
|
|
19
|
+
dev_name: # where a branch's tables are built, for `datawright review --diff`: {schema}, {table} and {name} fill in. e.g.
|
|
20
|
+
# {schema}_dev.{table} dbt_ci.{table} analytics_pr.{schema}__{table} (query_command's user must read both copies)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""datawright connector: Dagster assets, their dependencies and the schedules that materialize them.
|
|
2
|
+
|
|
3
|
+
Reads @asset / @dg.asset functions (key from name, key or key_prefix), their upstream from deps=[...], ins={...: AssetIn(...)}
|
|
4
|
+
and function parameters, AssetSpec(key, deps=...) declarations, and define_asset_job + ScheduleDefinition schedules.
|
|
5
|
+
Asset keys become dataset names joined with dots (["ANALYTICS", "orders"] -> analytics.orders), which line up with
|
|
6
|
+
warehouse tables named schema.table. dbt models under @dbt_assets need nothing extra: datawright reads their SQL. Static only:
|
|
7
|
+
assets built in loops or factories from runtime config are missed. Copy to .data-context/connectors/.
|
|
8
|
+
"""
|
|
9
|
+
import ast
|
|
10
|
+
|
|
11
|
+
ROOT = None
|
|
12
|
+
SKIP_PARAMS = {"context", "config", "self"}
|
|
13
|
+
_repo_jobs = None # job variable -> (name, selected keys), across the repo: schedules often live apart from their jobs
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def match(path):
|
|
17
|
+
return path.endswith(".py")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def extract(path, text):
|
|
21
|
+
if "dagster" not in text:
|
|
22
|
+
return
|
|
23
|
+
tree = ast.parse(text)
|
|
24
|
+
keys = {} # asset function name -> its key, for deps that name a function
|
|
25
|
+
assets = []
|
|
26
|
+
for n in ast.walk(tree):
|
|
27
|
+
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
28
|
+
dec = next((d for d in n.decorator_list if callee(d) in ("asset", "multi_asset")), None)
|
|
29
|
+
if dec is not None:
|
|
30
|
+
kw = {k.arg: k.value for k in dec.keywords if k.arg} if isinstance(dec, ast.Call) else {}
|
|
31
|
+
key = key_of(kw.get("key")) or ".".join(strings(kw.get("key_prefix")) + [lit(kw.get("name")) or n.name])
|
|
32
|
+
keys[n.name] = key
|
|
33
|
+
assets.append((n, kw, key))
|
|
34
|
+
for n, kw, key in assets:
|
|
35
|
+
yield {"produces": key, "line": n.lineno, "kind": "dagster asset"}
|
|
36
|
+
ups = [dep(d, keys) for d in (kw.get("deps").elts if isinstance(kw.get("deps"), (ast.List, ast.Tuple)) else [])]
|
|
37
|
+
ins = kw.get("ins")
|
|
38
|
+
named = {}
|
|
39
|
+
if isinstance(ins, ast.Dict):
|
|
40
|
+
for k, v in zip(ins.keys, ins.values):
|
|
41
|
+
if isinstance(k, ast.Constant) and isinstance(v, ast.Call):
|
|
42
|
+
a = {x.arg: x.value for x in v.keywords if x.arg}
|
|
43
|
+
named[k.value] = key_of(a.get("key")) or ".".join(strings(a.get("key_prefix")) + [k.value])
|
|
44
|
+
params = [a.arg for a in n.args.args if a.arg not in SKIP_PARAMS]
|
|
45
|
+
ups += [named.get(p) or keys.get(p, p) for p in params] # a parameter is an upstream asset unless it's a resource or config
|
|
46
|
+
for u in filter(None, ups):
|
|
47
|
+
yield {"edge": [u, key], "kind": "dagster asset"}
|
|
48
|
+
for n in ast.walk(tree): # AssetSpec("key", deps=[...]): assets defined by other systems, with their lineage
|
|
49
|
+
if isinstance(n, ast.Call) and callee(n) == "AssetSpec":
|
|
50
|
+
kw = {k.arg: k.value for k in n.keywords if k.arg}
|
|
51
|
+
key = key_of(n.args[0] if n.args else kw.get("key"))
|
|
52
|
+
if key:
|
|
53
|
+
yield {"dataset": key, "kind": "dagster asset", "description": lit(kw.get("description"))}
|
|
54
|
+
for d in kw.get("deps").elts if isinstance(kw.get("deps"), (ast.List, ast.Tuple)) else []:
|
|
55
|
+
if u := dep(d, keys):
|
|
56
|
+
yield {"edge": [u, key], "kind": "dagster asset"}
|
|
57
|
+
jobs = file_jobs(tree, keys)
|
|
58
|
+
for n in ast.walk(tree):
|
|
59
|
+
if isinstance(n, ast.Call) and callee(n) == "ScheduleDefinition":
|
|
60
|
+
kw = {k.arg: k.value for k in n.keywords if k.arg}
|
|
61
|
+
name = kw["job"].id if isinstance(kw.get("job"), ast.Name) else None
|
|
62
|
+
job = jobs.get(name) or repo_jobs().get(name) if name else None
|
|
63
|
+
if job and lit(kw.get("cron_schedule")):
|
|
64
|
+
for key in job[1]:
|
|
65
|
+
yield {"run": {"dag": job[0], "task": key, "schedule": lit(kw["cron_schedule"]), "dataset": key}, "line": n.lineno}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def file_jobs(tree, keys):
|
|
69
|
+
"""define_asset_job assignments in one module: variable -> (job name, selected asset keys)."""
|
|
70
|
+
jobs = {}
|
|
71
|
+
for n in ast.walk(tree):
|
|
72
|
+
if isinstance(n, ast.Assign) and isinstance(n.value, ast.Call) and callee(n.value) == "define_asset_job":
|
|
73
|
+
kw = {k.arg: k.value for k in n.value.keywords if k.arg}
|
|
74
|
+
name = lit(n.value.args[0]) if n.value.args else lit(kw.get("name"))
|
|
75
|
+
sel = kw.get("selection")
|
|
76
|
+
picked = [dep(x, keys) for x in sel.elts] if isinstance(sel, (ast.List, ast.Tuple)) else [dep(sel, keys)] if sel is not None else []
|
|
77
|
+
for t in n.targets:
|
|
78
|
+
if isinstance(t, ast.Name):
|
|
79
|
+
jobs[t.id] = (name or t.id, [p for p in picked if p])
|
|
80
|
+
return jobs
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def repo_jobs():
|
|
84
|
+
global _repo_jobs
|
|
85
|
+
if _repo_jobs is None:
|
|
86
|
+
_repo_jobs = {}
|
|
87
|
+
for f in ROOT.rglob("*.py") if ROOT else []:
|
|
88
|
+
if "/." not in str(f) and "define_asset_job" in (text := f.read_text(errors="replace")):
|
|
89
|
+
try:
|
|
90
|
+
_repo_jobs.update(file_jobs(ast.parse(text), {}))
|
|
91
|
+
except SyntaxError:
|
|
92
|
+
pass
|
|
93
|
+
return _repo_jobs
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def callee(n):
|
|
97
|
+
f = n.func if isinstance(n, ast.Call) else n
|
|
98
|
+
return f.attr if isinstance(f, ast.Attribute) else f.id if isinstance(f, ast.Name) else ""
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def lit(n):
|
|
102
|
+
return n.value if isinstance(n, ast.Constant) and isinstance(n.value, str) else None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def strings(n):
|
|
106
|
+
return [lit(n)] if lit(n) else [x.value for x in n.elts if isinstance(x, ast.Constant)] if isinstance(n, (ast.List, ast.Tuple)) else []
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def key_of(n):
|
|
110
|
+
"""AssetKey(["a", "b"]), AssetKey("a"), ["a", "b"] or "a/b" as a dotted name."""
|
|
111
|
+
if isinstance(n, ast.Call) and callee(n) == "AssetKey" and n.args:
|
|
112
|
+
n = n.args[0]
|
|
113
|
+
parts = strings(n) if isinstance(n, (ast.List, ast.Tuple)) else (lit(n) or "").split("/") if lit(n) else []
|
|
114
|
+
return ".".join(parts).lower() or None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def dep(n, keys):
|
|
118
|
+
"""A deps or selection entry: a key, an asset function's name, or another asset's function."""
|
|
119
|
+
if isinstance(n, ast.Name):
|
|
120
|
+
return keys.get(n.id, n.id)
|
|
121
|
+
return key_of(n)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""datawright connector: dbt schema files (models/**/*.yml, version 2), for repos that don't commit a compiled manifest.
|
|
2
|
+
|
|
3
|
+
Reads model, source, seed and snapshot descriptions and their columns (name, data_type, description). Lineage still comes
|
|
4
|
+
from the models' SQL, which datawright reads with ref() and source() resolved; with target/manifest.json present, datawright reads
|
|
5
|
+
that instead and this adds little. Copy to .data-context/connectors/.
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def match(path):
|
|
11
|
+
return path.endswith((".yml", ".yaml")) and not re.search(r"(^|/)(dbt_project|profiles|packages|dependencies|selectors)\.ya?ml$", path)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def extract(path, text):
|
|
15
|
+
import yaml
|
|
16
|
+
for doc in yaml.safe_load_all(text): # files may hold several documents
|
|
17
|
+
if isinstance(doc, dict):
|
|
18
|
+
yield from document(doc)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def document(doc):
|
|
22
|
+
for kind in ("models", "seeds", "snapshots"):
|
|
23
|
+
for m in doc.get(kind) or []:
|
|
24
|
+
if isinstance(m, dict) and m.get("name"):
|
|
25
|
+
yield from table(m["name"], m, kind[:-1])
|
|
26
|
+
for src in doc.get("sources") or []: # source('src', 'tbl') reads as src.tbl
|
|
27
|
+
for tbl in (src.get("tables") or []) if isinstance(src, dict) else []:
|
|
28
|
+
if isinstance(tbl, dict) and tbl.get("name"):
|
|
29
|
+
yield from table(f"{src['name']}.{tbl['name']}", tbl, "dbt source")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def table(name, spec, kind):
|
|
33
|
+
yield {"dataset": name, "description": spec.get("description"), "kind": kind}
|
|
34
|
+
for c in spec.get("columns") or []:
|
|
35
|
+
if isinstance(c, dict) and c.get("name"):
|
|
36
|
+
yield {"column": [name, c["name"], c.get("data_type"), c.get("description")]}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""datawright connector: Kafka Connect connector configs (JSON, as posted to the Connect REST API, or .properties).
|
|
2
|
+
|
|
3
|
+
Source connectors (Debezium CDC: table.include.list with topic.prefix or database.server.name) give table -> topic lineage;
|
|
4
|
+
sink connectors (topics or topics.regex, with table.name.format or a database/table option) give topic -> table lineage.
|
|
5
|
+
Topics are named kafka:<topic>, like the topics datawright finds in Flink and Spark jobs, so both sides connect.
|
|
6
|
+
Copy to .data-context/connectors/ and adjust FILES to where your configs live.
|
|
7
|
+
"""
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
FILES = r"connect|connector|debezium|sink|source" # a regex on the path of .json / .properties files to read
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def match(path):
|
|
15
|
+
return path.endswith((".json", ".properties")) and re.search(FILES, path, re.I) is not None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def extract(path, text):
|
|
19
|
+
cfg = json.loads(text) if path.endswith(".json") else dict(
|
|
20
|
+
(k.strip(), v.strip()) for k, _, v in (ln.partition("=") for ln in text.splitlines() if "=" in ln and not ln.lstrip().startswith("#")))
|
|
21
|
+
cfg = cfg.get("config", cfg) # the REST API wraps it: {"name": ..., "config": {...}}
|
|
22
|
+
cls = str(cfg.get("connector.class", ""))
|
|
23
|
+
if not cls:
|
|
24
|
+
return
|
|
25
|
+
prefix = cfg.get("topic.prefix") or cfg.get("database.server.name")
|
|
26
|
+
if prefix and cfg.get("table.include.list"): # Debezium: one topic per captured table, <prefix>.<db>.<table>
|
|
27
|
+
for table in filter(None, (t.strip() for t in cfg["table.include.list"].split(","))):
|
|
28
|
+
yield {"edge": [table, f"kafka:{prefix}.{table}"], "kind": "topic"}
|
|
29
|
+
topics = [t.strip() for t in str(cfg.get("topics", "")).split(",") if t.strip()]
|
|
30
|
+
target = cfg.get("table.name.format") or cfg.get("table") or cfg.get("collection")
|
|
31
|
+
db = cfg.get("database") or cfg.get("database-name")
|
|
32
|
+
for topic in topics: # sinks: each topic lands in the table its format names (${topic} by default)
|
|
33
|
+
table = (target or "${topic}").replace("${topic}", topic)
|
|
34
|
+
yield {"edge": [f"kafka:{topic}", f"{db}.{table}" if db and "." not in table else table], "kind": "kafka connect sink"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""datawright connector: lineage written by hand in YAML or JSON, the files teams keep for OpenMetadata, DataHub or their own docs.
|
|
2
|
+
|
|
3
|
+
Finds every list of edges, at any depth, written as from/to, source/target or upstream/downstream. Each side is a name
|
|
4
|
+
or an object with fqn, name or id (and optionally type, which becomes the dataset kind). Example:
|
|
5
|
+
|
|
6
|
+
edges:
|
|
7
|
+
- from: {type: table, fqn: warehouse.ods.orders}
|
|
8
|
+
to: {type: apiEndpoint, fqn: api.orders.list}
|
|
9
|
+
|
|
10
|
+
Copy to .data-context/connectors/ and adjust the settings below.
|
|
11
|
+
"""
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
|
|
15
|
+
FILES = r"lineage|\.lineage\.|edges" # which .yaml/.yml/.json paths hold lineage (a regex on the path)
|
|
16
|
+
DROP_PARTS = 0 # leading name parts to drop, e.g. 1 for OpenMetadata's service prefix
|
|
17
|
+
TYPE_PREFIX = {"topic": "kafka:"} # name prefixes by type, so these line up with what datawright finds in code
|
|
18
|
+
RENAME = [] # (regex, replacement) pairs for your naming, e.g. [(r"\$\{env\}_?", "")]
|
|
19
|
+
PAIRS = (("from", "to"), ("source", "target"), ("upstream", "downstream"))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def match(path):
|
|
23
|
+
return path.endswith((".yaml", ".yml", ".json")) and re.search(FILES, path, re.I) is not None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def extract(path, text):
|
|
27
|
+
if path.endswith(".json"):
|
|
28
|
+
doc = json.loads(text)
|
|
29
|
+
else:
|
|
30
|
+
import yaml
|
|
31
|
+
doc = list(yaml.safe_load_all(text)) # files may hold several documents
|
|
32
|
+
for item in walk(doc):
|
|
33
|
+
for a, b in PAIRS:
|
|
34
|
+
if a in item and b in item and (src := name(item[a])) and (dst := name(item[b])):
|
|
35
|
+
kind = item[b].get("type") if isinstance(item[b], dict) else None
|
|
36
|
+
yield {"edge": [src, dst], "kind": kind or "lineage yaml"}
|
|
37
|
+
break
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def walk(x):
|
|
41
|
+
if isinstance(x, dict):
|
|
42
|
+
yield x
|
|
43
|
+
for v in x.values():
|
|
44
|
+
yield from walk(v)
|
|
45
|
+
elif isinstance(x, list):
|
|
46
|
+
for v in x:
|
|
47
|
+
yield from walk(v)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def name(side):
|
|
51
|
+
n = side if isinstance(side, str) else next((side.get(k) for k in ("fqn", "fullyQualifiedName", "name", "id") if side.get(k)), None) \
|
|
52
|
+
if isinstance(side, dict) else None
|
|
53
|
+
if not n:
|
|
54
|
+
return None
|
|
55
|
+
n = ".".join(str(n).split(".")[DROP_PARTS:])
|
|
56
|
+
for rx, to in RENAME:
|
|
57
|
+
n = re.sub(rx, to, n)
|
|
58
|
+
kind = side.get("type") if isinstance(side, dict) else None
|
|
59
|
+
return TYPE_PREFIX.get(kind, "") + n
|