datawright-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datawright.py +3839 -0
- datawright_cli-0.1.0.dist-info/METADATA +129 -0
- datawright_cli-0.1.0.dist-info/RECORD +17 -0
- datawright_cli-0.1.0.dist-info/WHEEL +4 -0
- datawright_cli-0.1.0.dist-info/entry_points.txt +4 -0
- datawright_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- datawright_kit/assets/bi_export.py +98 -0
- datawright_kit/assets/config.yml +20 -0
- datawright_kit/assets/connectors/dagster.py +121 -0
- datawright_kit/assets/connectors/dbt_yaml.py +36 -0
- datawright_kit/assets/connectors/kafka_connect.py +34 -0
- datawright_kit/assets/connectors/lineage_yaml.py +59 -0
- datawright_kit/assets/connectors/lookml.py +151 -0
- datawright_kit/assets/connectors/metabase.py +140 -0
- datawright_kit/assets/connectors/mybatis.py +77 -0
- datawright_kit/assets/connectors/odcs.py +53 -0
- datawright_kit/assets/connectors/superset.py +171 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""datawright connector: Looker's LookML, so dashboards show up in `impact` and `review`.
|
|
2
|
+
|
|
3
|
+
Views (`sql_table_name`, or the tables a `derived_table` reads) become `looker:view.<view>`, with a
|
|
4
|
+
column per dimension and measure traced to the warehouse column it reads. Explores (`looker:explore.<explore>`) read
|
|
5
|
+
their base and joined views; LookML dashboards (`*.dashboard.lookml`) read their explores, and each tile's fields are
|
|
6
|
+
column edges, so `datawright impact mart.orders.revenue` reaches `looker:dashboard.<dashboard>.<tile>`.
|
|
7
|
+
Dashboards built in the Looker UI live in the instance, not the repo: export them through its API instead.
|
|
8
|
+
"""
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
|
|
14
|
+
TOKEN = re.compile(r"(\+?\w+)\s*:\s*|\}|#[^\n]*")
|
|
15
|
+
REF = re.compile(r"\$\{(\w+)(?:\.(\w+))?\}(?:\.(\w+))?")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def match(path):
|
|
19
|
+
return path.endswith((".lkml", ".lookml"))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parse(text):
|
|
23
|
+
"""LookML as nested [key, name or value, children or None]."""
|
|
24
|
+
root, i = [], 0
|
|
25
|
+
stack = [root]
|
|
26
|
+
while m := TOKEN.search(text, i):
|
|
27
|
+
tok, i = m.group(0), m.end()
|
|
28
|
+
if tok.startswith("#"):
|
|
29
|
+
continue
|
|
30
|
+
if tok == "}":
|
|
31
|
+
if len(stack) > 1:
|
|
32
|
+
stack.pop()
|
|
33
|
+
continue
|
|
34
|
+
key = m.group(1)
|
|
35
|
+
if key.startswith("sql") or key.endswith("_sql") or key == "html": # free text up to ";;"
|
|
36
|
+
end = text.find(";;", i)
|
|
37
|
+
end = len(text) if end < 0 else end
|
|
38
|
+
stack[-1].append([key, text[i:end].strip(), None])
|
|
39
|
+
i = end + 2
|
|
40
|
+
elif b := re.match(r"(\+?\w*)\s*\{", text[i:]):
|
|
41
|
+
node = [key, b.group(1), []]
|
|
42
|
+
stack[-1].append(node)
|
|
43
|
+
stack.append(node[2])
|
|
44
|
+
i += b.end()
|
|
45
|
+
else:
|
|
46
|
+
v = re.match(r'"[^"]*"|\[[^\]]*\]|[^\s}\]]+', text[i:])
|
|
47
|
+
stack[-1].append([key, v.group(0).strip('"') if v else "", None])
|
|
48
|
+
i += v.end() if v else 0
|
|
49
|
+
return root
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def get(node, key):
|
|
53
|
+
return next((v for k, v, c in node or () if k == key and c is None), None)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def table(name):
|
|
57
|
+
return re.sub(r'[`"\[\]]', "", name).strip().lower()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def constants():
|
|
61
|
+
"""`@{NAME}` values from the project's manifest.lkml. ROOT is set by datawright."""
|
|
62
|
+
try:
|
|
63
|
+
text = next(Path(ROOT).rglob("manifest.lkml")).read_text()
|
|
64
|
+
except (NameError, StopIteration, OSError):
|
|
65
|
+
return {}
|
|
66
|
+
return {n: get(c, "value") or "" for k, n, c in parse(text) if k == "constant" and c is not None}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def extract(path, text):
|
|
70
|
+
consts = constants()
|
|
71
|
+
text = re.sub(r"@\{(\w+)\}", lambda m: consts.get(m[1], m[0]), text)
|
|
72
|
+
if path.endswith(".lookml"): # LookML dashboards are YAML
|
|
73
|
+
yield from dashboards(text)
|
|
74
|
+
return
|
|
75
|
+
for key, name, body in parse(text):
|
|
76
|
+
name = name.lstrip("+")
|
|
77
|
+
if key == "view" and body is not None:
|
|
78
|
+
yield from view(name, body)
|
|
79
|
+
elif key == "explore" and body is not None:
|
|
80
|
+
base = get(body, "from") or get(body, "view_name") or name
|
|
81
|
+
yield {"edge": [f"looker:view.{base}", f"looker:explore.{name}"], "kind": "looker explore", "source_kind": "looker view"}
|
|
82
|
+
for k, j, jb in body:
|
|
83
|
+
if k == "join" and jb is not None:
|
|
84
|
+
yield {"edge": [f"looker:view.{get(jb, 'from') or j}", f"looker:explore.{name}"], "kind": "looker explore", "source_kind": "looker view"}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def view(name, body):
|
|
88
|
+
me = f"looker:view.{name}"
|
|
89
|
+
src = get(body, "sql_table_name")
|
|
90
|
+
derived = next((c for k, _, c in body if k == "derived_table" and c is not None), None)
|
|
91
|
+
if src:
|
|
92
|
+
src = table(src)
|
|
93
|
+
yield {"edge": [src, me], "kind": "looker view"}
|
|
94
|
+
elif derived is not None:
|
|
95
|
+
sql = re.sub(r"\{%\s*condition.*?endcondition\s*%\}", "1=1", get(derived, "sql") or "", flags=re.S) # a templated filter
|
|
96
|
+
sql = re.sub(r"\{%.*?%\}|\{\{.*?\}\}", "", sql, flags=re.S)
|
|
97
|
+
for v in re.findall(r"\$\{(\w+)\.SQL_TABLE_NAME\}", sql):
|
|
98
|
+
yield {"edge": [f"looker:view.{v}", me], "kind": "looker view", "source_kind": "looker view"}
|
|
99
|
+
for t in tables(re.sub(r"\$\{\w+\.SQL_TABLE_NAME\}", "looker_pdt", sql)):
|
|
100
|
+
yield {"edge": [t, me], "kind": "looker view"}
|
|
101
|
+
for k, e, _ in derived:
|
|
102
|
+
if k == "explore_source": # a native derived table queries an explore
|
|
103
|
+
yield {"edge": [f"looker:explore.{e}", me], "kind": "looker view", "source_kind": "looker explore"}
|
|
104
|
+
yield {"dataset": me, "kind": "looker view", "description": get(body, "label")}
|
|
105
|
+
for k, field, fb in body:
|
|
106
|
+
if k not in ("dimension", "measure", "dimension_group") or fb is None:
|
|
107
|
+
continue
|
|
108
|
+
names = [f"{field}_{t}" for t in re.findall(r"\w+", get(fb, "timeframes") or "")] if k == "dimension_group" else []
|
|
109
|
+
for f in names or [field]:
|
|
110
|
+
yield {"column": [me, f, get(fb, "type"), get(fb, "description") or get(fb, "label")]}
|
|
111
|
+
for v, col, tcol in REF.findall(get(fb, "sql") or ""):
|
|
112
|
+
if v == "TABLE" and tcol:
|
|
113
|
+
origin = f"{src or me}.{tcol}"
|
|
114
|
+
elif v != "TABLE" and not tcol:
|
|
115
|
+
origin = f"looker:view.{v}.{col}" if col else f"{me}.{v}"
|
|
116
|
+
else:
|
|
117
|
+
continue
|
|
118
|
+
if origin.lower() != f"{me}.{f}".lower():
|
|
119
|
+
yield {"column_edge": [origin, f"{me}.{f}"]}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def tables(sql):
|
|
123
|
+
"""Tables a derived table's SQL reads. ponytail: tables only, so its fields don't trace to warehouse columns."""
|
|
124
|
+
import sqlglot
|
|
125
|
+
from sqlglot import exp
|
|
126
|
+
for dialect in (None, "bigquery"): # backticked `project.dataset.table` needs bigquery's reader
|
|
127
|
+
try:
|
|
128
|
+
tree = sqlglot.parse_one(sql, read=dialect)
|
|
129
|
+
break
|
|
130
|
+
except Exception:
|
|
131
|
+
tree = None
|
|
132
|
+
if tree is None:
|
|
133
|
+
return []
|
|
134
|
+
ctes = {c.alias_or_name.lower() for c in tree.find_all(exp.CTE)}
|
|
135
|
+
names = {table(".".join(p.name for p in t.parts)) for t in tree.find_all(exp.Table)}
|
|
136
|
+
return sorted(names - ctes - {"looker_pdt"})
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def dashboards(text):
|
|
140
|
+
for d in yaml.safe_load(text) or []:
|
|
141
|
+
if not isinstance(d, dict) or not d.get("dashboard"):
|
|
142
|
+
continue
|
|
143
|
+
me = f"looker:dashboard.{d['dashboard']}"
|
|
144
|
+
yield {"dataset": me, "kind": "looker dashboard", "description": d.get("title")}
|
|
145
|
+
for el in d.get("elements") or []:
|
|
146
|
+
if el.get("explore"):
|
|
147
|
+
yield {"edge": [f"looker:explore.{el['explore']}", me], "kind": "looker dashboard", "source_kind": "looker explore"}
|
|
148
|
+
tile = el.get("name") or el.get("title")
|
|
149
|
+
for f in el.get("fields") or []:
|
|
150
|
+
if tile and "." in str(f):
|
|
151
|
+
yield {"column_edge": [f"looker:view.{f}", f"{me}.{tile}"]}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""datawright connector: Metabase questions, models and dashboards, so they show up in `impact` and `review`.
|
|
2
|
+
|
|
3
|
+
Reads .data-context/imports/metabase/metabase.json, written by `uv run <data-context skill>/assets/bi_export.py metabase`
|
|
4
|
+
(read-only API calls). Questions become `metabase:question.<name>` (models `metabase:model.<name>`) reading the tables
|
|
5
|
+
their query builder or SQL names, and the questions they build on; dashboards `metabase:dashboard.<name>` with a column
|
|
6
|
+
per card, so `datawright impact mart.orders.revenue` reaches `metabase:dashboard.sales.<card>`.
|
|
7
|
+
ponytail: in SQL questions, any column of a table it reads that the SQL names counts as used (any clause).
|
|
8
|
+
"""
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def match(path):
|
|
14
|
+
return path.endswith("metabase.json")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def slug(s):
|
|
18
|
+
return re.sub(r"\W+", "_", str(s).strip().lower()).strip("_")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def field_ids(node, out):
|
|
22
|
+
"""Field ids in a query, legacy (["field", 12, opts]) or staged (["field", {opts}, 12])."""
|
|
23
|
+
if isinstance(node, list):
|
|
24
|
+
if len(node) >= 2 and node[0] == "field":
|
|
25
|
+
out.update(x for x in node[1:3] if isinstance(x, int))
|
|
26
|
+
for x in node:
|
|
27
|
+
field_ids(x, out)
|
|
28
|
+
elif isinstance(node, dict):
|
|
29
|
+
for x in node.values():
|
|
30
|
+
field_ids(x, out)
|
|
31
|
+
return out
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def sources(node, out):
|
|
35
|
+
"""source-table values anywhere in a query: table ids, and "card__N" for questions built on questions."""
|
|
36
|
+
if isinstance(node, dict):
|
|
37
|
+
for k, v in node.items():
|
|
38
|
+
if k in ("source-table", "source-card"):
|
|
39
|
+
out.add(f"card__{v}" if k == "source-card" else v)
|
|
40
|
+
sources(v, out)
|
|
41
|
+
elif isinstance(node, list):
|
|
42
|
+
for x in node:
|
|
43
|
+
sources(x, out)
|
|
44
|
+
return out
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def field_names(node, out):
|
|
48
|
+
"""Columns of a base question named in a query: ["field", "REVENUE", opts] or ["field", opts, "REVENUE"]."""
|
|
49
|
+
if isinstance(node, list):
|
|
50
|
+
if len(node) >= 2 and node[0] == "field":
|
|
51
|
+
out.update(x.lower() for x in node[1:3] if isinstance(x, str))
|
|
52
|
+
for x in node:
|
|
53
|
+
field_names(x, out)
|
|
54
|
+
elif isinstance(node, dict):
|
|
55
|
+
for x in node.values():
|
|
56
|
+
field_names(x, out)
|
|
57
|
+
return out
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def native_sql(q):
|
|
61
|
+
if q.get("type") == "native":
|
|
62
|
+
return (q.get("native") or {}).get("query") or ""
|
|
63
|
+
return " ".join(s.get("native", "") for s in q.get("stages") or [] if s.get("lib/type") == "mbql.stage/native")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def extract(path, text):
|
|
67
|
+
doc = json.loads(text)
|
|
68
|
+
tables = {t["id"]: ".".join(x for x in (t.get("schema"), t["name"]) if x).lower() for t in doc.get("tables") or []}
|
|
69
|
+
fields = {f["id"]: (tables[t["id"]], f["name"].lower()) for t in doc.get("tables") or [] for f in t.get("fields") or []}
|
|
70
|
+
cols_of = {}
|
|
71
|
+
for tid, col in fields.values():
|
|
72
|
+
cols_of.setdefault(tid, set()).add(col)
|
|
73
|
+
cards = [c for c in doc.get("cards") or [] if not c.get("archived")]
|
|
74
|
+
taken = {}
|
|
75
|
+
for c in cards:
|
|
76
|
+
taken[slug(c["name"])] = taken.get(slug(c["name"]), 0) + 1
|
|
77
|
+
name = {c["id"]: f"metabase:{'model' if c.get('type') == 'model' else 'question'}.{slug(c['name'])}"
|
|
78
|
+
+ (f"_{c['id']}" if taken[slug(c["name"])] > 1 else "") for c in cards}
|
|
79
|
+
used = {}
|
|
80
|
+
for c in sorted(cards, key=lambda c: c["id"]): # a question's base questions come first, so their columns are known
|
|
81
|
+
me, q = name[c["id"]], c.get("dataset_query") or {}
|
|
82
|
+
kind = "metabase model" if c.get("type") == "model" else "metabase question"
|
|
83
|
+
yield {"dataset": me, "kind": kind, "description": c.get("description")}
|
|
84
|
+
reads, cols, parents = set(), set(), set()
|
|
85
|
+
for s in sources(q, set()):
|
|
86
|
+
if isinstance(s, int) and s in tables:
|
|
87
|
+
reads.add(tables[s])
|
|
88
|
+
elif isinstance(s, str) and s.startswith("card__") and s[6:].isdigit() and int(s[6:]) in name:
|
|
89
|
+
parents.add(int(s[6:]))
|
|
90
|
+
cols |= {fields[f] for f in field_ids(q, set()) if f in fields}
|
|
91
|
+
sql = native_sql(q)
|
|
92
|
+
if sql: # strip Metabase's template syntax, then read it like any SQL
|
|
93
|
+
parents |= {int(r) for r in re.findall(r"\{\{\s*#(\d+)", sql) if int(r) in name}
|
|
94
|
+
sql = re.sub(r"\{\{\s*#\d+[^}]*\}\}", "metabase_card", re.sub(r"\[\[.*?\]\]", "", sql, flags=re.S))
|
|
95
|
+
sql = re.sub(r"\{\{[^}]*\}\}", "1", sql)
|
|
96
|
+
found = tables_in(sql)
|
|
97
|
+
for f in found - {"metabase_card"}: # as Metabase knows it (schema.table), when the SQL leaves the schema out
|
|
98
|
+
known = [t for t in tables.values() if t == f or t.endswith("." + f) or f.endswith("." + t)]
|
|
99
|
+
reads.add(known[0] if len(known) == 1 else f)
|
|
100
|
+
words = set(re.findall(r"[a-z_]\w*", sql.lower()))
|
|
101
|
+
cols |= {(t, w) for t in reads for w in cols_of.get(t, set()) & words}
|
|
102
|
+
for t in sorted(reads):
|
|
103
|
+
yield {"edge": [t, me], "kind": kind}
|
|
104
|
+
for col in c.get("columns") or []:
|
|
105
|
+
yield {"column": [me, str(col).lower()]}
|
|
106
|
+
for t, col in sorted(cols):
|
|
107
|
+
yield {"column_edge": [f"{t}.{col}", f"{me}.{col}"]}
|
|
108
|
+
named = field_names(q, set()) # a base question's columns, referred to by name
|
|
109
|
+
for pid in sorted(parents):
|
|
110
|
+
yield {"edge": [name[pid], me], "kind": kind, "source_kind": "metabase question"}
|
|
111
|
+
via = {n for n in named if n in used.get(pid, set())} if named else used.get(pid, set()) # none named: all pass through
|
|
112
|
+
for col in sorted(via):
|
|
113
|
+
yield {"column_edge": [f"{name[pid]}.{col}", f"{me}.{col}"]}
|
|
114
|
+
cols.add((name[pid], col))
|
|
115
|
+
used[c["id"]] = {col for _, col in cols}
|
|
116
|
+
for d in doc.get("dashboards") or []:
|
|
117
|
+
me = f"metabase:dashboard.{slug(d['name'])}"
|
|
118
|
+
yield {"dataset": me, "kind": "metabase dashboard", "description": d.get("description") or d["name"]}
|
|
119
|
+
for cid in d.get("cards") or []:
|
|
120
|
+
if cid in name:
|
|
121
|
+
tile = name[cid].split(".", 1)[1]
|
|
122
|
+
yield {"edge": [name[cid], me], "kind": "metabase dashboard", "source_kind": "metabase question"}
|
|
123
|
+
yield {"column": [me, tile]}
|
|
124
|
+
for col in sorted(used.get(cid, ())):
|
|
125
|
+
yield {"column_edge": [f"{name[cid]}.{col}", f"{me}.{tile}"]}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def tables_in(sql):
|
|
129
|
+
import sqlglot
|
|
130
|
+
from sqlglot import exp
|
|
131
|
+
for dialect in (None, "postgres", "mysql", "bigquery"):
|
|
132
|
+
try:
|
|
133
|
+
tree = sqlglot.parse_one(sql, read=dialect)
|
|
134
|
+
break
|
|
135
|
+
except Exception:
|
|
136
|
+
tree = None
|
|
137
|
+
if tree is None:
|
|
138
|
+
return set()
|
|
139
|
+
ctes = {c.alias_or_name.lower() for c in tree.find_all(exp.CTE)}
|
|
140
|
+
return {".".join(p.name for p in t.parts).lower() for t in tree.find_all(exp.Table)} - ctes
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""datawright connector: MyBatis mapper XML, the SQL behind Java services (<select>, <insert>, <update>, <delete>).
|
|
2
|
+
|
|
3
|
+
Dynamic SQL is rendered to plain SQL: every <if> is included, <where>/<set>/<trim> add their keyword and drop the leading
|
|
4
|
+
AND/OR or trailing comma, <foreach> keeps one item, <include> pulls in its <sql> fragment, #{param} becomes a value and
|
|
5
|
+
${param} an unknown. A statement with <choose> is rendered once per branch, so every table any branch reads is seen.
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
import xml.etree.ElementTree as ET
|
|
9
|
+
|
|
10
|
+
FILES = r"mapper" # which .xml paths hold mappers (a regex on the path)
|
|
11
|
+
STATEMENTS = ("select", "insert", "update", "delete")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def match(path):
|
|
15
|
+
return path.endswith(".xml") and re.search(FILES, path, re.I) is not None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def extract(path, text):
|
|
19
|
+
try:
|
|
20
|
+
root = ET.fromstring(re.sub(r"<!DOCTYPE[^>]*>", "", text))
|
|
21
|
+
except ET.ParseError:
|
|
22
|
+
return
|
|
23
|
+
if root.tag != "mapper":
|
|
24
|
+
return
|
|
25
|
+
fragments = {e.get("id"): e for e in root.iter("sql") if e.get("id")}
|
|
26
|
+
for el in root:
|
|
27
|
+
if el.tag not in STATEMENTS:
|
|
28
|
+
continue
|
|
29
|
+
line = text.count("\n", 0, max(text.find(f'id="{el.get("id")}"'), 0)) + 1
|
|
30
|
+
branches = max([len([c for c in ch if c.tag in ("when", "otherwise")]) for ch in el.iter("choose")] or [1])
|
|
31
|
+
seen = set()
|
|
32
|
+
for pick in range(min(branches, 12)):
|
|
33
|
+
sql = re.sub(r"--[^\n]*", "", render(el, fragments, pick)) # before joining lines, or a comment eats the rest
|
|
34
|
+
sql = re.sub(r"\s+", " ", sql).strip()
|
|
35
|
+
sql = re.sub(r"#\{[^}]*\}", "1", sql)
|
|
36
|
+
sql = re.sub(r"\$\{[^}]*\}", "jinja_expr", sql)
|
|
37
|
+
if sql and sql not in seen:
|
|
38
|
+
seen.add(sql)
|
|
39
|
+
yield {"sql": sql, "line": line}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def render(el, fragments, pick, depth=0):
|
|
43
|
+
"""The SQL text an element stands for, with one branch of each <choose> (the pick-th, or its last)."""
|
|
44
|
+
if depth > 20:
|
|
45
|
+
return ""
|
|
46
|
+
parts = [el.text or ""]
|
|
47
|
+
for ch in el:
|
|
48
|
+
inner = lambda e=ch: render(e, fragments, pick, depth + 1)
|
|
49
|
+
if ch.tag == "include":
|
|
50
|
+
ref = (ch.get("refid") or "").rsplit(".", 1)[-1]
|
|
51
|
+
parts.append(render(fragments[ref], fragments, pick, depth + 1) if ref in fragments else "")
|
|
52
|
+
elif ch.tag == "where":
|
|
53
|
+
body = re.sub(r"^\s*(AND|OR)\b", "", inner(), flags=re.I)
|
|
54
|
+
parts.append(f" WHERE {body} " if body.strip() else " ")
|
|
55
|
+
elif ch.tag == "set":
|
|
56
|
+
parts.append(" SET " + re.sub(r",\s*$", "", inner().rstrip()) + " ")
|
|
57
|
+
elif ch.tag == "trim":
|
|
58
|
+
body = inner()
|
|
59
|
+
for o in (ch.get("prefixOverrides") or "").split("|"):
|
|
60
|
+
if o.strip() and body.lstrip().upper().startswith(o.strip().upper()):
|
|
61
|
+
body = body.lstrip()[len(o.strip()):]
|
|
62
|
+
for o in (ch.get("suffixOverrides") or "").split("|"):
|
|
63
|
+
if o.strip() and body.rstrip().upper().endswith(o.strip().upper()):
|
|
64
|
+
body = body.rstrip()[: -len(o.strip())]
|
|
65
|
+
parts.append(f" {ch.get('prefix') or ''} {body} {ch.get('suffix') or ''} ")
|
|
66
|
+
elif ch.tag == "foreach":
|
|
67
|
+
parts.append(f" {ch.get('open') or ''}{inner()}{ch.get('close') or ''} ")
|
|
68
|
+
elif ch.tag == "choose":
|
|
69
|
+
options = [c for c in ch if c.tag in ("when", "otherwise")]
|
|
70
|
+
if options:
|
|
71
|
+
parts.append(" " + render(options[min(pick, len(options) - 1)], fragments, pick, depth + 1) + " ")
|
|
72
|
+
elif ch.tag in ("if", "when", "otherwise"):
|
|
73
|
+
parts.append(" " + inner() + " ")
|
|
74
|
+
elif ch.tag != "bind":
|
|
75
|
+
parts.append(inner())
|
|
76
|
+
parts.append(ch.tail or "")
|
|
77
|
+
return "".join(parts)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""datawright connector: data contracts in the Open Data Contract Standard (ODCS v3, as written by datacontract-cli), plus the older
|
|
2
|
+
Data Contract Specification. Each contract's tables get the contracted columns, types and required flags, a description and an
|
|
3
|
+
owner; `datawright review` then flags a change that stops producing a required column, and cards show the contract.
|
|
4
|
+
Copy to .data-context/connectors/ and adjust FILES to where your contracts live.
|
|
5
|
+
"""
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
FILES = r"contract|odcs" # a regex on the path of .yaml/.yml files to read
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def match(path):
|
|
12
|
+
return path.endswith((".yaml", ".yml")) and re.search(FILES, path, re.I) is not None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def extract(path, text):
|
|
16
|
+
import yaml
|
|
17
|
+
for doc in yaml.safe_load_all(text):
|
|
18
|
+
if isinstance(doc, dict) and (doc.get("kind") == "DataContract" or "dataContractSpecification" in doc):
|
|
19
|
+
yield from contract(doc)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def contract(doc):
|
|
23
|
+
owner = text_of(doc.get("owner")) or text_of((doc.get("info") or {}).get("owner")) or team(doc.get("team"))
|
|
24
|
+
prefix = qualifier(doc.get("servers"))
|
|
25
|
+
tables = doc.get("schema") if isinstance(doc.get("schema"), list) else [ # ODCS: schema is a list; the older spec keys models by name
|
|
26
|
+
{"name": k, **v} for k, v in (doc.get("models") or {}).items() if isinstance(v, dict)]
|
|
27
|
+
for tbl in tables:
|
|
28
|
+
if not isinstance(tbl, dict) or not (tbl.get("physicalName") or tbl.get("name")):
|
|
29
|
+
continue
|
|
30
|
+
name = ".".join(filter(None, [prefix, tbl.get("physicalName") or tbl.get("name")]))
|
|
31
|
+
yield {"dataset": name, "description": text_of(tbl.get("description")) or text_of(doc.get("description")), "owner": owner}
|
|
32
|
+
props = tbl.get("properties") or [{"name": k, **v} for k, v in (tbl.get("fields") or {}).items() if isinstance(v, dict)]
|
|
33
|
+
for c in props:
|
|
34
|
+
if isinstance(c, dict) and c.get("name"):
|
|
35
|
+
typ = c.get("physicalType") or c.get("logicalType") or c.get("type")
|
|
36
|
+
required = c.get("required") or c.get("primaryKey") or c.get("primary")
|
|
37
|
+
yield {"contract": [name, c["name"], typ, required]}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def text_of(v): # ODCS v3 descriptions are objects ({purpose, usage, limitations})
|
|
41
|
+
return v if isinstance(v, str) else (v.get("purpose") or v.get("usage")) if isinstance(v, dict) else None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def team(t):
|
|
45
|
+
return t.get("name") if isinstance(t, dict) else None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def qualifier(servers):
|
|
49
|
+
"""The schema (or dataset) of the only server, so contract names line up with the warehouse."""
|
|
50
|
+
items = servers if isinstance(servers, list) else list(servers.values()) if isinstance(servers, dict) else []
|
|
51
|
+
if len(items) == 1 and isinstance(items[0], dict):
|
|
52
|
+
return items[0].get("schema") or items[0].get("dataset")
|
|
53
|
+
return None
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""datawright connector: Apache Superset (and Preset) dashboards, so they show up in `impact` and `review`.
|
|
2
|
+
|
|
3
|
+
Reads Superset's export format: the ZIP from Dashboards > Export (or `superset export-dashboards`, or the REST API)
|
|
4
|
+
dropped into .data-context/imports/, which datawright unpacks; or the YAML files kept in git by `preset-cli superset sync`.
|
|
5
|
+
Datasets become `superset:dataset.<table>`, read from their table or, for virtual datasets, from the tables their SQL
|
|
6
|
+
reads; charts `superset:chart.<name>` with the columns and metrics they use; dashboards `superset:dashboard.<name>`
|
|
7
|
+
with a column per chart. So `datawright impact mart.orders.revenue` reaches `superset:dashboard.sales.revenue_by_day`.
|
|
8
|
+
"""
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import yaml
|
|
14
|
+
|
|
15
|
+
KINDS = ("charts", "datasets", "dashboards")
|
|
16
|
+
COLUMN_KEYS = ("groupby", "columns", "all_columns", "x_axis", "granularity_sqla", "series", "entity", "order_by_cols",
|
|
17
|
+
"metric", "metrics", "metric_2", "percent_metrics", "secondary_metric", "size", "x", "y", "timeseries_limit_metric")
|
|
18
|
+
_index = {}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def match(path):
|
|
22
|
+
p = Path(path)
|
|
23
|
+
return p.suffix in (".yaml", ".yml") and (bool(set(p.parts) & set(KINDS)) or p.name in ("dashboard.yaml", "dataset.yaml"))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def slug(s):
|
|
27
|
+
return re.sub(r"\W+", "_", str(s).strip().lower()).strip("_")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def root_of(path):
|
|
31
|
+
"""The export's top folder: the one holding charts/, datasets/ and dashboards/."""
|
|
32
|
+
p = Path(ROOT, path)
|
|
33
|
+
for i, part in enumerate(p.parts):
|
|
34
|
+
if part in KINDS:
|
|
35
|
+
return Path(*p.parts[:i])
|
|
36
|
+
return p.parent
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load(text):
|
|
40
|
+
try:
|
|
41
|
+
doc = yaml.safe_load(text)
|
|
42
|
+
except yaml.YAMLError:
|
|
43
|
+
return {}
|
|
44
|
+
return doc if isinstance(doc, dict) else {}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def params(chart):
|
|
48
|
+
p = chart.get("params") or {}
|
|
49
|
+
if isinstance(p, str): # older exports keep params as a JSON string
|
|
50
|
+
try:
|
|
51
|
+
p = json.loads(p)
|
|
52
|
+
except ValueError:
|
|
53
|
+
p = {}
|
|
54
|
+
return p if isinstance(p, dict) else {}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def dataset_name(d):
|
|
58
|
+
return ".".join(x for x in (d.get("schema"), d.get("table_name")) if x).lower()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def index(root):
|
|
62
|
+
"""uuid -> what it is, across one export: datasets with their columns and metrics, charts with what they use."""
|
|
63
|
+
if root in _index:
|
|
64
|
+
return _index[root]
|
|
65
|
+
docs = [load(f.read_text(errors="replace")) for f in sorted(root.rglob("*.y*ml"))]
|
|
66
|
+
ix = {}
|
|
67
|
+
for d in docs:
|
|
68
|
+
if d.get("table_name") and d.get("uuid"):
|
|
69
|
+
cols = {str(c.get("column_name")).lower() for c in d.get("columns") or [] if c.get("column_name")}
|
|
70
|
+
metrics = {str(m.get("metric_name")).lower() for m in d.get("metrics") or [] if m.get("metric_name")}
|
|
71
|
+
ix[d["uuid"]] = {"name": f"superset:dataset.{dataset_name(d)}", "cols": cols, "metrics": metrics}
|
|
72
|
+
for d in docs:
|
|
73
|
+
if d.get("slice_name") and d.get("uuid"):
|
|
74
|
+
ds = ix.get(d.get("dataset_uuid"), {})
|
|
75
|
+
ix[d["uuid"]] = {"name": f"superset:chart.{slug(d['slice_name'])}", "dataset": ds.get("name"),
|
|
76
|
+
"uses": uses(params(d), ds.get("cols", set()), ds.get("metrics", set()))}
|
|
77
|
+
_index[root] = ix
|
|
78
|
+
return ix
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def uses(p, cols, metrics):
|
|
82
|
+
"""Dataset columns and metrics a chart's form data refers to."""
|
|
83
|
+
out = set()
|
|
84
|
+
|
|
85
|
+
def add(v):
|
|
86
|
+
if isinstance(v, str):
|
|
87
|
+
if v.lower() in cols | metrics or not cols:
|
|
88
|
+
out.add(v.lower())
|
|
89
|
+
elif isinstance(v, dict):
|
|
90
|
+
if isinstance(v.get("column"), dict):
|
|
91
|
+
add(v["column"].get("column_name"))
|
|
92
|
+
if v.get("subject"):
|
|
93
|
+
add(v["subject"])
|
|
94
|
+
for w in re.findall(r"[A-Za-z_]\w*", str(v.get("sqlExpression") or "")):
|
|
95
|
+
if w.lower() in cols | metrics:
|
|
96
|
+
out.add(w.lower())
|
|
97
|
+
elif isinstance(v, list):
|
|
98
|
+
for x in v:
|
|
99
|
+
add(x)
|
|
100
|
+
for k in COLUMN_KEYS:
|
|
101
|
+
add(p.get(k))
|
|
102
|
+
add(p.get("adhoc_filters"))
|
|
103
|
+
return sorted(out)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def extract(path, text):
|
|
107
|
+
d = load(text)
|
|
108
|
+
if d.get("table_name") and "database_uuid" in d:
|
|
109
|
+
yield from dataset(d)
|
|
110
|
+
elif d.get("slice_name"):
|
|
111
|
+
c = index(root_of(path)).get(d.get("uuid")) or {}
|
|
112
|
+
me = f"superset:chart.{slug(d['slice_name'])}"
|
|
113
|
+
yield {"dataset": me, "kind": "superset chart", "description": d.get("description") or d.get("viz_type")}
|
|
114
|
+
if c.get("dataset"):
|
|
115
|
+
yield {"edge": [c["dataset"], me], "kind": "superset chart", "source_kind": "superset dataset"}
|
|
116
|
+
for col in c["uses"]:
|
|
117
|
+
yield {"column": [me, col]}
|
|
118
|
+
yield {"column_edge": [f"{c['dataset']}.{col}", f"{me}.{col}"]}
|
|
119
|
+
elif d.get("dashboard_title"):
|
|
120
|
+
me = f"superset:dashboard.{slug(d.get('slug') or d['dashboard_title'])}"
|
|
121
|
+
yield {"dataset": me, "kind": "superset dashboard", "description": d["dashboard_title"]}
|
|
122
|
+
ix = index(root_of(path))
|
|
123
|
+
for v in (d.get("position") or {}).values():
|
|
124
|
+
if isinstance(v, dict) and v.get("type") == "CHART" and (c := ix.get((v.get("meta") or {}).get("uuid"))):
|
|
125
|
+
tile = c["name"].split(".", 1)[1]
|
|
126
|
+
yield {"edge": [c["name"], me], "kind": "superset dashboard", "source_kind": "superset chart"}
|
|
127
|
+
yield {"column": [me, tile]}
|
|
128
|
+
for col in c["uses"]:
|
|
129
|
+
yield {"column_edge": [f"{c['name']}.{col}", f"{me}.{tile}"]}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def dataset(d):
|
|
133
|
+
me = f"superset:dataset.{dataset_name(d)}"
|
|
134
|
+
sql = d.get("sql")
|
|
135
|
+
yield {"dataset": me, "kind": "superset dataset", "description": d.get("description")}
|
|
136
|
+
if sql: # a virtual dataset: the tables its SQL reads. ponytail: tables only, its columns don't trace further
|
|
137
|
+
for t in tables(sql):
|
|
138
|
+
yield {"edge": [t, me], "kind": "superset dataset"}
|
|
139
|
+
else:
|
|
140
|
+
yield {"edge": [dataset_name(d), me], "kind": "superset dataset"}
|
|
141
|
+
cols = {str(c["column_name"]).lower(): c for c in d.get("columns") or [] if c.get("column_name")}
|
|
142
|
+
for name, c in cols.items():
|
|
143
|
+
yield {"column": [me, name, c.get("type"), c.get("description") or c.get("verbose_name")]}
|
|
144
|
+
if c.get("expression"): # a calculated column: from the columns its expression names
|
|
145
|
+
for w in set(re.findall(r"[A-Za-z_]\w*", c["expression"].lower())) & set(cols) - {name}:
|
|
146
|
+
yield {"column_edge": [f"{me}.{w}", f"{me}.{name}"]}
|
|
147
|
+
elif not sql:
|
|
148
|
+
yield {"column_edge": [f"{dataset_name(d)}.{name}", f"{me}.{name}"]}
|
|
149
|
+
for m in d.get("metrics") or []:
|
|
150
|
+
if m.get("metric_name"):
|
|
151
|
+
name = str(m["metric_name"]).lower()
|
|
152
|
+
yield {"column": [me, name, "metric", m.get("description") or m.get("expression")]}
|
|
153
|
+
for w in set(re.findall(r"[A-Za-z_]\w*", str(m.get("expression") or "").lower())) & set(cols):
|
|
154
|
+
yield {"column_edge": [f"{me}.{w}", f"{me}.{name}"]}
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def tables(sql):
|
|
158
|
+
"""Tables a query reads, with Jinja templating removed."""
|
|
159
|
+
import sqlglot
|
|
160
|
+
from sqlglot import exp
|
|
161
|
+
sql = re.sub(r"\{%.*?%\}|\{\{.*?\}\}", "1", sql, flags=re.S)
|
|
162
|
+
for dialect in (None, "bigquery", "mysql"):
|
|
163
|
+
try:
|
|
164
|
+
tree = sqlglot.parse_one(sql, read=dialect)
|
|
165
|
+
break
|
|
166
|
+
except Exception:
|
|
167
|
+
tree = None
|
|
168
|
+
if tree is None:
|
|
169
|
+
return []
|
|
170
|
+
ctes = {c.alias_or_name.lower() for c in tree.find_all(exp.CTE)}
|
|
171
|
+
return sorted({".".join(p.name for p in t.parts).lower().replace("`", "") for t in tree.find_all(exp.Table)} - ctes)
|