datawright-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,151 @@
1
+ """datawright connector: Looker's LookML, so dashboards show up in `impact` and `review`.
2
+
3
+ Views (`sql_table_name`, or the tables a `derived_table` reads) become `looker:view.<view>`, with a
4
+ column per dimension and measure traced to the warehouse column it reads. Explores (`looker:explore.<explore>`) read
5
+ their base and joined views; LookML dashboards (`*.dashboard.lookml`) read their explores, and each tile's fields are
6
+ column edges, so `datawright impact mart.orders.revenue` reaches `looker:dashboard.<dashboard>.<tile>`.
7
+ Dashboards built in the Looker UI live in the instance, not the repo: export them through its API instead.
8
+ """
9
+ import re
10
+ from pathlib import Path
11
+
12
+ import yaml
13
+
14
+ TOKEN = re.compile(r"(\+?\w+)\s*:\s*|\}|#[^\n]*")
15
+ REF = re.compile(r"\$\{(\w+)(?:\.(\w+))?\}(?:\.(\w+))?")
16
+
17
+
18
+ def match(path):
19
+ return path.endswith((".lkml", ".lookml"))
20
+
21
+
22
+ def parse(text):
23
+ """LookML as nested [key, name or value, children or None]."""
24
+ root, i = [], 0
25
+ stack = [root]
26
+ while m := TOKEN.search(text, i):
27
+ tok, i = m.group(0), m.end()
28
+ if tok.startswith("#"):
29
+ continue
30
+ if tok == "}":
31
+ if len(stack) > 1:
32
+ stack.pop()
33
+ continue
34
+ key = m.group(1)
35
+ if key.startswith("sql") or key.endswith("_sql") or key == "html": # free text up to ";;"
36
+ end = text.find(";;", i)
37
+ end = len(text) if end < 0 else end
38
+ stack[-1].append([key, text[i:end].strip(), None])
39
+ i = end + 2
40
+ elif b := re.match(r"(\+?\w*)\s*\{", text[i:]):
41
+ node = [key, b.group(1), []]
42
+ stack[-1].append(node)
43
+ stack.append(node[2])
44
+ i += b.end()
45
+ else:
46
+ v = re.match(r'"[^"]*"|\[[^\]]*\]|[^\s}\]]+', text[i:])
47
+ stack[-1].append([key, v.group(0).strip('"') if v else "", None])
48
+ i += v.end() if v else 0
49
+ return root
50
+
51
+
52
+ def get(node, key):
53
+ return next((v for k, v, c in node or () if k == key and c is None), None)
54
+
55
+
56
+ def table(name):
57
+ return re.sub(r'[`"\[\]]', "", name).strip().lower()
58
+
59
+
60
+ def constants():
61
+ """`@{NAME}` values from the project's manifest.lkml. ROOT is set by datawright."""
62
+ try:
63
+ text = next(Path(ROOT).rglob("manifest.lkml")).read_text()
64
+ except (NameError, StopIteration, OSError):
65
+ return {}
66
+ return {n: get(c, "value") or "" for k, n, c in parse(text) if k == "constant" and c is not None}
67
+
68
+
69
+ def extract(path, text):
70
+ consts = constants()
71
+ text = re.sub(r"@\{(\w+)\}", lambda m: consts.get(m[1], m[0]), text)
72
+ if path.endswith(".lookml"): # LookML dashboards are YAML
73
+ yield from dashboards(text)
74
+ return
75
+ for key, name, body in parse(text):
76
+ name = name.lstrip("+")
77
+ if key == "view" and body is not None:
78
+ yield from view(name, body)
79
+ elif key == "explore" and body is not None:
80
+ base = get(body, "from") or get(body, "view_name") or name
81
+ yield {"edge": [f"looker:view.{base}", f"looker:explore.{name}"], "kind": "looker explore", "source_kind": "looker view"}
82
+ for k, j, jb in body:
83
+ if k == "join" and jb is not None:
84
+ yield {"edge": [f"looker:view.{get(jb, 'from') or j}", f"looker:explore.{name}"], "kind": "looker explore", "source_kind": "looker view"}
85
+
86
+
87
+ def view(name, body):
88
+ me = f"looker:view.{name}"
89
+ src = get(body, "sql_table_name")
90
+ derived = next((c for k, _, c in body if k == "derived_table" and c is not None), None)
91
+ if src:
92
+ src = table(src)
93
+ yield {"edge": [src, me], "kind": "looker view"}
94
+ elif derived is not None:
95
+ sql = re.sub(r"\{%\s*condition.*?endcondition\s*%\}", "1=1", get(derived, "sql") or "", flags=re.S) # a templated filter
96
+ sql = re.sub(r"\{%.*?%\}|\{\{.*?\}\}", "", sql, flags=re.S)
97
+ for v in re.findall(r"\$\{(\w+)\.SQL_TABLE_NAME\}", sql):
98
+ yield {"edge": [f"looker:view.{v}", me], "kind": "looker view", "source_kind": "looker view"}
99
+ for t in tables(re.sub(r"\$\{\w+\.SQL_TABLE_NAME\}", "looker_pdt", sql)):
100
+ yield {"edge": [t, me], "kind": "looker view"}
101
+ for k, e, _ in derived:
102
+ if k == "explore_source": # a native derived table queries an explore
103
+ yield {"edge": [f"looker:explore.{e}", me], "kind": "looker view", "source_kind": "looker explore"}
104
+ yield {"dataset": me, "kind": "looker view", "description": get(body, "label")}
105
+ for k, field, fb in body:
106
+ if k not in ("dimension", "measure", "dimension_group") or fb is None:
107
+ continue
108
+ names = [f"{field}_{t}" for t in re.findall(r"\w+", get(fb, "timeframes") or "")] if k == "dimension_group" else []
109
+ for f in names or [field]:
110
+ yield {"column": [me, f, get(fb, "type"), get(fb, "description") or get(fb, "label")]}
111
+ for v, col, tcol in REF.findall(get(fb, "sql") or ""):
112
+ if v == "TABLE" and tcol:
113
+ origin = f"{src or me}.{tcol}"
114
+ elif v != "TABLE" and not tcol:
115
+ origin = f"looker:view.{v}.{col}" if col else f"{me}.{v}"
116
+ else:
117
+ continue
118
+ if origin.lower() != f"{me}.{f}".lower():
119
+ yield {"column_edge": [origin, f"{me}.{f}"]}
120
+
121
+
122
+ def tables(sql):
123
+ """Tables a derived table's SQL reads. ponytail: tables only, so its fields don't trace to warehouse columns."""
124
+ import sqlglot
125
+ from sqlglot import exp
126
+ for dialect in (None, "bigquery"): # backticked `project.dataset.table` needs bigquery's reader
127
+ try:
128
+ tree = sqlglot.parse_one(sql, read=dialect)
129
+ break
130
+ except Exception:
131
+ tree = None
132
+ if tree is None:
133
+ return []
134
+ ctes = {c.alias_or_name.lower() for c in tree.find_all(exp.CTE)}
135
+ names = {table(".".join(p.name for p in t.parts)) for t in tree.find_all(exp.Table)}
136
+ return sorted(names - ctes - {"looker_pdt"})
137
+
138
+
139
+ def dashboards(text):
140
+ for d in yaml.safe_load(text) or []:
141
+ if not isinstance(d, dict) or not d.get("dashboard"):
142
+ continue
143
+ me = f"looker:dashboard.{d['dashboard']}"
144
+ yield {"dataset": me, "kind": "looker dashboard", "description": d.get("title")}
145
+ for el in d.get("elements") or []:
146
+ if el.get("explore"):
147
+ yield {"edge": [f"looker:explore.{el['explore']}", me], "kind": "looker dashboard", "source_kind": "looker explore"}
148
+ tile = el.get("name") or el.get("title")
149
+ for f in el.get("fields") or []:
150
+ if tile and "." in str(f):
151
+ yield {"column_edge": [f"looker:view.{f}", f"{me}.{tile}"]}
@@ -0,0 +1,140 @@
1
+ """datawright connector: Metabase questions, models and dashboards, so they show up in `impact` and `review`.
2
+
3
+ Reads .data-context/imports/metabase/metabase.json, written by `uv run <data-context skill>/assets/bi_export.py metabase`
4
+ (read-only API calls). Questions become `metabase:question.<name>` (models `metabase:model.<name>`) reading the tables
5
+ their query builder or SQL names, and the questions they build on; dashboards `metabase:dashboard.<name>` with a column
6
+ per card, so `datawright impact mart.orders.revenue` reaches `metabase:dashboard.sales.<card>`.
7
+ ponytail: in SQL questions, any column of a table it reads that the SQL names counts as used (any clause).
8
+ """
9
+ import json
10
+ import re
11
+
12
+
13
+ def match(path):
14
+ return path.endswith("metabase.json")
15
+
16
+
17
+ def slug(s):
18
+ return re.sub(r"\W+", "_", str(s).strip().lower()).strip("_")
19
+
20
+
21
+ def field_ids(node, out):
22
+ """Field ids in a query, legacy (["field", 12, opts]) or staged (["field", {opts}, 12])."""
23
+ if isinstance(node, list):
24
+ if len(node) >= 2 and node[0] == "field":
25
+ out.update(x for x in node[1:3] if isinstance(x, int))
26
+ for x in node:
27
+ field_ids(x, out)
28
+ elif isinstance(node, dict):
29
+ for x in node.values():
30
+ field_ids(x, out)
31
+ return out
32
+
33
+
34
+ def sources(node, out):
35
+ """source-table values anywhere in a query: table ids, and "card__N" for questions built on questions."""
36
+ if isinstance(node, dict):
37
+ for k, v in node.items():
38
+ if k in ("source-table", "source-card"):
39
+ out.add(f"card__{v}" if k == "source-card" else v)
40
+ sources(v, out)
41
+ elif isinstance(node, list):
42
+ for x in node:
43
+ sources(x, out)
44
+ return out
45
+
46
+
47
+ def field_names(node, out):
48
+ """Columns of a base question named in a query: ["field", "REVENUE", opts] or ["field", opts, "REVENUE"]."""
49
+ if isinstance(node, list):
50
+ if len(node) >= 2 and node[0] == "field":
51
+ out.update(x.lower() for x in node[1:3] if isinstance(x, str))
52
+ for x in node:
53
+ field_names(x, out)
54
+ elif isinstance(node, dict):
55
+ for x in node.values():
56
+ field_names(x, out)
57
+ return out
58
+
59
+
60
+ def native_sql(q):
61
+ if q.get("type") == "native":
62
+ return (q.get("native") or {}).get("query") or ""
63
+ return " ".join(s.get("native", "") for s in q.get("stages") or [] if s.get("lib/type") == "mbql.stage/native")
64
+
65
+
66
+ def extract(path, text):
67
+ doc = json.loads(text)
68
+ tables = {t["id"]: ".".join(x for x in (t.get("schema"), t["name"]) if x).lower() for t in doc.get("tables") or []}
69
+ fields = {f["id"]: (tables[t["id"]], f["name"].lower()) for t in doc.get("tables") or [] for f in t.get("fields") or []}
70
+ cols_of = {}
71
+ for tid, col in fields.values():
72
+ cols_of.setdefault(tid, set()).add(col)
73
+ cards = [c for c in doc.get("cards") or [] if not c.get("archived")]
74
+ taken = {}
75
+ for c in cards:
76
+ taken[slug(c["name"])] = taken.get(slug(c["name"]), 0) + 1
77
+ name = {c["id"]: f"metabase:{'model' if c.get('type') == 'model' else 'question'}.{slug(c['name'])}"
78
+ + (f"_{c['id']}" if taken[slug(c["name"])] > 1 else "") for c in cards}
79
+ used = {}
80
+ for c in sorted(cards, key=lambda c: c["id"]): # a question's base questions come first, so their columns are known
81
+ me, q = name[c["id"]], c.get("dataset_query") or {}
82
+ kind = "metabase model" if c.get("type") == "model" else "metabase question"
83
+ yield {"dataset": me, "kind": kind, "description": c.get("description")}
84
+ reads, cols, parents = set(), set(), set()
85
+ for s in sources(q, set()):
86
+ if isinstance(s, int) and s in tables:
87
+ reads.add(tables[s])
88
+ elif isinstance(s, str) and s.startswith("card__") and s[6:].isdigit() and int(s[6:]) in name:
89
+ parents.add(int(s[6:]))
90
+ cols |= {fields[f] for f in field_ids(q, set()) if f in fields}
91
+ sql = native_sql(q)
92
+ if sql: # strip Metabase's template syntax, then read it like any SQL
93
+ parents |= {int(r) for r in re.findall(r"\{\{\s*#(\d+)", sql) if int(r) in name}
94
+ sql = re.sub(r"\{\{\s*#\d+[^}]*\}\}", "metabase_card", re.sub(r"\[\[.*?\]\]", "", sql, flags=re.S))
95
+ sql = re.sub(r"\{\{[^}]*\}\}", "1", sql)
96
+ found = tables_in(sql)
97
+ for f in found - {"metabase_card"}: # as Metabase knows it (schema.table), when the SQL leaves the schema out
98
+ known = [t for t in tables.values() if t == f or t.endswith("." + f) or f.endswith("." + t)]
99
+ reads.add(known[0] if len(known) == 1 else f)
100
+ words = set(re.findall(r"[a-z_]\w*", sql.lower()))
101
+ cols |= {(t, w) for t in reads for w in cols_of.get(t, set()) & words}
102
+ for t in sorted(reads):
103
+ yield {"edge": [t, me], "kind": kind}
104
+ for col in c.get("columns") or []:
105
+ yield {"column": [me, str(col).lower()]}
106
+ for t, col in sorted(cols):
107
+ yield {"column_edge": [f"{t}.{col}", f"{me}.{col}"]}
108
+ named = field_names(q, set()) # a base question's columns, referred to by name
109
+ for pid in sorted(parents):
110
+ yield {"edge": [name[pid], me], "kind": kind, "source_kind": "metabase question"}
111
+ via = {n for n in named if n in used.get(pid, set())} if named else used.get(pid, set()) # none named: all pass through
112
+ for col in sorted(via):
113
+ yield {"column_edge": [f"{name[pid]}.{col}", f"{me}.{col}"]}
114
+ cols.add((name[pid], col))
115
+ used[c["id"]] = {col for _, col in cols}
116
+ for d in doc.get("dashboards") or []:
117
+ me = f"metabase:dashboard.{slug(d['name'])}"
118
+ yield {"dataset": me, "kind": "metabase dashboard", "description": d.get("description") or d["name"]}
119
+ for cid in d.get("cards") or []:
120
+ if cid in name:
121
+ tile = name[cid].split(".", 1)[1]
122
+ yield {"edge": [name[cid], me], "kind": "metabase dashboard", "source_kind": "metabase question"}
123
+ yield {"column": [me, tile]}
124
+ for col in sorted(used.get(cid, ())):
125
+ yield {"column_edge": [f"{name[cid]}.{col}", f"{me}.{tile}"]}
126
+
127
+
128
+ def tables_in(sql):
129
+ import sqlglot
130
+ from sqlglot import exp
131
+ for dialect in (None, "postgres", "mysql", "bigquery"):
132
+ try:
133
+ tree = sqlglot.parse_one(sql, read=dialect)
134
+ break
135
+ except Exception:
136
+ tree = None
137
+ if tree is None:
138
+ return set()
139
+ ctes = {c.alias_or_name.lower() for c in tree.find_all(exp.CTE)}
140
+ return {".".join(p.name for p in t.parts).lower() for t in tree.find_all(exp.Table)} - ctes
@@ -0,0 +1,77 @@
1
+ """datawright connector: MyBatis mapper XML, the SQL behind Java services (<select>, <insert>, <update>, <delete>).
2
+
3
+ Dynamic SQL is rendered to plain SQL: every <if> is included, <where>/<set>/<trim> add their keyword and drop the leading
4
+ AND/OR or trailing comma, <foreach> keeps one item, <include> pulls in its <sql> fragment, #{param} becomes a value and
5
+ ${param} an unknown. A statement with <choose> is rendered once per branch, so every table any branch reads is seen.
6
+ """
7
+ import re
8
+ import xml.etree.ElementTree as ET
9
+
10
+ FILES = r"mapper" # which .xml paths hold mappers (a regex on the path)
11
+ STATEMENTS = ("select", "insert", "update", "delete")
12
+
13
+
14
+ def match(path):
15
+ return path.endswith(".xml") and re.search(FILES, path, re.I) is not None
16
+
17
+
18
+ def extract(path, text):
19
+ try:
20
+ root = ET.fromstring(re.sub(r"<!DOCTYPE[^>]*>", "", text))
21
+ except ET.ParseError:
22
+ return
23
+ if root.tag != "mapper":
24
+ return
25
+ fragments = {e.get("id"): e for e in root.iter("sql") if e.get("id")}
26
+ for el in root:
27
+ if el.tag not in STATEMENTS:
28
+ continue
29
+ line = text.count("\n", 0, max(text.find(f'id="{el.get("id")}"'), 0)) + 1
30
+ branches = max([len([c for c in ch if c.tag in ("when", "otherwise")]) for ch in el.iter("choose")] or [1])
31
+ seen = set()
32
+ for pick in range(min(branches, 12)):
33
+ sql = re.sub(r"--[^\n]*", "", render(el, fragments, pick)) # before joining lines, or a comment eats the rest
34
+ sql = re.sub(r"\s+", " ", sql).strip()
35
+ sql = re.sub(r"#\{[^}]*\}", "1", sql)
36
+ sql = re.sub(r"\$\{[^}]*\}", "jinja_expr", sql)
37
+ if sql and sql not in seen:
38
+ seen.add(sql)
39
+ yield {"sql": sql, "line": line}
40
+
41
+
42
+ def render(el, fragments, pick, depth=0):
43
+ """The SQL text an element stands for, with one branch of each <choose> (the pick-th, or its last)."""
44
+ if depth > 20:
45
+ return ""
46
+ parts = [el.text or ""]
47
+ for ch in el:
48
+ inner = lambda e=ch: render(e, fragments, pick, depth + 1)
49
+ if ch.tag == "include":
50
+ ref = (ch.get("refid") or "").rsplit(".", 1)[-1]
51
+ parts.append(render(fragments[ref], fragments, pick, depth + 1) if ref in fragments else "")
52
+ elif ch.tag == "where":
53
+ body = re.sub(r"^\s*(AND|OR)\b", "", inner(), flags=re.I)
54
+ parts.append(f" WHERE {body} " if body.strip() else " ")
55
+ elif ch.tag == "set":
56
+ parts.append(" SET " + re.sub(r",\s*$", "", inner().rstrip()) + " ")
57
+ elif ch.tag == "trim":
58
+ body = inner()
59
+ for o in (ch.get("prefixOverrides") or "").split("|"):
60
+ if o.strip() and body.lstrip().upper().startswith(o.strip().upper()):
61
+ body = body.lstrip()[len(o.strip()):]
62
+ for o in (ch.get("suffixOverrides") or "").split("|"):
63
+ if o.strip() and body.rstrip().upper().endswith(o.strip().upper()):
64
+ body = body.rstrip()[: -len(o.strip())]
65
+ parts.append(f" {ch.get('prefix') or ''} {body} {ch.get('suffix') or ''} ")
66
+ elif ch.tag == "foreach":
67
+ parts.append(f" {ch.get('open') or ''}{inner()}{ch.get('close') or ''} ")
68
+ elif ch.tag == "choose":
69
+ options = [c for c in ch if c.tag in ("when", "otherwise")]
70
+ if options:
71
+ parts.append(" " + render(options[min(pick, len(options) - 1)], fragments, pick, depth + 1) + " ")
72
+ elif ch.tag in ("if", "when", "otherwise"):
73
+ parts.append(" " + inner() + " ")
74
+ elif ch.tag != "bind":
75
+ parts.append(inner())
76
+ parts.append(ch.tail or "")
77
+ return "".join(parts)
@@ -0,0 +1,53 @@
1
+ """datawright connector: data contracts in the Open Data Contract Standard (ODCS v3, as written by datacontract-cli), plus the older
2
+ Data Contract Specification. Each contract's tables get the contracted columns, types and required flags, a description and an
3
+ owner; `datawright review` then flags a change that stops producing a required column, and cards show the contract.
4
+ Copy to .data-context/connectors/ and adjust FILES to where your contracts live.
5
+ """
6
+ import re
7
+
8
+ FILES = r"contract|odcs" # a regex on the path of .yaml/.yml files to read
9
+
10
+
11
+ def match(path):
12
+ return path.endswith((".yaml", ".yml")) and re.search(FILES, path, re.I) is not None
13
+
14
+
15
+ def extract(path, text):
16
+ import yaml
17
+ for doc in yaml.safe_load_all(text):
18
+ if isinstance(doc, dict) and (doc.get("kind") == "DataContract" or "dataContractSpecification" in doc):
19
+ yield from contract(doc)
20
+
21
+
22
+ def contract(doc):
23
+ owner = text_of(doc.get("owner")) or text_of((doc.get("info") or {}).get("owner")) or team(doc.get("team"))
24
+ prefix = qualifier(doc.get("servers"))
25
+ tables = doc.get("schema") if isinstance(doc.get("schema"), list) else [ # ODCS: schema is a list; the older spec keys models by name
26
+ {"name": k, **v} for k, v in (doc.get("models") or {}).items() if isinstance(v, dict)]
27
+ for tbl in tables:
28
+ if not isinstance(tbl, dict) or not (tbl.get("physicalName") or tbl.get("name")):
29
+ continue
30
+ name = ".".join(filter(None, [prefix, tbl.get("physicalName") or tbl.get("name")]))
31
+ yield {"dataset": name, "description": text_of(tbl.get("description")) or text_of(doc.get("description")), "owner": owner}
32
+ props = tbl.get("properties") or [{"name": k, **v} for k, v in (tbl.get("fields") or {}).items() if isinstance(v, dict)]
33
+ for c in props:
34
+ if isinstance(c, dict) and c.get("name"):
35
+ typ = c.get("physicalType") or c.get("logicalType") or c.get("type")
36
+ required = c.get("required") or c.get("primaryKey") or c.get("primary")
37
+ yield {"contract": [name, c["name"], typ, required]}
38
+
39
+
40
+ def text_of(v): # ODCS v3 descriptions are objects ({purpose, usage, limitations})
41
+ return v if isinstance(v, str) else (v.get("purpose") or v.get("usage")) if isinstance(v, dict) else None
42
+
43
+
44
+ def team(t):
45
+ return t.get("name") if isinstance(t, dict) else None
46
+
47
+
48
+ def qualifier(servers):
49
+ """The schema (or dataset) of the only server, so contract names line up with the warehouse."""
50
+ items = servers if isinstance(servers, list) else list(servers.values()) if isinstance(servers, dict) else []
51
+ if len(items) == 1 and isinstance(items[0], dict):
52
+ return items[0].get("schema") or items[0].get("dataset")
53
+ return None
@@ -0,0 +1,171 @@
1
+ """datawright connector: Apache Superset (and Preset) dashboards, so they show up in `impact` and `review`.
2
+
3
+ Reads Superset's export format: the ZIP from Dashboards > Export (or `superset export-dashboards`, or the REST API)
4
+ dropped into .data-context/imports/, which datawright unpacks; or the YAML files kept in git by `preset-cli superset sync`.
5
+ Datasets become `superset:dataset.<table>`, read from their table or, for virtual datasets, from the tables their SQL
6
+ reads; charts `superset:chart.<name>` with the columns and metrics they use; dashboards `superset:dashboard.<name>`
7
+ with a column per chart. So `datawright impact mart.orders.revenue` reaches `superset:dashboard.sales.revenue_by_day`.
8
+ """
9
+ import json
10
+ import re
11
+ from pathlib import Path
12
+
13
+ import yaml
14
+
15
+ KINDS = ("charts", "datasets", "dashboards")
16
+ COLUMN_KEYS = ("groupby", "columns", "all_columns", "x_axis", "granularity_sqla", "series", "entity", "order_by_cols",
17
+ "metric", "metrics", "metric_2", "percent_metrics", "secondary_metric", "size", "x", "y", "timeseries_limit_metric")
18
+ _index = {}
19
+
20
+
21
+ def match(path):
22
+ p = Path(path)
23
+ return p.suffix in (".yaml", ".yml") and (bool(set(p.parts) & set(KINDS)) or p.name in ("dashboard.yaml", "dataset.yaml"))
24
+
25
+
26
+ def slug(s):
27
+ return re.sub(r"\W+", "_", str(s).strip().lower()).strip("_")
28
+
29
+
30
+ def root_of(path):
31
+ """The export's top folder: the one holding charts/, datasets/ and dashboards/."""
32
+ p = Path(ROOT, path)
33
+ for i, part in enumerate(p.parts):
34
+ if part in KINDS:
35
+ return Path(*p.parts[:i])
36
+ return p.parent
37
+
38
+
39
+ def load(text):
40
+ try:
41
+ doc = yaml.safe_load(text)
42
+ except yaml.YAMLError:
43
+ return {}
44
+ return doc if isinstance(doc, dict) else {}
45
+
46
+
47
+ def params(chart):
48
+ p = chart.get("params") or {}
49
+ if isinstance(p, str): # older exports keep params as a JSON string
50
+ try:
51
+ p = json.loads(p)
52
+ except ValueError:
53
+ p = {}
54
+ return p if isinstance(p, dict) else {}
55
+
56
+
57
+ def dataset_name(d):
58
+ return ".".join(x for x in (d.get("schema"), d.get("table_name")) if x).lower()
59
+
60
+
61
+ def index(root):
62
+ """uuid -> what it is, across one export: datasets with their columns and metrics, charts with what they use."""
63
+ if root in _index:
64
+ return _index[root]
65
+ docs = [load(f.read_text(errors="replace")) for f in sorted(root.rglob("*.y*ml"))]
66
+ ix = {}
67
+ for d in docs:
68
+ if d.get("table_name") and d.get("uuid"):
69
+ cols = {str(c.get("column_name")).lower() for c in d.get("columns") or [] if c.get("column_name")}
70
+ metrics = {str(m.get("metric_name")).lower() for m in d.get("metrics") or [] if m.get("metric_name")}
71
+ ix[d["uuid"]] = {"name": f"superset:dataset.{dataset_name(d)}", "cols": cols, "metrics": metrics}
72
+ for d in docs:
73
+ if d.get("slice_name") and d.get("uuid"):
74
+ ds = ix.get(d.get("dataset_uuid"), {})
75
+ ix[d["uuid"]] = {"name": f"superset:chart.{slug(d['slice_name'])}", "dataset": ds.get("name"),
76
+ "uses": uses(params(d), ds.get("cols", set()), ds.get("metrics", set()))}
77
+ _index[root] = ix
78
+ return ix
79
+
80
+
81
+ def uses(p, cols, metrics):
82
+ """Dataset columns and metrics a chart's form data refers to."""
83
+ out = set()
84
+
85
+ def add(v):
86
+ if isinstance(v, str):
87
+ if v.lower() in cols | metrics or not cols:
88
+ out.add(v.lower())
89
+ elif isinstance(v, dict):
90
+ if isinstance(v.get("column"), dict):
91
+ add(v["column"].get("column_name"))
92
+ if v.get("subject"):
93
+ add(v["subject"])
94
+ for w in re.findall(r"[A-Za-z_]\w*", str(v.get("sqlExpression") or "")):
95
+ if w.lower() in cols | metrics:
96
+ out.add(w.lower())
97
+ elif isinstance(v, list):
98
+ for x in v:
99
+ add(x)
100
+ for k in COLUMN_KEYS:
101
+ add(p.get(k))
102
+ add(p.get("adhoc_filters"))
103
+ return sorted(out)
104
+
105
+
106
+ def extract(path, text):
107
+ d = load(text)
108
+ if d.get("table_name") and "database_uuid" in d:
109
+ yield from dataset(d)
110
+ elif d.get("slice_name"):
111
+ c = index(root_of(path)).get(d.get("uuid")) or {}
112
+ me = f"superset:chart.{slug(d['slice_name'])}"
113
+ yield {"dataset": me, "kind": "superset chart", "description": d.get("description") or d.get("viz_type")}
114
+ if c.get("dataset"):
115
+ yield {"edge": [c["dataset"], me], "kind": "superset chart", "source_kind": "superset dataset"}
116
+ for col in c["uses"]:
117
+ yield {"column": [me, col]}
118
+ yield {"column_edge": [f"{c['dataset']}.{col}", f"{me}.{col}"]}
119
+ elif d.get("dashboard_title"):
120
+ me = f"superset:dashboard.{slug(d.get('slug') or d['dashboard_title'])}"
121
+ yield {"dataset": me, "kind": "superset dashboard", "description": d["dashboard_title"]}
122
+ ix = index(root_of(path))
123
+ for v in (d.get("position") or {}).values():
124
+ if isinstance(v, dict) and v.get("type") == "CHART" and (c := ix.get((v.get("meta") or {}).get("uuid"))):
125
+ tile = c["name"].split(".", 1)[1]
126
+ yield {"edge": [c["name"], me], "kind": "superset dashboard", "source_kind": "superset chart"}
127
+ yield {"column": [me, tile]}
128
+ for col in c["uses"]:
129
+ yield {"column_edge": [f"{c['name']}.{col}", f"{me}.{tile}"]}
130
+
131
+
132
+ def dataset(d):
133
+ me = f"superset:dataset.{dataset_name(d)}"
134
+ sql = d.get("sql")
135
+ yield {"dataset": me, "kind": "superset dataset", "description": d.get("description")}
136
+ if sql: # a virtual dataset: the tables its SQL reads. ponytail: tables only, its columns don't trace further
137
+ for t in tables(sql):
138
+ yield {"edge": [t, me], "kind": "superset dataset"}
139
+ else:
140
+ yield {"edge": [dataset_name(d), me], "kind": "superset dataset"}
141
+ cols = {str(c["column_name"]).lower(): c for c in d.get("columns") or [] if c.get("column_name")}
142
+ for name, c in cols.items():
143
+ yield {"column": [me, name, c.get("type"), c.get("description") or c.get("verbose_name")]}
144
+ if c.get("expression"): # a calculated column: from the columns its expression names
145
+ for w in set(re.findall(r"[A-Za-z_]\w*", c["expression"].lower())) & set(cols) - {name}:
146
+ yield {"column_edge": [f"{me}.{w}", f"{me}.{name}"]}
147
+ elif not sql:
148
+ yield {"column_edge": [f"{dataset_name(d)}.{name}", f"{me}.{name}"]}
149
+ for m in d.get("metrics") or []:
150
+ if m.get("metric_name"):
151
+ name = str(m["metric_name"]).lower()
152
+ yield {"column": [me, name, "metric", m.get("description") or m.get("expression")]}
153
+ for w in set(re.findall(r"[A-Za-z_]\w*", str(m.get("expression") or "").lower())) & set(cols):
154
+ yield {"column_edge": [f"{me}.{w}", f"{me}.{name}"]}
155
+
156
+
157
+ def tables(sql):
158
+ """Tables a query reads, with Jinja templating removed."""
159
+ import sqlglot
160
+ from sqlglot import exp
161
+ sql = re.sub(r"\{%.*?%\}|\{\{.*?\}\}", "1", sql, flags=re.S)
162
+ for dialect in (None, "bigquery", "mysql"):
163
+ try:
164
+ tree = sqlglot.parse_one(sql, read=dialect)
165
+ break
166
+ except Exception:
167
+ tree = None
168
+ if tree is None:
169
+ return []
170
+ ctes = {c.alias_or_name.lower() for c in tree.find_all(exp.CTE)}
171
+ return sorted({".".join(p.name for p in t.parts).lower().replace("`", "") for t in tree.find_all(exp.Table)} - ctes)