data-graf 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_graf-0.1.0.dev0.dist-info/METADATA +86 -0
- data_graf-0.1.0.dev0.dist-info/RECORD +15 -0
- data_graf-0.1.0.dev0.dist-info/WHEEL +4 -0
- data_graf-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- data_graf-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
- data_graf-0.1.0.dev0.dist-info/licenses/NOTICE +10 -0
- datagraf/__init__.py +17 -0
- datagraf/checks.py +114 -0
- datagraf/cli.py +231 -0
- datagraf/export.py +41 -0
- datagraf/lineage.py +212 -0
- datagraf/model.py +201 -0
- datagraf/schemas/osi-0.1.1.schema.json +344 -0
- datagraf/schemas/ossie-0.3.0.dev0.schema.json +398 -0
- datagraf/sqlite.py +134 -0
datagraf/cli.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""`dgr`: validate, explore, export and profile an Ossie semantic model.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 ok, 1 the model has errors (validate / check / profile), 2 a usage or load error.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import click
|
|
12
|
+
import yaml
|
|
13
|
+
|
|
14
|
+
from . import __version__
|
|
15
|
+
from . import checks as C
|
|
16
|
+
from . import lineage as L
|
|
17
|
+
from .model import ModelError, load, read_document
|
|
18
|
+
|
|
19
|
+
MODEL = click.argument("model", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
|
20
|
+
PICK = click.option("--name", "model_name", default=None, help="Which model, when a 0.1.x file holds several.")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _load(path: Path, name: str | None):
|
|
24
|
+
try:
|
|
25
|
+
m = load(path, name)
|
|
26
|
+
except ModelError as e:
|
|
27
|
+
raise click.ClickException(str(e)) from None
|
|
28
|
+
return m, L.build(m)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _print_problems(problems: list[dict]) -> None:
|
|
32
|
+
for p in problems:
|
|
33
|
+
where = f" {p['node']}" if p["node"] else ""
|
|
34
|
+
click.echo(f" {p['severity']:7} {p['code']}{where}: {p['message']}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@click.group()
|
|
38
|
+
@click.version_option(__version__, prog_name="dgr")
|
|
39
|
+
def main() -> None:
|
|
40
|
+
"""Lineage, checks and profiling for Apache Ossie semantic models."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@main.command()
|
|
44
|
+
@MODEL
|
|
45
|
+
@PICK
|
|
46
|
+
@click.option("--strict", is_flag=True, help="Fail on warnings too.")
|
|
47
|
+
def validate(model: Path, model_name: str | None, strict: bool) -> None:
|
|
48
|
+
"""The JSON Schema for the file's Ossie version, then references, joins and gaps."""
|
|
49
|
+
try:
|
|
50
|
+
doc = read_document(model)
|
|
51
|
+
except ModelError as e:
|
|
52
|
+
raise click.ClickException(str(e)) from None
|
|
53
|
+
problems = C.schema_problems(doc)
|
|
54
|
+
m, G = _load(model, model_name)
|
|
55
|
+
problems += C.model_problems(m, G)
|
|
56
|
+
errors, warnings = C.summarize(problems)
|
|
57
|
+
click.echo(f"{model}: {m.name} (Ossie {m.version or '?'}): {errors} error(s), {warnings} warning(s)")
|
|
58
|
+
_print_problems(problems)
|
|
59
|
+
sys.exit(1 if errors or (strict and warnings) else 0)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@main.command()
|
|
63
|
+
@MODEL
|
|
64
|
+
@PICK
|
|
65
|
+
def check(model: Path, model_name: str | None) -> None:
|
|
66
|
+
"""References, joins and gaps only (no schema)."""
|
|
67
|
+
m, G = _load(model, model_name)
|
|
68
|
+
problems = C.model_problems(m, G)
|
|
69
|
+
errors, warnings = C.summarize(problems)
|
|
70
|
+
click.echo(f"{m.name}: {errors} error(s), {warnings} warning(s)")
|
|
71
|
+
_print_problems(problems)
|
|
72
|
+
sys.exit(1 if errors else 0)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@main.command()
|
|
76
|
+
@MODEL
|
|
77
|
+
@PICK
|
|
78
|
+
def show(model: Path, model_name: str | None) -> None:
|
|
79
|
+
"""Datasets, fields, metrics and joins."""
|
|
80
|
+
m, G = _load(model, model_name)
|
|
81
|
+
click.echo(f"{m.name} (Ossie {m.version or '?'})")
|
|
82
|
+
if m.description:
|
|
83
|
+
click.echo(f" {m.description}")
|
|
84
|
+
n_fields = sum(len(d.fields) for d in m.datasets)
|
|
85
|
+
click.echo(f" {len(m.datasets)} datasets, {n_fields} fields, {len(m.all_metrics())} metrics, {len(m.relationships)} joins")
|
|
86
|
+
for d in m.datasets:
|
|
87
|
+
dims = sum(f.is_dimension for f in d.fields)
|
|
88
|
+
click.echo(f"\n dataset {d.name} <- {d.source} ({len(d.fields)} fields, {dims} dimensions"
|
|
89
|
+
+ (f", key {', '.join(d.primary_key)}" if d.primary_key else "") + ")")
|
|
90
|
+
for mm in d.metrics:
|
|
91
|
+
click.echo(f" metric {mm.name} = {mm.sql}")
|
|
92
|
+
if m.metrics:
|
|
93
|
+
click.echo("\n model metrics")
|
|
94
|
+
for mm in m.metrics:
|
|
95
|
+
click.echo(f" {mm.name} = {mm.sql}")
|
|
96
|
+
if m.relationships:
|
|
97
|
+
click.echo("\n joins (many -> one)")
|
|
98
|
+
for r in m.relationships:
|
|
99
|
+
click.echo(f" {r.from_dataset}({', '.join(r.from_columns)}) -> {r.to_dataset}({', '.join(r.to_columns)})")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _node(G, name: str) -> str:
|
|
103
|
+
try:
|
|
104
|
+
return L.resolve(G, name)
|
|
105
|
+
except KeyError as e:
|
|
106
|
+
raise click.ClickException(e.args[0]) from None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@main.command()
|
|
110
|
+
@MODEL
|
|
111
|
+
@click.argument("name")
|
|
112
|
+
@PICK
|
|
113
|
+
def lineage(model: Path, name: str, model_name: str | None) -> None:
|
|
114
|
+
"""What a metric or field is made of, down to source columns. NAME is `total_sales`, `store_sales.ss_quantity`, or an id."""
|
|
115
|
+
_, G = _load(model, model_name)
|
|
116
|
+
click.echo("\n".join(L.outline(G, _node(G, name), up=True)))
|
|
117
|
+
cols = sorted(n for n in L.upstream(G, _node(G, name)) if G.nodes[n]["kind"] == "column")
|
|
118
|
+
click.echo(f"\n{len(cols)} source column(s): {', '.join(c[len('column:'):] for c in cols) or '-'}")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@main.command()
|
|
122
|
+
@MODEL
|
|
123
|
+
@click.argument("name")
|
|
124
|
+
@PICK
|
|
125
|
+
def impact(model: Path, name: str, model_name: str | None) -> None:
|
|
126
|
+
"""What depends on a column, field, dataset or metric: what a change to it can break."""
|
|
127
|
+
_, G = _load(model, model_name)
|
|
128
|
+
node = _node(G, name)
|
|
129
|
+
click.echo("\n".join(L.outline(G, node, up=False)))
|
|
130
|
+
hit = sorted(n for n in L.downstream(G, node) if G.nodes[n]["kind"] == "metric" and n != node)
|
|
131
|
+
click.echo(f"\n{len(hit)} metric(s) affected: {', '.join(h[len('metric:'):] for h in hit) or '-'}")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@main.command()
|
|
135
|
+
@MODEL
|
|
136
|
+
@PICK
|
|
137
|
+
@click.option("--format", "fmt", type=click.Choice(["zinet", "json"]), default="zinet", show_default=True)
|
|
138
|
+
@click.option("-o", "--out", type=click.Path(dir_okay=False, path_type=Path), help="Write here instead of stdout.")
|
|
139
|
+
def export(model: Path, model_name: str | None, fmt: str, out: Path | None) -> None:
|
|
140
|
+
"""The lineage graph as zinet graph.yaml or JSON."""
|
|
141
|
+
_, G = _load(model, model_name)
|
|
142
|
+
if fmt == "json":
|
|
143
|
+
from .export import to_json
|
|
144
|
+
text = to_json(G)
|
|
145
|
+
else:
|
|
146
|
+
from zinet import to_yaml
|
|
147
|
+
from .export import to_zinet
|
|
148
|
+
text = to_yaml(to_zinet(G))
|
|
149
|
+
if out:
|
|
150
|
+
out.write_text(text, encoding="utf-8")
|
|
151
|
+
click.echo(f"wrote {out}")
|
|
152
|
+
else:
|
|
153
|
+
click.echo(text)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@main.command()
|
|
157
|
+
@MODEL
|
|
158
|
+
@PICK
|
|
159
|
+
@click.option("--format", "fmt", type=click.Choice(["html", "mermaid", "ascii", "svg", "png"]), default="html", show_default=True)
|
|
160
|
+
@click.option("-o", "--out", type=click.Path(dir_okay=False, path_type=Path), help="Required for html, svg and png.")
|
|
161
|
+
def render(model: Path, model_name: str | None, fmt: str, out: Path | None) -> None:
|
|
162
|
+
"""Draw the lineage graph through zinet (svg and png need Graphviz)."""
|
|
163
|
+
from zinet import render as zrender
|
|
164
|
+
from .export import to_zinet
|
|
165
|
+
_, G = _load(model, model_name)
|
|
166
|
+
if fmt in ("html", "svg", "png") and out is None:
|
|
167
|
+
raise click.UsageError(f"--format {fmt} needs -o FILE")
|
|
168
|
+
try:
|
|
169
|
+
data = zrender(to_zinet(G), fmt)
|
|
170
|
+
except RuntimeError as e:
|
|
171
|
+
raise click.ClickException(str(e)) from None
|
|
172
|
+
if out:
|
|
173
|
+
out.write_bytes(data)
|
|
174
|
+
click.echo(f"wrote {out}")
|
|
175
|
+
else:
|
|
176
|
+
click.echo(data.decode("utf-8"))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@main.command()
|
|
180
|
+
@click.option("--sqlite", "db", required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
|
181
|
+
@click.option("--name", default=None, help="Model name (default: the database file's stem).")
|
|
182
|
+
@click.option("-o", "--out", type=click.Path(dir_okay=False, path_type=Path), help="Write here instead of stdout.")
|
|
183
|
+
def draft(db: Path, name: str | None, out: Path | None) -> None:
|
|
184
|
+
"""An Ossie skeleton from a SQLite catalogue: datasets, fields, keys, joins. No descriptions or metrics."""
|
|
185
|
+
from .sqlite import draft as do_draft
|
|
186
|
+
text = "# Drafted by data-graf (dgr) from a SQLite catalogue: add descriptions, ai_context and metrics.\n" + \
|
|
187
|
+
yaml.safe_dump(do_draft(db, name), sort_keys=False, allow_unicode=True)
|
|
188
|
+
if out:
|
|
189
|
+
out.write_text(text, encoding="utf-8")
|
|
190
|
+
click.echo(f"wrote {out}")
|
|
191
|
+
else:
|
|
192
|
+
click.echo(text)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@main.command()
|
|
196
|
+
@MODEL
|
|
197
|
+
@click.option("--sqlite", "db", required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
|
198
|
+
@PICK
|
|
199
|
+
@click.option("--json", "as_json", is_flag=True, help="Print the profile as JSON.")
|
|
200
|
+
def profile(model: Path, db: Path, model_name: str | None, as_json: bool) -> None:
|
|
201
|
+
"""Profile the data behind a model, read-only: rows, nulls, distinct values, ranges, dataset metric values."""
|
|
202
|
+
from .sqlite import profile as do_profile
|
|
203
|
+
m, _ = _load(model, model_name)
|
|
204
|
+
try:
|
|
205
|
+
prof = do_profile(m, db)
|
|
206
|
+
except FileNotFoundError as e:
|
|
207
|
+
raise click.ClickException(str(e)) from None
|
|
208
|
+
failed = 0
|
|
209
|
+
if as_json:
|
|
210
|
+
click.echo(json.dumps(prof, indent=2, default=str))
|
|
211
|
+
for d in prof["datasets"]:
|
|
212
|
+
failed += bool(d.get("error")) + sum(bool(x.get("error")) for x in d["fields"] + d["metrics"])
|
|
213
|
+
if as_json:
|
|
214
|
+
continue
|
|
215
|
+
if d.get("error"):
|
|
216
|
+
click.echo(f"\n{d['name']}: ERROR {d['error']}")
|
|
217
|
+
continue
|
|
218
|
+
click.echo(f"\n{d['name']} ({d['table']}, {d['rows']} rows)")
|
|
219
|
+
for f in d["fields"]:
|
|
220
|
+
if f.get("error"):
|
|
221
|
+
click.echo(f" {f['name']:28} ERROR {f['error']}")
|
|
222
|
+
else:
|
|
223
|
+
nulls = "" if f["nulls"] is None else f"{f['nulls']:>8} null"
|
|
224
|
+
click.echo(f" {f['name']:28} {f['distinct']:>8} distinct {nulls} {f['min']!s:.20} .. {f['max']!s:.20}")
|
|
225
|
+
for me in d["metrics"]:
|
|
226
|
+
click.echo(f" metric {me['name']:21} " + (f"ERROR {me['error']}" if me.get("error") else f"= {me['value']}"))
|
|
227
|
+
sys.exit(1 if failed else 0)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
if __name__ == "__main__":
|
|
231
|
+
main()
|
datagraf/export.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""The lineage graph as zinet `graph.yaml` (for zinet's validate / render / slice) or as JSON.
|
|
2
|
+
|
|
3
|
+
zinet layers: sources and columns are primitives (they depend on nothing); datasets, fields and metrics are concepts;
|
|
4
|
+
the model itself is the one application, depending on every metric (or every dataset, when it has no metrics).
|
|
5
|
+
zinet ids are plain names, so `kind:name` becomes `kind__name` with dots kept.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
import networkx as nx
|
|
13
|
+
from zinet import EdgeType, Graph, Node
|
|
14
|
+
|
|
15
|
+
_EDGE_TYPE = {"reads": "data_flow", "derives": "data_flow", "contains": "depends_on", "computes": "depends_on"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def zinet_id(node: str) -> str:
|
|
19
|
+
return re.sub(r"[^\w.\-]", "_", node.replace(":", "__", 1))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def to_zinet(G: nx.DiGraph) -> Graph:
|
|
23
|
+
prims, concepts = [], []
|
|
24
|
+
for n, data in G.nodes(data=True):
|
|
25
|
+
deps = sorted(G.predecessors(n))
|
|
26
|
+
meta = {k: v for k, v in data.items() if v not in (None, "", [], False) and k != "label"}
|
|
27
|
+
node = Node(id=zinet_id(n), label=f"{data['kind']} {n.split(':', 1)[1]}", deps=[zinet_id(d) for d in deps],
|
|
28
|
+
edge_types={zinet_id(d): EdgeType(_EDGE_TYPE[G.edges[d, n]["rel"]]) for d in deps}, meta=meta)
|
|
29
|
+
(prims if data["kind"] in ("source", "column") and not deps else concepts).append(node)
|
|
30
|
+
tops = [n for n, k in G.nodes(data="kind") if k == "metric" and G.out_degree(n) == 0] or \
|
|
31
|
+
[n for n, k in G.nodes(data="kind") if k == "dataset"]
|
|
32
|
+
app = Node(id=zinet_id(f"model:{G.graph['name']}"), label=f"model {G.graph['name']}", deps=[zinet_id(t) for t in sorted(tops)],
|
|
33
|
+
meta={"kind": "model", "version": G.graph.get("version"), "joins": G.graph["joins"]})
|
|
34
|
+
return Graph(primitives=prims, concepts=concepts, applications=[app])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def to_json(G: nx.DiGraph) -> str:
|
|
38
|
+
return json.dumps({"name": G.graph["name"], "version": G.graph.get("version"),
|
|
39
|
+
"nodes": [{"id": n, **d} for n, d in G.nodes(data=True)],
|
|
40
|
+
"edges": [{"from": a, "to": b, **d} for a, b, d in G.edges(data=True)],
|
|
41
|
+
"joins": G.graph["joins"]}, indent=2)
|
datagraf/lineage.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""The lineage graph of a semantic model: source tables and columns -> datasets and fields -> metrics.
|
|
2
|
+
|
|
3
|
+
Edges point from what is needed to what needs it (zinet's convention), with `rel` one of
|
|
4
|
+
`reads` (source -> dataset), `contains` (dataset -> field), `derives` (column -> field), `computes` (field or metric -> metric).
|
|
5
|
+
Relationships are not dependencies: they are kept as many-to-one joins in `G.graph["joins"]`.
|
|
6
|
+
|
|
7
|
+
References are read from each expression's SQL (sqlglot), by the scoping rules of the spec:
|
|
8
|
+
a field names columns of its dataset's source; a dataset-scoped metric names that dataset's fields (or its other metrics);
|
|
9
|
+
a model-scoped metric names `dataset.field` or `dataset.metric`, or another model-scoped metric by its bare name.
|
|
10
|
+
What cannot be resolved is recorded in `G.graph["problems"]` (read by `checks`), never guessed.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
import networkx as nx
|
|
17
|
+
import sqlglot
|
|
18
|
+
from sqlglot import exp
|
|
19
|
+
|
|
20
|
+
from .model import SemanticModel
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Ref:
|
|
25
|
+
qualifier: str | None # `store_sales` in `store_sales.ss_quantity`; None when unqualified
|
|
26
|
+
name: str
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sql_refs(sql: str) -> tuple[list[Ref], str | None]:
|
|
30
|
+
"""The column references in an expression, and a parse error (or None). `COUNT(*)` names no column."""
|
|
31
|
+
try:
|
|
32
|
+
tree = sqlglot.parse_one(sql)
|
|
33
|
+
except (sqlglot.errors.ParseError, sqlglot.errors.TokenError) as e:
|
|
34
|
+
return [], str(e).splitlines()[0][:200]
|
|
35
|
+
refs = []
|
|
36
|
+
for col in tree.find_all(exp.Column):
|
|
37
|
+
parts = [p.name for p in col.parts]
|
|
38
|
+
if not parts or not parts[-1]:
|
|
39
|
+
continue
|
|
40
|
+
refs.append(Ref(".".join(parts[:-1]) or None, parts[-1]))
|
|
41
|
+
return list(dict.fromkeys(refs)), None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def source_id(source: str) -> str:
|
|
45
|
+
return f"source:{source}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def column_id(source: str, column: str) -> str:
|
|
49
|
+
return f"column:{source}.{column}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def dataset_id(name: str) -> str:
|
|
53
|
+
return f"dataset:{name}"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def field_id(dataset: str, name: str) -> str:
|
|
57
|
+
return f"field:{dataset}.{name}"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def metric_id(name: str, dataset: str | None = None) -> str:
|
|
61
|
+
return f"metric:{dataset}.{name}" if dataset else f"metric:{name}"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def build(model: SemanticModel) -> nx.DiGraph:
|
|
65
|
+
G = nx.DiGraph(name=model.name, version=model.version, joins=[], problems=[])
|
|
66
|
+
problems = G.graph["problems"]
|
|
67
|
+
|
|
68
|
+
def problem(code: str, node: str, message: str, severity: str = "error") -> None:
|
|
69
|
+
problems.append({"severity": severity, "code": code, "node": node, "message": message})
|
|
70
|
+
|
|
71
|
+
for d in model.datasets:
|
|
72
|
+
did = dataset_id(d.name)
|
|
73
|
+
G.add_node(did, kind="dataset", label=d.name, description=d.description, source=d.source,
|
|
74
|
+
primary_key=d.primary_key, synonyms=d.synonyms)
|
|
75
|
+
if d.source:
|
|
76
|
+
G.add_node(source_id(d.source), kind="source", label=d.source)
|
|
77
|
+
G.add_edge(source_id(d.source), did, rel="reads")
|
|
78
|
+
for f in d.fields:
|
|
79
|
+
fid = field_id(d.name, f.name)
|
|
80
|
+
G.add_node(fid, kind="field", label=f.name, dataset=d.name, sql=f.sql, datatype=f.datatype,
|
|
81
|
+
description=f.description, dimension=f.is_dimension, time=f.is_time, synonyms=f.synonyms)
|
|
82
|
+
G.add_edge(did, fid, rel="contains")
|
|
83
|
+
if f.sql is None:
|
|
84
|
+
problem("OPAQUE_EXPRESSION", fid, "field expression is in no SQL dialect read here; its columns are unknown", "warning")
|
|
85
|
+
continue
|
|
86
|
+
refs, err = sql_refs(f.sql)
|
|
87
|
+
if err:
|
|
88
|
+
problem("SQL_PARSE", fid, f"cannot parse {f.sql!r}: {err}", "warning")
|
|
89
|
+
for r in refs: # a field names columns of its dataset's source
|
|
90
|
+
cid = column_id(d.source, r.name)
|
|
91
|
+
G.add_node(cid, kind="column", label=r.name, source=d.source)
|
|
92
|
+
G.add_edge(cid, fid, rel="derives")
|
|
93
|
+
|
|
94
|
+
def metric_node(m) -> str:
|
|
95
|
+
mid = metric_id(m.name, m.dataset)
|
|
96
|
+
G.add_node(mid, kind="metric", label=m.name, dataset=m.dataset, sql=m.sql, datatype=m.datatype,
|
|
97
|
+
description=m.description, instructions=m.instructions, synonyms=m.synonyms, datasets=[])
|
|
98
|
+
return mid
|
|
99
|
+
|
|
100
|
+
for d in model.datasets:
|
|
101
|
+
for m in d.metrics:
|
|
102
|
+
mid = metric_node(m)
|
|
103
|
+
G.nodes[mid]["datasets"] = [d.name]
|
|
104
|
+
if m.sql is None:
|
|
105
|
+
problem("OPAQUE_EXPRESSION", mid, "metric expression is in no SQL dialect read here", "warning")
|
|
106
|
+
continue
|
|
107
|
+
refs, err = sql_refs(m.sql)
|
|
108
|
+
if err:
|
|
109
|
+
problem("SQL_PARSE", mid, f"cannot parse {m.sql!r}: {err}", "warning")
|
|
110
|
+
for r in refs:
|
|
111
|
+
if r.qualifier in (None, d.name):
|
|
112
|
+
if d.field_named(r.name):
|
|
113
|
+
G.add_edge(field_id(d.name, r.name), mid, rel="computes")
|
|
114
|
+
elif d.metric_named(r.name) and r.name != m.name:
|
|
115
|
+
G.add_edge(metric_id(r.name, d.name), mid, rel="computes")
|
|
116
|
+
else:
|
|
117
|
+
problem("UNKNOWN_REFERENCE", mid, f"{r.name!r} is not a field of dataset {d.name!r}")
|
|
118
|
+
else: # `source.column` of a measure relation: a column of the dataset's source
|
|
119
|
+
cid = column_id(d.source, r.name)
|
|
120
|
+
G.add_node(cid, kind="column", label=r.name, source=d.source)
|
|
121
|
+
G.add_edge(cid, mid, rel="computes")
|
|
122
|
+
|
|
123
|
+
for m in model.metrics:
|
|
124
|
+
mid = metric_node(m)
|
|
125
|
+
if m.sql is None:
|
|
126
|
+
problem("OPAQUE_EXPRESSION", mid, "metric expression is in no SQL dialect read here", "warning")
|
|
127
|
+
continue
|
|
128
|
+
refs, err = sql_refs(m.sql)
|
|
129
|
+
if err:
|
|
130
|
+
problem("SQL_PARSE", mid, f"cannot parse {m.sql!r}: {err}", "warning")
|
|
131
|
+
used = []
|
|
132
|
+
for r in refs:
|
|
133
|
+
if r.qualifier is None:
|
|
134
|
+
if model.metric(r.name) and r.name != m.name:
|
|
135
|
+
G.add_edge(metric_id(r.name), mid, rel="computes")
|
|
136
|
+
else:
|
|
137
|
+
problem("UNKNOWN_REFERENCE", mid, f"{r.name!r}: a model-scoped metric names `dataset.field`, "
|
|
138
|
+
"`dataset.metric` or another model-scoped metric")
|
|
139
|
+
continue
|
|
140
|
+
d = model.dataset(r.qualifier)
|
|
141
|
+
if d is None:
|
|
142
|
+
problem("UNKNOWN_REFERENCE", mid, f"{r.qualifier}.{r.name}: no dataset named {r.qualifier!r}")
|
|
143
|
+
continue
|
|
144
|
+
used.append(d.name)
|
|
145
|
+
if d.field_named(r.name):
|
|
146
|
+
G.add_edge(field_id(d.name, r.name), mid, rel="computes")
|
|
147
|
+
elif d.metric_named(r.name):
|
|
148
|
+
G.add_edge(metric_id(r.name, d.name), mid, rel="computes")
|
|
149
|
+
else:
|
|
150
|
+
problem("UNKNOWN_REFERENCE", mid, f"{r.qualifier}.{r.name}: dataset {d.name!r} has no field or metric {r.name!r}")
|
|
151
|
+
G.nodes[mid]["datasets"] = list(dict.fromkeys(used))
|
|
152
|
+
|
|
153
|
+
for rel in model.relationships:
|
|
154
|
+
G.graph["joins"].append({"name": rel.name, "from": rel.from_dataset, "to": rel.to_dataset,
|
|
155
|
+
"from_columns": rel.from_columns, "to_columns": rel.to_columns})
|
|
156
|
+
return G
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def join_graph(G: nx.DiGraph) -> nx.Graph:
|
|
160
|
+
"""Datasets joined by relationships, undirected (a join path may be walked either way)."""
|
|
161
|
+
J = nx.Graph()
|
|
162
|
+
J.add_nodes_from(n[len("dataset:"):] for n, k in G.nodes(data="kind") if k == "dataset")
|
|
163
|
+
for j in G.graph["joins"]:
|
|
164
|
+
if J.has_node(j["from"]) and J.has_node(j["to"]):
|
|
165
|
+
J.add_edge(j["from"], j["to"], name=j["name"])
|
|
166
|
+
return J
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def resolve(G: nx.DiGraph, name: str) -> str:
|
|
170
|
+
"""A node id from what a person types: an id (`metric:total_sales`), or a name (`total_sales`, `store_sales.ss_quantity`,
|
|
171
|
+
`store_sales`). Raises KeyError when nothing or more than one node matches."""
|
|
172
|
+
if name in G:
|
|
173
|
+
return name
|
|
174
|
+
hits = [n for n in G if n.split(":", 1)[1] == name]
|
|
175
|
+
if not hits:
|
|
176
|
+
hits = [n for n in G if n.split(":", 1)[1].lower() == name.lower()]
|
|
177
|
+
if len(hits) == 1:
|
|
178
|
+
return hits[0]
|
|
179
|
+
if not hits:
|
|
180
|
+
raise KeyError(f"no node named {name!r}")
|
|
181
|
+
raise KeyError(f"{name!r} is ambiguous: {', '.join(sorted(hits))}")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def upstream(G: nx.DiGraph, node: str) -> nx.DiGraph:
|
|
185
|
+
"""The node and everything it is made of."""
|
|
186
|
+
return G.subgraph(nx.ancestors(G, node) | {node}).copy()
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def downstream(G: nx.DiGraph, node: str) -> nx.DiGraph:
|
|
190
|
+
"""The node and everything that depends on it."""
|
|
191
|
+
return G.subgraph(nx.descendants(G, node) | {node}).copy()
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def outline(G: nx.DiGraph, root: str, *, up: bool) -> list[str]:
|
|
195
|
+
"""An indented outline from `root`, walking toward the sources (up) or toward the metrics (down)."""
|
|
196
|
+
lines: list[str] = []
|
|
197
|
+
seen: set[str] = set()
|
|
198
|
+
|
|
199
|
+
def walk(n: str, depth: int) -> None:
|
|
200
|
+
mark = " (see above)" if n in seen else ""
|
|
201
|
+
sql = G.nodes[n].get("sql")
|
|
202
|
+
extra = f" = {sql}" if sql and G.nodes[n].get("kind") in ("metric", "field") and sql != G.nodes[n].get("label") else ""
|
|
203
|
+
lines.append(f"{' ' * depth}{n}{extra}{mark}")
|
|
204
|
+
if mark:
|
|
205
|
+
return
|
|
206
|
+
seen.add(n)
|
|
207
|
+
nxt = G.predecessors(n) if up else G.successors(n)
|
|
208
|
+
for m in sorted(nxt):
|
|
209
|
+
walk(m, depth + 1)
|
|
210
|
+
|
|
211
|
+
walk(root, 0)
|
|
212
|
+
return lines
|
datagraf/model.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Reading an Ossie semantic model (0.1.x and 0.3.x) into plain dataclasses.
|
|
2
|
+
|
|
3
|
+
0.1.x wraps models in `semantic_model: [...]` and writes an expression as `{dialects: [{dialect, expression}]}`.
|
|
4
|
+
0.3.x puts one model at the root and writes `{sql: ..., dialects: [...]}`, with `sql` authoritative. Both are read the same way.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import yaml
|
|
14
|
+
|
|
15
|
+
SQL_DIALECTS = ("OSSIE_SQL_2026", "ANSI_SQL") # dialects whose expressions are read as SQL; any other is opaque
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ModelError(ValueError):
|
|
19
|
+
"""The file is not an Ossie semantic model (unreadable, or missing what every version requires)."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def expression_sql(expr: Any) -> str | None:
|
|
23
|
+
"""The SQL of an Ossie expression: `sql`, else the first OSSIE_SQL_2026 or ANSI_SQL entry of `dialects`; None if the
|
|
24
|
+
expression is only in another dialect (opaque here) or missing."""
|
|
25
|
+
if isinstance(expr, str):
|
|
26
|
+
return expr
|
|
27
|
+
if not isinstance(expr, dict):
|
|
28
|
+
return None
|
|
29
|
+
if expr.get("sql"):
|
|
30
|
+
return str(expr["sql"])
|
|
31
|
+
entries = [d for d in expr.get("dialects") or [] if isinstance(d, dict)]
|
|
32
|
+
for wanted in SQL_DIALECTS:
|
|
33
|
+
for d in entries:
|
|
34
|
+
if str(d.get("dialect", "")).upper() == wanted and d.get("expression"):
|
|
35
|
+
return str(d["expression"])
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _ai_text(ai: Any) -> tuple[str, list[str]]:
|
|
40
|
+
"""`ai_context` as (instructions, synonyms): it is a string or an object with instructions / synonyms / examples."""
|
|
41
|
+
if isinstance(ai, str):
|
|
42
|
+
return ai, []
|
|
43
|
+
if isinstance(ai, dict):
|
|
44
|
+
return str(ai.get("instructions") or ""), [str(s) for s in ai.get("synonyms") or []]
|
|
45
|
+
return "", []
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class Field:
|
|
50
|
+
name: str
|
|
51
|
+
sql: str | None
|
|
52
|
+
datatype: str | None = None
|
|
53
|
+
description: str = ""
|
|
54
|
+
is_dimension: bool = False
|
|
55
|
+
is_time: bool = False
|
|
56
|
+
instructions: str = ""
|
|
57
|
+
synonyms: list[str] = field(default_factory=list)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class Metric:
|
|
62
|
+
name: str
|
|
63
|
+
sql: str | None
|
|
64
|
+
dataset: str | None = None # the dataset of a dataset-scoped metric; None for a model-scoped one
|
|
65
|
+
datatype: str | None = None
|
|
66
|
+
description: str = ""
|
|
67
|
+
instructions: str = ""
|
|
68
|
+
synonyms: list[str] = field(default_factory=list)
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def key(self) -> str:
|
|
72
|
+
return f"{self.dataset}.{self.name}" if self.dataset else self.name
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class Dataset:
|
|
77
|
+
name: str
|
|
78
|
+
source: str
|
|
79
|
+
primary_key: list[str] = field(default_factory=list)
|
|
80
|
+
description: str = ""
|
|
81
|
+
instructions: str = ""
|
|
82
|
+
synonyms: list[str] = field(default_factory=list)
|
|
83
|
+
fields: list[Field] = field(default_factory=list)
|
|
84
|
+
metrics: list[Metric] = field(default_factory=list)
|
|
85
|
+
|
|
86
|
+
def field_named(self, name: str) -> Field | None:
|
|
87
|
+
return next((f for f in self.fields if f.name == name), None)
|
|
88
|
+
|
|
89
|
+
def metric_named(self, name: str) -> Metric | None:
|
|
90
|
+
return next((m for m in self.metrics if m.name == name), None)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass
|
|
94
|
+
class Relationship:
|
|
95
|
+
name: str
|
|
96
|
+
from_dataset: str
|
|
97
|
+
to_dataset: str
|
|
98
|
+
from_columns: list[str]
|
|
99
|
+
to_columns: list[str]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class SemanticModel:
|
|
104
|
+
name: str
|
|
105
|
+
version: str
|
|
106
|
+
description: str = ""
|
|
107
|
+
instructions: str = ""
|
|
108
|
+
datasets: list[Dataset] = field(default_factory=list)
|
|
109
|
+
relationships: list[Relationship] = field(default_factory=list)
|
|
110
|
+
metrics: list[Metric] = field(default_factory=list) # model-scoped
|
|
111
|
+
path: Path | None = None
|
|
112
|
+
|
|
113
|
+
def dataset(self, name: str) -> Dataset | None:
|
|
114
|
+
return next((d for d in self.datasets if d.name == name), None)
|
|
115
|
+
|
|
116
|
+
def metric(self, name: str) -> Metric | None:
|
|
117
|
+
return next((m for m in self.metrics if m.name == name), None)
|
|
118
|
+
|
|
119
|
+
def all_metrics(self) -> list[Metric]:
|
|
120
|
+
return [m for d in self.datasets for m in d.metrics] + self.metrics
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def read_document(path: str | Path) -> dict:
|
|
124
|
+
"""The raw document (YAML or JSON) as a mapping."""
|
|
125
|
+
p = Path(path)
|
|
126
|
+
try:
|
|
127
|
+
text = p.read_text(encoding="utf-8")
|
|
128
|
+
data = json.loads(text) if p.suffix == ".json" else yaml.safe_load(text)
|
|
129
|
+
except (OSError, ValueError, yaml.YAMLError) as e:
|
|
130
|
+
raise ModelError(f"cannot read {p}: {e}") from None
|
|
131
|
+
if not isinstance(data, dict):
|
|
132
|
+
raise ModelError(f"{p}: not a mapping at the top level")
|
|
133
|
+
return data
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def raw_models(doc: dict) -> list[dict]:
|
|
137
|
+
"""The model mappings of a document: the `semantic_model` list (0.1.x) or the document itself (0.3.x)."""
|
|
138
|
+
if "semantic_model" in doc:
|
|
139
|
+
models = doc["semantic_model"]
|
|
140
|
+
if isinstance(models, dict):
|
|
141
|
+
models = [models]
|
|
142
|
+
if not isinstance(models, list) or not models:
|
|
143
|
+
raise ModelError("`semantic_model` must be a non-empty list of models")
|
|
144
|
+
return models
|
|
145
|
+
if "datasets" not in doc:
|
|
146
|
+
raise ModelError("neither `datasets` (Ossie 0.2+) nor `semantic_model` (0.1.x) found")
|
|
147
|
+
return [doc]
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _metric(raw: dict, dataset: str | None) -> Metric:
|
|
151
|
+
instructions, synonyms = _ai_text(raw.get("ai_context"))
|
|
152
|
+
return Metric(name=str(raw.get("name", "")), sql=expression_sql(raw.get("expression")), dataset=dataset,
|
|
153
|
+
datatype=raw.get("datatype"), description=str(raw.get("description") or ""),
|
|
154
|
+
instructions=instructions, synonyms=synonyms)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _model(raw: dict, version: str, path: Path | None) -> SemanticModel:
|
|
158
|
+
instructions, _ = _ai_text(raw.get("ai_context"))
|
|
159
|
+
datasets = []
|
|
160
|
+
for d in raw.get("datasets") or []:
|
|
161
|
+
d_instr, d_syn = _ai_text(d.get("ai_context"))
|
|
162
|
+
fields = []
|
|
163
|
+
for f in d.get("fields") or []:
|
|
164
|
+
f_instr, f_syn = _ai_text(f.get("ai_context"))
|
|
165
|
+
dim = f.get("dimension")
|
|
166
|
+
is_dim = isinstance(dim, dict)
|
|
167
|
+
is_time = bool(dim.get("is_time")) if is_dim and "is_time" in dim else (
|
|
168
|
+
is_dim and f.get("datatype") in ("Date", "Time", "DateTime", "DateTimeTz"))
|
|
169
|
+
fields.append(Field(name=str(f.get("name", "")), sql=expression_sql(f.get("expression")), datatype=f.get("datatype"),
|
|
170
|
+
description=str(f.get("description") or ""), is_dimension=is_dim, is_time=is_time,
|
|
171
|
+
instructions=f_instr, synonyms=f_syn))
|
|
172
|
+
name = str(d.get("name", ""))
|
|
173
|
+
datasets.append(Dataset(name=name, source=str(d.get("source", "")), primary_key=list(d.get("primary_key") or []),
|
|
174
|
+
description=str(d.get("description") or ""), instructions=d_instr, synonyms=d_syn,
|
|
175
|
+
fields=fields, metrics=[_metric(m, name) for m in d.get("metrics") or []]))
|
|
176
|
+
rels = [Relationship(name=str(r.get("name", "")), from_dataset=str(r.get("from", "")), to_dataset=str(r.get("to", "")),
|
|
177
|
+
from_columns=list(r.get("from_columns") or []), to_columns=list(r.get("to_columns") or []))
|
|
178
|
+
for r in raw.get("relationships") or []]
|
|
179
|
+
return SemanticModel(name=str(raw.get("name", "")), version=version, description=str(raw.get("description") or ""),
|
|
180
|
+
instructions=instructions, datasets=datasets, relationships=rels,
|
|
181
|
+
metrics=[_metric(m, None) for m in raw.get("metrics") or []], path=path)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def load_models(path: str | Path) -> list[SemanticModel]:
|
|
185
|
+
"""Every semantic model in a file (one for 0.3.x; one or more for 0.1.x)."""
|
|
186
|
+
doc = read_document(path)
|
|
187
|
+
version = str(doc.get("version", ""))
|
|
188
|
+
return [_model(m, version, Path(path)) for m in raw_models(doc)]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def load(path: str | Path, name: str | None = None) -> SemanticModel:
|
|
192
|
+
"""The semantic model in a file; `name` picks one when a 0.1.x document holds several."""
|
|
193
|
+
models = load_models(path)
|
|
194
|
+
if name is None:
|
|
195
|
+
if len(models) > 1:
|
|
196
|
+
raise ModelError(f"{path} holds {len(models)} models ({', '.join(m.name for m in models)}); pick one by name")
|
|
197
|
+
return models[0]
|
|
198
|
+
for m in models:
|
|
199
|
+
if m.name == name:
|
|
200
|
+
return m
|
|
201
|
+
raise ModelError(f"no model named {name!r} in {path} (has: {', '.join(m.name for m in models)})")
|