data-graf 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
datagraf/cli.py ADDED
@@ -0,0 +1,231 @@
1
+ """`dgr`: validate, explore, export and profile an Ossie semantic model.
2
+
3
+ Exit codes: 0 ok, 1 the model has errors (validate / check / profile), 2 a usage or load error.
4
+ """
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ import click
12
+ import yaml
13
+
14
+ from . import __version__
15
+ from . import checks as C
16
+ from . import lineage as L
17
+ from .model import ModelError, load, read_document
18
+
19
+ MODEL = click.argument("model", type=click.Path(exists=True, dir_okay=False, path_type=Path))
20
+ PICK = click.option("--name", "model_name", default=None, help="Which model, when a 0.1.x file holds several.")
21
+
22
+
23
+ def _load(path: Path, name: str | None):
24
+ try:
25
+ m = load(path, name)
26
+ except ModelError as e:
27
+ raise click.ClickException(str(e)) from None
28
+ return m, L.build(m)
29
+
30
+
31
+ def _print_problems(problems: list[dict]) -> None:
32
+ for p in problems:
33
+ where = f" {p['node']}" if p["node"] else ""
34
+ click.echo(f" {p['severity']:7} {p['code']}{where}: {p['message']}")
35
+
36
+
37
+ @click.group()
38
+ @click.version_option(__version__, prog_name="dgr")
39
+ def main() -> None:
40
+ """Lineage, checks and profiling for Apache Ossie semantic models."""
41
+
42
+
43
+ @main.command()
44
+ @MODEL
45
+ @PICK
46
+ @click.option("--strict", is_flag=True, help="Fail on warnings too.")
47
+ def validate(model: Path, model_name: str | None, strict: bool) -> None:
48
+ """The JSON Schema for the file's Ossie version, then references, joins and gaps."""
49
+ try:
50
+ doc = read_document(model)
51
+ except ModelError as e:
52
+ raise click.ClickException(str(e)) from None
53
+ problems = C.schema_problems(doc)
54
+ m, G = _load(model, model_name)
55
+ problems += C.model_problems(m, G)
56
+ errors, warnings = C.summarize(problems)
57
+ click.echo(f"{model}: {m.name} (Ossie {m.version or '?'}): {errors} error(s), {warnings} warning(s)")
58
+ _print_problems(problems)
59
+ sys.exit(1 if errors or (strict and warnings) else 0)
60
+
61
+
62
+ @main.command()
63
+ @MODEL
64
+ @PICK
65
+ def check(model: Path, model_name: str | None) -> None:
66
+ """References, joins and gaps only (no schema)."""
67
+ m, G = _load(model, model_name)
68
+ problems = C.model_problems(m, G)
69
+ errors, warnings = C.summarize(problems)
70
+ click.echo(f"{m.name}: {errors} error(s), {warnings} warning(s)")
71
+ _print_problems(problems)
72
+ sys.exit(1 if errors else 0)
73
+
74
+
75
+ @main.command()
76
+ @MODEL
77
+ @PICK
78
+ def show(model: Path, model_name: str | None) -> None:
79
+ """Datasets, fields, metrics and joins."""
80
+ m, G = _load(model, model_name)
81
+ click.echo(f"{m.name} (Ossie {m.version or '?'})")
82
+ if m.description:
83
+ click.echo(f" {m.description}")
84
+ n_fields = sum(len(d.fields) for d in m.datasets)
85
+ click.echo(f" {len(m.datasets)} datasets, {n_fields} fields, {len(m.all_metrics())} metrics, {len(m.relationships)} joins")
86
+ for d in m.datasets:
87
+ dims = sum(f.is_dimension for f in d.fields)
88
+ click.echo(f"\n dataset {d.name} <- {d.source} ({len(d.fields)} fields, {dims} dimensions"
89
+ + (f", key {', '.join(d.primary_key)}" if d.primary_key else "") + ")")
90
+ for mm in d.metrics:
91
+ click.echo(f" metric {mm.name} = {mm.sql}")
92
+ if m.metrics:
93
+ click.echo("\n model metrics")
94
+ for mm in m.metrics:
95
+ click.echo(f" {mm.name} = {mm.sql}")
96
+ if m.relationships:
97
+ click.echo("\n joins (many -> one)")
98
+ for r in m.relationships:
99
+ click.echo(f" {r.from_dataset}({', '.join(r.from_columns)}) -> {r.to_dataset}({', '.join(r.to_columns)})")
100
+
101
+
102
+ def _node(G, name: str) -> str:
103
+ try:
104
+ return L.resolve(G, name)
105
+ except KeyError as e:
106
+ raise click.ClickException(e.args[0]) from None
107
+
108
+
109
+ @main.command()
110
+ @MODEL
111
+ @click.argument("name")
112
+ @PICK
113
+ def lineage(model: Path, name: str, model_name: str | None) -> None:
114
+ """What a metric or field is made of, down to source columns. NAME is `total_sales`, `store_sales.ss_quantity`, or an id."""
115
+ _, G = _load(model, model_name)
116
+ click.echo("\n".join(L.outline(G, _node(G, name), up=True)))
117
+ cols = sorted(n for n in L.upstream(G, _node(G, name)) if G.nodes[n]["kind"] == "column")
118
+ click.echo(f"\n{len(cols)} source column(s): {', '.join(c[len('column:'):] for c in cols) or '-'}")
119
+
120
+
121
+ @main.command()
122
+ @MODEL
123
+ @click.argument("name")
124
+ @PICK
125
+ def impact(model: Path, name: str, model_name: str | None) -> None:
126
+ """What depends on a column, field, dataset or metric: what a change to it can break."""
127
+ _, G = _load(model, model_name)
128
+ node = _node(G, name)
129
+ click.echo("\n".join(L.outline(G, node, up=False)))
130
+ hit = sorted(n for n in L.downstream(G, node) if G.nodes[n]["kind"] == "metric" and n != node)
131
+ click.echo(f"\n{len(hit)} metric(s) affected: {', '.join(h[len('metric:'):] for h in hit) or '-'}")
132
+
133
+
134
+ @main.command()
135
+ @MODEL
136
+ @PICK
137
+ @click.option("--format", "fmt", type=click.Choice(["zinet", "json"]), default="zinet", show_default=True)
138
+ @click.option("-o", "--out", type=click.Path(dir_okay=False, path_type=Path), help="Write here instead of stdout.")
139
+ def export(model: Path, model_name: str | None, fmt: str, out: Path | None) -> None:
140
+ """The lineage graph as zinet graph.yaml or JSON."""
141
+ _, G = _load(model, model_name)
142
+ if fmt == "json":
143
+ from .export import to_json
144
+ text = to_json(G)
145
+ else:
146
+ from zinet import to_yaml
147
+ from .export import to_zinet
148
+ text = to_yaml(to_zinet(G))
149
+ if out:
150
+ out.write_text(text, encoding="utf-8")
151
+ click.echo(f"wrote {out}")
152
+ else:
153
+ click.echo(text)
154
+
155
+
156
+ @main.command()
157
+ @MODEL
158
+ @PICK
159
+ @click.option("--format", "fmt", type=click.Choice(["html", "mermaid", "ascii", "svg", "png"]), default="html", show_default=True)
160
+ @click.option("-o", "--out", type=click.Path(dir_okay=False, path_type=Path), help="Required for html, svg and png.")
161
+ def render(model: Path, model_name: str | None, fmt: str, out: Path | None) -> None:
162
+ """Draw the lineage graph through zinet (svg and png need Graphviz)."""
163
+ from zinet import render as zrender
164
+ from .export import to_zinet
165
+ _, G = _load(model, model_name)
166
+ if fmt in ("html", "svg", "png") and out is None:
167
+ raise click.UsageError(f"--format {fmt} needs -o FILE")
168
+ try:
169
+ data = zrender(to_zinet(G), fmt)
170
+ except RuntimeError as e:
171
+ raise click.ClickException(str(e)) from None
172
+ if out:
173
+ out.write_bytes(data)
174
+ click.echo(f"wrote {out}")
175
+ else:
176
+ click.echo(data.decode("utf-8"))
177
+
178
+
179
+ @main.command()
180
+ @click.option("--sqlite", "db", required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
181
+ @click.option("--name", default=None, help="Model name (default: the database file's stem).")
182
+ @click.option("-o", "--out", type=click.Path(dir_okay=False, path_type=Path), help="Write here instead of stdout.")
183
+ def draft(db: Path, name: str | None, out: Path | None) -> None:
184
+ """An Ossie skeleton from a SQLite catalogue: datasets, fields, keys, joins. No descriptions or metrics."""
185
+ from .sqlite import draft as do_draft
186
+ text = "# Drafted by data-graf (dgr) from a SQLite catalogue: add descriptions, ai_context and metrics.\n" + \
187
+ yaml.safe_dump(do_draft(db, name), sort_keys=False, allow_unicode=True)
188
+ if out:
189
+ out.write_text(text, encoding="utf-8")
190
+ click.echo(f"wrote {out}")
191
+ else:
192
+ click.echo(text)
193
+
194
+
195
+ @main.command()
196
+ @MODEL
197
+ @click.option("--sqlite", "db", required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
198
+ @PICK
199
+ @click.option("--json", "as_json", is_flag=True, help="Print the profile as JSON.")
200
+ def profile(model: Path, db: Path, model_name: str | None, as_json: bool) -> None:
201
+ """Profile the data behind a model, read-only: rows, nulls, distinct values, ranges, dataset metric values."""
202
+ from .sqlite import profile as do_profile
203
+ m, _ = _load(model, model_name)
204
+ try:
205
+ prof = do_profile(m, db)
206
+ except FileNotFoundError as e:
207
+ raise click.ClickException(str(e)) from None
208
+ failed = 0
209
+ if as_json:
210
+ click.echo(json.dumps(prof, indent=2, default=str))
211
+ for d in prof["datasets"]:
212
+ failed += bool(d.get("error")) + sum(bool(x.get("error")) for x in d["fields"] + d["metrics"])
213
+ if as_json:
214
+ continue
215
+ if d.get("error"):
216
+ click.echo(f"\n{d['name']}: ERROR {d['error']}")
217
+ continue
218
+ click.echo(f"\n{d['name']} ({d['table']}, {d['rows']} rows)")
219
+ for f in d["fields"]:
220
+ if f.get("error"):
221
+ click.echo(f" {f['name']:28} ERROR {f['error']}")
222
+ else:
223
+ nulls = "" if f["nulls"] is None else f"{f['nulls']:>8} null"
224
+ click.echo(f" {f['name']:28} {f['distinct']:>8} distinct {nulls} {f['min']!s:.20} .. {f['max']!s:.20}")
225
+ for me in d["metrics"]:
226
+ click.echo(f" metric {me['name']:21} " + (f"ERROR {me['error']}" if me.get("error") else f"= {me['value']}"))
227
+ sys.exit(1 if failed else 0)
228
+
229
+
230
+ if __name__ == "__main__":
231
+ main()
datagraf/export.py ADDED
@@ -0,0 +1,41 @@
1
+ """The lineage graph as zinet `graph.yaml` (for zinet's validate / render / slice) or as JSON.
2
+
3
+ zinet layers: sources and columns are primitives (they depend on nothing); datasets, fields and metrics are concepts;
4
+ the model itself is the one application, depending on every metric (or every dataset, when it has no metrics).
5
+ zinet ids are plain names, so `kind:name` becomes `kind__name` with dots kept.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import re
11
+
12
+ import networkx as nx
13
+ from zinet import EdgeType, Graph, Node
14
+
15
+ _EDGE_TYPE = {"reads": "data_flow", "derives": "data_flow", "contains": "depends_on", "computes": "depends_on"}
16
+
17
+
18
+ def zinet_id(node: str) -> str:
19
+ return re.sub(r"[^\w.\-]", "_", node.replace(":", "__", 1))
20
+
21
+
22
+ def to_zinet(G: nx.DiGraph) -> Graph:
23
+ prims, concepts = [], []
24
+ for n, data in G.nodes(data=True):
25
+ deps = sorted(G.predecessors(n))
26
+ meta = {k: v for k, v in data.items() if v not in (None, "", [], False) and k != "label"}
27
+ node = Node(id=zinet_id(n), label=f"{data['kind']} {n.split(':', 1)[1]}", deps=[zinet_id(d) for d in deps],
28
+ edge_types={zinet_id(d): EdgeType(_EDGE_TYPE[G.edges[d, n]["rel"]]) for d in deps}, meta=meta)
29
+ (prims if data["kind"] in ("source", "column") and not deps else concepts).append(node)
30
+ tops = [n for n, k in G.nodes(data="kind") if k == "metric" and G.out_degree(n) == 0] or \
31
+ [n for n, k in G.nodes(data="kind") if k == "dataset"]
32
+ app = Node(id=zinet_id(f"model:{G.graph['name']}"), label=f"model {G.graph['name']}", deps=[zinet_id(t) for t in sorted(tops)],
33
+ meta={"kind": "model", "version": G.graph.get("version"), "joins": G.graph["joins"]})
34
+ return Graph(primitives=prims, concepts=concepts, applications=[app])
35
+
36
+
37
+ def to_json(G: nx.DiGraph) -> str:
38
+ return json.dumps({"name": G.graph["name"], "version": G.graph.get("version"),
39
+ "nodes": [{"id": n, **d} for n, d in G.nodes(data=True)],
40
+ "edges": [{"from": a, "to": b, **d} for a, b, d in G.edges(data=True)],
41
+ "joins": G.graph["joins"]}, indent=2)
datagraf/lineage.py ADDED
@@ -0,0 +1,212 @@
1
+ """The lineage graph of a semantic model: source tables and columns -> datasets and fields -> metrics.
2
+
3
+ Edges point from what is needed to what needs it (zinet's convention), with `rel` one of
4
+ `reads` (source -> dataset), `contains` (dataset -> field), `derives` (column -> field), `computes` (field or metric -> metric).
5
+ Relationships are not dependencies: they are kept as many-to-one joins in `G.graph["joins"]`.
6
+
7
+ References are read from each expression's SQL (sqlglot), by the scoping rules of the spec:
8
+ a field names columns of its dataset's source; a dataset-scoped metric names that dataset's fields (or its other metrics);
9
+ a model-scoped metric names `dataset.field` or `dataset.metric`, or another model-scoped metric by its bare name.
10
+ What cannot be resolved is recorded in `G.graph["problems"]` (read by `checks`), never guessed.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ from dataclasses import dataclass
15
+
16
+ import networkx as nx
17
+ import sqlglot
18
+ from sqlglot import exp
19
+
20
+ from .model import SemanticModel
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Ref:
25
+ qualifier: str | None # `store_sales` in `store_sales.ss_quantity`; None when unqualified
26
+ name: str
27
+
28
+
29
+ def sql_refs(sql: str) -> tuple[list[Ref], str | None]:
30
+ """The column references in an expression, and a parse error (or None). `COUNT(*)` names no column."""
31
+ try:
32
+ tree = sqlglot.parse_one(sql)
33
+ except (sqlglot.errors.ParseError, sqlglot.errors.TokenError) as e:
34
+ return [], str(e).splitlines()[0][:200]
35
+ refs = []
36
+ for col in tree.find_all(exp.Column):
37
+ parts = [p.name for p in col.parts]
38
+ if not parts or not parts[-1]:
39
+ continue
40
+ refs.append(Ref(".".join(parts[:-1]) or None, parts[-1]))
41
+ return list(dict.fromkeys(refs)), None
42
+
43
+
44
+ def source_id(source: str) -> str:
45
+ return f"source:{source}"
46
+
47
+
48
+ def column_id(source: str, column: str) -> str:
49
+ return f"column:{source}.{column}"
50
+
51
+
52
+ def dataset_id(name: str) -> str:
53
+ return f"dataset:{name}"
54
+
55
+
56
+ def field_id(dataset: str, name: str) -> str:
57
+ return f"field:{dataset}.{name}"
58
+
59
+
60
+ def metric_id(name: str, dataset: str | None = None) -> str:
61
+ return f"metric:{dataset}.{name}" if dataset else f"metric:{name}"
62
+
63
+
64
+ def build(model: SemanticModel) -> nx.DiGraph:
65
+ G = nx.DiGraph(name=model.name, version=model.version, joins=[], problems=[])
66
+ problems = G.graph["problems"]
67
+
68
+ def problem(code: str, node: str, message: str, severity: str = "error") -> None:
69
+ problems.append({"severity": severity, "code": code, "node": node, "message": message})
70
+
71
+ for d in model.datasets:
72
+ did = dataset_id(d.name)
73
+ G.add_node(did, kind="dataset", label=d.name, description=d.description, source=d.source,
74
+ primary_key=d.primary_key, synonyms=d.synonyms)
75
+ if d.source:
76
+ G.add_node(source_id(d.source), kind="source", label=d.source)
77
+ G.add_edge(source_id(d.source), did, rel="reads")
78
+ for f in d.fields:
79
+ fid = field_id(d.name, f.name)
80
+ G.add_node(fid, kind="field", label=f.name, dataset=d.name, sql=f.sql, datatype=f.datatype,
81
+ description=f.description, dimension=f.is_dimension, time=f.is_time, synonyms=f.synonyms)
82
+ G.add_edge(did, fid, rel="contains")
83
+ if f.sql is None:
84
+ problem("OPAQUE_EXPRESSION", fid, "field expression is in no SQL dialect read here; its columns are unknown", "warning")
85
+ continue
86
+ refs, err = sql_refs(f.sql)
87
+ if err:
88
+ problem("SQL_PARSE", fid, f"cannot parse {f.sql!r}: {err}", "warning")
89
+ for r in refs: # a field names columns of its dataset's source
90
+ cid = column_id(d.source, r.name)
91
+ G.add_node(cid, kind="column", label=r.name, source=d.source)
92
+ G.add_edge(cid, fid, rel="derives")
93
+
94
+ def metric_node(m) -> str:
95
+ mid = metric_id(m.name, m.dataset)
96
+ G.add_node(mid, kind="metric", label=m.name, dataset=m.dataset, sql=m.sql, datatype=m.datatype,
97
+ description=m.description, instructions=m.instructions, synonyms=m.synonyms, datasets=[])
98
+ return mid
99
+
100
+ for d in model.datasets:
101
+ for m in d.metrics:
102
+ mid = metric_node(m)
103
+ G.nodes[mid]["datasets"] = [d.name]
104
+ if m.sql is None:
105
+ problem("OPAQUE_EXPRESSION", mid, "metric expression is in no SQL dialect read here", "warning")
106
+ continue
107
+ refs, err = sql_refs(m.sql)
108
+ if err:
109
+ problem("SQL_PARSE", mid, f"cannot parse {m.sql!r}: {err}", "warning")
110
+ for r in refs:
111
+ if r.qualifier in (None, d.name):
112
+ if d.field_named(r.name):
113
+ G.add_edge(field_id(d.name, r.name), mid, rel="computes")
114
+ elif d.metric_named(r.name) and r.name != m.name:
115
+ G.add_edge(metric_id(r.name, d.name), mid, rel="computes")
116
+ else:
117
+ problem("UNKNOWN_REFERENCE", mid, f"{r.name!r} is not a field of dataset {d.name!r}")
118
+ else: # `source.column` of a measure relation: a column of the dataset's source
119
+ cid = column_id(d.source, r.name)
120
+ G.add_node(cid, kind="column", label=r.name, source=d.source)
121
+ G.add_edge(cid, mid, rel="computes")
122
+
123
+ for m in model.metrics:
124
+ mid = metric_node(m)
125
+ if m.sql is None:
126
+ problem("OPAQUE_EXPRESSION", mid, "metric expression is in no SQL dialect read here", "warning")
127
+ continue
128
+ refs, err = sql_refs(m.sql)
129
+ if err:
130
+ problem("SQL_PARSE", mid, f"cannot parse {m.sql!r}: {err}", "warning")
131
+ used = []
132
+ for r in refs:
133
+ if r.qualifier is None:
134
+ if model.metric(r.name) and r.name != m.name:
135
+ G.add_edge(metric_id(r.name), mid, rel="computes")
136
+ else:
137
+ problem("UNKNOWN_REFERENCE", mid, f"{r.name!r}: a model-scoped metric names `dataset.field`, "
138
+ "`dataset.metric` or another model-scoped metric")
139
+ continue
140
+ d = model.dataset(r.qualifier)
141
+ if d is None:
142
+ problem("UNKNOWN_REFERENCE", mid, f"{r.qualifier}.{r.name}: no dataset named {r.qualifier!r}")
143
+ continue
144
+ used.append(d.name)
145
+ if d.field_named(r.name):
146
+ G.add_edge(field_id(d.name, r.name), mid, rel="computes")
147
+ elif d.metric_named(r.name):
148
+ G.add_edge(metric_id(r.name, d.name), mid, rel="computes")
149
+ else:
150
+ problem("UNKNOWN_REFERENCE", mid, f"{r.qualifier}.{r.name}: dataset {d.name!r} has no field or metric {r.name!r}")
151
+ G.nodes[mid]["datasets"] = list(dict.fromkeys(used))
152
+
153
+ for rel in model.relationships:
154
+ G.graph["joins"].append({"name": rel.name, "from": rel.from_dataset, "to": rel.to_dataset,
155
+ "from_columns": rel.from_columns, "to_columns": rel.to_columns})
156
+ return G
157
+
158
+
159
+ def join_graph(G: nx.DiGraph) -> nx.Graph:
160
+ """Datasets joined by relationships, undirected (a join path may be walked either way)."""
161
+ J = nx.Graph()
162
+ J.add_nodes_from(n[len("dataset:"):] for n, k in G.nodes(data="kind") if k == "dataset")
163
+ for j in G.graph["joins"]:
164
+ if J.has_node(j["from"]) and J.has_node(j["to"]):
165
+ J.add_edge(j["from"], j["to"], name=j["name"])
166
+ return J
167
+
168
+
169
+ def resolve(G: nx.DiGraph, name: str) -> str:
170
+ """A node id from what a person types: an id (`metric:total_sales`), or a name (`total_sales`, `store_sales.ss_quantity`,
171
+ `store_sales`). Raises KeyError when nothing or more than one node matches."""
172
+ if name in G:
173
+ return name
174
+ hits = [n for n in G if n.split(":", 1)[1] == name]
175
+ if not hits:
176
+ hits = [n for n in G if n.split(":", 1)[1].lower() == name.lower()]
177
+ if len(hits) == 1:
178
+ return hits[0]
179
+ if not hits:
180
+ raise KeyError(f"no node named {name!r}")
181
+ raise KeyError(f"{name!r} is ambiguous: {', '.join(sorted(hits))}")
182
+
183
+
184
+ def upstream(G: nx.DiGraph, node: str) -> nx.DiGraph:
185
+ """The node and everything it is made of."""
186
+ return G.subgraph(nx.ancestors(G, node) | {node}).copy()
187
+
188
+
189
+ def downstream(G: nx.DiGraph, node: str) -> nx.DiGraph:
190
+ """The node and everything that depends on it."""
191
+ return G.subgraph(nx.descendants(G, node) | {node}).copy()
192
+
193
+
194
+ def outline(G: nx.DiGraph, root: str, *, up: bool) -> list[str]:
195
+ """An indented outline from `root`, walking toward the sources (up) or toward the metrics (down)."""
196
+ lines: list[str] = []
197
+ seen: set[str] = set()
198
+
199
+ def walk(n: str, depth: int) -> None:
200
+ mark = " (see above)" if n in seen else ""
201
+ sql = G.nodes[n].get("sql")
202
+ extra = f" = {sql}" if sql and G.nodes[n].get("kind") in ("metric", "field") and sql != G.nodes[n].get("label") else ""
203
+ lines.append(f"{' ' * depth}{n}{extra}{mark}")
204
+ if mark:
205
+ return
206
+ seen.add(n)
207
+ nxt = G.predecessors(n) if up else G.successors(n)
208
+ for m in sorted(nxt):
209
+ walk(m, depth + 1)
210
+
211
+ walk(root, 0)
212
+ return lines
datagraf/model.py ADDED
@@ -0,0 +1,201 @@
1
+ """Reading an Ossie semantic model (0.1.x and 0.3.x) into plain dataclasses.
2
+
3
+ 0.1.x wraps models in `semantic_model: [...]` and writes an expression as `{dialects: [{dialect, expression}]}`.
4
+ 0.3.x puts one model at the root and writes `{sql: ..., dialects: [...]}`, with `sql` authoritative. Both are read the same way.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ from dataclasses import dataclass, field
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ import yaml
14
+
15
+ SQL_DIALECTS = ("OSSIE_SQL_2026", "ANSI_SQL") # dialects whose expressions are read as SQL; any other is opaque
16
+
17
+
18
+ class ModelError(ValueError):
19
+ """The file is not an Ossie semantic model (unreadable, or missing what every version requires)."""
20
+
21
+
22
+ def expression_sql(expr: Any) -> str | None:
23
+ """The SQL of an Ossie expression: `sql`, else the first OSSIE_SQL_2026 or ANSI_SQL entry of `dialects`; None if the
24
+ expression is only in another dialect (opaque here) or missing."""
25
+ if isinstance(expr, str):
26
+ return expr
27
+ if not isinstance(expr, dict):
28
+ return None
29
+ if expr.get("sql"):
30
+ return str(expr["sql"])
31
+ entries = [d for d in expr.get("dialects") or [] if isinstance(d, dict)]
32
+ for wanted in SQL_DIALECTS:
33
+ for d in entries:
34
+ if str(d.get("dialect", "")).upper() == wanted and d.get("expression"):
35
+ return str(d["expression"])
36
+ return None
37
+
38
+
39
+ def _ai_text(ai: Any) -> tuple[str, list[str]]:
40
+ """`ai_context` as (instructions, synonyms): it is a string or an object with instructions / synonyms / examples."""
41
+ if isinstance(ai, str):
42
+ return ai, []
43
+ if isinstance(ai, dict):
44
+ return str(ai.get("instructions") or ""), [str(s) for s in ai.get("synonyms") or []]
45
+ return "", []
46
+
47
+
48
+ @dataclass
49
+ class Field:
50
+ name: str
51
+ sql: str | None
52
+ datatype: str | None = None
53
+ description: str = ""
54
+ is_dimension: bool = False
55
+ is_time: bool = False
56
+ instructions: str = ""
57
+ synonyms: list[str] = field(default_factory=list)
58
+
59
+
60
+ @dataclass
61
+ class Metric:
62
+ name: str
63
+ sql: str | None
64
+ dataset: str | None = None # the dataset of a dataset-scoped metric; None for a model-scoped one
65
+ datatype: str | None = None
66
+ description: str = ""
67
+ instructions: str = ""
68
+ synonyms: list[str] = field(default_factory=list)
69
+
70
+ @property
71
+ def key(self) -> str:
72
+ return f"{self.dataset}.{self.name}" if self.dataset else self.name
73
+
74
+
75
+ @dataclass
76
+ class Dataset:
77
+ name: str
78
+ source: str
79
+ primary_key: list[str] = field(default_factory=list)
80
+ description: str = ""
81
+ instructions: str = ""
82
+ synonyms: list[str] = field(default_factory=list)
83
+ fields: list[Field] = field(default_factory=list)
84
+ metrics: list[Metric] = field(default_factory=list)
85
+
86
+ def field_named(self, name: str) -> Field | None:
87
+ return next((f for f in self.fields if f.name == name), None)
88
+
89
+ def metric_named(self, name: str) -> Metric | None:
90
+ return next((m for m in self.metrics if m.name == name), None)
91
+
92
+
93
+ @dataclass
94
+ class Relationship:
95
+ name: str
96
+ from_dataset: str
97
+ to_dataset: str
98
+ from_columns: list[str]
99
+ to_columns: list[str]
100
+
101
+
102
+ @dataclass
103
+ class SemanticModel:
104
+ name: str
105
+ version: str
106
+ description: str = ""
107
+ instructions: str = ""
108
+ datasets: list[Dataset] = field(default_factory=list)
109
+ relationships: list[Relationship] = field(default_factory=list)
110
+ metrics: list[Metric] = field(default_factory=list) # model-scoped
111
+ path: Path | None = None
112
+
113
+ def dataset(self, name: str) -> Dataset | None:
114
+ return next((d for d in self.datasets if d.name == name), None)
115
+
116
+ def metric(self, name: str) -> Metric | None:
117
+ return next((m for m in self.metrics if m.name == name), None)
118
+
119
+ def all_metrics(self) -> list[Metric]:
120
+ return [m for d in self.datasets for m in d.metrics] + self.metrics
121
+
122
+
123
+ def read_document(path: str | Path) -> dict:
124
+ """The raw document (YAML or JSON) as a mapping."""
125
+ p = Path(path)
126
+ try:
127
+ text = p.read_text(encoding="utf-8")
128
+ data = json.loads(text) if p.suffix == ".json" else yaml.safe_load(text)
129
+ except (OSError, ValueError, yaml.YAMLError) as e:
130
+ raise ModelError(f"cannot read {p}: {e}") from None
131
+ if not isinstance(data, dict):
132
+ raise ModelError(f"{p}: not a mapping at the top level")
133
+ return data
134
+
135
+
136
+ def raw_models(doc: dict) -> list[dict]:
137
+ """The model mappings of a document: the `semantic_model` list (0.1.x) or the document itself (0.3.x)."""
138
+ if "semantic_model" in doc:
139
+ models = doc["semantic_model"]
140
+ if isinstance(models, dict):
141
+ models = [models]
142
+ if not isinstance(models, list) or not models:
143
+ raise ModelError("`semantic_model` must be a non-empty list of models")
144
+ return models
145
+ if "datasets" not in doc:
146
+ raise ModelError("neither `datasets` (Ossie 0.2+) nor `semantic_model` (0.1.x) found")
147
+ return [doc]
148
+
149
+
150
+ def _metric(raw: dict, dataset: str | None) -> Metric:
151
+ instructions, synonyms = _ai_text(raw.get("ai_context"))
152
+ return Metric(name=str(raw.get("name", "")), sql=expression_sql(raw.get("expression")), dataset=dataset,
153
+ datatype=raw.get("datatype"), description=str(raw.get("description") or ""),
154
+ instructions=instructions, synonyms=synonyms)
155
+
156
+
157
+ def _model(raw: dict, version: str, path: Path | None) -> SemanticModel:
158
+ instructions, _ = _ai_text(raw.get("ai_context"))
159
+ datasets = []
160
+ for d in raw.get("datasets") or []:
161
+ d_instr, d_syn = _ai_text(d.get("ai_context"))
162
+ fields = []
163
+ for f in d.get("fields") or []:
164
+ f_instr, f_syn = _ai_text(f.get("ai_context"))
165
+ dim = f.get("dimension")
166
+ is_dim = isinstance(dim, dict)
167
+ is_time = bool(dim.get("is_time")) if is_dim and "is_time" in dim else (
168
+ is_dim and f.get("datatype") in ("Date", "Time", "DateTime", "DateTimeTz"))
169
+ fields.append(Field(name=str(f.get("name", "")), sql=expression_sql(f.get("expression")), datatype=f.get("datatype"),
170
+ description=str(f.get("description") or ""), is_dimension=is_dim, is_time=is_time,
171
+ instructions=f_instr, synonyms=f_syn))
172
+ name = str(d.get("name", ""))
173
+ datasets.append(Dataset(name=name, source=str(d.get("source", "")), primary_key=list(d.get("primary_key") or []),
174
+ description=str(d.get("description") or ""), instructions=d_instr, synonyms=d_syn,
175
+ fields=fields, metrics=[_metric(m, name) for m in d.get("metrics") or []]))
176
+ rels = [Relationship(name=str(r.get("name", "")), from_dataset=str(r.get("from", "")), to_dataset=str(r.get("to", "")),
177
+ from_columns=list(r.get("from_columns") or []), to_columns=list(r.get("to_columns") or []))
178
+ for r in raw.get("relationships") or []]
179
+ return SemanticModel(name=str(raw.get("name", "")), version=version, description=str(raw.get("description") or ""),
180
+ instructions=instructions, datasets=datasets, relationships=rels,
181
+ metrics=[_metric(m, None) for m in raw.get("metrics") or []], path=path)
182
+
183
+
184
+ def load_models(path: str | Path) -> list[SemanticModel]:
185
+ """Every semantic model in a file (one for 0.3.x; one or more for 0.1.x)."""
186
+ doc = read_document(path)
187
+ version = str(doc.get("version", ""))
188
+ return [_model(m, version, Path(path)) for m in raw_models(doc)]
189
+
190
+
191
+ def load(path: str | Path, name: str | None = None) -> SemanticModel:
192
+ """The semantic model in a file; `name` picks one when a 0.1.x document holds several."""
193
+ models = load_models(path)
194
+ if name is None:
195
+ if len(models) > 1:
196
+ raise ModelError(f"{path} holds {len(models)} models ({', '.join(m.name for m in models)}); pick one by name")
197
+ return models[0]
198
+ for m in models:
199
+ if m.name == name:
200
+ return m
201
+ raise ModelError(f"no model named {name!r} in {path} (has: {', '.join(m.name for m in models)})")