lexis-cli 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lexis/__init__.py +10 -0
- lexis/_vendor/ossie/NOTICE.md +14 -0
- lexis/_vendor/ossie/__init__.py +44 -0
- lexis/_vendor/ossie/models.py +224 -0
- lexis/cli.py +298 -0
- lexis/demo_data.py +87 -0
- lexis/dispatch.py +66 -0
- lexis/mcp_server.py +158 -0
- lexis/parser.py +19 -0
- lexis/resolved_model.py +128 -0
- lexis/retail_demo_data.py +615 -0
- lexis/sml/__init__.py +10 -0
- lexis/sml/_common.py +224 -0
- lexis/sml/emit.py +403 -0
- lexis/sml/models.py +172 -0
- lexis/sml/parse.py +382 -0
- lexis/transpilers/base.py +11 -0
- lexis/transpilers/cube.py +119 -0
- lexis/transpilers/dbt_ossie.py +67 -0
- lexis/transpilers/mcp.py +182 -0
- lexis/transpilers/snowflake_semantic_view.py +126 -0
- lexis/transpilers/sql/__init__.py +22 -0
- lexis/transpilers/sql/base.py +172 -0
- lexis/transpilers/sql/bigquery.py +8 -0
- lexis/transpilers/sql/databricks.py +8 -0
- lexis/transpilers/sql/duckdb.py +8 -0
- lexis/transpilers/sql/postgres.py +10 -0
- lexis/transpilers/sql/snowflake.py +8 -0
- lexis_cli-0.4.2.dist-info/METADATA +883 -0
- lexis_cli-0.4.2.dist-info/RECORD +33 -0
- lexis_cli-0.4.2.dist-info/WHEEL +4 -0
- lexis_cli-0.4.2.dist-info/entry_points.txt +2 -0
- lexis_cli-0.4.2.dist-info/licenses/LICENSE +202 -0
lexis/sml/_common.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""Shared helpers for the Ossie <-> SML converter.
|
|
2
|
+
|
|
3
|
+
Mirrors the conventions Apache Ossie's other bidirectional converters already
|
|
4
|
+
established (third_party/ossie/converters/{omni,orionbelt,databricks}/src/*/_common.py):
|
|
5
|
+
a `custom_extensions` stash protocol keyed by vendor name, a `ConversionError` for
|
|
6
|
+
clean error messages, and a dialect-preference table. See SML_OSSIE_CONVERTER_PLAN.md
|
|
7
|
+
at the repo root for the full data-model mapping this implements.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
from typing import Any, NamedTuple, Optional
|
|
13
|
+
|
|
14
|
+
from lexis._vendor.ossie import OssieCustomExtension, OssieDataType, OssieDialect
|
|
15
|
+
|
|
16
|
+
VENDOR = "SML"
|
|
17
|
+
|
|
18
|
+
# Bump when the shape of a stashed `data` blob changes.
|
|
19
|
+
STASH_VERSION = 1
|
|
20
|
+
|
|
21
|
+
# SML SQL dialects this converter maps to/from a corresponding Ossie dialect.
|
|
22
|
+
# Postgresql and Iris have no Ossie enum slot (see SML_OSSIE_CONVERTER_PLAN.md,
|
|
23
|
+
# Edge Case #9): never emitted on Ossie -> SML; preserved via stash on SML -> Ossie
|
|
24
|
+
# (parse.py, phase 2).
|
|
25
|
+
SML_TO_OSSIE_DIALECT = {
|
|
26
|
+
"Snowflake": OssieDialect.SNOWFLAKE,
|
|
27
|
+
"DatabricksSQL": OssieDialect.DATABRICKS,
|
|
28
|
+
"BigQuery": OssieDialect.BIGQUERY,
|
|
29
|
+
}
|
|
30
|
+
OSSIE_TO_SML_DIALECT = {v: k for k, v in SML_TO_OSSIE_DIALECT.items()}
|
|
31
|
+
|
|
32
|
+
# OssieDataType -> a plausible SML `data_type` string. SML's docs show free-form
|
|
33
|
+
# examples (e.g. "decimal(7,2)") rather than a strict enum, so this is a
|
|
34
|
+
# reasonable default, not a spec-mandated mapping.
|
|
35
|
+
OSSIE_TO_SML_DATATYPE = {
|
|
36
|
+
OssieDataType.STRING: "string",
|
|
37
|
+
OssieDataType.INTEGER: "int",
|
|
38
|
+
OssieDataType.DECIMAL: "decimal(18,2)",
|
|
39
|
+
OssieDataType.FLOAT: "double",
|
|
40
|
+
OssieDataType.BOOLEAN: "boolean",
|
|
41
|
+
OssieDataType.DATE: "date",
|
|
42
|
+
OssieDataType.TIME: "time",
|
|
43
|
+
OssieDataType.DATE_TIME: "datetime",
|
|
44
|
+
OssieDataType.DATE_TIME_TZ: "datetime",
|
|
45
|
+
OssieDataType.OPAQUE: "string",
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def sml_datatype_to_ossie(data_type: Optional[str]) -> Optional[OssieDataType]:
|
|
50
|
+
"""Best-effort reverse of OSSIE_TO_SML_DATATYPE. SML's `data_type` is
|
|
51
|
+
free-form (e.g. "decimal(7,2)", "varchar(50)") rather than a strict enum, so
|
|
52
|
+
this matches on a lowercased prefix rather than exact equality. Returns None
|
|
53
|
+
(rather than guessing) for anything unrecognized - the Ossie field is simply
|
|
54
|
+
emitted with no `datatype`."""
|
|
55
|
+
if not data_type:
|
|
56
|
+
return None
|
|
57
|
+
lowered = data_type.strip().lower()
|
|
58
|
+
if lowered.startswith(("decimal", "numeric")):
|
|
59
|
+
return OssieDataType.DECIMAL
|
|
60
|
+
if lowered.startswith(("double", "float", "real")):
|
|
61
|
+
return OssieDataType.FLOAT
|
|
62
|
+
if lowered.startswith(("int", "bigint", "smallint", "tinyint")):
|
|
63
|
+
return OssieDataType.INTEGER
|
|
64
|
+
if lowered.startswith(("bool",)):
|
|
65
|
+
return OssieDataType.BOOLEAN
|
|
66
|
+
if lowered.startswith("datetime") or lowered.startswith("timestamp"):
|
|
67
|
+
return OssieDataType.DATE_TIME
|
|
68
|
+
if lowered.startswith("date"):
|
|
69
|
+
return OssieDataType.DATE
|
|
70
|
+
if lowered.startswith("time"):
|
|
71
|
+
return OssieDataType.TIME
|
|
72
|
+
if lowered.startswith(("string", "varchar", "char", "text")):
|
|
73
|
+
return OssieDataType.STRING
|
|
74
|
+
return None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class ConversionError(Exception):
|
|
78
|
+
"""Raised when an input cannot be converted."""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def require(obj: dict, key: str, what: str) -> Any:
|
|
82
|
+
"""Return `obj[key]`, or raise a clean ConversionError if it's missing/empty."""
|
|
83
|
+
if not isinstance(obj, dict) or key not in obj or obj[key] is None:
|
|
84
|
+
raise ConversionError(f"{what} is missing required {key!r}")
|
|
85
|
+
value = obj[key]
|
|
86
|
+
if isinstance(value, str) and not value.strip():
|
|
87
|
+
raise ConversionError(f"{what} has an empty {key!r}")
|
|
88
|
+
return value
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def read_stash(obj: Any) -> dict:
|
|
92
|
+
"""Return the SML `custom_extensions` stash dict on an Ossie object (anything
|
|
93
|
+
with a `custom_extensions` attribute - Dataset/Field/Relationship/Metric/
|
|
94
|
+
SemanticModel), or {} if absent. Strips the `_v` version marker.
|
|
95
|
+
|
|
96
|
+
Phase 1 (Ossie -> SML) never encounters real stash data yet, since nothing has
|
|
97
|
+
written an SML stash - this exists so Phase 2's SML -> Ossie -> SML round trip
|
|
98
|
+
can reconstruct richer SML shapes once `parse.py` starts writing one.
|
|
99
|
+
"""
|
|
100
|
+
for ext in getattr(obj, "custom_extensions", None) or []:
|
|
101
|
+
if ext.vendor_name == VENDOR:
|
|
102
|
+
try:
|
|
103
|
+
data = json.loads(ext.data)
|
|
104
|
+
except json.JSONDecodeError as e:
|
|
105
|
+
raise ConversionError(f"SML custom_extensions data is not valid JSON: {e}") from e
|
|
106
|
+
data.pop("_v", None)
|
|
107
|
+
return data
|
|
108
|
+
return {}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def make_stash_extension(data: dict) -> OssieCustomExtension:
|
|
112
|
+
"""Build an OssieCustomExtension carrying `data` under the SML vendor tag."""
|
|
113
|
+
payload = {"_v": STASH_VERSION, **data}
|
|
114
|
+
return OssieCustomExtension(vendor_name=VENDOR, data=json.dumps(payload))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# calculation_method (SML) <-> SQL aggregate function name. Extends the same
|
|
118
|
+
# pattern already used for the Cube.js emitter (transpilers/cube.py's
|
|
119
|
+
# _SIMPLE_AGGREGATE_RE) with SML's fuller calculation_method vocabulary.
|
|
120
|
+
# `sum distinct` and `estimated count distinct` have no single ANSI function and
|
|
121
|
+
# are intentionally left unmapped (see SML_OSSIE_CONVERTER_PLAN.md Edge Cases).
|
|
122
|
+
CALC_METHOD_TO_SQL_FUNC = {
|
|
123
|
+
"sum": "SUM",
|
|
124
|
+
"average": "AVG",
|
|
125
|
+
"count non-null": "COUNT",
|
|
126
|
+
"maximum": "MAX",
|
|
127
|
+
"minimum": "MIN",
|
|
128
|
+
"stddev_pop": "STDDEV_POP",
|
|
129
|
+
"stddev_samp": "STDDEV_SAMP",
|
|
130
|
+
"var_pop": "VAR_POP",
|
|
131
|
+
"var_samp": "VAR_SAMP",
|
|
132
|
+
"percentile": "PERCENTILE_CONT",
|
|
133
|
+
"count_if": "COUNT_IF",
|
|
134
|
+
}
|
|
135
|
+
_SQL_FUNC_TO_CALC_METHOD = {v: k for k, v in CALC_METHOD_TO_SQL_FUNC.items()}
|
|
136
|
+
|
|
137
|
+
_SIMPLE_AGGREGATE_RE = re.compile(
|
|
138
|
+
r"^(SUM|AVG|COUNT|MIN|MAX|STDDEV_POP|STDDEV_SAMP|VAR_POP|VAR_SAMP|COUNT_IF)"
|
|
139
|
+
r"\s*\(\s*(DISTINCT\s+)?([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\s*\)$",
|
|
140
|
+
re.IGNORECASE,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def decompose_simple_aggregate(expr: str) -> Optional[tuple[str, str, str]]:
|
|
145
|
+
"""Match `FUNC([DISTINCT] dataset.column)` and return
|
|
146
|
+
`(calculation_method, dataset, column)`, or None if `expr` isn't that shape."""
|
|
147
|
+
match = _SIMPLE_AGGREGATE_RE.match(expr.strip())
|
|
148
|
+
if not match:
|
|
149
|
+
return None
|
|
150
|
+
func, distinct, dataset, column = match.groups()
|
|
151
|
+
func = func.upper()
|
|
152
|
+
if distinct:
|
|
153
|
+
if func != "COUNT":
|
|
154
|
+
return None # DISTINCT on a non-COUNT aggregate has no SML calculation_method
|
|
155
|
+
method = "count distinct"
|
|
156
|
+
else:
|
|
157
|
+
method = _SQL_FUNC_TO_CALC_METHOD.get(func)
|
|
158
|
+
if method is None:
|
|
159
|
+
return None
|
|
160
|
+
return method, dataset, column
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def synthesize_aggregate_sql(calculation_method: str, dataset: str, column: str) -> Optional[str]:
|
|
164
|
+
"""Reverse of decompose_simple_aggregate: build `FUNC(dataset.column)` ANSI SQL
|
|
165
|
+
for a calculation_method. Returns None for methods with no single ANSI function."""
|
|
166
|
+
if calculation_method == "count distinct":
|
|
167
|
+
return f"COUNT(DISTINCT {dataset}.{column})"
|
|
168
|
+
func = CALC_METHOD_TO_SQL_FUNC.get(calculation_method)
|
|
169
|
+
return f"{func}({dataset}.{column})" if func else None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class RatioAggregate(NamedTuple):
|
|
173
|
+
"""A `numerator / denominator` metric where both sides are simple
|
|
174
|
+
`FUNC(dataset.column)` aggregates - the exact shape SML's own metric_calc docs
|
|
175
|
+
use for ratio metrics (`"[Measures].[a]/[Measures].[b]"`)."""
|
|
176
|
+
|
|
177
|
+
numerator: tuple[str, str, str] # (calculation_method, dataset, column)
|
|
178
|
+
denominator: tuple[str, str, str]
|
|
179
|
+
denominator_nullif_guard_dropped: bool
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
_NULLIF_ZERO_RE = re.compile(r"^NULLIF\s*\(\s*(.+?)\s*,\s*0\s*\)$", re.IGNORECASE)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _split_top_level_slash(expr: str) -> Optional[tuple[str, str]]:
|
|
186
|
+
"""Split `expr` on the first `/` that isn't nested inside parentheses.
|
|
187
|
+
Returns None if there's no such split point (so `A / B / C` correctly fails to
|
|
188
|
+
decompose here rather than silently dropping the third term)."""
|
|
189
|
+
depth = 0
|
|
190
|
+
for i, ch in enumerate(expr):
|
|
191
|
+
if ch == "(":
|
|
192
|
+
depth += 1
|
|
193
|
+
elif ch == ")":
|
|
194
|
+
depth -= 1
|
|
195
|
+
elif ch == "/" and depth == 0:
|
|
196
|
+
return expr[:i], expr[i + 1 :]
|
|
197
|
+
return None
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def decompose_ratio_aggregate(expr: str) -> Optional[RatioAggregate]:
|
|
201
|
+
"""Match `AGG(dataset.column) / AGG(dataset.column)`, optionally with a
|
|
202
|
+
`NULLIF(..., 0)` divide-by-zero guard around the denominator (a common SQL
|
|
203
|
+
idiom with no MDX equivalent - MDX division already returns blank/error on
|
|
204
|
+
divide-by-zero). Returns None if `expr` isn't that shape."""
|
|
205
|
+
split = _split_top_level_slash(expr.strip())
|
|
206
|
+
if split is None:
|
|
207
|
+
return None
|
|
208
|
+
left, right = split[0].strip(), split[1].strip()
|
|
209
|
+
|
|
210
|
+
numerator = decompose_simple_aggregate(left)
|
|
211
|
+
if numerator is None:
|
|
212
|
+
return None
|
|
213
|
+
|
|
214
|
+
nullif_guard = False
|
|
215
|
+
nullif_match = _NULLIF_ZERO_RE.match(right)
|
|
216
|
+
if nullif_match:
|
|
217
|
+
right = nullif_match.group(1)
|
|
218
|
+
nullif_guard = True
|
|
219
|
+
|
|
220
|
+
denominator = decompose_simple_aggregate(right)
|
|
221
|
+
if denominator is None:
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
return RatioAggregate(numerator=numerator, denominator=denominator, denominator_nullif_guard_dropped=nullif_guard)
|
lexis/sml/emit.py
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
"""Ossie -> SML (Semantic Modeling Language) emitter.
|
|
2
|
+
|
|
3
|
+
SML (github.com/semanticdatalayer/SML) models a semantic layer as many small
|
|
4
|
+
YAML files (one per object) discriminated by `object_type`, resolved through a
|
|
5
|
+
global `unique_name` registry - not a single document. This emitter produces
|
|
6
|
+
that file set from one OssieDocument.
|
|
7
|
+
|
|
8
|
+
v1 is the "documented supported subset" (see SML_OSSIE_CONVERTER_PLAN.md at the
|
|
9
|
+
repo root): datasets, a synthesized single-level dimension per relationship
|
|
10
|
+
target, relationships, and metrics that decompose into a plain
|
|
11
|
+
`AGG(dataset.column)` shape. A metric that is a ratio of two such aggregates
|
|
12
|
+
(`AGG(a.b) / AGG(c.d)`, optionally with a `NULLIF(..., 0)` divide-by-zero guard
|
|
13
|
+
on the denominator) becomes an SML `metric_calc` with a synthesized MDX division
|
|
14
|
+
expression - the exact shape SML's own docs use for ratio metrics - referencing
|
|
15
|
+
two plain `metric` objects (reusing an existing one with the same aggregate
|
|
16
|
+
shape when available, e.g. a `total_sales` metric doubling as a ratio's
|
|
17
|
+
numerator, rather than duplicating it). A metric that doesn't decompose either
|
|
18
|
+
way is preserved verbatim (never faked as arbitrary MDX) under a small
|
|
19
|
+
`x_lexis_unconverted_metrics` escape hatch on the emitted `model.yml` - SML
|
|
20
|
+
itself has no vendor-extension slot for an object it can't represent at all -
|
|
21
|
+
so a later `parse_sml_repo()` call restores it exactly rather than losing it
|
|
22
|
+
silently, mirroring the "no silent loss" convention Ossie's other bidirectional
|
|
23
|
+
converters use (see `SML_OSSIE_CONVERTER_PLAN.md`'s Phase 3 section).
|
|
24
|
+
|
|
25
|
+
If `document` was itself produced by `parse_sml_repo()`, this emitter is
|
|
26
|
+
stash-aware: a dimension whose full original shape (hierarchies, levels,
|
|
27
|
+
calculation_groups, ...) was stashed under `custom_extensions` on the way in is
|
|
28
|
+
re-emitted verbatim instead of re-flattened into a fresh single-level
|
|
29
|
+
synthesis, so an SML -> Ossie -> SML round trip doesn't lose hierarchy
|
|
30
|
+
structure it never needed to lose.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from dataclasses import dataclass
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
import yaml
|
|
37
|
+
|
|
38
|
+
from lexis._vendor.ossie import OssieDataset, OssieDialect, OssieDocument
|
|
39
|
+
from lexis.resolved_model import MissingExpressionError, ResolvedModel
|
|
40
|
+
from lexis.sml._common import (
|
|
41
|
+
OSSIE_TO_SML_DATATYPE,
|
|
42
|
+
decompose_ratio_aggregate,
|
|
43
|
+
decompose_simple_aggregate,
|
|
44
|
+
read_stash,
|
|
45
|
+
)
|
|
46
|
+
from lexis.sml.models import (
|
|
47
|
+
SmlCatalog,
|
|
48
|
+
SmlConnection,
|
|
49
|
+
SmlDataset,
|
|
50
|
+
SmlDatasetColumn,
|
|
51
|
+
SmlDimension,
|
|
52
|
+
SmlHierarchy,
|
|
53
|
+
SmlLevel,
|
|
54
|
+
SmlLevelAttribute,
|
|
55
|
+
SmlMetric,
|
|
56
|
+
SmlMetricCalc,
|
|
57
|
+
SmlModel,
|
|
58
|
+
SmlModelMetricRef,
|
|
59
|
+
SmlModelRelationship,
|
|
60
|
+
SmlRelationshipEnd,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class SmlEmitResult:
|
|
66
|
+
files: dict[str, str]
|
|
67
|
+
warnings: list[str]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _dump(obj) -> str:
|
|
71
|
+
data = obj.model_dump(by_alias=True, exclude_none=True, mode="json")
|
|
72
|
+
return yaml.dump(data, sort_keys=False, default_flow_style=False, allow_unicode=True)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _split_source(dataset: OssieDataset, warnings: list[str]) -> tuple[str, str, str] | None:
|
|
76
|
+
"""Split `database.schema.table`; warn and return None if source isn't 3-part
|
|
77
|
+
(mirrors dbt_ossie.py's `_source_shape_warnings` check on the same convention)."""
|
|
78
|
+
parts = dataset.source.split(".")
|
|
79
|
+
if len(parts) != 3 or not all(parts):
|
|
80
|
+
warnings.append(
|
|
81
|
+
f"LOSSY: dataset {dataset.name!r} source {dataset.source!r} is not "
|
|
82
|
+
"'database.schema.table' - SML requires a connection + table name; "
|
|
83
|
+
"this dataset was excluded from the SML output."
|
|
84
|
+
)
|
|
85
|
+
return None
|
|
86
|
+
return parts[0], parts[1], parts[2]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _emit_dimension(
|
|
90
|
+
dataset: OssieDataset, warnings: list[str]
|
|
91
|
+
) -> tuple[dict[str, str], str | None]:
|
|
92
|
+
"""Synthesize a single-level SML dimension for `dataset`.
|
|
93
|
+
|
|
94
|
+
v1 flattens every Ossie field into one level's key/secondary attributes
|
|
95
|
+
rather than attempting to infer a natural multi-level hierarchy - Ossie has
|
|
96
|
+
no data (level order, grouping) that would justify one. Hand-edit the
|
|
97
|
+
emitted SML if a richer hierarchy is wanted. See SML_OSSIE_CONVERTER_PLAN.md.
|
|
98
|
+
"""
|
|
99
|
+
dim_name = f"{dataset.name} Dimension"
|
|
100
|
+
fields = dataset.fields or []
|
|
101
|
+
if not fields:
|
|
102
|
+
warnings.append(f"LOSSY: dataset {dataset.name!r} has no fields; dimension skipped.")
|
|
103
|
+
return {}, None
|
|
104
|
+
|
|
105
|
+
key_field = dataset.primary_key[0] if dataset.primary_key else fields[0].name
|
|
106
|
+
is_time = any(f.is_time_dimension() for f in fields)
|
|
107
|
+
|
|
108
|
+
# Per SML's dimension.md spec, dimension-level `level_attributes` holds only
|
|
109
|
+
# the one joinable key attribute per level ("Only level attributes can be
|
|
110
|
+
# used to define relationships between datasets and other dimensions") -
|
|
111
|
+
# our single synthesized level has exactly one. Every other field is a
|
|
112
|
+
# non-joinable secondary attribute, inlined directly under the level rather
|
|
113
|
+
# than duplicated at the dimension level.
|
|
114
|
+
key_attribute = SmlLevelAttribute(
|
|
115
|
+
unique_name=dim_name,
|
|
116
|
+
dataset=dataset.name,
|
|
117
|
+
name_column=key_field,
|
|
118
|
+
key_columns=[key_field],
|
|
119
|
+
is_unique_key=True,
|
|
120
|
+
)
|
|
121
|
+
secondary_attributes = [
|
|
122
|
+
SmlLevelAttribute(
|
|
123
|
+
unique_name=f"{dataset.name} {f.name}",
|
|
124
|
+
dataset=dataset.name,
|
|
125
|
+
name_column=f.name,
|
|
126
|
+
key_columns=[f.name],
|
|
127
|
+
)
|
|
128
|
+
for f in fields
|
|
129
|
+
if f.name != key_field
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
hierarchy = SmlHierarchy(
|
|
133
|
+
unique_name=dim_name,
|
|
134
|
+
levels=[SmlLevel(unique_name=dim_name, secondary_attributes=secondary_attributes or None)],
|
|
135
|
+
)
|
|
136
|
+
dimension = SmlDimension(
|
|
137
|
+
unique_name=dim_name,
|
|
138
|
+
type="time" if is_time else "standard",
|
|
139
|
+
hierarchies=[hierarchy],
|
|
140
|
+
level_attributes=[key_attribute],
|
|
141
|
+
)
|
|
142
|
+
return {f"dimensions/{dim_name}.yml": _dump(dimension)}, dim_name
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _dump_raw(data: dict) -> str:
|
|
146
|
+
return yaml.dump(data, sort_keys=False, default_flow_style=False, allow_unicode=True)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _leaf_level_of(raw_dimension: dict) -> str:
|
|
150
|
+
"""The dimension's grain-key level unique_name - what a relationship's
|
|
151
|
+
`to.level` must reference. Falls back to the dimension's own unique_name,
|
|
152
|
+
matching `_emit_dimension`'s own single-level synthesis convention, if no
|
|
153
|
+
`level_attributes` entry is marked `is_unique_key`."""
|
|
154
|
+
for attr in raw_dimension.get("level_attributes") or []:
|
|
155
|
+
if attr.get("is_unique_key"):
|
|
156
|
+
return attr["unique_name"]
|
|
157
|
+
return raw_dimension.get("unique_name")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _stashed_dimensions_by_dataset(stashed_dimensions: dict[str, dict]) -> dict[str, str]:
|
|
161
|
+
"""Reverse-index the stash: physical dataset name -> the (first) stashed
|
|
162
|
+
dimension unique_name whose `level_attributes` names it. A snowflaked
|
|
163
|
+
dimension spanning multiple datasets is looked up by whichever of its
|
|
164
|
+
datasets a relationship or degenerate-dimension pass asks about."""
|
|
165
|
+
by_dataset: dict[str, str] = {}
|
|
166
|
+
for dim_name, raw_dim in stashed_dimensions.items():
|
|
167
|
+
for attr in raw_dim.get("level_attributes") or []:
|
|
168
|
+
ds = attr.get("dataset")
|
|
169
|
+
if ds:
|
|
170
|
+
by_dataset.setdefault(ds, dim_name)
|
|
171
|
+
return by_dataset
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _metric_ref_for_aggregate(
|
|
175
|
+
aggregate: tuple[str, str, str],
|
|
176
|
+
aggregate_index: dict[tuple[str, str, str], str],
|
|
177
|
+
files: dict[str, str],
|
|
178
|
+
metric_refs: list[SmlModelMetricRef],
|
|
179
|
+
) -> str:
|
|
180
|
+
"""Return the unique_name of an SML metric matching `aggregate`
|
|
181
|
+
(calculation_method, dataset, column) - reusing an existing metric with that
|
|
182
|
+
exact shape (from `aggregate_index`, populated from every metric that already
|
|
183
|
+
decomposes to a plain aggregate) if one exists, else synthesizing and emitting
|
|
184
|
+
a new helper metric for it. Used to build a ratio metric_calc's MDX operands."""
|
|
185
|
+
if aggregate in aggregate_index:
|
|
186
|
+
return aggregate_index[aggregate]
|
|
187
|
+
calc_method, ds_name, column = aggregate
|
|
188
|
+
name = f"{ds_name}.{column} {calc_method}"
|
|
189
|
+
files[f"metrics/{name}.yml"] = _dump(
|
|
190
|
+
SmlMetric(unique_name=name, label=name, calculation_method=calc_method, dataset=ds_name, column=column)
|
|
191
|
+
)
|
|
192
|
+
metric_refs.append(SmlModelMetricRef(unique_name=name))
|
|
193
|
+
aggregate_index[aggregate] = name
|
|
194
|
+
return name
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def emit_sml_files(document: OssieDocument) -> SmlEmitResult:
|
|
198
|
+
"""Render `document`'s first semantic model as an SML file set."""
|
|
199
|
+
semantic_model = document.semantic_model[0]
|
|
200
|
+
model = ResolvedModel.build(semantic_model)
|
|
201
|
+
warnings: list[str] = []
|
|
202
|
+
files: dict[str, str] = {}
|
|
203
|
+
|
|
204
|
+
stash = read_stash(semantic_model)
|
|
205
|
+
stashed_dimensions: dict[str, dict] = stash.get("dimensions", {})
|
|
206
|
+
stashed_dim_by_dataset = _stashed_dimensions_by_dataset(stashed_dimensions)
|
|
207
|
+
emitted_dimension_names: set[str] = set()
|
|
208
|
+
|
|
209
|
+
# --- connections: one per unique (database, schema) pair ---
|
|
210
|
+
dataset_parts: dict[str, tuple[str, str, str]] = {}
|
|
211
|
+
connection_names: dict[tuple[str, str], str] = {}
|
|
212
|
+
for dataset in semantic_model.datasets:
|
|
213
|
+
parts = _split_source(dataset, warnings)
|
|
214
|
+
if parts is None:
|
|
215
|
+
continue
|
|
216
|
+
dataset_parts[dataset.name] = parts
|
|
217
|
+
db, schema, _table = parts
|
|
218
|
+
key = (db, schema)
|
|
219
|
+
if key not in connection_names:
|
|
220
|
+
conn_name = f"Connection - {db}.{schema}"
|
|
221
|
+
connection_names[key] = conn_name
|
|
222
|
+
conn = SmlConnection(unique_name=conn_name, database=db, schema_name=schema)
|
|
223
|
+
files[f"connections/{conn_name}.yml"] = _dump(conn)
|
|
224
|
+
|
|
225
|
+
# --- datasets (+ a dimension for every relationship target: re-emitted
|
|
226
|
+
# verbatim from the stash if this dataset was originally parsed from one,
|
|
227
|
+
# else freshly synthesized as a single flat level) ---
|
|
228
|
+
dimension_targets = {rel.to for rel in model.relationships}
|
|
229
|
+
dimension_key: dict[str, tuple[str, str]] = {} # dataset -> (dimension unique_name, leaf level unique_name)
|
|
230
|
+
for dataset in semantic_model.datasets:
|
|
231
|
+
if dataset.name not in dataset_parts:
|
|
232
|
+
continue
|
|
233
|
+
db, schema, table = dataset_parts[dataset.name]
|
|
234
|
+
|
|
235
|
+
columns = []
|
|
236
|
+
for f in dataset.fields or []:
|
|
237
|
+
try:
|
|
238
|
+
sql = model.resolve_expression(f.expression, OssieDialect.ANSI_SQL)
|
|
239
|
+
except MissingExpressionError:
|
|
240
|
+
warnings.append(
|
|
241
|
+
f"LOSSY: field {dataset.name}.{f.name!r} has no ANSI_SQL expression; "
|
|
242
|
+
"column excluded from the SML dataset."
|
|
243
|
+
)
|
|
244
|
+
continue
|
|
245
|
+
data_type = OSSIE_TO_SML_DATATYPE.get(f.datatype) if f.datatype else None
|
|
246
|
+
columns.append(
|
|
247
|
+
SmlDatasetColumn(name=f.name, data_type=data_type, sql=None if sql == f.name else sql)
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
sml_dataset = SmlDataset(
|
|
251
|
+
unique_name=dataset.name,
|
|
252
|
+
label=dataset.name,
|
|
253
|
+
description=dataset.description,
|
|
254
|
+
connection_id=connection_names[(db, schema)],
|
|
255
|
+
table=table,
|
|
256
|
+
columns=columns,
|
|
257
|
+
)
|
|
258
|
+
files[f"datasets/{dataset.name}.yml"] = _dump(sml_dataset)
|
|
259
|
+
|
|
260
|
+
if dataset.name in dimension_targets:
|
|
261
|
+
stashed_name = stashed_dim_by_dataset.get(dataset.name)
|
|
262
|
+
if stashed_name:
|
|
263
|
+
raw_dim = stashed_dimensions[stashed_name]
|
|
264
|
+
if stashed_name not in emitted_dimension_names:
|
|
265
|
+
files[f"dimensions/{stashed_name}.yml"] = _dump_raw(raw_dim)
|
|
266
|
+
emitted_dimension_names.add(stashed_name)
|
|
267
|
+
dimension_key[dataset.name] = (stashed_name, _leaf_level_of(raw_dim))
|
|
268
|
+
else:
|
|
269
|
+
dim_files, dim_name = _emit_dimension(dataset, warnings)
|
|
270
|
+
files.update(dim_files)
|
|
271
|
+
if dim_name:
|
|
272
|
+
dimension_key[dataset.name] = (dim_name, dim_name)
|
|
273
|
+
|
|
274
|
+
# A degenerate/shared-degenerate dimension (keyed on a fact dataset's own
|
|
275
|
+
# column) has no relationship pointing at it - `dimension_targets` above
|
|
276
|
+
# never catches it - so re-emit any stashed dimension not already handled,
|
|
277
|
+
# as long as its backing dataset(s) are still part of this document.
|
|
278
|
+
for dim_name, raw_dim in stashed_dimensions.items():
|
|
279
|
+
if dim_name in emitted_dimension_names:
|
|
280
|
+
continue
|
|
281
|
+
backing_datasets = {a.get("dataset") for a in raw_dim.get("level_attributes") or []}
|
|
282
|
+
if backing_datasets & dataset_parts.keys():
|
|
283
|
+
files[f"dimensions/{dim_name}.yml"] = _dump_raw(raw_dim)
|
|
284
|
+
emitted_dimension_names.add(dim_name)
|
|
285
|
+
|
|
286
|
+
# --- metrics ---
|
|
287
|
+
# First pass: resolve every metric's expression and record which ones decompose
|
|
288
|
+
# to a plain aggregate, so a ratio metric processed below can reuse one as an
|
|
289
|
+
# operand (e.g. `total_sales`) regardless of the two metrics' declaration order.
|
|
290
|
+
resolved: list[tuple] = []
|
|
291
|
+
aggregate_index: dict[tuple[str, str, str], str] = {}
|
|
292
|
+
unconverted_metrics: list[dict[str, Any]] = []
|
|
293
|
+
for metric in semantic_model.metrics or []:
|
|
294
|
+
try:
|
|
295
|
+
expr = model.resolve_expression(metric.expression, OssieDialect.ANSI_SQL)
|
|
296
|
+
except MissingExpressionError:
|
|
297
|
+
warnings.append(
|
|
298
|
+
f"LOSSY: metric {metric.name!r} has no ANSI_SQL expression; preserved verbatim "
|
|
299
|
+
"(not a queryable SML metric/metric_calc, but round-trips back to Ossie)."
|
|
300
|
+
)
|
|
301
|
+
unconverted_metrics.append(metric.model_dump(exclude_none=True, mode="json"))
|
|
302
|
+
resolved.append((metric, None))
|
|
303
|
+
continue
|
|
304
|
+
resolved.append((metric, expr))
|
|
305
|
+
decomposed = decompose_simple_aggregate(expr)
|
|
306
|
+
if decomposed is not None and decomposed not in aggregate_index:
|
|
307
|
+
# First declaration wins if two metrics share the same aggregate shape,
|
|
308
|
+
# so which one a later ratio metric_calc references is deterministic
|
|
309
|
+
# and follows document order rather than depending on dict overwrite order.
|
|
310
|
+
aggregate_index[decomposed] = metric.name
|
|
311
|
+
|
|
312
|
+
metric_refs = []
|
|
313
|
+
for metric, expr in resolved:
|
|
314
|
+
if expr is None:
|
|
315
|
+
continue
|
|
316
|
+
|
|
317
|
+
decomposed = decompose_simple_aggregate(expr)
|
|
318
|
+
if decomposed is not None:
|
|
319
|
+
calc_method, ds_name, column = decomposed
|
|
320
|
+
sml_metric = SmlMetric(
|
|
321
|
+
unique_name=metric.name,
|
|
322
|
+
label=metric.name,
|
|
323
|
+
description=metric.description,
|
|
324
|
+
calculation_method=calc_method,
|
|
325
|
+
dataset=ds_name,
|
|
326
|
+
column=column,
|
|
327
|
+
)
|
|
328
|
+
files[f"metrics/{metric.name}.yml"] = _dump(sml_metric)
|
|
329
|
+
metric_refs.append(SmlModelMetricRef(unique_name=metric.name))
|
|
330
|
+
continue
|
|
331
|
+
|
|
332
|
+
ratio = decompose_ratio_aggregate(expr)
|
|
333
|
+
if ratio is not None:
|
|
334
|
+
numerator_ref = _metric_ref_for_aggregate(ratio.numerator, aggregate_index, files, metric_refs)
|
|
335
|
+
denominator_ref = _metric_ref_for_aggregate(ratio.denominator, aggregate_index, files, metric_refs)
|
|
336
|
+
metric_calc = SmlMetricCalc(
|
|
337
|
+
unique_name=metric.name,
|
|
338
|
+
label=metric.name,
|
|
339
|
+
description=metric.description,
|
|
340
|
+
expression=f"[Measures].[{numerator_ref}]/[Measures].[{denominator_ref}]",
|
|
341
|
+
)
|
|
342
|
+
files[f"metrics/{metric.name}.yml"] = _dump(metric_calc)
|
|
343
|
+
metric_refs.append(SmlModelMetricRef(unique_name=metric.name))
|
|
344
|
+
if ratio.denominator_nullif_guard_dropped:
|
|
345
|
+
warnings.append(
|
|
346
|
+
f"metric {metric.name!r}: the SQL denominator's NULLIF(..., 0) divide-by-zero "
|
|
347
|
+
"guard has no MDX equivalent (MDX division already returns blank/error on "
|
|
348
|
+
"divide-by-zero) and was dropped from the emitted metric_calc; behavior at a "
|
|
349
|
+
"zero denominator may differ."
|
|
350
|
+
)
|
|
351
|
+
continue
|
|
352
|
+
|
|
353
|
+
warnings.append(
|
|
354
|
+
f"LOSSY: metric {metric.name!r} expression {expr!r} is not a simple AGG(dataset.column) "
|
|
355
|
+
"aggregate or a ratio of two such aggregates - preserved verbatim (not a queryable SML "
|
|
356
|
+
"metric/metric_calc, but round-trips back to Ossie) rather than faked as arbitrary MDX."
|
|
357
|
+
)
|
|
358
|
+
unconverted_metrics.append(metric.model_dump(exclude_none=True, mode="json"))
|
|
359
|
+
|
|
360
|
+
# --- model-level relationships (fact -> dimension) ---
|
|
361
|
+
relationships = []
|
|
362
|
+
for rel in model.relationships:
|
|
363
|
+
target = dimension_key.get(rel.to)
|
|
364
|
+
if target is None:
|
|
365
|
+
warnings.append(
|
|
366
|
+
f"LOSSY: relationship {rel.name!r} targets dataset {rel.to!r}, which has "
|
|
367
|
+
"no synthesized dimension; relationship excluded from SML output."
|
|
368
|
+
)
|
|
369
|
+
continue
|
|
370
|
+
dim_name, leaf_level = target
|
|
371
|
+
relationships.append(
|
|
372
|
+
SmlModelRelationship(
|
|
373
|
+
unique_name=rel.name,
|
|
374
|
+
from_end=SmlRelationshipEnd(dataset=rel.from_dataset, join_columns=rel.from_columns),
|
|
375
|
+
to=SmlRelationshipEnd(dimension=dim_name, level=leaf_level),
|
|
376
|
+
)
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
sml_model_data: dict[str, Any] = dict(
|
|
380
|
+
unique_name=semantic_model.name,
|
|
381
|
+
label=semantic_model.name,
|
|
382
|
+
description=semantic_model.description,
|
|
383
|
+
relationships=relationships,
|
|
384
|
+
metrics=metric_refs,
|
|
385
|
+
)
|
|
386
|
+
if unconverted_metrics:
|
|
387
|
+
# No SML object has a vendor-extension slot to stash an unconvertible
|
|
388
|
+
# metric on, unlike Ossie's own `custom_extensions` - this small,
|
|
389
|
+
# clearly-namespaced escape hatch on `model.yml` is this converter's
|
|
390
|
+
# own, restored by `parse.py`'s matching `x_lexis_unconverted_metrics` read.
|
|
391
|
+
sml_model_data["x_lexis_unconverted_metrics"] = unconverted_metrics
|
|
392
|
+
sml_model = SmlModel(**sml_model_data)
|
|
393
|
+
files[f"models/{semantic_model.name}.yml"] = _dump(sml_model)
|
|
394
|
+
|
|
395
|
+
# --- catalog ---
|
|
396
|
+
catalog = SmlCatalog(
|
|
397
|
+
unique_name=semantic_model.name,
|
|
398
|
+
label=semantic_model.name,
|
|
399
|
+
description=semantic_model.description,
|
|
400
|
+
)
|
|
401
|
+
files["catalog.yml"] = _dump(catalog)
|
|
402
|
+
|
|
403
|
+
return SmlEmitResult(files=files, warnings=warnings)
|