metricflow-sql 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metricflow_sql/__init__.py +8 -0
- metricflow_sql/compiler/__init__.py +0 -0
- metricflow_sql/compiler/catalog/__init__.py +0 -0
- metricflow_sql/compiler/catalog/cluster.py +152 -0
- metricflow_sql/compiler/catalog/information_schema.py +47 -0
- metricflow_sql/compiler/catalog/metadata.py +65 -0
- metricflow_sql/compiler/catalog/passthrough.py +184 -0
- metricflow_sql/compiler/catalog/protocols.py +67 -0
- metricflow_sql/compiler/catalog/semantic.py +59 -0
- metricflow_sql/compiler/compiler.py +297 -0
- metricflow_sql/compiler/rules/__init__.py +0 -0
- metricflow_sql/compiler/rules/expand_constellations.py +790 -0
- metricflow_sql/compiler/rules/expand_datasets.py +1208 -0
- metricflow_sql/compiler/rules/expand_metadata.py +88 -0
- metricflow_sql/compiler/rules/isolate_fragments.py +244 -0
- metricflow_sql/compiler/rules/passthrough_tables.py +36 -0
- metricflow_sql/compiler/table_arguments.py +35 -0
- metricflow_sql/converters/__init__.py +0 -0
- metricflow_sql/converters/lookml.py +1041 -0
- metricflow_sql/objects/__init__.py +0 -0
- metricflow_sql/objects/base.py +308 -0
- metricflow_sql/objects/constellation.py +210 -0
- metricflow_sql/objects/dataset.py +275 -0
- metricflow_sql/objects/entity.py +131 -0
- metricflow_sql/objects/reference.py +35 -0
- metricflow_sql/objects/wiki.py +101 -0
- metricflow_sql/py.typed +0 -0
- metricflow_sql/registry/__init__.py +0 -0
- metricflow_sql/registry/memory.py +409 -0
- metricflow_sql/registry/operation.py +25 -0
- metricflow_sql/registry/reference_resolution.py +68 -0
- metricflow_sql/registry/registry.py +225 -0
- metricflow_sql/registry/type_resolver.py +460 -0
- metricflow_sql/schemas/__init__.py +0 -0
- metricflow_sql/schemas/bundle.py +327 -0
- metricflow_sql/schemas/fragment.schema.json +1165 -0
- metricflow_sql/schemas/generate.py +70 -0
- metricflow_sql/schemas/load.py +333 -0
- metricflow_sql/schemas/v1/__init__.py +51 -0
- metricflow_sql/schemas/v1/fragment.py +544 -0
- metricflow_sql/server/__init__.py +4 -0
- metricflow_sql/server/app.py +33 -0
- metricflow_sql/server/compile.py +69 -0
- metricflow_sql/server/errors.py +10 -0
- metricflow_sql/server/serve.py +161 -0
- metricflow_sql/utils/__init__.py +0 -0
- metricflow_sql/utils/composed_set.py +32 -0
- metricflow_sql/utils/normalize.py +74 -0
- metricflow_sql/utils/ordered_set.py +24 -0
- metricflow_sql/utils/sql.py +226 -0
- metricflow_sql/utils/template.py +57 -0
- metricflow_sql-0.1.1.dist-info/METADATA +30 -0
- metricflow_sql-0.1.1.dist-info/RECORD +56 -0
- metricflow_sql-0.1.1.dist-info/WHEEL +4 -0
- metricflow_sql-0.1.1.dist-info/entry_points.txt +2 -0
- metricflow_sql-0.1.1.dist-info/licenses/LICENSE +29 -0
|
@@ -0,0 +1,544 @@
|
|
|
1
|
+
"""v1 of the fragment schema."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
from typing import Annotated, Any, Literal
|
|
7
|
+
|
|
8
|
+
from msgspec import Meta, Struct, field
|
|
9
|
+
|
|
10
|
+
type SqlExpr = str
|
|
11
|
+
type DataType = str
|
|
12
|
+
|
|
13
|
+
# Names of collection members: a dataset, a calculation, etc.
|
|
14
|
+
_NAME_DESCRIPTION = "The name, unique among its collection's."
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class _Model(Struct, frozen=True, kw_only=True, forbid_unknown_fields=True):
|
|
18
|
+
"""Base for all fragment objects.
|
|
19
|
+
|
|
20
|
+
`forbid` so a typo'd field fails at parse time instead of silently
|
|
21
|
+
dropping data - forward compatibility is handled by versioned modules
|
|
22
|
+
(`v1/`, `v2/`, ...), not by ignoring unknown fields.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
meta: dict[str, Any] = field(default_factory=dict)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class _Labeled(_Model, frozen=True, kw_only=True):
|
|
29
|
+
label: str | None = None
|
|
30
|
+
description: str | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _check_unique(names: Iterable[str], where: str) -> None:
|
|
34
|
+
"""Rejects a collection naming a member twice, e.g. two datasets named `orders`."""
|
|
35
|
+
seen: set[str] = set()
|
|
36
|
+
for name in names:
|
|
37
|
+
if name in seen:
|
|
38
|
+
raise ValueError(f"`{where}` has '{name}' twice")
|
|
39
|
+
seen.add(name)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class FragmentMetadata(_Model, frozen=True, kw_only=True):
|
|
43
|
+
"""Header describing the production of this fragment."""
|
|
44
|
+
|
|
45
|
+
generator_name: Annotated[
|
|
46
|
+
str | None, Meta(description="The name of the tool that generated this fragment.")
|
|
47
|
+
] = None
|
|
48
|
+
generator_version: Annotated[
|
|
49
|
+
str | None, Meta(description="The version of the tool that generated this fragment.")
|
|
50
|
+
] = None
|
|
51
|
+
generated_at: Annotated[str | None, Meta(description="When the fragment was generated.")] = None
|
|
52
|
+
env: Annotated[str | None, Meta(description="Environment")] = None
|
|
53
|
+
invocation_id: Annotated[
|
|
54
|
+
str | None, Meta(description="The invocation that generated this fragment, if applicable.")
|
|
55
|
+
] = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Column(_Labeled, frozen=True, kw_only=True):
|
|
59
|
+
"""A concrete column in the underlying database."""
|
|
60
|
+
|
|
61
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
62
|
+
datatype: Annotated[DataType | None, Meta(description="The column's data type.")] = None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Calculation(_Labeled, frozen=True, kw_only=True):
|
|
66
|
+
"""A derived column on the dataset.
|
|
67
|
+
|
|
68
|
+
`expr` is a SQL expression, where columns reference other columns,
|
|
69
|
+
calculations, or relationships.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
73
|
+
expr: Annotated[SqlExpr, Meta(description="A SQL expression, relative to the dataset.")]
|
|
74
|
+
datatype: Annotated[
|
|
75
|
+
DataType | None,
|
|
76
|
+
Meta(description="Optionally define the datatype. If left null, this will be inferred."),
|
|
77
|
+
] = None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class RelationshipKeyColumn(_Model, frozen=True, kw_only=True):
|
|
81
|
+
"""How a relationship binds one of its target's fields."""
|
|
82
|
+
|
|
83
|
+
name: Annotated[
|
|
84
|
+
str,
|
|
85
|
+
Meta(
|
|
86
|
+
description=(
|
|
87
|
+
"A key column of the target entity, or a key part or field of the target "
|
|
88
|
+
"dataset, unique among the key's."
|
|
89
|
+
)
|
|
90
|
+
),
|
|
91
|
+
]
|
|
92
|
+
expr: Annotated[SqlExpr, Meta(description="A SQL expression, relative to the dataset.")]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class Relationship(_Labeled, frozen=True, kw_only=True):
|
|
96
|
+
"""A relationship to an entity or a dataset, its target.
|
|
97
|
+
|
|
98
|
+
A key binding all of the target's key is many-to-one, and any other is
|
|
99
|
+
one-to-many.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
103
|
+
target: Annotated[
|
|
104
|
+
str,
|
|
105
|
+
Meta(
|
|
106
|
+
description=(
|
|
107
|
+
"The name of the related entity or dataset. An entity is looked up first, "
|
|
108
|
+
"then a dataset."
|
|
109
|
+
)
|
|
110
|
+
),
|
|
111
|
+
]
|
|
112
|
+
key: Annotated[
|
|
113
|
+
list[RelationshipKeyColumn],
|
|
114
|
+
Meta(description="Expressions on this dataset for the target's fields."),
|
|
115
|
+
] = field(default_factory=list)
|
|
116
|
+
cardinality: Annotated[
|
|
117
|
+
Literal["many_to_one", "one_to_many"] | None,
|
|
118
|
+
Meta(
|
|
119
|
+
description=(
|
|
120
|
+
"Optionally declare the cardinality. If left null, this will be inferred from "
|
|
121
|
+
"the key, and it must agree with it otherwise."
|
|
122
|
+
)
|
|
123
|
+
),
|
|
124
|
+
] = None
|
|
125
|
+
|
|
126
|
+
def __post_init__(self) -> None:
|
|
127
|
+
_check_unique((column.name for column in self.key), f"relationships.{self.name}.key")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# One struct, not a tagged union: msgspec requires the tag, and `type` defaults to `table`.
|
|
131
|
+
class DatasetSource(_Labeled, frozen=True, kw_only=True):
|
|
132
|
+
"""A physical table, or a query."""
|
|
133
|
+
|
|
134
|
+
type: Annotated[
|
|
135
|
+
Literal["table", "sql"],
|
|
136
|
+
Meta(description="`table` reads `location`; `sql` reads the rows `sql` returns."),
|
|
137
|
+
] = "table"
|
|
138
|
+
location: Annotated[
|
|
139
|
+
str | None,
|
|
140
|
+
Meta(description="The fully-qualified physical table this dataset reads from."),
|
|
141
|
+
] = None
|
|
142
|
+
sql: Annotated[
|
|
143
|
+
SqlExpr | None,
|
|
144
|
+
Meta(
|
|
145
|
+
description=(
|
|
146
|
+
"A query whose rows this dataset reads, e.g. a filtered table. A Jinja "
|
|
147
|
+
"template over the dataset's parameters."
|
|
148
|
+
)
|
|
149
|
+
),
|
|
150
|
+
] = None
|
|
151
|
+
columns: Annotated[list[Column], Meta(description="Columns of the table or query's rows.")] = (
|
|
152
|
+
field(default_factory=list)
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def __post_init__(self) -> None:
|
|
156
|
+
_check_unique((column.name for column in self.columns), "columns")
|
|
157
|
+
match self.type:
|
|
158
|
+
case "table" if self.location is None or self.sql is not None:
|
|
159
|
+
raise ValueError("A `table` source requires `location` and no `sql`")
|
|
160
|
+
case "sql" if self.sql is None or self.location is not None:
|
|
161
|
+
raise ValueError("A `sql` source requires `sql` and no `location`")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class Parameter(_Model, frozen=True, kw_only=True):
|
|
165
|
+
"""A value a query passes on its root table function, e.g. `ds(min_quantity => 90)`."""
|
|
166
|
+
|
|
167
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
168
|
+
datatype: Annotated[DataType | None, Meta(description="The parameter's data type.")] = None
|
|
169
|
+
default: Annotated[
|
|
170
|
+
SqlExpr | None,
|
|
171
|
+
Meta(description="A constant used when a query passes nothing; NULL when absent."),
|
|
172
|
+
] = None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class KeyParameter(_Model, frozen=True, kw_only=True):
|
|
176
|
+
"""A placeholder of a key condition, e.g. `:as_of`."""
|
|
177
|
+
|
|
178
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
179
|
+
datatype: Annotated[DataType | None, Meta(description="The parameter's data type.")] = None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class KeyCondition(_Model, frozen=True, kw_only=True):
|
|
183
|
+
"""A key part that identifies rows by a condition rather than a field's value.
|
|
184
|
+
|
|
185
|
+
For example, `:as_of >= valid_from AND :as_of < valid_to` for versions valid
|
|
186
|
+
over a period.
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
params: Annotated[
|
|
190
|
+
list[KeyParameter] | None,
|
|
191
|
+
Meta(
|
|
192
|
+
description=(
|
|
193
|
+
"The condition's placeholders. If left null, they are inferred from `on`, "
|
|
194
|
+
"and they must be exactly its placeholders otherwise."
|
|
195
|
+
)
|
|
196
|
+
),
|
|
197
|
+
] = None
|
|
198
|
+
on: Annotated[
|
|
199
|
+
SqlExpr,
|
|
200
|
+
Meta(
|
|
201
|
+
description=(
|
|
202
|
+
"A SQL condition, where columns are relative to the dataset and values a "
|
|
203
|
+
"relationship binds are `:`-prefixed placeholders, which can't be named like "
|
|
204
|
+
"the dataset's fields or parameters."
|
|
205
|
+
)
|
|
206
|
+
),
|
|
207
|
+
]
|
|
208
|
+
|
|
209
|
+
def __post_init__(self) -> None:
|
|
210
|
+
if self.params is not None:
|
|
211
|
+
_check_unique((param.name for param in self.params), "key.params")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
type KeyPart = str | KeyCondition
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class Dataset(_Labeled, frozen=True, kw_only=True):
|
|
218
|
+
"""Physical dataset - serves as the input to the semantic layer graph"""
|
|
219
|
+
|
|
220
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
221
|
+
source: Annotated[
|
|
222
|
+
DatasetSource | None,
|
|
223
|
+
Meta(
|
|
224
|
+
description=(
|
|
225
|
+
"Source of this dataset (a physical table or a query). Omit it to add "
|
|
226
|
+
"calculations and relationships to a dataset whose source another fragment "
|
|
227
|
+
"of the bundle defines."
|
|
228
|
+
)
|
|
229
|
+
),
|
|
230
|
+
] = None
|
|
231
|
+
parameters: Annotated[
|
|
232
|
+
list[Parameter],
|
|
233
|
+
Meta(
|
|
234
|
+
description=(
|
|
235
|
+
"Values a query passes, read by calculations as `:name` and by a `sql` source's "
|
|
236
|
+
"template."
|
|
237
|
+
)
|
|
238
|
+
),
|
|
239
|
+
] = field(default_factory=list)
|
|
240
|
+
key: Annotated[
|
|
241
|
+
list[KeyPart],
|
|
242
|
+
Meta(
|
|
243
|
+
description=(
|
|
244
|
+
"What identifies a row: names of fields, whose values identify rows, and "
|
|
245
|
+
"conditions, e.g. `[user_id, {on: ':as_of >= valid_from AND :as_of < valid_to'}]`. "
|
|
246
|
+
"Relationships binding all of it are many-to-one. Requires `source`."
|
|
247
|
+
)
|
|
248
|
+
),
|
|
249
|
+
] = field(default_factory=list)
|
|
250
|
+
calculations: Annotated[list[Calculation], Meta(description="Derived columns.")] = field(
|
|
251
|
+
default_factory=list
|
|
252
|
+
)
|
|
253
|
+
relationships: Annotated[
|
|
254
|
+
list[Relationship], Meta(description="References to entities and datasets.")
|
|
255
|
+
] = field(default_factory=list)
|
|
256
|
+
|
|
257
|
+
def __post_init__(self) -> None:
|
|
258
|
+
where = f"datasets.{self.name}"
|
|
259
|
+
_check_unique((parameter.name for parameter in self.parameters), f"{where}.parameters")
|
|
260
|
+
_check_unique(
|
|
261
|
+
(calculation.name for calculation in self.calculations), f"{where}.calculations"
|
|
262
|
+
)
|
|
263
|
+
_check_unique(
|
|
264
|
+
(relationship.name for relationship in self.relationships), f"{where}.relationships"
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class EntityKeyColumn(_Model, frozen=True, kw_only=True):
|
|
269
|
+
"""Specification for an individual entity key column."""
|
|
270
|
+
|
|
271
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
272
|
+
datatype: Annotated[DataType | None, Meta(description="The key column's data type.")] = None
|
|
273
|
+
optional: Annotated[
|
|
274
|
+
bool,
|
|
275
|
+
Meta(description="Whether dataset relationships may omit referencing this key column."),
|
|
276
|
+
] = False
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class Attribute(_Labeled, frozen=True, kw_only=True):
|
|
280
|
+
"""A property of an entity."""
|
|
281
|
+
|
|
282
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
283
|
+
expr: Annotated[
|
|
284
|
+
SqlExpr,
|
|
285
|
+
Meta(
|
|
286
|
+
description=(
|
|
287
|
+
"SQL expression, where qualified columns reference fields from a source "
|
|
288
|
+
"and unqualified columns reference other attributes."
|
|
289
|
+
)
|
|
290
|
+
),
|
|
291
|
+
]
|
|
292
|
+
datatype: Annotated[
|
|
293
|
+
DataType | None,
|
|
294
|
+
Meta(description="Optionally define the datatype. If left null, this will be inferred."),
|
|
295
|
+
] = None
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
class Source(_Model, frozen=True, kw_only=True):
|
|
299
|
+
"""A dataset sourcing the entity.
|
|
300
|
+
|
|
301
|
+
Entities have attributes that are sourced from many datasets - a source
|
|
302
|
+
defines such a dataset.
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
dataset: Annotated[
|
|
306
|
+
str, Meta(description="The name of the dataset, unique among the entity's sources.")
|
|
307
|
+
]
|
|
308
|
+
on: Annotated[
|
|
309
|
+
SqlExpr,
|
|
310
|
+
Meta(
|
|
311
|
+
description=(
|
|
312
|
+
"A SQL expression that defines how the dataset implements the entity's key. "
|
|
313
|
+
"Used as the ON clause of JOIN expressions: columns are relative to the "
|
|
314
|
+
"dataset, and entity key values from the other side of the join are "
|
|
315
|
+
"referencable with `:`-prefixed variables."
|
|
316
|
+
)
|
|
317
|
+
),
|
|
318
|
+
]
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
class Entity(_Labeled, frozen=True, kw_only=True):
|
|
322
|
+
"""An identifiable, referencable business object.
|
|
323
|
+
|
|
324
|
+
Entities are like virtual datasets, composed from many datasets.
|
|
325
|
+
Attributes act like an entity's columns.
|
|
326
|
+
"""
|
|
327
|
+
|
|
328
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
329
|
+
key: Annotated[
|
|
330
|
+
list[EntityKeyColumn],
|
|
331
|
+
Meta(
|
|
332
|
+
description=(
|
|
333
|
+
"The interface that sources must implement and dataset relationships reference."
|
|
334
|
+
)
|
|
335
|
+
),
|
|
336
|
+
] = field(default_factory=list)
|
|
337
|
+
attributes: Annotated[list[Attribute], Meta(description="Properties of the entity.")] = field(
|
|
338
|
+
default_factory=list
|
|
339
|
+
)
|
|
340
|
+
sources: Annotated[
|
|
341
|
+
list[Source], Meta(description="Datasets implementing the entity's key.")
|
|
342
|
+
] = field(default_factory=list)
|
|
343
|
+
|
|
344
|
+
def __post_init__(self) -> None:
|
|
345
|
+
where = f"entities.{self.name}"
|
|
346
|
+
_check_unique((column.name for column in self.key), f"{where}.key")
|
|
347
|
+
_check_unique((attribute.name for attribute in self.attributes), f"{where}.attributes")
|
|
348
|
+
_check_unique((source.dataset for source in self.sources), f"{where}.sources")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
type ImportRule = str
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
class Import(_Model, frozen=True, kw_only=True):
|
|
355
|
+
"""Specification for how one constellation imports another."""
|
|
356
|
+
|
|
357
|
+
constellation: Annotated[str, Meta(description="The name of the constellation to import from.")]
|
|
358
|
+
fields: Annotated[
|
|
359
|
+
ImportRule | Annotated[list[ImportRule], Meta(min_length=1)],
|
|
360
|
+
Meta(
|
|
361
|
+
description=(
|
|
362
|
+
"Which fields to import: `*`, a field name, `!name` to exclude a field, or a list "
|
|
363
|
+
"of these, e.g. `['*', '!refunds']`."
|
|
364
|
+
)
|
|
365
|
+
),
|
|
366
|
+
]
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
class Role(_Labeled, frozen=True, kw_only=True):
|
|
370
|
+
"""A qualitative property that describes metrics - metrics can be sliced
|
|
371
|
+
and diced by roles.
|
|
372
|
+
|
|
373
|
+
Each dataset contributing to a constellation can assign fields to a role.
|
|
374
|
+
"""
|
|
375
|
+
|
|
376
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
377
|
+
datatype: Annotated[
|
|
378
|
+
DataType | None,
|
|
379
|
+
Meta(
|
|
380
|
+
description=(
|
|
381
|
+
"A physical data type, or `REF(<name>)` for a reference to an entity or dataset, "
|
|
382
|
+
"in which case datasets must implement the role with a relationship."
|
|
383
|
+
)
|
|
384
|
+
),
|
|
385
|
+
] = None
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
class Metric(_Labeled, frozen=True, kw_only=True):
|
|
389
|
+
"""An aggregatable measurement.
|
|
390
|
+
|
|
391
|
+
Metrics are typically quantitative, but they can be any aggregatable
|
|
392
|
+
value. Metrics can span multiple datasets.
|
|
393
|
+
"""
|
|
394
|
+
|
|
395
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
396
|
+
expr: Annotated[
|
|
397
|
+
SqlExpr,
|
|
398
|
+
Meta(
|
|
399
|
+
description=(
|
|
400
|
+
"A SQL expression, where qualified columns reference fields from datasets "
|
|
401
|
+
"and unqualified columns reference other metrics or roles."
|
|
402
|
+
)
|
|
403
|
+
),
|
|
404
|
+
]
|
|
405
|
+
datatype: Annotated[
|
|
406
|
+
DataType | None,
|
|
407
|
+
Meta(description="Optionally define the datatype. If left null, this will be inferred."),
|
|
408
|
+
] = None
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
class Assignment(_Labeled, frozen=True, kw_only=True):
|
|
412
|
+
"""Defines how a dataset provides a role.
|
|
413
|
+
|
|
414
|
+
This is akin to "dimension conforming" in traditional dimensional
|
|
415
|
+
modeling, allowing cross-table analysis - queries reference roles, and the
|
|
416
|
+
physical location of roles are resolved per dataset.
|
|
417
|
+
"""
|
|
418
|
+
|
|
419
|
+
name: Annotated[
|
|
420
|
+
str, Meta(description="The name of the role, unique among the dataset's assignments.")
|
|
421
|
+
]
|
|
422
|
+
expr: Annotated[
|
|
423
|
+
SqlExpr,
|
|
424
|
+
Meta(description="A SQL expression, where columns resolve relative to the dataset."),
|
|
425
|
+
]
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
class DatasetAssignment(_Model, frozen=True, kw_only=True):
|
|
429
|
+
"""The roles a dataset provides."""
|
|
430
|
+
|
|
431
|
+
dataset: Annotated[
|
|
432
|
+
str, Meta(description="The name of the dataset, unique among the constellation's.")
|
|
433
|
+
]
|
|
434
|
+
roles: Annotated[list[Assignment], Meta(description="How the dataset provides each role.")] = (
|
|
435
|
+
field(default_factory=list)
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
def __post_init__(self) -> None:
|
|
439
|
+
_check_unique((assignment.name for assignment in self.roles), f"{self.dataset}.roles")
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
class Constellation(_Labeled, frozen=True, kw_only=True):
|
|
443
|
+
"""A group of fact tables connected by conformed dimensions.
|
|
444
|
+
|
|
445
|
+
Constellations capture higher-order business domains and enable
|
|
446
|
+
multi-fact queries. Constellations can include configuration from other
|
|
447
|
+
constellations.
|
|
448
|
+
"""
|
|
449
|
+
|
|
450
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
451
|
+
imports: Annotated[
|
|
452
|
+
list[Import], Meta(description="Other constellations to import fields from.")
|
|
453
|
+
] = field(default_factory=list)
|
|
454
|
+
roles: Annotated[list[Role], Meta(description="Roles of the constellation.")] = field(
|
|
455
|
+
default_factory=list
|
|
456
|
+
)
|
|
457
|
+
metrics: Annotated[list[Metric], Meta(description="Metrics of the constellation.")] = field(
|
|
458
|
+
default_factory=list
|
|
459
|
+
)
|
|
460
|
+
assignments: Annotated[
|
|
461
|
+
list[DatasetAssignment], Meta(description="How datasets provide roles, by dataset.")
|
|
462
|
+
] = field(default_factory=list)
|
|
463
|
+
|
|
464
|
+
def __post_init__(self) -> None:
|
|
465
|
+
where = f"constellations.{self.name}"
|
|
466
|
+
_check_unique((role.name for role in self.roles), f"{where}.roles")
|
|
467
|
+
_check_unique((metric.name for metric in self.metrics), f"{where}.metrics")
|
|
468
|
+
_check_unique(
|
|
469
|
+
(assignment.dataset for assignment in self.assignments), f"{where}.assignments"
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
class Page(_Labeled, frozen=True, kw_only=True):
|
|
474
|
+
"""A markdown wiki page.
|
|
475
|
+
|
|
476
|
+
Links to other objects are markdown links with a reference as the URL, e.g.
|
|
477
|
+
`[revenue](metric:sales:revenue)` or `[onboarding](page:guides/onboarding)`.
|
|
478
|
+
"""
|
|
479
|
+
|
|
480
|
+
name: Annotated[str, Meta(description=_NAME_DESCRIPTION)]
|
|
481
|
+
content: Annotated[
|
|
482
|
+
str | None, Meta(description="The page's markdown, inline. Exclusive with `path`.")
|
|
483
|
+
] = None
|
|
484
|
+
path: Annotated[
|
|
485
|
+
str | None,
|
|
486
|
+
Meta(
|
|
487
|
+
description=(
|
|
488
|
+
"Path to a markdown file holding the page's content, relative to this "
|
|
489
|
+
"fragment file within its bundle. Exclusive with `content`."
|
|
490
|
+
)
|
|
491
|
+
),
|
|
492
|
+
] = None
|
|
493
|
+
|
|
494
|
+
def __post_init__(self) -> None:
|
|
495
|
+
if (self.content is None) == (self.path is None):
|
|
496
|
+
raise ValueError("A page requires exactly one of `content` or `path`")
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
class Fragment(
|
|
500
|
+
_Model,
|
|
501
|
+
frozen=True,
|
|
502
|
+
kw_only=True,
|
|
503
|
+
tag_field="schema_version",
|
|
504
|
+
tag="v1",
|
|
505
|
+
):
|
|
506
|
+
"""
|
|
507
|
+
A piece of a bundle's semantic graph: the fragments of a bundle are merged into it.
|
|
508
|
+
|
|
509
|
+
Datasets, entities and constellations are named like a warehouse's tables: `sales.orders` is
|
|
510
|
+
`orders` in the `sales` namespace, and a name without a namespace is in `public`. A reference
|
|
511
|
+
without a namespace, e.g. a relationship's `entity: customer`, is in the namespace of the
|
|
512
|
+
object that declares it. A dataset and a constellation can't share a name.
|
|
513
|
+
"""
|
|
514
|
+
|
|
515
|
+
bundle: Annotated[
|
|
516
|
+
str,
|
|
517
|
+
Meta(
|
|
518
|
+
description="The name of the bundle this fragment is part of, e.g. a dbt project's "
|
|
519
|
+
"name. Every fragment in a bundle has the same one: it identifies the bundle when "
|
|
520
|
+
"it's published. It's a segment of the bundle's URL, so it can't contain a `/`.",
|
|
521
|
+
min_length=1,
|
|
522
|
+
pattern=r"^[^/]+$",
|
|
523
|
+
),
|
|
524
|
+
]
|
|
525
|
+
dialect: Annotated[
|
|
526
|
+
str, Meta(description="The default SQL dialect used to parse expressions and datatypes.")
|
|
527
|
+
]
|
|
528
|
+
generation_meta: Annotated[
|
|
529
|
+
FragmentMetadata, Meta(description="Header describing the production of this fragment.")
|
|
530
|
+
] = field(default_factory=FragmentMetadata)
|
|
531
|
+
datasets: Annotated[list[Dataset], Meta(description="Datasets.")] = field(default_factory=list)
|
|
532
|
+
entities: Annotated[list[Entity], Meta(description="Entities.")] = field(default_factory=list)
|
|
533
|
+
constellations: Annotated[list[Constellation], Meta(description="Constellations.")] = field(
|
|
534
|
+
default_factory=list
|
|
535
|
+
)
|
|
536
|
+
pages: Annotated[list[Page], Meta(description="Wiki pages.")] = field(default_factory=list)
|
|
537
|
+
|
|
538
|
+
def __post_init__(self) -> None:
|
|
539
|
+
_check_unique((dataset.name for dataset in self.datasets), "datasets")
|
|
540
|
+
_check_unique((entity.name for entity in self.entities), "entities")
|
|
541
|
+
_check_unique(
|
|
542
|
+
(constellation.name for constellation in self.constellations), "constellations"
|
|
543
|
+
)
|
|
544
|
+
_check_unique((page.name for page in self.pages), "pages")
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from starlette.applications import Starlette
|
|
6
|
+
from starlette.concurrency import run_in_threadpool
|
|
7
|
+
|
|
8
|
+
from metricflow_sql.compiler.compiler import Compiler, Plan
|
|
9
|
+
from metricflow_sql.server.compile import compile_routes
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def create_compile_app(compiler: Compiler) -> Starlette:
|
|
15
|
+
"""
|
|
16
|
+
A server of `/v1/compile` alone (see `metricflow_sql.server.compile`), with no warehouse.
|
|
17
|
+
|
|
18
|
+
It compiles with `compiler` on Starlette's threadpool, so concurrent compiles share it, and
|
|
19
|
+
its registry must be safe to read from several threads, e.g. an `InMemoryRegistry` that's
|
|
20
|
+
no longer written.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
async def compile_sql(sql: str, *, input_dialect: str, output_dialect: str) -> Plan:
|
|
24
|
+
return await run_in_threadpool(
|
|
25
|
+
compiler.compile,
|
|
26
|
+
sql,
|
|
27
|
+
read_only=True,
|
|
28
|
+
input_dialect=input_dialect,
|
|
29
|
+
output_dialect=output_dialect,
|
|
30
|
+
with_tables=True,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
return Starlette(routes=compile_routes(lambda request: compile_sql))
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Compiles semantic SQL to warehouse SQL without running it, e.g. for dbt to build.
|
|
2
|
+
|
|
3
|
+
POST /v1/compile {
|
|
4
|
+
"sql": "SELECT AGG(revenue) FROM sales",
|
|
5
|
+
"input_dialect": "bigquery",
|
|
6
|
+
"output_dialect": "bigquery"
|
|
7
|
+
}
|
|
8
|
+
-> 200 {"sql": "SELECT SUM(amount) ...", "tables": [["warehouse", "main", "sales"]]}
|
|
9
|
+
or 400 {"error": "..."}
|
|
10
|
+
|
|
11
|
+
`input_dialect` is the SQL's, and `output_dialect` the compiled SQL's, e.g. both a warehouse's
|
|
12
|
+
for dbt: no Trino in the path. `tables` are the physical tables the compiled SQL reads, by
|
|
13
|
+
their parts as written, e.g. for dbt to order the model after the models that build them.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
from collections.abc import Awaitable, Callable
|
|
20
|
+
from typing import Protocol
|
|
21
|
+
|
|
22
|
+
from sqlglot.errors import SqlglotError
|
|
23
|
+
from starlette.requests import Request
|
|
24
|
+
from starlette.responses import JSONResponse
|
|
25
|
+
from starlette.routing import BaseRoute, Route
|
|
26
|
+
|
|
27
|
+
from metricflow_sql.compiler.compiler import Plan
|
|
28
|
+
from metricflow_sql.server.errors import UserError
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Compile(Protocol):
|
|
34
|
+
"""Compiles `sql` as `Compiler.compile` does `with_tables`."""
|
|
35
|
+
|
|
36
|
+
def __call__(self, sql: str, *, input_dialect: str, output_dialect: str) -> Awaitable[Plan]: ...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# A request's `Compile`, e.g. against the environment the request names.
|
|
40
|
+
CompileFor = Callable[[Request], Compile]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def compile_routes(compile_for: CompileFor) -> list[BaseRoute]:
|
|
44
|
+
"""
|
|
45
|
+
The `/v1/compile` route, compiling with `compile_for(request)`.
|
|
46
|
+
|
|
47
|
+
`compile_for` is called before the body is read, so an exception it raises fails the request
|
|
48
|
+
rather than being answered as a bad body.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
async def post_compile(request: Request) -> JSONResponse:
|
|
52
|
+
compile_sql = compile_for(request)
|
|
53
|
+
body = await request.json()
|
|
54
|
+
try:
|
|
55
|
+
plan = await compile_sql(
|
|
56
|
+
body["sql"],
|
|
57
|
+
input_dialect=body["input_dialect"],
|
|
58
|
+
output_dialect=body["output_dialect"],
|
|
59
|
+
)
|
|
60
|
+
except KeyError as e:
|
|
61
|
+
return JSONResponse({"error": f"Missing `{e.args[0]}`"}, status_code=400)
|
|
62
|
+
except (SqlglotError, UserError, ValueError) as e:
|
|
63
|
+
logger.info("compile failed: %s", e)
|
|
64
|
+
return JSONResponse({"error": str(e)}, status_code=400)
|
|
65
|
+
logger.info("Compiled:\n%s\n=>\n%s", body["sql"].strip(), plan.out_sql)
|
|
66
|
+
assert plan.tables is not None, "compiled `with_tables`"
|
|
67
|
+
return JSONResponse({"sql": plan.out_sql, "tables": [list(table) for table in plan.tables]})
|
|
68
|
+
|
|
69
|
+
return [Route("/v1/compile", post_compile, methods=["POST"])]
|