metricflow-sql 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. metricflow_sql/__init__.py +8 -0
  2. metricflow_sql/compiler/__init__.py +0 -0
  3. metricflow_sql/compiler/catalog/__init__.py +0 -0
  4. metricflow_sql/compiler/catalog/cluster.py +152 -0
  5. metricflow_sql/compiler/catalog/information_schema.py +47 -0
  6. metricflow_sql/compiler/catalog/metadata.py +65 -0
  7. metricflow_sql/compiler/catalog/passthrough.py +184 -0
  8. metricflow_sql/compiler/catalog/protocols.py +67 -0
  9. metricflow_sql/compiler/catalog/semantic.py +59 -0
  10. metricflow_sql/compiler/compiler.py +297 -0
  11. metricflow_sql/compiler/rules/__init__.py +0 -0
  12. metricflow_sql/compiler/rules/expand_constellations.py +790 -0
  13. metricflow_sql/compiler/rules/expand_datasets.py +1208 -0
  14. metricflow_sql/compiler/rules/expand_metadata.py +88 -0
  15. metricflow_sql/compiler/rules/isolate_fragments.py +244 -0
  16. metricflow_sql/compiler/rules/passthrough_tables.py +36 -0
  17. metricflow_sql/compiler/table_arguments.py +35 -0
  18. metricflow_sql/converters/__init__.py +0 -0
  19. metricflow_sql/converters/lookml.py +1041 -0
  20. metricflow_sql/objects/__init__.py +0 -0
  21. metricflow_sql/objects/base.py +308 -0
  22. metricflow_sql/objects/constellation.py +210 -0
  23. metricflow_sql/objects/dataset.py +275 -0
  24. metricflow_sql/objects/entity.py +131 -0
  25. metricflow_sql/objects/reference.py +35 -0
  26. metricflow_sql/objects/wiki.py +101 -0
  27. metricflow_sql/py.typed +0 -0
  28. metricflow_sql/registry/__init__.py +0 -0
  29. metricflow_sql/registry/memory.py +409 -0
  30. metricflow_sql/registry/operation.py +25 -0
  31. metricflow_sql/registry/reference_resolution.py +68 -0
  32. metricflow_sql/registry/registry.py +225 -0
  33. metricflow_sql/registry/type_resolver.py +460 -0
  34. metricflow_sql/schemas/__init__.py +0 -0
  35. metricflow_sql/schemas/bundle.py +327 -0
  36. metricflow_sql/schemas/fragment.schema.json +1165 -0
  37. metricflow_sql/schemas/generate.py +70 -0
  38. metricflow_sql/schemas/load.py +333 -0
  39. metricflow_sql/schemas/v1/__init__.py +51 -0
  40. metricflow_sql/schemas/v1/fragment.py +544 -0
  41. metricflow_sql/server/__init__.py +4 -0
  42. metricflow_sql/server/app.py +33 -0
  43. metricflow_sql/server/compile.py +69 -0
  44. metricflow_sql/server/errors.py +10 -0
  45. metricflow_sql/server/serve.py +161 -0
  46. metricflow_sql/utils/__init__.py +0 -0
  47. metricflow_sql/utils/composed_set.py +32 -0
  48. metricflow_sql/utils/normalize.py +74 -0
  49. metricflow_sql/utils/ordered_set.py +24 -0
  50. metricflow_sql/utils/sql.py +226 -0
  51. metricflow_sql/utils/template.py +57 -0
  52. metricflow_sql-0.1.1.dist-info/METADATA +30 -0
  53. metricflow_sql-0.1.1.dist-info/RECORD +56 -0
  54. metricflow_sql-0.1.1.dist-info/WHEEL +4 -0
  55. metricflow_sql-0.1.1.dist-info/entry_points.txt +2 -0
  56. metricflow_sql-0.1.1.dist-info/licenses/LICENSE +29 -0
@@ -0,0 +1,70 @@
1
+ """Generates the JSON Schema artifact for fragment documents.
2
+
3
+ The artifact is checked into the repo so consumers can validate fragments without
4
+ installing the package. Regenerate it after changing the schema models:
5
+
6
+ uv run python -m metricflow_sql.schemas.generate
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import inspect
12
+ import json
13
+ import logging
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ import msgspec
18
+
19
+ import metricflow_sql.schemas as schemas
20
+ from metricflow_sql.schemas.load import Fragment
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+ # Lives beside the schema modules; the fragment document is versioned internally
25
+ # via the `schema_version` discriminator, so one artifact covers all versions.
26
+ ARTIFACT = Path(schemas.__file__).parent / "fragment.schema.json"
27
+
28
+
29
+ def fragment_jsonschema() -> dict[str, Any]:
30
+ """JSON Schema for fragment documents, derived from the versioned structs."""
31
+ # `msgspec.json.schema` is typed as returning `Any`.
32
+ jsonschema: dict[str, Any] = msgspec.json.schema(Fragment)
33
+ return {key: _clean_descriptions(value) for key, value in jsonschema.items()}
34
+
35
+
36
+ type JsonValue = dict[str, JsonValue] | list[JsonValue] | str | int | float | bool | None
37
+
38
+
39
+ def _clean_descriptions(node: JsonValue) -> JsonValue:
40
+ """Dedents `description` fields.
41
+
42
+ msgspec uses struct docstrings verbatim as descriptions, indentation and all;
43
+ `inspect.cleandoc` strips the class-body indentation.
44
+ """
45
+ match node:
46
+ case dict():
47
+ return {
48
+ key: inspect.cleandoc(value)
49
+ if key == "description" and type(value) is str
50
+ else _clean_descriptions(value)
51
+ for key, value in node.items()
52
+ }
53
+ case list():
54
+ return [_clean_descriptions(item) for item in node]
55
+ case _:
56
+ return node
57
+
58
+
59
+ def serialize(jsonschema: dict[str, Any]) -> str:
60
+ """Renders a JSON Schema deterministically, matching the artifact format."""
61
+ return json.dumps(jsonschema, indent=2, sort_keys=True) + "\n"
62
+
63
+
64
+ def main() -> None:
65
+ ARTIFACT.write_text(serialize(fragment_jsonschema()))
66
+ logger.info("Wrote %s", ARTIFACT)
67
+
68
+
69
+ if __name__ == "__main__":
70
+ main()
@@ -0,0 +1,333 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from collections.abc import Callable
5
+ from dataclasses import dataclass
6
+ from typing import Any, assert_never
7
+
8
+ import msgspec
9
+ from sqlglot import exp
10
+ from sqlglot.dialects.dialect import NormalizationStrategy
11
+
12
+ from metricflow_sql.objects.base import DEFAULT_NAMESPACE, BaseObject, in_namespace
13
+ from metricflow_sql.objects.constellation import (
14
+ Assignment,
15
+ Constellation,
16
+ Import,
17
+ ImportRule,
18
+ Metric,
19
+ Role,
20
+ import_rule_from_str,
21
+ )
22
+ from metricflow_sql.objects.dataset import (
23
+ Calculation,
24
+ Cardinality,
25
+ Column,
26
+ DatasetKey,
27
+ KeyCondition,
28
+ KeyParameter,
29
+ Parameter,
30
+ Relationship,
31
+ Table,
32
+ View,
33
+ )
34
+ from metricflow_sql.objects.entity import Attribute, Entity, KeyColumn, Source
35
+ from metricflow_sql.objects.wiki import Page
36
+ from metricflow_sql.schemas import v1
37
+ from metricflow_sql.utils.normalize import read_identifier, read_table, strategy_of
38
+ from metricflow_sql.utils.sql import parse_sql_with_source
39
+
40
+ logger = logging.getLogger(__name__)
41
+
42
+ # Versioned fragment types, discriminated by the `schema_version` tag field.
43
+ # Add future versions here, e.g. `Fragment = v1.Fragment | v2.Fragment`.
44
+ Fragment = v1.Fragment
45
+
46
+
47
+ def _import_rules(
48
+ fields: str | list[str], name: Callable[[str], exp.Identifier]
49
+ ) -> tuple[ImportRule, ...]:
50
+ """
51
+ Parses an import's `fields`, one rule or a list of them, e.g. `['*', '!refunds']`, naming
52
+ fields by `name`.
53
+ """
54
+ match fields:
55
+ case str():
56
+ return (import_rule_from_str(fields, name),)
57
+ case _:
58
+ return tuple(import_rule_from_str(text, name) for text in fields)
59
+
60
+
61
+ @dataclass(frozen=True)
62
+ class Loader:
63
+ """
64
+ Loads fragments into objects, whose names are normalized by `normalization_strategy`, the
65
+ registry's, else by the fragment dialect's own, e.g. `LOWERCASE` for postgres.
66
+ """
67
+
68
+ normalization_strategy: NormalizationStrategy | None = None
69
+
70
+ def load_dict(self, fragment: dict[str, Any]) -> list[BaseObject]:
71
+ parsed = msgspec.convert(fragment, Fragment)
72
+
73
+ match parsed:
74
+ case v1.Fragment():
75
+ return self.load_v1(parsed)
76
+ case _:
77
+ assert_never(parsed)
78
+
79
+ def load(self, fragment: Fragment) -> list[BaseObject]:
80
+ match fragment:
81
+ case v1.Fragment():
82
+ return self.load_v1(fragment)
83
+ case _:
84
+ assert_never(fragment)
85
+
86
+ def load_v1(self, fragment: v1.Fragment) -> list[BaseObject]:
87
+ objects: list[BaseObject] = []
88
+ dialect = fragment.dialect
89
+ strategy = self.normalization_strategy or strategy_of(dialect)
90
+
91
+ def n(text: str) -> exp.Identifier:
92
+ return read_identifier(text, dialect, strategy)
93
+
94
+ def reference(text: str, referrer: exp.Table) -> exp.Table:
95
+ return in_namespace(read_table(text, dialect, strategy), referrer.args["db"])
96
+
97
+ default_namespace = n(DEFAULT_NAMESPACE)
98
+
99
+ def table(text: str) -> exp.Table:
100
+ return in_namespace(read_table(text, dialect, strategy), default_namespace)
101
+
102
+ def sql(text: str) -> exp.Expr:
103
+ """An expression, whose names are normalized."""
104
+ return parse_sql_with_source(text, dialect=dialect, normalization_strategy=strategy)
105
+
106
+ def build_datatype(datatype: str | None) -> exp.DataType | None:
107
+ """Parse a datatype, including REF targets normalized as registry names."""
108
+ if datatype is None:
109
+ return None
110
+ return parse_sql_with_source(
111
+ datatype, dialect=dialect, into=exp.DataType, normalization_strategy=strategy
112
+ )
113
+
114
+ def build_key(dataset: v1.Dataset) -> DatasetKey | None:
115
+ """
116
+ Builds a dataset's key. It's validated against the dataset's fields once every
117
+ fragment's are merged, as several fragments can contribute to one dataset.
118
+ """
119
+ if not dataset.key:
120
+ return None
121
+ parts: list[exp.Identifier | KeyCondition] = []
122
+ for part in dataset.key:
123
+ match part:
124
+ case str():
125
+ parts.append(n(part))
126
+ case v1.KeyCondition():
127
+ on = sql(part.on)
128
+ # In order of appearance, e.g. `:start <= :as_of` gives start, as_of,
129
+ # normalized like other names, e.g. `:as_of` names snowflake's `AS_OF`.
130
+ placeholders = dict.fromkeys(
131
+ n(placeholder.name) for placeholder in on.find_all(exp.Placeholder)
132
+ )
133
+ if part.params is None:
134
+ parameters = {name: KeyParameter() for name in placeholders}
135
+ else:
136
+ parameters = {
137
+ n(param.name): KeyParameter(datatype=build_datatype(param.datatype))
138
+ for param in part.params
139
+ }
140
+ parts.append(KeyCondition(on=on, parameters=parameters))
141
+ case _:
142
+ assert_never(part)
143
+ return DatasetKey(parts=tuple(parts))
144
+
145
+ def build_dataset(dataset: v1.Dataset) -> Table | View:
146
+ name = table(dataset.name)
147
+ source = dataset.source
148
+ assert source is not None
149
+ parameters = {
150
+ n(parameter.name): Parameter(
151
+ datatype=build_datatype(parameter.datatype),
152
+ default=(sql(parameter.default) if parameter.default is not None else None),
153
+ )
154
+ for parameter in dataset.parameters
155
+ }
156
+ if source.type == "sql":
157
+ assert source.sql is not None
158
+ view = View(
159
+ name=name,
160
+ sql=source.sql,
161
+ dialect=dialect,
162
+ parameters=parameters,
163
+ key=build_key(dataset),
164
+ label=dataset.label,
165
+ description=dataset.description,
166
+ )
167
+ # Render once with the defaults, so a broken template fails at load.
168
+ view.from_({})
169
+ return view
170
+ assert source.location is not None
171
+ return Table(
172
+ name=name,
173
+ location=parse_sql_with_source(
174
+ source.location,
175
+ dialect=dialect,
176
+ into=exp.Table,
177
+ normalization_strategy=strategy,
178
+ ),
179
+ parameters=parameters,
180
+ key=build_key(dataset),
181
+ label=dataset.label,
182
+ description=dataset.description,
183
+ )
184
+
185
+ for dataset in fragment.datasets:
186
+ name = table(dataset.name)
187
+ # Without a source, the dataset only extends one another fragment defines.
188
+ if dataset.source is None:
189
+ if dataset.parameters:
190
+ raise ValueError(
191
+ f"Dataset '{name.sql(dialect)}' declares parameters but no source"
192
+ )
193
+ if dataset.key:
194
+ raise ValueError(f"Dataset '{name.sql(dialect)}' declares a key but no source")
195
+ else:
196
+ objects.append(build_dataset(dataset))
197
+ objects.extend(
198
+ Column(
199
+ dataset=name,
200
+ name=n(column.name),
201
+ datatype=build_datatype(column.datatype),
202
+ label=column.label,
203
+ description=column.description,
204
+ )
205
+ for column in (dataset.source.columns if dataset.source else [])
206
+ )
207
+ objects.extend(
208
+ Calculation(
209
+ dataset=name,
210
+ name=n(calculation.name),
211
+ expr=sql(calculation.expr),
212
+ datatype=build_datatype(calculation.datatype),
213
+ label=calculation.label,
214
+ description=calculation.description,
215
+ )
216
+ for calculation in dataset.calculations
217
+ )
218
+ objects.extend(
219
+ Relationship(
220
+ dataset=name,
221
+ name=n(relationship.name),
222
+ target=reference(relationship.target, name),
223
+ key={
224
+ n(key_column.name): sql(key_column.expr) for key_column in relationship.key
225
+ },
226
+ cardinality=(
227
+ Cardinality(relationship.cardinality) if relationship.cardinality else None
228
+ ),
229
+ label=relationship.label,
230
+ description=relationship.description,
231
+ )
232
+ for relationship in dataset.relationships
233
+ )
234
+
235
+ for entity in fragment.entities:
236
+ name = table(entity.name)
237
+ objects.append(
238
+ Entity(
239
+ name=name,
240
+ key={
241
+ n(key_column.name): KeyColumn(
242
+ datatype=build_datatype(key_column.datatype),
243
+ optional=key_column.optional,
244
+ )
245
+ for key_column in entity.key
246
+ },
247
+ )
248
+ )
249
+ objects.extend(
250
+ Source(
251
+ dataset=reference(source.dataset, name),
252
+ entity=name,
253
+ on=sql(source.on),
254
+ )
255
+ for source in entity.sources
256
+ )
257
+ objects.extend(
258
+ Attribute(
259
+ entity=name,
260
+ name=n(attribute.name),
261
+ expr=sql(attribute.expr),
262
+ datatype=build_datatype(attribute.datatype),
263
+ label=attribute.label,
264
+ description=attribute.description,
265
+ )
266
+ for attribute in entity.attributes
267
+ )
268
+
269
+ for constellation in fragment.constellations:
270
+ name = table(constellation.name)
271
+ objects.append(
272
+ Constellation(
273
+ name=name,
274
+ label=constellation.label,
275
+ description=constellation.description,
276
+ imports=[
277
+ Import(
278
+ constellation=reference(import_.constellation, name),
279
+ fields=_import_rules(import_.fields, n),
280
+ )
281
+ for import_ in constellation.imports
282
+ ],
283
+ )
284
+ )
285
+ objects.extend(
286
+ Role(
287
+ constellation=name,
288
+ name=n(role.name),
289
+ datatype=build_datatype(role.datatype),
290
+ label=role.label,
291
+ description=role.description,
292
+ )
293
+ for role in constellation.roles
294
+ )
295
+ objects.extend(
296
+ Metric(
297
+ constellation=name,
298
+ name=n(metric.name),
299
+ expr=sql(metric.expr),
300
+ datatype=build_datatype(metric.datatype),
301
+ label=metric.label,
302
+ description=metric.description,
303
+ )
304
+ for metric in constellation.metrics
305
+ )
306
+ for dataset_assignment in constellation.assignments:
307
+ objects.extend(
308
+ Assignment(
309
+ constellation=name,
310
+ dataset=reference(dataset_assignment.dataset, name),
311
+ role=n(assignment.name),
312
+ expr=sql(assignment.expr),
313
+ )
314
+ for assignment in dataset_assignment.roles
315
+ )
316
+
317
+ for page in fragment.pages:
318
+ if page.content is None:
319
+ raise ValueError(
320
+ f"Page '{page.name}' has content at `{page.path}`; load it through a `Bundle`"
321
+ )
322
+ objects.append(
323
+ Page(
324
+ name=page.name,
325
+ content=page.content,
326
+ dialect=dialect,
327
+ normalization_strategy=strategy,
328
+ label=page.label,
329
+ description=page.description,
330
+ )
331
+ )
332
+
333
+ return objects
@@ -0,0 +1,51 @@
1
+ """v1 of the fragment schema."""
2
+
3
+ from metricflow_sql.schemas.v1.fragment import (
4
+ Assignment,
5
+ Attribute,
6
+ Calculation,
7
+ Column,
8
+ Constellation,
9
+ Dataset,
10
+ DatasetAssignment,
11
+ DatasetSource,
12
+ Entity,
13
+ EntityKeyColumn,
14
+ Fragment,
15
+ FragmentMetadata,
16
+ Import,
17
+ KeyCondition,
18
+ KeyParameter,
19
+ Metric,
20
+ Page,
21
+ Parameter,
22
+ Relationship,
23
+ RelationshipKeyColumn,
24
+ Role,
25
+ Source,
26
+ )
27
+
28
+ __all__ = [
29
+ "Assignment",
30
+ "Calculation",
31
+ "Column",
32
+ "Constellation",
33
+ "Dataset",
34
+ "DatasetSource",
35
+ "Entity",
36
+ "Attribute",
37
+ "EntityKeyColumn",
38
+ "Source",
39
+ "Import",
40
+ "KeyCondition",
41
+ "KeyParameter",
42
+ "FragmentMetadata",
43
+ "Metric",
44
+ "Page",
45
+ "Parameter",
46
+ "Relationship",
47
+ "RelationshipKeyColumn",
48
+ "Role",
49
+ "DatasetAssignment",
50
+ "Fragment",
51
+ ]