metricflow-sql 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. metricflow_sql/__init__.py +8 -0
  2. metricflow_sql/compiler/__init__.py +0 -0
  3. metricflow_sql/compiler/catalog/__init__.py +0 -0
  4. metricflow_sql/compiler/catalog/cluster.py +152 -0
  5. metricflow_sql/compiler/catalog/information_schema.py +47 -0
  6. metricflow_sql/compiler/catalog/metadata.py +65 -0
  7. metricflow_sql/compiler/catalog/passthrough.py +184 -0
  8. metricflow_sql/compiler/catalog/protocols.py +67 -0
  9. metricflow_sql/compiler/catalog/semantic.py +59 -0
  10. metricflow_sql/compiler/compiler.py +297 -0
  11. metricflow_sql/compiler/rules/__init__.py +0 -0
  12. metricflow_sql/compiler/rules/expand_constellations.py +790 -0
  13. metricflow_sql/compiler/rules/expand_datasets.py +1208 -0
  14. metricflow_sql/compiler/rules/expand_metadata.py +88 -0
  15. metricflow_sql/compiler/rules/isolate_fragments.py +244 -0
  16. metricflow_sql/compiler/rules/passthrough_tables.py +36 -0
  17. metricflow_sql/compiler/table_arguments.py +35 -0
  18. metricflow_sql/converters/__init__.py +0 -0
  19. metricflow_sql/converters/lookml.py +1041 -0
  20. metricflow_sql/objects/__init__.py +0 -0
  21. metricflow_sql/objects/base.py +308 -0
  22. metricflow_sql/objects/constellation.py +210 -0
  23. metricflow_sql/objects/dataset.py +275 -0
  24. metricflow_sql/objects/entity.py +131 -0
  25. metricflow_sql/objects/reference.py +35 -0
  26. metricflow_sql/objects/wiki.py +101 -0
  27. metricflow_sql/py.typed +0 -0
  28. metricflow_sql/registry/__init__.py +0 -0
  29. metricflow_sql/registry/memory.py +409 -0
  30. metricflow_sql/registry/operation.py +25 -0
  31. metricflow_sql/registry/reference_resolution.py +68 -0
  32. metricflow_sql/registry/registry.py +225 -0
  33. metricflow_sql/registry/type_resolver.py +460 -0
  34. metricflow_sql/schemas/__init__.py +0 -0
  35. metricflow_sql/schemas/bundle.py +327 -0
  36. metricflow_sql/schemas/fragment.schema.json +1165 -0
  37. metricflow_sql/schemas/generate.py +70 -0
  38. metricflow_sql/schemas/load.py +333 -0
  39. metricflow_sql/schemas/v1/__init__.py +51 -0
  40. metricflow_sql/schemas/v1/fragment.py +544 -0
  41. metricflow_sql/server/__init__.py +4 -0
  42. metricflow_sql/server/app.py +33 -0
  43. metricflow_sql/server/compile.py +69 -0
  44. metricflow_sql/server/errors.py +10 -0
  45. metricflow_sql/server/serve.py +161 -0
  46. metricflow_sql/utils/__init__.py +0 -0
  47. metricflow_sql/utils/composed_set.py +32 -0
  48. metricflow_sql/utils/normalize.py +74 -0
  49. metricflow_sql/utils/ordered_set.py +24 -0
  50. metricflow_sql/utils/sql.py +226 -0
  51. metricflow_sql/utils/template.py +57 -0
  52. metricflow_sql-0.1.1.dist-info/METADATA +30 -0
  53. metricflow_sql-0.1.1.dist-info/RECORD +56 -0
  54. metricflow_sql-0.1.1.dist-info/WHEEL +4 -0
  55. metricflow_sql-0.1.1.dist-info/entry_points.txt +2 -0
  56. metricflow_sql-0.1.1.dist-info/licenses/LICENSE +29 -0
@@ -0,0 +1,327 @@
1
+ """
2
+ Bundles: fragment files plus the files they reference, e.g. markdown for pages,
3
+ transported and stored as a zip archive.
4
+
5
+ A bundle might look like:
6
+
7
+ fragment.yaml # datasets and entities
8
+ sales/fragment.yaml # the sales constellation, and pages with `path: wiki/revenue.md`
9
+ sales/wiki/revenue.md
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import dataclasses
15
+ import functools
16
+ import hashlib
17
+ import io
18
+ import json
19
+ import logging
20
+ import posixpath
21
+ import zipfile
22
+ from collections.abc import Mapping
23
+ from dataclasses import dataclass
24
+ from pathlib import Path
25
+ from typing import Any, assert_never
26
+
27
+ import msgspec
28
+ from sqlglot.dialects.dialect import NormalizationStrategy
29
+
30
+ from metricflow_sql.objects.base import BaseObject, Id
31
+ from metricflow_sql.schemas import v1
32
+ from metricflow_sql.schemas.load import Fragment, Loader
33
+ from metricflow_sql.utils.normalize import strategy_of
34
+
35
+ logger = logging.getLogger(__name__)
36
+
37
+ # Files with these suffixes are fragment documents; any other file is only read when a
38
+ # fragment references it, e.g. by a page's `path`. YAML requires the `yaml` extra.
39
+ JSON_SUFFIXES = (".json",)
40
+ YAML_SUFFIXES = (".yaml", ".yml")
41
+ FRAGMENT_SUFFIXES = JSON_SUFFIXES + YAML_SUFFIXES
42
+
43
+ # Path of the single fragment in a bundle made by `Bundle.from_fragment`.
44
+ DEFAULT_FRAGMENT_PATH = "fragment.json"
45
+
46
+ # Default size limit for a bundle: 1 GiB. Checked before the files are read.
47
+ MAX_BUNDLE_BYTES = 1 << 30
48
+
49
+ # Fixed timestamp for archive entries, so equal bundles produce identical archives.
50
+ _ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0)
51
+
52
+
53
+ class BundleTooLarge(ValueError):
54
+ """A bundle's files total more than the allowed bytes."""
55
+
56
+ def __init__(self, size: int, max_bytes: int) -> None:
57
+ super().__init__(f"Bundle is {size} bytes, over the limit of {max_bytes}")
58
+
59
+
60
+ @dataclass(frozen=True)
61
+ class Bundle:
62
+ """
63
+ Fragment files and the files they reference, keyed by POSIX path within the bundle.
64
+
65
+ Each fragment is a complete document, loaded on its own, so fragments may use different
66
+ dialects. Several fragments may define the same object, e.g. each team adding its own
67
+ sources and attributes to a shared entity: `load_bundle` merges them.
68
+ """
69
+
70
+ files: Mapping[str, bytes]
71
+
72
+ def __post_init__(self) -> None:
73
+ for path in self.files:
74
+ _check_path(path)
75
+
76
+ @classmethod
77
+ def from_fragment(cls, fragment: Fragment) -> Bundle:
78
+ """Bundle a single, self-contained fragment document."""
79
+ return cls({DEFAULT_FRAGMENT_PATH: msgspec.json.encode(fragment, order="deterministic")})
80
+
81
+ @classmethod
82
+ def from_zip(cls, data: bytes, max_bytes: int = MAX_BUNDLE_BYTES) -> Bundle:
83
+ """
84
+ Read a bundle from a zip archive, skipping hidden files (see `_is_hidden`).
85
+
86
+ Raises `BundleTooLarge` if the uncompressed files add up to more than `max_bytes`.
87
+ """
88
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
89
+ size = sum(
90
+ info.file_size for info in archive.infolist() if not _is_hidden(info.filename)
91
+ )
92
+ if size > max_bytes:
93
+ raise BundleTooLarge(size, max_bytes)
94
+ return cls(
95
+ {
96
+ info.filename: archive.read(info)
97
+ for info in archive.infolist()
98
+ if not info.is_dir() and not _is_hidden(info.filename)
99
+ }
100
+ )
101
+
102
+ @classmethod
103
+ def from_directory(cls, root: Path, max_bytes: int = MAX_BUNDLE_BYTES) -> Bundle:
104
+ """
105
+ Read a bundle from the files under `root`, skipping hidden files (see `_is_hidden`).
106
+
107
+ Raises `BundleTooLarge` if the files add up to more than `max_bytes`.
108
+ """
109
+ paths: dict[str, Path] = {}
110
+ for file in sorted(root.rglob("*")):
111
+ path = file.relative_to(root).as_posix()
112
+ if file.is_file() and not _is_hidden(path):
113
+ paths[path] = file
114
+ size = sum(file.stat().st_size for file in paths.values())
115
+ if size > max_bytes:
116
+ raise BundleTooLarge(size, max_bytes)
117
+ return cls({path: file.read_bytes() for path, file in paths.items()})
118
+
119
+ def to_zip(self) -> bytes:
120
+ buffer = io.BytesIO()
121
+ with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive:
122
+ for path in sorted(self.files):
123
+ info = zipfile.ZipInfo(path, date_time=_ZIP_DATE_TIME)
124
+ info.compress_type = zipfile.ZIP_DEFLATED
125
+ archive.writestr(info, self.files[path])
126
+ return buffer.getvalue()
127
+
128
+ def sha(self) -> str:
129
+ """
130
+ Deterministic hash of the bundle's paths and contents, except each fragment's
131
+ `generation_meta`: it says how the bundle was made, e.g. when, and so changes with every
132
+ build of the same content. Fragments are hashed as parsed, so formatting doesn't count
133
+ either.
134
+ """
135
+ return self._sha
136
+
137
+ @functools.cached_property
138
+ def _sha(self) -> str:
139
+ digest = hashlib.sha256()
140
+ for path in sorted(self.files):
141
+ content = _hashed_content(path, self.files[path])
142
+ # Length-prefix each entry, so entry boundaries are unambiguous.
143
+ digest.update(f"{path}\0{len(content)}\0".encode())
144
+ digest.update(content)
145
+ return digest.hexdigest()
146
+
147
+ def fragments(self) -> dict[str, Fragment]:
148
+ """
149
+ Fragment documents by path, with page content read from the files they reference.
150
+
151
+ Parsed once per bundle: the documents are shared by every call, so don't mutate them.
152
+ """
153
+ return dict(self._fragments)
154
+
155
+ @functools.cached_property
156
+ def _fragments(self) -> dict[str, Fragment]:
157
+ return {
158
+ path: self._resolve(path, _decode_fragment(path, content))
159
+ for path, content in sorted(self.files.items())
160
+ if path.endswith(FRAGMENT_SUFFIXES)
161
+ }
162
+
163
+ def _resolve(self, fragment_path: str, fragment: Fragment) -> Fragment:
164
+ match fragment:
165
+ case v1.Fragment():
166
+ pages = [self._resolve_page(fragment_path, page) for page in fragment.pages]
167
+ return msgspec.structs.replace(fragment, pages=pages)
168
+ case _:
169
+ assert_never(fragment)
170
+
171
+ def _resolve_page(self, fragment_path: str, page: v1.Page) -> v1.Page:
172
+ if page.path is None:
173
+ return page
174
+ # Paths escaping the bundle, e.g. `../x.md`, normalize to paths no file can have.
175
+ path = posixpath.normpath(posixpath.join(posixpath.dirname(fragment_path), page.path))
176
+ content = self.files.get(path)
177
+ if content is None:
178
+ raise ValueError(
179
+ f"Page '{page.name}' in `{fragment_path}` references missing file `{page.path}`"
180
+ )
181
+ return msgspec.structs.replace(page, content=content.decode(), path=None)
182
+
183
+
184
+ def bundle_name(bundle: Bundle) -> str:
185
+ """
186
+ The bundle's name, which every fragment in it gives.
187
+
188
+ Raises `ValueError` if the bundle has no fragments, or they disagree on the name.
189
+ """
190
+ fragments = bundle.fragments()
191
+ if not fragments:
192
+ raise ValueError(f"Bundle has no fragment files ({', '.join(FRAGMENT_SUFFIXES)})")
193
+ names = sorted({fragment.bundle for fragment in fragments.values()})
194
+ if len(names) > 1:
195
+ raise ValueError(f"Bundle fragments disagree on the bundle's name: {', '.join(names)}")
196
+ return names[0]
197
+
198
+
199
+ def bundle_dialect(bundle: Bundle) -> str:
200
+ """The main dialect: the first fragment's."""
201
+ fragments = bundle.fragments()
202
+ if not fragments:
203
+ raise ValueError(f"Bundle has no fragment files ({', '.join(FRAGMENT_SUFFIXES)})")
204
+ return next(iter(fragments.values())).dialect
205
+
206
+
207
+ def bundle_normalization_strategy(bundle: Bundle) -> NormalizationStrategy:
208
+ return strategy_of(bundle_dialect(bundle))
209
+
210
+
211
+ def load_bundle(
212
+ bundle: Bundle, normalization_strategy: NormalizationStrategy | None = None
213
+ ) -> list[BaseObject]:
214
+ """
215
+ Load the objects of every fragment in `bundle`, merging objects several fragments define.
216
+
217
+ Names are normalized by `normalization_strategy`, the registry's, else by that of the bundle's
218
+ first fragment, so every fragment's are normalized alike.
219
+
220
+ Definitions of the same object merge field by field: a field one definition leaves
221
+ unset (`None` or empty, e.g. an entity without a `key`) takes the other's value.
222
+
223
+ Raises `ValueError` if the bundle has no fragments, or if two definitions of an object
224
+ conflict: they're of different types, e.g. a table and a view, or set a field to
225
+ different values.
226
+ """
227
+ fragments = bundle.fragments()
228
+ if not fragments:
229
+ raise ValueError(f"Bundle has no fragment files ({', '.join(FRAGMENT_SUFFIXES)})")
230
+ loader = Loader(normalization_strategy or bundle_normalization_strategy(bundle))
231
+ objects: dict[Id, BaseObject] = {}
232
+ origins: dict[Id, list[str]] = {}
233
+ for path, fragment in fragments.items():
234
+ for obj in loader.load(fragment):
235
+ existing = objects.get(obj.id)
236
+ objects[obj.id] = (
237
+ obj if existing is None else _merge(existing, obj, origins[obj.id], path)
238
+ )
239
+ origins.setdefault(obj.id, []).append(path)
240
+ return list(objects.values())
241
+
242
+
243
+ def _merge(existing: BaseObject, new: BaseObject, origins: list[str], path: str) -> BaseObject:
244
+ """Merges two definitions of the same object, as `load_bundle` describes."""
245
+ defined_by = f"{', '.join(f'`{origin}`' for origin in origins)} and `{path}`"
246
+ name = existing.id.text()
247
+ if type(existing) is not type(new):
248
+ raise ValueError(
249
+ f"`{name}` is defined by {defined_by} as different kinds of object: "
250
+ f"{type(existing).__name__} and {type(new).__name__}"
251
+ )
252
+ # Objects are dataclasses: read their declared fields, not e.g. cached properties.
253
+ old_values, new_values = vars(existing), vars(new)
254
+ changes: dict[str, Any] = {}
255
+ for field_name in (field.name for field in dataclasses.fields(existing)):
256
+ old_value, new_value = old_values[field_name], new_values[field_name]
257
+ if old_value == new_value or _is_unset(new_value):
258
+ continue
259
+ if not _is_unset(old_value):
260
+ raise ValueError(f"`{name}` is defined by {defined_by} with different `{field_name}`")
261
+ changes[field_name] = new_value
262
+ return dataclasses.replace(existing, **changes) if changes else existing
263
+
264
+
265
+ def _is_unset(value: Any) -> bool:
266
+ """Whether a definition left a field unset: `None`, or an empty collection like `{}`."""
267
+ return value is None or (type(value) in (dict, list, tuple, set) and not value)
268
+
269
+
270
+ def _hashed_content(path: str, content: bytes) -> bytes:
271
+ """
272
+ What `Bundle.sha` hashes of a file: its content, but for a fragment, its document without
273
+ `generation_meta`, in a fixed format. A fragment that doesn't parse is hashed as it is.
274
+ """
275
+ if not path.endswith(FRAGMENT_SUFFIXES):
276
+ return content
277
+ try:
278
+ if path.endswith(JSON_SUFFIXES):
279
+ document = msgspec.json.decode(content)
280
+ else:
281
+ document = _load_yaml(path, content)
282
+ except (msgspec.MsgspecError, ValueError, ImportError):
283
+ return content
284
+ if type(document) is dict:
285
+ document.pop("generation_meta", None)
286
+ # `default=str` for the dates and times YAML has.
287
+ return json.dumps(document, sort_keys=True, separators=(",", ":"), default=str).encode()
288
+
289
+
290
+ def _check_path(path: str) -> None:
291
+ """Rejects paths that aren't normalized and relative, e.g. `/a`, `a/../b` or `./a`."""
292
+ if path.startswith("/") or posixpath.normpath(path) != path or path.split("/")[0] == "..":
293
+ raise ValueError(f"Invalid bundle path: `{path}`")
294
+
295
+
296
+ def _is_hidden(path: str) -> bool:
297
+ """Whether `path` is under a dot-file or a `__MACOSX` dir, e.g. ones macOS adds to zips."""
298
+ return any(part.startswith(".") or part == "__MACOSX" for part in path.split("/"))
299
+
300
+
301
+ def _decode_fragment(path: str, content: bytes) -> Fragment:
302
+ try:
303
+ if path.endswith(JSON_SUFFIXES):
304
+ return msgspec.json.decode(content, type=Fragment)
305
+ return msgspec.convert(_load_yaml(path, content), Fragment)
306
+ except msgspec.MsgspecError as error:
307
+ raise ValueError(f"Invalid fragment `{path}`: {error}") from error
308
+
309
+
310
+ def _load_yaml(path: str, content: bytes) -> Any:
311
+ """
312
+ Parse YAML with `ruamel.yaml`, an optional dependency imported only when needed.
313
+
314
+ Raises `ImportError` if it isn't installed, and `ValueError` for invalid YAML.
315
+ """
316
+ try:
317
+ from ruamel.yaml import YAML
318
+ from ruamel.yaml.error import YAMLError
319
+ except ImportError as error:
320
+ raise ImportError(
321
+ f"Reading the YAML fragment `{path}` requires ruamel-yaml; "
322
+ "install the `metricflow-sql[yaml]` extra"
323
+ ) from error
324
+ try:
325
+ return YAML(typ="safe").load(content.decode())
326
+ except YAMLError as error:
327
+ raise ValueError(f"Invalid fragment `{path}`: {error}") from error