metricflow-sql 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metricflow_sql/__init__.py +8 -0
- metricflow_sql/compiler/__init__.py +0 -0
- metricflow_sql/compiler/catalog/__init__.py +0 -0
- metricflow_sql/compiler/catalog/cluster.py +152 -0
- metricflow_sql/compiler/catalog/information_schema.py +47 -0
- metricflow_sql/compiler/catalog/metadata.py +65 -0
- metricflow_sql/compiler/catalog/passthrough.py +184 -0
- metricflow_sql/compiler/catalog/protocols.py +67 -0
- metricflow_sql/compiler/catalog/semantic.py +59 -0
- metricflow_sql/compiler/compiler.py +297 -0
- metricflow_sql/compiler/rules/__init__.py +0 -0
- metricflow_sql/compiler/rules/expand_constellations.py +790 -0
- metricflow_sql/compiler/rules/expand_datasets.py +1208 -0
- metricflow_sql/compiler/rules/expand_metadata.py +88 -0
- metricflow_sql/compiler/rules/isolate_fragments.py +244 -0
- metricflow_sql/compiler/rules/passthrough_tables.py +36 -0
- metricflow_sql/compiler/table_arguments.py +35 -0
- metricflow_sql/converters/__init__.py +0 -0
- metricflow_sql/converters/lookml.py +1041 -0
- metricflow_sql/objects/__init__.py +0 -0
- metricflow_sql/objects/base.py +308 -0
- metricflow_sql/objects/constellation.py +210 -0
- metricflow_sql/objects/dataset.py +275 -0
- metricflow_sql/objects/entity.py +131 -0
- metricflow_sql/objects/reference.py +35 -0
- metricflow_sql/objects/wiki.py +101 -0
- metricflow_sql/py.typed +0 -0
- metricflow_sql/registry/__init__.py +0 -0
- metricflow_sql/registry/memory.py +409 -0
- metricflow_sql/registry/operation.py +25 -0
- metricflow_sql/registry/reference_resolution.py +68 -0
- metricflow_sql/registry/registry.py +225 -0
- metricflow_sql/registry/type_resolver.py +460 -0
- metricflow_sql/schemas/__init__.py +0 -0
- metricflow_sql/schemas/bundle.py +327 -0
- metricflow_sql/schemas/fragment.schema.json +1165 -0
- metricflow_sql/schemas/generate.py +70 -0
- metricflow_sql/schemas/load.py +333 -0
- metricflow_sql/schemas/v1/__init__.py +51 -0
- metricflow_sql/schemas/v1/fragment.py +544 -0
- metricflow_sql/server/__init__.py +4 -0
- metricflow_sql/server/app.py +33 -0
- metricflow_sql/server/compile.py +69 -0
- metricflow_sql/server/errors.py +10 -0
- metricflow_sql/server/serve.py +161 -0
- metricflow_sql/utils/__init__.py +0 -0
- metricflow_sql/utils/composed_set.py +32 -0
- metricflow_sql/utils/normalize.py +74 -0
- metricflow_sql/utils/ordered_set.py +24 -0
- metricflow_sql/utils/sql.py +226 -0
- metricflow_sql/utils/template.py +57 -0
- metricflow_sql-0.1.1.dist-info/METADATA +30 -0
- metricflow_sql-0.1.1.dist-info/RECORD +56 -0
- metricflow_sql-0.1.1.dist-info/WHEEL +4 -0
- metricflow_sql-0.1.1.dist-info/entry_points.txt +2 -0
- metricflow_sql-0.1.1.dist-info/licenses/LICENSE +29 -0
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bundles: fragment files plus the files they reference, e.g. markdown for pages,
|
|
3
|
+
transported and stored as a zip archive.
|
|
4
|
+
|
|
5
|
+
A bundle might look like:
|
|
6
|
+
|
|
7
|
+
fragment.yaml # datasets and entities
|
|
8
|
+
sales/fragment.yaml # the sales constellation, and pages with `path: wiki/revenue.md`
|
|
9
|
+
sales/wiki/revenue.md
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import dataclasses
|
|
15
|
+
import functools
|
|
16
|
+
import hashlib
|
|
17
|
+
import io
|
|
18
|
+
import json
|
|
19
|
+
import logging
|
|
20
|
+
import posixpath
|
|
21
|
+
import zipfile
|
|
22
|
+
from collections.abc import Mapping
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any, assert_never
|
|
26
|
+
|
|
27
|
+
import msgspec
|
|
28
|
+
from sqlglot.dialects.dialect import NormalizationStrategy
|
|
29
|
+
|
|
30
|
+
from metricflow_sql.objects.base import BaseObject, Id
|
|
31
|
+
from metricflow_sql.schemas import v1
|
|
32
|
+
from metricflow_sql.schemas.load import Fragment, Loader
|
|
33
|
+
from metricflow_sql.utils.normalize import strategy_of
|
|
34
|
+
|
|
35
|
+
logger = logging.getLogger(__name__)
|
|
36
|
+
|
|
37
|
+
# Files with these suffixes are fragment documents; any other file is only read when a
|
|
38
|
+
# fragment references it, e.g. by a page's `path`. YAML requires the `yaml` extra.
|
|
39
|
+
JSON_SUFFIXES = (".json",)
|
|
40
|
+
YAML_SUFFIXES = (".yaml", ".yml")
|
|
41
|
+
FRAGMENT_SUFFIXES = JSON_SUFFIXES + YAML_SUFFIXES
|
|
42
|
+
|
|
43
|
+
# Path of the single fragment in a bundle made by `Bundle.from_fragment`.
|
|
44
|
+
DEFAULT_FRAGMENT_PATH = "fragment.json"
|
|
45
|
+
|
|
46
|
+
# Default size limit for a bundle: 1 GiB. Checked before the files are read.
|
|
47
|
+
MAX_BUNDLE_BYTES = 1 << 30
|
|
48
|
+
|
|
49
|
+
# Fixed timestamp for archive entries, so equal bundles produce identical archives.
|
|
50
|
+
_ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class BundleTooLarge(ValueError):
|
|
54
|
+
"""A bundle's files total more than the allowed bytes."""
|
|
55
|
+
|
|
56
|
+
def __init__(self, size: int, max_bytes: int) -> None:
|
|
57
|
+
super().__init__(f"Bundle is {size} bytes, over the limit of {max_bytes}")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class Bundle:
|
|
62
|
+
"""
|
|
63
|
+
Fragment files and the files they reference, keyed by POSIX path within the bundle.
|
|
64
|
+
|
|
65
|
+
Each fragment is a complete document, loaded on its own, so fragments may use different
|
|
66
|
+
dialects. Several fragments may define the same object, e.g. each team adding its own
|
|
67
|
+
sources and attributes to a shared entity: `load_bundle` merges them.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
files: Mapping[str, bytes]
|
|
71
|
+
|
|
72
|
+
def __post_init__(self) -> None:
|
|
73
|
+
for path in self.files:
|
|
74
|
+
_check_path(path)
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def from_fragment(cls, fragment: Fragment) -> Bundle:
|
|
78
|
+
"""Bundle a single, self-contained fragment document."""
|
|
79
|
+
return cls({DEFAULT_FRAGMENT_PATH: msgspec.json.encode(fragment, order="deterministic")})
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def from_zip(cls, data: bytes, max_bytes: int = MAX_BUNDLE_BYTES) -> Bundle:
|
|
83
|
+
"""
|
|
84
|
+
Read a bundle from a zip archive, skipping hidden files (see `_is_hidden`).
|
|
85
|
+
|
|
86
|
+
Raises `BundleTooLarge` if the uncompressed files add up to more than `max_bytes`.
|
|
87
|
+
"""
|
|
88
|
+
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
|
89
|
+
size = sum(
|
|
90
|
+
info.file_size for info in archive.infolist() if not _is_hidden(info.filename)
|
|
91
|
+
)
|
|
92
|
+
if size > max_bytes:
|
|
93
|
+
raise BundleTooLarge(size, max_bytes)
|
|
94
|
+
return cls(
|
|
95
|
+
{
|
|
96
|
+
info.filename: archive.read(info)
|
|
97
|
+
for info in archive.infolist()
|
|
98
|
+
if not info.is_dir() and not _is_hidden(info.filename)
|
|
99
|
+
}
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
@classmethod
|
|
103
|
+
def from_directory(cls, root: Path, max_bytes: int = MAX_BUNDLE_BYTES) -> Bundle:
|
|
104
|
+
"""
|
|
105
|
+
Read a bundle from the files under `root`, skipping hidden files (see `_is_hidden`).
|
|
106
|
+
|
|
107
|
+
Raises `BundleTooLarge` if the files add up to more than `max_bytes`.
|
|
108
|
+
"""
|
|
109
|
+
paths: dict[str, Path] = {}
|
|
110
|
+
for file in sorted(root.rglob("*")):
|
|
111
|
+
path = file.relative_to(root).as_posix()
|
|
112
|
+
if file.is_file() and not _is_hidden(path):
|
|
113
|
+
paths[path] = file
|
|
114
|
+
size = sum(file.stat().st_size for file in paths.values())
|
|
115
|
+
if size > max_bytes:
|
|
116
|
+
raise BundleTooLarge(size, max_bytes)
|
|
117
|
+
return cls({path: file.read_bytes() for path, file in paths.items()})
|
|
118
|
+
|
|
119
|
+
def to_zip(self) -> bytes:
|
|
120
|
+
buffer = io.BytesIO()
|
|
121
|
+
with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
|
122
|
+
for path in sorted(self.files):
|
|
123
|
+
info = zipfile.ZipInfo(path, date_time=_ZIP_DATE_TIME)
|
|
124
|
+
info.compress_type = zipfile.ZIP_DEFLATED
|
|
125
|
+
archive.writestr(info, self.files[path])
|
|
126
|
+
return buffer.getvalue()
|
|
127
|
+
|
|
128
|
+
def sha(self) -> str:
|
|
129
|
+
"""
|
|
130
|
+
Deterministic hash of the bundle's paths and contents, except each fragment's
|
|
131
|
+
`generation_meta`: it says how the bundle was made, e.g. when, and so changes with every
|
|
132
|
+
build of the same content. Fragments are hashed as parsed, so formatting doesn't count
|
|
133
|
+
either.
|
|
134
|
+
"""
|
|
135
|
+
return self._sha
|
|
136
|
+
|
|
137
|
+
@functools.cached_property
|
|
138
|
+
def _sha(self) -> str:
|
|
139
|
+
digest = hashlib.sha256()
|
|
140
|
+
for path in sorted(self.files):
|
|
141
|
+
content = _hashed_content(path, self.files[path])
|
|
142
|
+
# Length-prefix each entry, so entry boundaries are unambiguous.
|
|
143
|
+
digest.update(f"{path}\0{len(content)}\0".encode())
|
|
144
|
+
digest.update(content)
|
|
145
|
+
return digest.hexdigest()
|
|
146
|
+
|
|
147
|
+
def fragments(self) -> dict[str, Fragment]:
|
|
148
|
+
"""
|
|
149
|
+
Fragment documents by path, with page content read from the files they reference.
|
|
150
|
+
|
|
151
|
+
Parsed once per bundle: the documents are shared by every call, so don't mutate them.
|
|
152
|
+
"""
|
|
153
|
+
return dict(self._fragments)
|
|
154
|
+
|
|
155
|
+
@functools.cached_property
|
|
156
|
+
def _fragments(self) -> dict[str, Fragment]:
|
|
157
|
+
return {
|
|
158
|
+
path: self._resolve(path, _decode_fragment(path, content))
|
|
159
|
+
for path, content in sorted(self.files.items())
|
|
160
|
+
if path.endswith(FRAGMENT_SUFFIXES)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
def _resolve(self, fragment_path: str, fragment: Fragment) -> Fragment:
|
|
164
|
+
match fragment:
|
|
165
|
+
case v1.Fragment():
|
|
166
|
+
pages = [self._resolve_page(fragment_path, page) for page in fragment.pages]
|
|
167
|
+
return msgspec.structs.replace(fragment, pages=pages)
|
|
168
|
+
case _:
|
|
169
|
+
assert_never(fragment)
|
|
170
|
+
|
|
171
|
+
def _resolve_page(self, fragment_path: str, page: v1.Page) -> v1.Page:
|
|
172
|
+
if page.path is None:
|
|
173
|
+
return page
|
|
174
|
+
# Paths escaping the bundle, e.g. `../x.md`, normalize to paths no file can have.
|
|
175
|
+
path = posixpath.normpath(posixpath.join(posixpath.dirname(fragment_path), page.path))
|
|
176
|
+
content = self.files.get(path)
|
|
177
|
+
if content is None:
|
|
178
|
+
raise ValueError(
|
|
179
|
+
f"Page '{page.name}' in `{fragment_path}` references missing file `{page.path}`"
|
|
180
|
+
)
|
|
181
|
+
return msgspec.structs.replace(page, content=content.decode(), path=None)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def bundle_name(bundle: Bundle) -> str:
|
|
185
|
+
"""
|
|
186
|
+
The bundle's name, which every fragment in it gives.
|
|
187
|
+
|
|
188
|
+
Raises `ValueError` if the bundle has no fragments, or they disagree on the name.
|
|
189
|
+
"""
|
|
190
|
+
fragments = bundle.fragments()
|
|
191
|
+
if not fragments:
|
|
192
|
+
raise ValueError(f"Bundle has no fragment files ({', '.join(FRAGMENT_SUFFIXES)})")
|
|
193
|
+
names = sorted({fragment.bundle for fragment in fragments.values()})
|
|
194
|
+
if len(names) > 1:
|
|
195
|
+
raise ValueError(f"Bundle fragments disagree on the bundle's name: {', '.join(names)}")
|
|
196
|
+
return names[0]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def bundle_dialect(bundle: Bundle) -> str:
|
|
200
|
+
"""The main dialect: the first fragment's."""
|
|
201
|
+
fragments = bundle.fragments()
|
|
202
|
+
if not fragments:
|
|
203
|
+
raise ValueError(f"Bundle has no fragment files ({', '.join(FRAGMENT_SUFFIXES)})")
|
|
204
|
+
return next(iter(fragments.values())).dialect
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def bundle_normalization_strategy(bundle: Bundle) -> NormalizationStrategy:
|
|
208
|
+
return strategy_of(bundle_dialect(bundle))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def load_bundle(
|
|
212
|
+
bundle: Bundle, normalization_strategy: NormalizationStrategy | None = None
|
|
213
|
+
) -> list[BaseObject]:
|
|
214
|
+
"""
|
|
215
|
+
Load the objects of every fragment in `bundle`, merging objects several fragments define.
|
|
216
|
+
|
|
217
|
+
Names are normalized by `normalization_strategy`, the registry's, else by that of the bundle's
|
|
218
|
+
first fragment, so every fragment's are normalized alike.
|
|
219
|
+
|
|
220
|
+
Definitions of the same object merge field by field: a field one definition leaves
|
|
221
|
+
unset (`None` or empty, e.g. an entity without a `key`) takes the other's value.
|
|
222
|
+
|
|
223
|
+
Raises `ValueError` if the bundle has no fragments, or if two definitions of an object
|
|
224
|
+
conflict: they're of different types, e.g. a table and a view, or set a field to
|
|
225
|
+
different values.
|
|
226
|
+
"""
|
|
227
|
+
fragments = bundle.fragments()
|
|
228
|
+
if not fragments:
|
|
229
|
+
raise ValueError(f"Bundle has no fragment files ({', '.join(FRAGMENT_SUFFIXES)})")
|
|
230
|
+
loader = Loader(normalization_strategy or bundle_normalization_strategy(bundle))
|
|
231
|
+
objects: dict[Id, BaseObject] = {}
|
|
232
|
+
origins: dict[Id, list[str]] = {}
|
|
233
|
+
for path, fragment in fragments.items():
|
|
234
|
+
for obj in loader.load(fragment):
|
|
235
|
+
existing = objects.get(obj.id)
|
|
236
|
+
objects[obj.id] = (
|
|
237
|
+
obj if existing is None else _merge(existing, obj, origins[obj.id], path)
|
|
238
|
+
)
|
|
239
|
+
origins.setdefault(obj.id, []).append(path)
|
|
240
|
+
return list(objects.values())
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _merge(existing: BaseObject, new: BaseObject, origins: list[str], path: str) -> BaseObject:
|
|
244
|
+
"""Merges two definitions of the same object, as `load_bundle` describes."""
|
|
245
|
+
defined_by = f"{', '.join(f'`{origin}`' for origin in origins)} and `{path}`"
|
|
246
|
+
name = existing.id.text()
|
|
247
|
+
if type(existing) is not type(new):
|
|
248
|
+
raise ValueError(
|
|
249
|
+
f"`{name}` is defined by {defined_by} as different kinds of object: "
|
|
250
|
+
f"{type(existing).__name__} and {type(new).__name__}"
|
|
251
|
+
)
|
|
252
|
+
# Objects are dataclasses: read their declared fields, not e.g. cached properties.
|
|
253
|
+
old_values, new_values = vars(existing), vars(new)
|
|
254
|
+
changes: dict[str, Any] = {}
|
|
255
|
+
for field_name in (field.name for field in dataclasses.fields(existing)):
|
|
256
|
+
old_value, new_value = old_values[field_name], new_values[field_name]
|
|
257
|
+
if old_value == new_value or _is_unset(new_value):
|
|
258
|
+
continue
|
|
259
|
+
if not _is_unset(old_value):
|
|
260
|
+
raise ValueError(f"`{name}` is defined by {defined_by} with different `{field_name}`")
|
|
261
|
+
changes[field_name] = new_value
|
|
262
|
+
return dataclasses.replace(existing, **changes) if changes else existing
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _is_unset(value: Any) -> bool:
|
|
266
|
+
"""Whether a definition left a field unset: `None`, or an empty collection like `{}`."""
|
|
267
|
+
return value is None or (type(value) in (dict, list, tuple, set) and not value)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _hashed_content(path: str, content: bytes) -> bytes:
|
|
271
|
+
"""
|
|
272
|
+
What `Bundle.sha` hashes of a file: its content, but for a fragment, its document without
|
|
273
|
+
`generation_meta`, in a fixed format. A fragment that doesn't parse is hashed as it is.
|
|
274
|
+
"""
|
|
275
|
+
if not path.endswith(FRAGMENT_SUFFIXES):
|
|
276
|
+
return content
|
|
277
|
+
try:
|
|
278
|
+
if path.endswith(JSON_SUFFIXES):
|
|
279
|
+
document = msgspec.json.decode(content)
|
|
280
|
+
else:
|
|
281
|
+
document = _load_yaml(path, content)
|
|
282
|
+
except (msgspec.MsgspecError, ValueError, ImportError):
|
|
283
|
+
return content
|
|
284
|
+
if type(document) is dict:
|
|
285
|
+
document.pop("generation_meta", None)
|
|
286
|
+
# `default=str` for the dates and times YAML has.
|
|
287
|
+
return json.dumps(document, sort_keys=True, separators=(",", ":"), default=str).encode()
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _check_path(path: str) -> None:
|
|
291
|
+
"""Rejects paths that aren't normalized and relative, e.g. `/a`, `a/../b` or `./a`."""
|
|
292
|
+
if path.startswith("/") or posixpath.normpath(path) != path or path.split("/")[0] == "..":
|
|
293
|
+
raise ValueError(f"Invalid bundle path: `{path}`")
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _is_hidden(path: str) -> bool:
|
|
297
|
+
"""Whether `path` is under a dot-file or a `__MACOSX` dir, e.g. ones macOS adds to zips."""
|
|
298
|
+
return any(part.startswith(".") or part == "__MACOSX" for part in path.split("/"))
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _decode_fragment(path: str, content: bytes) -> Fragment:
|
|
302
|
+
try:
|
|
303
|
+
if path.endswith(JSON_SUFFIXES):
|
|
304
|
+
return msgspec.json.decode(content, type=Fragment)
|
|
305
|
+
return msgspec.convert(_load_yaml(path, content), Fragment)
|
|
306
|
+
except msgspec.MsgspecError as error:
|
|
307
|
+
raise ValueError(f"Invalid fragment `{path}`: {error}") from error
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _load_yaml(path: str, content: bytes) -> Any:
|
|
311
|
+
"""
|
|
312
|
+
Parse YAML with `ruamel.yaml`, an optional dependency imported only when needed.
|
|
313
|
+
|
|
314
|
+
Raises `ImportError` if it isn't installed, and `ValueError` for invalid YAML.
|
|
315
|
+
"""
|
|
316
|
+
try:
|
|
317
|
+
from ruamel.yaml import YAML
|
|
318
|
+
from ruamel.yaml.error import YAMLError
|
|
319
|
+
except ImportError as error:
|
|
320
|
+
raise ImportError(
|
|
321
|
+
f"Reading the YAML fragment `{path}` requires ruamel-yaml; "
|
|
322
|
+
"install the `metricflow-sql[yaml]` extra"
|
|
323
|
+
) from error
|
|
324
|
+
try:
|
|
325
|
+
return YAML(typ="safe").load(content.decode())
|
|
326
|
+
except YAMLError as error:
|
|
327
|
+
raise ValueError(f"Invalid fragment `{path}`: {error}") from error
|