dbt-metricflow-sql 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_metricflow_sql-0.1.1/.gitignore +218 -0
- dbt_metricflow_sql-0.1.1/LICENSE +29 -0
- dbt_metricflow_sql-0.1.1/PKG-INFO +20 -0
- dbt_metricflow_sql-0.1.1/PYPI.md +5 -0
- dbt_metricflow_sql-0.1.1/README.md +162 -0
- dbt_metricflow_sql-0.1.1/pyproject.toml +67 -0
- dbt_metricflow_sql-0.1.1/src/dbt_metricflow_sql/__init__.py +11 -0
- dbt_metricflow_sql-0.1.1/src/dbt_metricflow_sql/client.py +173 -0
- dbt_metricflow_sql-0.1.1/src/dbt_metricflow_sql/dependencies.py +56 -0
- dbt_metricflow_sql-0.1.1/src/dbt_metricflow_sql/fragment.schema.json +1165 -0
- dbt_metricflow_sql-0.1.1/src/dbt_metricflow_sql/plugin.py +452 -0
- dbt_metricflow_sql-0.1.1/src/dbt_metricflow_sql/semantics.py +440 -0
- dbt_metricflow_sql-0.1.1/src/dbt_metricflow_sql/server.py +128 -0
- dbt_metricflow_sql-0.1.1/uv.lock +3708 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
# Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
# poetry.lock
|
|
109
|
+
# poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
# pdm.lock
|
|
116
|
+
# pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
# pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# Redis
|
|
135
|
+
*.rdb
|
|
136
|
+
*.aof
|
|
137
|
+
*.pid
|
|
138
|
+
|
|
139
|
+
# RabbitMQ
|
|
140
|
+
mnesia/
|
|
141
|
+
rabbitmq/
|
|
142
|
+
rabbitmq-data/
|
|
143
|
+
|
|
144
|
+
# ActiveMQ
|
|
145
|
+
activemq-data/
|
|
146
|
+
|
|
147
|
+
# SageMath parsed files
|
|
148
|
+
*.sage.py
|
|
149
|
+
|
|
150
|
+
# Environments
|
|
151
|
+
.env
|
|
152
|
+
.envrc
|
|
153
|
+
.venv
|
|
154
|
+
env/
|
|
155
|
+
venv/
|
|
156
|
+
ENV/
|
|
157
|
+
env.bak/
|
|
158
|
+
venv.bak/
|
|
159
|
+
|
|
160
|
+
# Spyder project settings
|
|
161
|
+
.spyderproject
|
|
162
|
+
.spyproject
|
|
163
|
+
|
|
164
|
+
# Rope project settings
|
|
165
|
+
.ropeproject
|
|
166
|
+
|
|
167
|
+
# mkdocs documentation
|
|
168
|
+
/site
|
|
169
|
+
|
|
170
|
+
# mypy
|
|
171
|
+
.mypy_cache/
|
|
172
|
+
.dmypy.json
|
|
173
|
+
dmypy.json
|
|
174
|
+
|
|
175
|
+
# Pyre type checker
|
|
176
|
+
.pyre/
|
|
177
|
+
|
|
178
|
+
# pytype static type analyzer
|
|
179
|
+
.pytype/
|
|
180
|
+
|
|
181
|
+
# Cython debug symbols
|
|
182
|
+
cython_debug/
|
|
183
|
+
|
|
184
|
+
# PyCharm
|
|
185
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
186
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
188
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
|
+
.idea/
|
|
190
|
+
|
|
191
|
+
# Abstra
|
|
192
|
+
# Abstra is an AI-powered process automation framework.
|
|
193
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
194
|
+
# Learn more at https://abstra.io/docs
|
|
195
|
+
.abstra/
|
|
196
|
+
|
|
197
|
+
# Visual Studio Code
|
|
198
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
199
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
200
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
201
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
202
|
+
# .vscode/
|
|
203
|
+
# Temporary file for partial code execution
|
|
204
|
+
tempCodeRunnerFile.py
|
|
205
|
+
|
|
206
|
+
# Ruff stuff:
|
|
207
|
+
.ruff_cache/
|
|
208
|
+
|
|
209
|
+
# PyPI configuration file
|
|
210
|
+
.pypirc
|
|
211
|
+
|
|
212
|
+
# Marimo
|
|
213
|
+
marimo/_static/
|
|
214
|
+
marimo/_lsp/
|
|
215
|
+
__marimo__/
|
|
216
|
+
|
|
217
|
+
# Streamlit
|
|
218
|
+
.streamlit/secrets.toml
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Copyright (c) 2026 Fivetran, Inc.
|
|
2
|
+
|
|
3
|
+
By accessing or using this software, you agree to the following terms:
|
|
4
|
+
|
|
5
|
+
PERMITTED USE
|
|
6
|
+
This software is made available for internal use and evaluation purposes
|
|
7
|
+
only. You may install and use the software solely to access and interact
|
|
8
|
+
with Fivetran's services.
|
|
9
|
+
|
|
10
|
+
RESTRICTIONS
|
|
11
|
+
You may not use this software to:
|
|
12
|
+
- Build competing products or services
|
|
13
|
+
- Redistribute, sublicense, or transfer the software
|
|
14
|
+
|
|
15
|
+
INTELLECTUAL PROPERTY
|
|
16
|
+
All rights, title, and interest in the software remain exclusively with
|
|
17
|
+
Fivetran, Inc. No license to any intellectual property is granted
|
|
18
|
+
except as expressly stated herein.
|
|
19
|
+
|
|
20
|
+
NO WARRANTIES
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS". FIVETRAN, INC. MAKES NO WARRANTIES,
|
|
22
|
+
EXPRESS OR IMPLIED, AND HEREBY DISCLAIMS ALL IMPLIED WARRANTIES,
|
|
23
|
+
INCLUDING ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR
|
|
24
|
+
PURPOSE.
|
|
25
|
+
|
|
26
|
+
LIMITATION OF LIABILITY
|
|
27
|
+
FIVETRAN, INC. SHALL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND ARISING
|
|
28
|
+
FROM USE OF THIS SOFTWARE, INCLUDING DIRECT, INDIRECT, INCIDENTAL,
|
|
29
|
+
PUNITIVE, AND CONSEQUENTIAL DAMAGES.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dbt-metricflow-sql
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: dbt v1 plugin for MetricFlow
|
|
5
|
+
Project-URL: Documentation, https://congenial-bassoon-7zkjv8w.pages.github.io/
|
|
6
|
+
License-Expression: LicenseRef-Fivetran
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Requires-Python: >=3.9
|
|
10
|
+
Requires-Dist: dbt-core>=1.9
|
|
11
|
+
Requires-Dist: jsonschema>=4.19
|
|
12
|
+
Provides-Extra: sidecar
|
|
13
|
+
Requires-Dist: metricflow-sql-sidecar; extra == 'sidecar'
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# dbt-metricflow-sql
|
|
17
|
+
|
|
18
|
+
Something is cooking...
|
|
19
|
+
|
|
20
|
+
[Documentation](https://congenial-bassoon-7zkjv8w.pages.github.io/), for design partners with access.
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# dbt-metricflow-sql
|
|
2
|
+
|
|
3
|
+
A dbt-core plugin that builds a project's semantics into MetricFlow, and models written in
|
|
4
|
+
MetricFlow semantic SQL. Install it next to dbt and dbt loads it automatically.
|
|
5
|
+
|
|
6
|
+
It compiles MetricFlow models with a MetricFlow server: metricflow-sql's `metricflow-serve`, as
|
|
7
|
+
[metricflow-sql-sidecar](../metricflow-sql-sidecar/README.md) packages it, an executable with a
|
|
8
|
+
Python of its own, so it runs whatever dbt's Python is. A project with MetricFlow models needs
|
|
9
|
+
it, so install the plugin with its `sidecar` extra:
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
pip install 'dbt-metricflow-sql[sidecar]'
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
For another command, set the `metricflow_server` var, as a list of arguments or a string:
|
|
16
|
+
|
|
17
|
+
```yaml
|
|
18
|
+
# dbt_project.yml
|
|
19
|
+
vars:
|
|
20
|
+
metricflow_server: /opt/metricflow/bin/metricflow-serve
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
The plugin runs the server for each dbt command that compiles MetricFlow models, and stops it
|
|
24
|
+
once the command completes; its log is in `target/metricflow-server.log`. The command is run
|
|
25
|
+
from the project's directory, with `--bundle <directory> --catalog <project name>` appended: the
|
|
26
|
+
project's semantics, and the semantic catalog's name. Once it listens, it must print a JSON line
|
|
27
|
+
with its `port`, e.g. `{"host": "127.0.0.1", "port": 54321}`, to stdout, and it should exit when
|
|
28
|
+
its stdin closes.
|
|
29
|
+
|
|
30
|
+
## Semantics
|
|
31
|
+
|
|
32
|
+
Declare semantics in schema files: `datasets`, `entities`, `constellations` and `pages` at the
|
|
33
|
+
root, and dataset fields on models, seeds, snapshots and source tables. They're the objects of a
|
|
34
|
+
[v1 fragment](../metricflow-sql/src/metricflow_sql/schemas/v1/fragment.py), whose named
|
|
35
|
+
collections are dbt-style lists of objects with a `name`:
|
|
36
|
+
|
|
37
|
+
```yaml
|
|
38
|
+
models:
|
|
39
|
+
- name: orders # a dataset of the model's relation and columns
|
|
40
|
+
columns:
|
|
41
|
+
- name: amount
|
|
42
|
+
data_type: numeric
|
|
43
|
+
calculations:
|
|
44
|
+
- name: revenue
|
|
45
|
+
expr: SUM(amount)
|
|
46
|
+
|
|
47
|
+
- name: customers
|
|
48
|
+
dataset: true # a dataset of nothing but its columns
|
|
49
|
+
|
|
50
|
+
datasets:
|
|
51
|
+
- name: refunds
|
|
52
|
+
source: ref('refunds') # or source('<source>', '<table>')
|
|
53
|
+
|
|
54
|
+
constellations:
|
|
55
|
+
- name: sales
|
|
56
|
+
roles:
|
|
57
|
+
- name: customer
|
|
58
|
+
datatype: REF(customer)
|
|
59
|
+
metrics:
|
|
60
|
+
- name: revenue
|
|
61
|
+
expr: AGG(orders.revenue)
|
|
62
|
+
assignments:
|
|
63
|
+
- dataset: orders
|
|
64
|
+
roles:
|
|
65
|
+
- name: customer
|
|
66
|
+
expr: customer
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
A node is a dataset, named after it, if it sets `calculations`, `relationships`, `parameters` or
|
|
70
|
+
`key`, or `dataset: true`. Expressions are in the warehouse's dialect.
|
|
71
|
+
|
|
72
|
+
dbt validates schema files on some adapters (BigQuery, Databricks, Redshift, Snowflake), and
|
|
73
|
+
warns of keys it doesn't know as custom ones; the plugin declares these to it.
|
|
74
|
+
|
|
75
|
+
## Lifecycle
|
|
76
|
+
|
|
77
|
+
- **Parse** builds the semantics into a bundle in `target/metricflow/`, one fragment per
|
|
78
|
+
schema file, and validates it against the fragment JSON Schema. The root `fragment.json` holds
|
|
79
|
+
the bundle's `generation_meta`: the target and invocation that built it.
|
|
80
|
+
- **Compile**: the first time a `metricflow=true` model compiles, the plugin starts the server
|
|
81
|
+
over the bundle. Each such model's rendered SQL compiles there to warehouse SQL, which dbt
|
|
82
|
+
builds with any materialization:
|
|
83
|
+
|
|
84
|
+
```sql
|
|
85
|
+
{{ config(materialized='table', metricflow=true) }}
|
|
86
|
+
SELECT AGG(revenue) AS revenue FROM sales
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The semantic catalog is the project's, so name tables without it: `sales`, `finance.revenue`.
|
|
90
|
+
|
|
91
|
+
dbt can't see what semantic SQL reads, so as dbt builds its graph, each such model compiles
|
|
92
|
+
first: the models, seeds, snapshots and sources that build the tables its warehouse SQL
|
|
93
|
+
reads become its dependencies. It builds after them, and `+model` selects them. So every
|
|
94
|
+
command that builds the graph, e.g. `dbt ls`, runs the server when the project has
|
|
95
|
+
MetricFlow models.
|
|
96
|
+
- **Publish**: if the `metricflow_publish_url` var is set, `dbt run` and `dbt build` end by
|
|
97
|
+
sending the project's bundle to the MetricFlow server at that URL, e.g. production's, as the
|
|
98
|
+
next version of the bundle named for the project. It isn't the server the plugin runs to
|
|
99
|
+
compile models.
|
|
100
|
+
|
|
101
|
+
It happens after dbt's own `on-run-end` hooks, whether or not the run's models succeeded:
|
|
102
|
+
the semantics don't wait for every model to build, which in a large project may never
|
|
103
|
+
happen. dbt records it in `run_results.json` as an operation, `metricflow-publish`, so a
|
|
104
|
+
publish that fails fails the run. Publishing content the server already has changes nothing,
|
|
105
|
+
so a rerun is safe, and so is a build that changed no semantics.
|
|
106
|
+
|
|
107
|
+
dbt tells production from development only by the target a run uses, e.g.
|
|
108
|
+
`dbt build --target prod`, so the var is what decides where a run publishes. It's rendered like
|
|
109
|
+
any var, so it can read the environment and the target, and it publishes nothing if it renders
|
|
110
|
+
empty. Either of these keeps development builds from publishing:
|
|
111
|
+
|
|
112
|
+
```yaml
|
|
113
|
+
# dbt_project.yml
|
|
114
|
+
vars:
|
|
115
|
+
# The job that should publish sets METRICFLOW_URL...
|
|
116
|
+
metricflow_publish_url: "{{ env_var('METRICFLOW_URL', '') }}"
|
|
117
|
+
# ...or only production does, wherever it runs:
|
|
118
|
+
# metricflow_publish_url: "{{ env_var('METRICFLOW_URL') if target.name == 'prod' else '' }}"
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Other commands don't publish, and nor does a project without the var. For a one-off,
|
|
122
|
+
`--vars '{"metricflow_publish_url": "http://localhost:8080"}'` works too.
|
|
123
|
+
|
|
124
|
+
A server that authenticates a publish, e.g. dbt Cloud's, takes a token and a dbt Cloud project:
|
|
125
|
+
|
|
126
|
+
- The token is the `DBT_CLOUD_TOKEN` environment variable, the one dbt-core and Fusion read,
|
|
127
|
+
sent as `Authorization: Bearer`. It's an environment variable only, never a var: vars reach
|
|
128
|
+
dbt's artifacts, e.g. `manifest.json`. It may be a service token, a personal access token
|
|
129
|
+
(PAT) or a dbt OAuth access token. Use a service token for CI: a PAT stops working when its
|
|
130
|
+
user leaves the account or the token expires, and an OAuth access token is short-lived. A
|
|
131
|
+
user token publishes exactly as a service token does, but a query made with one runs on that
|
|
132
|
+
user's development credentials, not the Semantic Layer credential, so BI clients and CI
|
|
133
|
+
should query with a Semantic Layer service token.
|
|
134
|
+
- The project's id is `DBT_CLOUD_PROJECT_ID`, which dbt Cloud's job runs set, else
|
|
135
|
+
`project-id` in `dbt_project.yml`'s `dbt-cloud:` block, as Fusion reads it. It's sent as
|
|
136
|
+
`X-Dbt-Project-Id`. The server publishes to the environment of the project's Semantic Layer
|
|
137
|
+
configuration, so there's no environment to name, and answers with the catalog to query it
|
|
138
|
+
as, which the run's output shows.
|
|
139
|
+
|
|
140
|
+
Both go only with the plugin's requests to the server, never with the upload to the server's
|
|
141
|
+
storage, and both are optional: a server without authentication, e.g. a local one, needs
|
|
142
|
+
neither. The upload goes straight to the server's storage, e.g. S3, so that host must be
|
|
143
|
+
reachable from where dbt runs.
|
|
144
|
+
|
|
145
|
+
```yaml
|
|
146
|
+
# dbt_project.yml; the job's environment sets DBT_CLOUD_TOKEN and METRICFLOW_URL.
|
|
147
|
+
dbt-cloud:
|
|
148
|
+
project-id: 160
|
|
149
|
+
vars:
|
|
150
|
+
metricflow_publish_url: "{{ env_var('METRICFLOW_URL', '') if target.name == 'prod' else '' }}"
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Publish from a target that builds the tables the Semantic Layer environment reads, on the same
|
|
154
|
+
warehouse: the server refuses a bundle in another warehouse's dialect, but checks nothing else.
|
|
155
|
+
|
|
156
|
+
`DBT_CLOUD_TOKEN` is often set in CI for other dbt tools, so the plugin sends it only to an
|
|
157
|
+
`https://` URL, or an `http://` one whose host is `localhost` or a loopback IP, e.g.
|
|
158
|
+
`http://127.0.0.1:8080`; it never resolves a name to check. With the token set, any other URL
|
|
159
|
+
fails the publish, sending nothing. The plugin also follows no redirect, since following the
|
|
160
|
+
server's would send the token to the host it names. A redirected request fails the publish,
|
|
161
|
+
naming the redirect's host. That includes the upload's, e.g. S3's for a bucket in another
|
|
162
|
+
region.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dbt-metricflow-sql"
|
|
3
|
+
# A placeholder: a release sets its tag's version, with scripts/set_version.py.
|
|
4
|
+
version = "0.1.1"
|
|
5
|
+
description = "dbt v1 plugin for MetricFlow"
|
|
6
|
+
# Its PyPI page: README.md is for this repository.
|
|
7
|
+
readme = "PYPI.md"
|
|
8
|
+
# Fivetran's own, in LICENSE, which no SPDX identifier names.
|
|
9
|
+
license = "LicenseRef-Fivetran"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Programming Language :: Python :: 3",
|
|
13
|
+
]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"dbt-core>=1.9",
|
|
16
|
+
# Validates the fragments it builds; dbt-core requires it too.
|
|
17
|
+
"jsonschema>=4.19",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.optional-dependencies]
|
|
21
|
+
# The MetricFlow server it runs by default, which needs no Python of its own: only projects with
|
|
22
|
+
# MetricFlow models run one.
|
|
23
|
+
sidecar = ["metricflow-sql-sidecar"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Documentation = "https://congenial-bassoon-7zkjv8w.pages.github.io/"
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["hatchling"]
|
|
30
|
+
build-backend = "hatchling.build"
|
|
31
|
+
|
|
32
|
+
[tool.hatch.build.targets.wheel]
|
|
33
|
+
packages = ["src/dbt_metricflow_sql"]
|
|
34
|
+
|
|
35
|
+
# The workspace's `tests/dbt_metricflow_sql` run in this environment, against a real
|
|
36
|
+
# metricflow-sql server, which needs a newer Python than the plugin, and metricflow-service, which
|
|
37
|
+
# they publish to.
|
|
38
|
+
[dependency-groups]
|
|
39
|
+
dev = [
|
|
40
|
+
"dbt-duckdb>=1.9",
|
|
41
|
+
"duckdb>=1.5.6",
|
|
42
|
+
"metricflow-db[s3]",
|
|
43
|
+
"metricflow-service",
|
|
44
|
+
"metricflow-sql[server]",
|
|
45
|
+
"metricflow-sql-sidecar",
|
|
46
|
+
"mypy>=1.11",
|
|
47
|
+
"pytest>=8.0",
|
|
48
|
+
"ruamel-yaml>=0.18",
|
|
49
|
+
"trino>=0.334",
|
|
50
|
+
"types-jsonschema>=4.19",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
[tool.uv.dependency-groups]
|
|
54
|
+
dev = { requires-python = ">=3.12" }
|
|
55
|
+
|
|
56
|
+
[tool.uv.sources]
|
|
57
|
+
metricflow-db = { path = "../metricflow-db", editable = true }
|
|
58
|
+
metricflow-service = { path = "../metricflow-service", editable = true }
|
|
59
|
+
metricflow-sql = { path = "../metricflow-sql", editable = true }
|
|
60
|
+
metricflow-sql-sidecar = { path = "../metricflow-sql-sidecar" }
|
|
61
|
+
|
|
62
|
+
[tool.mypy]
|
|
63
|
+
python_version = "3.12"
|
|
64
|
+
files = ["src"]
|
|
65
|
+
# Read the service packages from source, as the workspace does.
|
|
66
|
+
mypy_path = "../metricflow-sql/src:../metricflow-service/src:../metricflow-sql-sidecar/src"
|
|
67
|
+
strict = true
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""A dbt plugin: dbt discovers it by its `dbt_` module prefix and `plugins` list."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from dbt_metricflow_sql.plugin import MetricFlowPlugin
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
plugins = [MetricFlowPlugin]
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Metricflow HTTP client"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.parse
|
|
9
|
+
import urllib.request
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from email.message import Message
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
_TIMEOUT_SECONDS = 10
|
|
18
|
+
# Bundles can be large, and the server reads one as it syncs it.
|
|
19
|
+
_PUBLISH_TIMEOUT_SECONDS = 300
|
|
20
|
+
# The header of a publish's response naming the catalog to query the bundle as, e.g. `env_220`.
|
|
21
|
+
CATALOG_HEADER = "X-Dbt-Catalog"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MetricFlowError(Exception):
|
|
25
|
+
"""A request to a MetricFlow server failed: it's unreachable, or it rejected the request."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
29
|
+
"""
|
|
30
|
+
Follows no redirect, so a 3xx raises `HTTPError`: urllib would otherwise send the redirect's
|
|
31
|
+
host the request's headers, e.g. `Authorization`.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def redirect_request(self, *args: Any, **kwargs: Any) -> None:
|
|
35
|
+
return None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# For the server's requests and the upload alike.
|
|
39
|
+
_OPENER = urllib.request.build_opener(_NoRedirectHandler)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class Compiled:
|
|
44
|
+
"""Warehouse SQL compiled from semantic SQL."""
|
|
45
|
+
|
|
46
|
+
sql: str
|
|
47
|
+
# The physical tables `sql` reads, by their parts, e.g. `("analytics", "main", "orders")`.
|
|
48
|
+
tables: tuple[tuple[str, ...], ...]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def compile_sql(url: str, sql: str, dialect: str) -> Compiled:
|
|
52
|
+
"""Metricflow SQL, in the warehouse's `dialect`, to warehouse SQL."""
|
|
53
|
+
response = _request(
|
|
54
|
+
url,
|
|
55
|
+
"POST",
|
|
56
|
+
"/v1/compile",
|
|
57
|
+
{"sql": sql, "input_dialect": dialect, "output_dialect": dialect},
|
|
58
|
+
).body
|
|
59
|
+
return Compiled(
|
|
60
|
+
sql=str(response["sql"]),
|
|
61
|
+
tables=tuple(tuple(str(part) for part in table) for table in response.get("tables", [])),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Published:
|
|
67
|
+
"""A bundle published to a MetricFlow server."""
|
|
68
|
+
|
|
69
|
+
name: str
|
|
70
|
+
version: int
|
|
71
|
+
# False if the server already had this bundle, as its current version.
|
|
72
|
+
changed: bool
|
|
73
|
+
# The catalog to query it as, if the server names one, e.g. `env_220`.
|
|
74
|
+
catalog: str | None = None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def publish_bundle(
|
|
78
|
+
url: str, name: str, archive: bytes, headers: Mapping[str, str] | None = None
|
|
79
|
+
) -> Published:
|
|
80
|
+
"""
|
|
81
|
+
Publishes the zip `archive` of a bundle to the server at `url`, as the next version of
|
|
82
|
+
the bundle `name`: uploads it through the URL the server gives, then asks the server to
|
|
83
|
+
publish it. `headers`, e.g. credentials, go with each request to the server, and never
|
|
84
|
+
with the upload.
|
|
85
|
+
"""
|
|
86
|
+
timeout = _PUBLISH_TIMEOUT_SECONDS
|
|
87
|
+
upload = _request(
|
|
88
|
+
url, "POST", "/api/v1/bundles/uploads", {"size": len(archive)}, timeout, headers
|
|
89
|
+
).body
|
|
90
|
+
upload_url = str(upload["url"])
|
|
91
|
+
# Only the headers the server gives: the URL is the store's, e.g. S3's, which mustn't see
|
|
92
|
+
# the server's credentials.
|
|
93
|
+
put = urllib.request.Request(
|
|
94
|
+
upload_url, data=archive, headers=dict(upload["headers"]), method=str(upload["method"])
|
|
95
|
+
)
|
|
96
|
+
# Named by its host: the URL itself carries a signature.
|
|
97
|
+
parts = urllib.parse.urlsplit(upload_url)
|
|
98
|
+
_send(put, f"{parts.scheme}://{parts.netloc}", timeout)
|
|
99
|
+
path = f"/api/v1/bundles/{urllib.parse.quote(name, safe='')}"
|
|
100
|
+
response = _request(url, "PUT", path, {"ref": upload["ref"]}, timeout, headers)
|
|
101
|
+
result = response.body
|
|
102
|
+
return Published(
|
|
103
|
+
name=str(result["name"]),
|
|
104
|
+
version=int(result["version"]),
|
|
105
|
+
changed=bool(result["changed"]),
|
|
106
|
+
catalog=response.headers.get(CATALOG_HEADER) or None,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True)
|
|
111
|
+
class _Response:
|
|
112
|
+
"""A JSON response from the server."""
|
|
113
|
+
|
|
114
|
+
body: dict[str, Any]
|
|
115
|
+
headers: Message
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _request(
|
|
119
|
+
url: str,
|
|
120
|
+
method: str,
|
|
121
|
+
path: str,
|
|
122
|
+
body: Mapping[str, Any],
|
|
123
|
+
timeout: float = _TIMEOUT_SECONDS,
|
|
124
|
+
headers: Mapping[str, str] | None = None,
|
|
125
|
+
) -> _Response:
|
|
126
|
+
parts = urllib.parse.urlsplit(url)
|
|
127
|
+
if parts.scheme not in ("http", "https") or not parts.netloc:
|
|
128
|
+
raise MetricFlowError(f"invalid URL `{url}`: it must be like http://localhost:8080")
|
|
129
|
+
request = urllib.request.Request(
|
|
130
|
+
f"{url.rstrip('/')}{path}",
|
|
131
|
+
data=json.dumps(body).encode(),
|
|
132
|
+
headers={**(headers or {}), "Content-Type": "application/json"},
|
|
133
|
+
method=method,
|
|
134
|
+
)
|
|
135
|
+
content, response_headers = _send(request, url, timeout)
|
|
136
|
+
return _Response(body=json.loads(content), headers=response_headers)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _send(request: urllib.request.Request, url: str, timeout: float) -> tuple[bytes, Message]:
|
|
140
|
+
"""
|
|
141
|
+
Sends `request`, returning the response's body and headers. A redirect is an error, naming its
|
|
142
|
+
host.
|
|
143
|
+
"""
|
|
144
|
+
try:
|
|
145
|
+
with _OPENER.open(request, timeout=timeout) as response:
|
|
146
|
+
content: bytes = response.read()
|
|
147
|
+
return content, response.headers
|
|
148
|
+
except urllib.error.HTTPError as e:
|
|
149
|
+
location = e.headers.get("Location")
|
|
150
|
+
if 300 <= e.code < 400 and location:
|
|
151
|
+
# Named by its host, as the upload's URL is: a `Location` can carry a signature. A
|
|
152
|
+
# relative one, e.g. `/other`, is on the request's host.
|
|
153
|
+
parts = urllib.parse.urlsplit(urllib.parse.urljoin(request.full_url, location))
|
|
154
|
+
raise MetricFlowError(
|
|
155
|
+
f"redirected ({e.code}) to {parts.scheme}://{parts.netloc}, not followed"
|
|
156
|
+
) from e
|
|
157
|
+
raise MetricFlowError(_error_message(e.read())) from e
|
|
158
|
+
# `URLError` (refused, DNS) and timeouts, which a read can raise unwrapped, are both `OSError`s.
|
|
159
|
+
except OSError as e:
|
|
160
|
+
raise MetricFlowError(f"unreachable at {url}: {e}") from e
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _error_message(content: bytes) -> str:
|
|
164
|
+
"""The message of an error response, e.g. `{"message": "..."}`, else its raw body."""
|
|
165
|
+
text = content.decode(errors="replace")
|
|
166
|
+
try:
|
|
167
|
+
body = json.loads(text)
|
|
168
|
+
except ValueError:
|
|
169
|
+
return text
|
|
170
|
+
message: Any | None = None
|
|
171
|
+
if type(body) is dict:
|
|
172
|
+
message = body.get("message", body.get("error"))
|
|
173
|
+
return text if message is None else str(message)
|