dbt-metricflow-sql 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/PKG-INFO +1 -1
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/README.md +50 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/pyproject.toml +1 -1
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/cli.py +59 -11
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/client.py +19 -5
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/llms-full.txt +63 -2
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/plugin.py +114 -19
- dbt_metricflow_sql-0.3.0/src/dbt_metricflow_sql/sl_config.py +1030 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/.gitignore +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/LICENSE +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/PYPI.md +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/__init__.py +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/dependencies.py +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/docs.py +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/fragment.schema.json +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/semantics.py +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/server.py +0 -0
- {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/uv.lock +0 -0
|
@@ -12,6 +12,9 @@ it, so install the plugin with its `sidecar` extra:
|
|
|
12
12
|
pip install 'dbt-metricflow-sql[sidecar]'
|
|
13
13
|
```
|
|
14
14
|
|
|
15
|
+
An install from source, e.g. `pip install -e`, has no docs for `dbtsl docs` until
|
|
16
|
+
`just bundle-docs` builds them: the wheel's `llms-full.txt` is a build artifact.
|
|
17
|
+
|
|
15
18
|
For another command, set the `metricflow_server` var, as a list of arguments or a string:
|
|
16
19
|
|
|
17
20
|
```yaml
|
|
@@ -188,6 +191,41 @@ warns of keys it doesn't know as custom ones; the plugin declares these to it.
|
|
|
188
191
|
naming the redirect's host. That includes the upload's, e.g. S3's for a bucket in another
|
|
189
192
|
region.
|
|
190
193
|
|
|
194
|
+
## Setting up dbt Cloud
|
|
195
|
+
|
|
196
|
+
Before the first publish, the dbt Cloud account needs its `experimental-semantic-layer` flag on,
|
|
197
|
+
which dbt Labs sets, and a **profile** for the environment to run as: a warehouse connection,
|
|
198
|
+
with a **login** (its credential), which a dbt Cloud group is **mapped** to, so its users run as
|
|
199
|
+
it, or a service token **granted**, as tokens ignore mappings. `dbtsl config` manages them, one
|
|
200
|
+
API call per command, mostly, on the account, host and token a publish uses:
|
|
201
|
+
|
|
202
|
+
```sh
|
|
203
|
+
dbtsl config status # everything set up, and every gap, each with the command filling it
|
|
204
|
+
dbtsl config profiles create prod --adapter-type bigquery --connection project_id=my-project
|
|
205
|
+
dbtsl config logins create --profile prod analytics_sa --keyfile ~/keys/sa.json
|
|
206
|
+
dbtsl config mappings create --profile prod --login analytics_sa --group Analysts --priority 100
|
|
207
|
+
dbtsl config grants create --profile prod --login analytics_sa --service-token ci
|
|
208
|
+
dbtsl publish --target prod --vars '{metricflow_publish_profile: prod}' # creates the environment
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
- `profiles`, `logins`, `mappings` and `grants` `list`, `create` and `delete`; profiles and
|
|
212
|
+
logins also `get` and `update`, keeping the fields not given. `environments` `list`,
|
|
213
|
+
`set-default` and `delete` the Semantic Layer's. Anything is named by its name or its id.
|
|
214
|
+
- `create --if-missing` returns the one with the same name, or group or token and login, instead
|
|
215
|
+
of failing, so a script can run it again.
|
|
216
|
+
- dbt Cloud checks the fields, and its errors are reported per field. Secrets come only from
|
|
217
|
+
files (`--keyfile`, `--private-key-file`) and environment variables (`--password-env`,
|
|
218
|
+
`--private-key-passphrase-env`, `--secret-env KEY=VAR`), go only in a login's create or update,
|
|
219
|
+
and are never printed.
|
|
220
|
+
- `--json` prints one JSON document on stdout, `"schema": "dbtsl.config/v1"`, its `result` or its
|
|
221
|
+
`error`'s stable `code`. Exit codes: 0 done, 2 usage, 3 `status` found gaps, 4 token, 5
|
|
222
|
+
unavailable or 404, 6 not found, 7 rejected.
|
|
223
|
+
- dbt Cloud answers these routes 404 alike whether the flag is off, the token lacks a permission,
|
|
224
|
+
or, for up to an hour after the flag is turned on, a server doesn't see it yet: a 404 is retried
|
|
225
|
+
for `--retry-for` seconds (default 120), which is safe, as nothing ran.
|
|
226
|
+
|
|
227
|
+
`dbtsl docs dbt-platform-setup` has the recipe.
|
|
228
|
+
|
|
191
229
|
## Querying
|
|
192
230
|
|
|
193
231
|
`dbtsl` also reads the semantic layer, from the command line:
|
|
@@ -209,6 +247,18 @@ server's default. `query` and `explain` use Trino's client protocol, `explain` a
|
|
|
209
247
|
`EXPLAIN <query>`, which compiles without running; `search` and `get-objects` use the server's
|
|
210
248
|
`/v1/search` and `/v1/objects`. SQL of `-` is read from stdin.
|
|
211
249
|
|
|
250
|
+
`query` and `explain` read their SQL in the target's dialect, e.g. BigQuery's
|
|
251
|
+
`DATE_TRUNC(date, YEAR)` for a BigQuery target, as a MetricFlow model's is, so a query reads the
|
|
252
|
+
same with `--local` as without. `--dialect` names another, e.g. `--dialect trino` for Trino's
|
|
253
|
+
`DATE_TRUNC('year', date)`: any of sqlglot's dialects, case-insensitively. The server answers an
|
|
254
|
+
unknown one with an error, which the command fails with. Either way, the SQL compiles to the
|
|
255
|
+
warehouse's dialect. The server is told the dialect as `X-Dbt-Dialect`; Trino clients that don't
|
|
256
|
+
send it are read as Trino's.
|
|
257
|
+
|
|
258
|
+
```sh
|
|
259
|
+
dbtsl query --dialect trino "SELECT DATE_TRUNC('year', date) AS year, AGG(revenue) FROM sales GROUP BY 1"
|
|
260
|
+
```
|
|
261
|
+
|
|
212
262
|
With `--local`, they parse the project, and read its own semantics instead, as `dbt build`
|
|
213
263
|
would compile them, on the server the plugin runs to compile MetricFlow models: `query` compiles the SQL there
|
|
214
264
|
and runs it on the target's warehouse, with dbt's own adapter, so the tables it reads must be
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dbt-metricflow-sql"
|
|
3
3
|
# A placeholder: a release sets its tag's version, with scripts/set_version.py.
|
|
4
|
-
version = "0.
|
|
4
|
+
version = "0.3.0"
|
|
5
5
|
description = "dbt v1 plugin for MetricFlow"
|
|
6
6
|
# Its PyPI page: README.md is for this repository.
|
|
7
7
|
readme = "PYPI.md"
|
|
@@ -12,12 +12,16 @@
|
|
|
12
12
|
- `dbtsl get-objects ID...`: prints the semantic layer's objects by id, as JSON.
|
|
13
13
|
- `dbtsl docs [PAGE]`: prints MetricFlow's docs, as Markdown, for AI agents: with no page, the
|
|
14
14
|
pages it has; with `--all`, every page.
|
|
15
|
+
- `dbtsl config profiles|logins|mappings|grants|environments|status`: sets up the dbt Cloud
|
|
16
|
+
account's Semantic Layer for the project to publish to (see `dbt_metricflow_sql.sl_config`).
|
|
15
17
|
|
|
16
18
|
Each of the last four reads the server and environment the project publishes to, with the same
|
|
17
19
|
vars and token, the environment naming the Trino catalog: it loads the project's and profile's
|
|
18
20
|
config, with the dbt arguments it's given, e.g. `--target prod`, without parsing the project, so
|
|
19
21
|
nothing else in the project, e.g. a model that doesn't parse, can fail it. Queries run on the
|
|
20
|
-
server, over Trino's protocol, and so does `explain`, as `EXPLAIN <query
|
|
22
|
+
server, over Trino's protocol, and so does `explain`, as `EXPLAIN <query>`, their SQL read in the
|
|
23
|
+
target's dialect, e.g. BigQuery's, as a MetricFlow model's is, or in the one `--dialect` names,
|
|
24
|
+
e.g. `--dialect trino`. With `--local`, it
|
|
21
25
|
parses the project, quietly, and reads its own semantics instead, on the server the plugin runs
|
|
22
26
|
to compile MetricFlow models; a query runs on the target's warehouse, through dbt's adapter. SQL
|
|
23
27
|
of `-` is read from stdin.
|
|
@@ -47,7 +51,7 @@ from dbt_common.clients.system import get_env
|
|
|
47
51
|
from dbt_common.context import set_invocation_context
|
|
48
52
|
from dbt_common.exceptions import DbtRuntimeError
|
|
49
53
|
|
|
50
|
-
from dbt_metricflow_sql import client, plugin
|
|
54
|
+
from dbt_metricflow_sql import client, plugin, sl_config
|
|
51
55
|
from dbt_metricflow_sql.client import MetricFlowError, QueryResult
|
|
52
56
|
from dbt_metricflow_sql.docs import DocsMissingError, DocsPage, read_docs, split_pages
|
|
53
57
|
from dbt_metricflow_sql.semantics import SemanticsError
|
|
@@ -77,8 +81,12 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
77
81
|
parser.error(f"unrecognized arguments: {' '.join(extra)}")
|
|
78
82
|
if args.command == DOCS:
|
|
79
83
|
return _docs(args.page, args.all)
|
|
84
|
+
if args.command == sl_config.CONFIG:
|
|
85
|
+
return sl_config.run(args, _load_config)
|
|
80
86
|
reader: _Reader | None = (
|
|
81
|
-
_local_reader(args.dbt_args
|
|
87
|
+
_local_reader(args.dbt_args, args.dialect)
|
|
88
|
+
if args.local
|
|
89
|
+
else _remote_reader(args.dbt_args, args.dialect)
|
|
82
90
|
)
|
|
83
91
|
if reader is None:
|
|
84
92
|
return 1
|
|
@@ -121,10 +129,12 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
121
129
|
),
|
|
122
130
|
)
|
|
123
131
|
query = _reading_command(commands, QUERY, "run semantic SQL, printing its result")
|
|
132
|
+
_add_dialect(query)
|
|
124
133
|
query.add_argument("sql", help="the semantic SQL to run, or `-` to read it from stdin")
|
|
125
134
|
explain = _reading_command(
|
|
126
135
|
commands, EXPLAIN, "compile semantic SQL, printing the warehouse SQL it compiles to"
|
|
127
136
|
)
|
|
137
|
+
_add_dialect(explain)
|
|
128
138
|
explain.add_argument("sql", help="the semantic SQL to compile, or `-` to read it from stdin")
|
|
129
139
|
search = _reading_command(commands, SEARCH, "search the semantic layer's objects")
|
|
130
140
|
search.add_argument("query", nargs="+", help="keywords, e.g. `return rate`")
|
|
@@ -145,9 +155,22 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
145
155
|
)
|
|
146
156
|
docs.add_argument("page", nargs="?", help="the page to print, e.g. `concepts`")
|
|
147
157
|
docs.add_argument("--all", action="store_true", help="print every page")
|
|
158
|
+
sl_config.add_parser(commands, _add_dbt_options)
|
|
148
159
|
return parser
|
|
149
160
|
|
|
150
161
|
|
|
162
|
+
def _add_dbt_options(command: argparse.ArgumentParser, options: Sequence[str]) -> None:
|
|
163
|
+
"""dbt's `options`, e.g. `--target`, collected into `dbt_args` in the order given."""
|
|
164
|
+
command.set_defaults(dbt_args=[])
|
|
165
|
+
for option in options:
|
|
166
|
+
command.add_argument(
|
|
167
|
+
option,
|
|
168
|
+
action=_PassToDbt,
|
|
169
|
+
metavar=option.lstrip("-").upper(),
|
|
170
|
+
help="as dbt's own option",
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
151
174
|
def _reading_command(
|
|
152
175
|
commands: argparse._SubParsersAction[argparse.ArgumentParser], name: str, summary: str
|
|
153
176
|
) -> argparse.ArgumentParser:
|
|
@@ -168,7 +191,8 @@ def _reading_command(
|
|
|
168
191
|
" queries on the target's warehouse with dbt's adapter"
|
|
169
192
|
),
|
|
170
193
|
)
|
|
171
|
-
|
|
194
|
+
# Only `query` and `explain` take `--dialect`.
|
|
195
|
+
command.set_defaults(dbt_args=[], dialect=None)
|
|
172
196
|
for option in _DBT_OPTIONS:
|
|
173
197
|
command.add_argument(
|
|
174
198
|
option,
|
|
@@ -179,6 +203,21 @@ def _reading_command(
|
|
|
179
203
|
return command
|
|
180
204
|
|
|
181
205
|
|
|
206
|
+
def _add_dialect(command: argparse.ArgumentParser) -> None:
|
|
207
|
+
"""`--dialect`, the dialect the command's SQL is written in, instead of the target's."""
|
|
208
|
+
command.add_argument(
|
|
209
|
+
"--dialect",
|
|
210
|
+
# Case-insensitive, e.g. `BigQuery`, as the server's statements are, though `/v1/compile`'s
|
|
211
|
+
# aren't. The server checks the name, so it's not checked here.
|
|
212
|
+
type=str.lower,
|
|
213
|
+
metavar="DIALECT",
|
|
214
|
+
help=(
|
|
215
|
+
"the SQL dialect the semantic SQL is written in, e.g. `trino` or `bigquery`"
|
|
216
|
+
" (default: the target's, e.g. `bigquery` for a BigQuery target)"
|
|
217
|
+
),
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
182
221
|
class _PassToDbt(argparse.Action):
|
|
183
222
|
"""Collects an option, e.g. `--target prod`, into `dbt_args`, in the order given."""
|
|
184
223
|
|
|
@@ -215,9 +254,16 @@ class _RemoteReader:
|
|
|
215
254
|
"""Reads the environment the project publishes to, on the server it publishes to."""
|
|
216
255
|
|
|
217
256
|
layer: plugin.SemanticLayer
|
|
257
|
+
# The dialect queries are written in, if not the target's (see `SemanticLayer.input_dialect`).
|
|
258
|
+
dialect: str | None = None
|
|
218
259
|
|
|
219
260
|
def query(self, sql: str) -> QueryResult:
|
|
220
|
-
return client.run_query(
|
|
261
|
+
return client.run_query(
|
|
262
|
+
self.layer.url,
|
|
263
|
+
sql,
|
|
264
|
+
self.layer.headers,
|
|
265
|
+
input_dialect=self.dialect or self.layer.input_dialect,
|
|
266
|
+
)
|
|
221
267
|
|
|
222
268
|
def explain(self, sql: str) -> str:
|
|
223
269
|
# The server compiles `EXPLAIN <query>` without running it, in the environment's dialect.
|
|
@@ -240,9 +286,11 @@ class _LocalReader:
|
|
|
240
286
|
manifest: Manifest
|
|
241
287
|
# The invocation's, as the plugin's own hooks find it.
|
|
242
288
|
config: RuntimeConfig
|
|
289
|
+
# The dialect queries are written in, if not the target's.
|
|
290
|
+
dialect: str | None = None
|
|
243
291
|
|
|
244
292
|
def query(self, sql: str) -> QueryResult:
|
|
245
|
-
compiled = plugin.compile_locally(self.manifest, self.config, sql)
|
|
293
|
+
compiled = plugin.compile_locally(self.manifest, self.config, sql, self.dialect)
|
|
246
294
|
if compiled.explain:
|
|
247
295
|
# As the server answers `EXPLAIN <query>`: its compiled SQL, without running it.
|
|
248
296
|
return QueryResult(
|
|
@@ -262,7 +310,7 @@ class _LocalReader:
|
|
|
262
310
|
)
|
|
263
311
|
|
|
264
312
|
def explain(self, sql: str) -> str:
|
|
265
|
-
return plugin.compile_locally(self.manifest, self.config, sql).sql
|
|
313
|
+
return plugin.compile_locally(self.manifest, self.config, sql, self.dialect).sql
|
|
266
314
|
|
|
267
315
|
def search(self, query: str, limit: int) -> tuple[client.SearchHit, ...]:
|
|
268
316
|
return client.search(plugin.local_url(self.manifest, self.config), query, limit)
|
|
@@ -271,13 +319,13 @@ class _LocalReader:
|
|
|
271
319
|
return client.get_objects(plugin.local_url(self.manifest, self.config), ids)
|
|
272
320
|
|
|
273
321
|
|
|
274
|
-
def _remote_reader(dbt_args: Sequence[str]) -> _RemoteReader | None:
|
|
322
|
+
def _remote_reader(dbt_args: Sequence[str], dialect: str | None = None) -> _RemoteReader | None:
|
|
275
323
|
"""
|
|
276
324
|
The reader of the server the project publishes to, from its config alone (see
|
|
277
325
|
`_load_config`), or None, reporting why, if it can't find the server.
|
|
278
326
|
"""
|
|
279
327
|
try:
|
|
280
|
-
return _RemoteReader(plugin.semantic_layer(_load_config(dbt_args)))
|
|
328
|
+
return _RemoteReader(plugin.semantic_layer(_load_config(dbt_args)), dialect)
|
|
281
329
|
except click.ClickException as error:
|
|
282
330
|
# dbt's own option's, e.g. `--vars` that isn't YAML.
|
|
283
331
|
print(error.format_message(), file=sys.stderr)
|
|
@@ -306,10 +354,10 @@ def _load_config(dbt_args: Sequence[str]) -> RuntimeConfig:
|
|
|
306
354
|
return RuntimeConfig.from_args(flags)
|
|
307
355
|
|
|
308
356
|
|
|
309
|
-
def _local_reader(dbt_args: Sequence[str]) -> _LocalReader | None:
|
|
357
|
+
def _local_reader(dbt_args: Sequence[str], dialect: str | None = None) -> _LocalReader | None:
|
|
310
358
|
"""The reader of the project `dbt parse` parses, quietly, or None if it fails, as dbt logs."""
|
|
311
359
|
parsed = _parse(dbt_args, quiet=True)
|
|
312
|
-
return None if parsed is None else _LocalReader(*parsed)
|
|
360
|
+
return None if parsed is None else _LocalReader(*parsed, dialect)
|
|
313
361
|
|
|
314
362
|
|
|
315
363
|
def _parse(dbt_args: Sequence[str], quiet: bool = False) -> tuple[Manifest, RuntimeConfig] | None:
|
|
@@ -26,6 +26,9 @@ PROFILE_HEADER = "X-Dbt-Profile"
|
|
|
26
26
|
# The header naming the catalog, i.e. the environment, a query, compile, search or lookup reads, as
|
|
27
27
|
# Trino's protocol names a session's catalog, e.g. `prod`. Without it, the server reads its default.
|
|
28
28
|
TRINO_CATALOG_HEADER = "X-Trino-Catalog"
|
|
29
|
+
# The header naming the dialect a query's semantic SQL is written in, e.g. `bigquery`. Without it,
|
|
30
|
+
# the server reads it as Trino's; it answers one it doesn't know with an error.
|
|
31
|
+
DIALECT_HEADER = "X-Dbt-Dialect"
|
|
29
32
|
# Trino's protocol requires a user; the server identifies the caller by its token instead.
|
|
30
33
|
_TRINO_USER_HEADER = "X-Trino-User"
|
|
31
34
|
_TRINO_USER = "dbtsl"
|
|
@@ -60,13 +63,16 @@ class Compiled:
|
|
|
60
63
|
explain: bool = False
|
|
61
64
|
|
|
62
65
|
|
|
63
|
-
def compile_sql(url: str, sql: str, dialect: str) -> Compiled:
|
|
64
|
-
"""
|
|
66
|
+
def compile_sql(url: str, sql: str, dialect: str, input_dialect: str | None = None) -> Compiled:
|
|
67
|
+
"""
|
|
68
|
+
Metricflow SQL, in `input_dialect` if given, else the warehouse's `dialect`, to warehouse SQL,
|
|
69
|
+
pretty-printed.
|
|
70
|
+
"""
|
|
65
71
|
response = _request(
|
|
66
72
|
url,
|
|
67
73
|
"POST",
|
|
68
74
|
"/v1/compile",
|
|
69
|
-
{"sql": sql, "input_dialect": dialect, "output_dialect": dialect},
|
|
75
|
+
{"sql": sql, "input_dialect": input_dialect or dialect, "output_dialect": dialect},
|
|
70
76
|
).body
|
|
71
77
|
return Compiled(
|
|
72
78
|
sql=str(response["sql"]),
|
|
@@ -175,14 +181,22 @@ class QueryResult:
|
|
|
175
181
|
numeric: tuple[bool, ...]
|
|
176
182
|
|
|
177
183
|
|
|
178
|
-
def run_query(
|
|
184
|
+
def run_query(
|
|
185
|
+
url: str,
|
|
186
|
+
sql: str,
|
|
187
|
+
headers: Mapping[str, str] | None = None,
|
|
188
|
+
input_dialect: str | None = None,
|
|
189
|
+
) -> QueryResult:
|
|
179
190
|
"""
|
|
180
191
|
Runs the semantic SQL `sql` on the server at `url`, over Trino's client protocol: submits it
|
|
181
192
|
to `/v1/statement`, then follows each response's `nextUri` until there is none, collecting its
|
|
182
193
|
rows. A `nextUri` is requested from `url`'s own scheme and host, whatever it names, so
|
|
183
|
-
`headers`, e.g. credentials, never go elsewhere.
|
|
194
|
+
`headers`, e.g. credentials, never go elsewhere. `sql` is read in `input_dialect`, e.g.
|
|
195
|
+
`bigquery`, if given, else as Trino's. Raises `MetricFlowError` if the query fails.
|
|
184
196
|
"""
|
|
185
197
|
request_headers = {**(headers or {}), _TRINO_USER_HEADER: _TRINO_USER}
|
|
198
|
+
if input_dialect:
|
|
199
|
+
request_headers[DIALECT_HEADER] = input_dialect
|
|
186
200
|
results = _request(
|
|
187
201
|
url,
|
|
188
202
|
"POST",
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
|
|
27
27
|
1. [Signup](https://www.getdbt.com/signup) if you don’t already have an account.
|
|
28
28
|
|
|
29
|
-
2.
|
|
29
|
+
2. Set up [dbt Platform CLI](https://docs.getdbt.com/docs/platform/dbt-cli-installation)
|
|
30
30
|
|
|
31
|
-
3.
|
|
31
|
+
3. [Configure the Semantic Layer in dbt Platform](../dbt-platform-setup/)
|
|
32
32
|
|
|
33
33
|
4. **Model your data**
|
|
34
34
|
|
|
@@ -63,6 +63,67 @@
|
|
|
63
63
|
|
|
64
64
|
To compile queries locally, use the `--local` flag on CLI commands:
|
|
65
65
|
|
|
66
|
+
# dbt Platform setup
|
|
67
|
+
|
|
68
|
+
> Set up dbt Cloud's Semantic Layer for a project with `dbtsl config`, from the command line.
|
|
69
|
+
|
|
70
|
+
Before a project publishes to dbt Cloud’s Semantic Layer for the first time, its account needs:
|
|
71
|
+
|
|
72
|
+
* a **profile**: a warehouse connection, which an environment runs as;
|
|
73
|
+
* a **login** on the profile: the warehouse credential queries run with;
|
|
74
|
+
* a **mapping** from a dbt Cloud group to the login, so the group’s users run as it. A user in several mapped groups runs as the login of the mapping with the highest priority;
|
|
75
|
+
* or a **grant** of the login to a service token. Service tokens ignore mappings.
|
|
76
|
+
|
|
77
|
+
dbt Labs turns on the account’s `experimental-semantic-layer` flag. Then `dbtsl config` sets up the rest, and the first `dbtsl publish` creates the environment on the profile. The account’s first environment becomes its default.
|
|
78
|
+
|
|
79
|
+
## The recipe
|
|
80
|
+
|
|
81
|
+
Start with `status`. It is read-only, and it reports every gap, each with the command that fills it. Its UPPERCASE arguments are placeholders to fill in.
|
|
82
|
+
|
|
83
|
+
```sh
|
|
84
|
+
dbtsl config status
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
1. Create the profile. Name it after the target that publishes, e.g. `prod`, or set the `metricflow_publish_profile` var to its name.
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
dbtsl config profiles create prod --adapter-type bigquery \
|
|
91
|
+
--connection project_id=my-project --if-missing
|
|
92
|
+
# Snowflake:
|
|
93
|
+
dbtsl config profiles create prod --adapter-type snowflake --connection account=ab12345 \
|
|
94
|
+
--connection database=ANALYTICS --connection warehouse=REPORTING --if-missing
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
2. Create a login with its secret. Secrets come only from files and environment variables, and are never printed.
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
dbtsl config logins create --profile prod analytics_sa --keyfile ~/keys/sa.json --if-missing
|
|
101
|
+
# Snowflake, with a key pair, or `--password-env VAR`:
|
|
102
|
+
dbtsl config logins create --profile prod reporter --login-detail user=REPORTER \
|
|
103
|
+
--login-detail role=REPORTER --private-key-file ~/keys/reporter.p8 \
|
|
104
|
+
--private-key-passphrase-env SNOWFLAKE_PASSPHRASE --if-missing
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
3. Map a group to the login, or grant the login to a service token, e.g. CI’s. Groups and service tokens are named by their name or id.
|
|
108
|
+
|
|
109
|
+
```sh
|
|
110
|
+
dbtsl config mappings create --profile prod --login analytics_sa \
|
|
111
|
+
--group Analysts --priority 100 --if-missing
|
|
112
|
+
dbtsl config grants create --profile prod --login analytics_sa --service-token ci --if-missing
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
4. Publish. This creates the environment on the profile.
|
|
116
|
+
|
|
117
|
+
```sh
|
|
118
|
+
dbtsl publish --target prod --vars '{metricflow_publish_profile: prod}'
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Then `dbtsl config status` reports `Ready.`
|
|
122
|
+
|
|
123
|
+
## Details
|
|
124
|
+
|
|
125
|
+
Each command’s options are in its help: `dbtsl config --help`, and e.g. `dbtsl config logins create --help`.
|
|
126
|
+
|
|
66
127
|
# Overview
|
|
67
128
|
|
|
68
129
|
MetricFlow is the engine that powers the dbt semantic layer.
|
|
@@ -150,9 +150,9 @@ class MetricflowManager:
|
|
|
150
150
|
# The server, once something needs it; or what kept it from starting.
|
|
151
151
|
self._server: ServerProcess | None = None
|
|
152
152
|
self._start_failure: MetricFlowError | SemanticsError | None = None
|
|
153
|
-
# Compilations by dialect and semantic SQL, so a model compiles once, as dbt builds its
|
|
153
|
+
# Compilations by dialect, input dialect and semantic SQL, so a model compiles once, as dbt builds its
|
|
154
154
|
# graph, however many times dbt compiles it.
|
|
155
|
-
self._compiled: dict[tuple[str, str], Compiled] = {}
|
|
155
|
+
self._compiled: dict[tuple[str, str | None, str], Compiled] = {}
|
|
156
156
|
# The physical tables each model's compiled SQL reads, by unique id.
|
|
157
157
|
self.tables: dict[str, tuple[tuple[str, ...], ...]] = {}
|
|
158
158
|
|
|
@@ -199,15 +199,22 @@ class MetricflowManager:
|
|
|
199
199
|
raise MetricFlowError(f"starting the server failed: {failure}") from failure
|
|
200
200
|
return server.url
|
|
201
201
|
|
|
202
|
-
def compile(
|
|
202
|
+
def compile(
|
|
203
|
+
self,
|
|
204
|
+
manifest: Manifest,
|
|
205
|
+
config: RuntimeConfig,
|
|
206
|
+
sql: str,
|
|
207
|
+
input_dialect: str | None = None,
|
|
208
|
+
) -> Compiled:
|
|
209
|
+
"""Compiles `sql`, in `input_dialect` if given, else the warehouse's, to the warehouse's."""
|
|
203
210
|
url = self.url(manifest, config)
|
|
204
211
|
# dbt's adapter types are sqlglot's dialect names: bigquery, duckdb, snowflake, ...
|
|
205
|
-
key = (config.credentials.type, sql)
|
|
212
|
+
key = (config.credentials.type, input_dialect, sql)
|
|
206
213
|
with self._lock:
|
|
207
214
|
compiled = self._compiled.get(key)
|
|
208
215
|
# Outside the lock, so models compile concurrently on dbt's threads.
|
|
209
216
|
if compiled is None:
|
|
210
|
-
compiled = client.compile_sql(url, sql, config.credentials.type)
|
|
217
|
+
compiled = client.compile_sql(url, sql, config.credentials.type, input_dialect)
|
|
211
218
|
with self._lock:
|
|
212
219
|
self._compiled[key] = compiled
|
|
213
220
|
return compiled
|
|
@@ -353,10 +360,20 @@ def _publish_target(config: RuntimeConfig, publishing: str) -> _PublishTarget |
|
|
|
353
360
|
|
|
354
361
|
def _dbt_cloud_target(config: RuntimeConfig) -> _PublishTarget:
|
|
355
362
|
"""
|
|
356
|
-
The Semantic Layer of the dbt Cloud account of the project's entry in `DBT_CLOUD_CONFIG
|
|
357
|
-
the entry's token.
|
|
358
|
-
|
|
359
|
-
|
|
363
|
+
The Semantic Layer of the dbt Cloud account of the project's entry in `DBT_CLOUD_CONFIG` (see
|
|
364
|
+
`_dbt_cloud_entry`), with the entry's token.
|
|
365
|
+
"""
|
|
366
|
+
entry = _dbt_cloud_entry(config)
|
|
367
|
+
host = str(entry.get("account-host") or "")
|
|
368
|
+
token = str(entry.get("token-value") or "").strip() or None
|
|
369
|
+
return _PublishTarget(_semantic_layer_url(host), token)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _dbt_cloud_entry(config: RuntimeConfig) -> dict[str, Any]:
|
|
373
|
+
"""
|
|
374
|
+
The project's entry in `DBT_CLOUD_CONFIG`: `dbt_project.yml`'s `dbt-cloud: project-id`, else
|
|
375
|
+
the file's active project. Raises `MetricFlowError` naming what's missing. Never echoes the
|
|
376
|
+
file, which holds tokens: YAML's errors quote the line they're on.
|
|
360
377
|
"""
|
|
361
378
|
path = DBT_CLOUD_CONFIG
|
|
362
379
|
try:
|
|
@@ -387,10 +404,8 @@ def _dbt_cloud_target(config: RuntimeConfig) -> _PublishTarget:
|
|
|
387
404
|
]
|
|
388
405
|
if not entries:
|
|
389
406
|
raise MetricFlowError(f"{path} has no project {str(project_id).strip()}")
|
|
390
|
-
entry = entries[0]
|
|
391
|
-
|
|
392
|
-
token = str(entry.get("token-value") or "").strip() or None
|
|
393
|
-
return _PublishTarget(_semantic_layer_url(host), token)
|
|
407
|
+
entry: dict[str, Any] = entries[0]
|
|
408
|
+
return entry
|
|
394
409
|
|
|
395
410
|
|
|
396
411
|
def _semantic_layer_url(account_host: str) -> str:
|
|
@@ -548,14 +563,18 @@ class SemanticLayer:
|
|
|
548
563
|
url: str
|
|
549
564
|
# The token, and the catalog naming the environment, if any.
|
|
550
565
|
headers: Mapping[str, str] = field(repr=False)
|
|
566
|
+
# The dialect queries' semantic SQL is written in: the target warehouse's, e.g. `bigquery`, as
|
|
567
|
+
# a MetricFlow model's is, so a query reads the same here as with `--local`.
|
|
568
|
+
input_dialect: str | None = None
|
|
551
569
|
|
|
552
570
|
|
|
553
571
|
def semantic_layer(config: RuntimeConfig) -> SemanticLayer:
|
|
554
572
|
"""
|
|
555
573
|
The server the project publishes to (see `_publish_target`), whatever the `metricflow_publish`
|
|
556
574
|
var says, with the headers reading the environment it publishes to there: the token (see
|
|
557
|
-
`_token_headers`), and the environment (see `_environment`) as the Trino catalog
|
|
558
|
-
`MetricFlowError` for a server it can't find, or a header
|
|
575
|
+
`_token_headers`), and the environment (see `_environment`) as the Trino catalog; queries are
|
|
576
|
+
read in the target's dialect. Raises `MetricFlowError` for a server it can't find, or a header
|
|
577
|
+
it can't send.
|
|
559
578
|
"""
|
|
560
579
|
target = _publish_target(config, _Publishing.REQUIRED)
|
|
561
580
|
assert target is not None, "a required publish always has a target"
|
|
@@ -565,7 +584,78 @@ def semantic_layer(config: RuntimeConfig) -> SemanticLayer:
|
|
|
565
584
|
((f"the `{ENVIRONMENT_VAR}` var", client.TRINO_CATALOG_HEADER, _environment(config)),),
|
|
566
585
|
)
|
|
567
586
|
)
|
|
568
|
-
|
|
587
|
+
# dbt's adapter types are sqlglot's dialect names: bigquery, duckdb, snowflake, ...
|
|
588
|
+
return SemanticLayer(url=target.url, headers=headers, input_dialect=config.credentials.type)
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
@dataclass(frozen=True)
|
|
592
|
+
class DbtCloudAccount:
|
|
593
|
+
"""The dbt Cloud account the project publishes to, for `dbtsl config` to set up."""
|
|
594
|
+
|
|
595
|
+
# dbt Cloud's API, e.g. `https://ab123.us1.dbt.com`.
|
|
596
|
+
api_url: str
|
|
597
|
+
account_id: int
|
|
598
|
+
# The headers authenticating to `api_url`: the bearer token.
|
|
599
|
+
api_headers: Mapping[str, str] = field(repr=False)
|
|
600
|
+
# Where the token came from, e.g. `DBT_CLOUD_TOKEN`; never the token itself.
|
|
601
|
+
token_source: str
|
|
602
|
+
# The server the project publishes to (see `_publish_target`), and the headers authenticating
|
|
603
|
+
# to it, as a publish sends them.
|
|
604
|
+
semantic_layer_url: str
|
|
605
|
+
semantic_layer_headers: Mapping[str, str] = field(repr=False)
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def dbt_cloud_account(
|
|
609
|
+
config: RuntimeConfig, account_host: str | None = None, account_id: str | None = None
|
|
610
|
+
) -> DbtCloudAccount:
|
|
611
|
+
"""
|
|
612
|
+
The dbt Cloud account of the project's entry in `DBT_CLOUD_CONFIG` (see `_dbt_cloud_entry`),
|
|
613
|
+
or of `account_host` and `account_id` where they're given, then not needing the file. The
|
|
614
|
+
token is `PUBLISH_TOKEN_ENV`, else the entry's. A host may name its scheme: `http://` only for
|
|
615
|
+
this machine's, e.g. a test's. Raises `MetricFlowError` naming what's missing.
|
|
616
|
+
"""
|
|
617
|
+
entry: Mapping[str, Any] = {}
|
|
618
|
+
if account_host is None or account_id is None:
|
|
619
|
+
entry = _dbt_cloud_entry(config)
|
|
620
|
+
host = (account_host or str(entry.get("account-host") or "")).strip().rstrip("/")
|
|
621
|
+
if not host:
|
|
622
|
+
raise MetricFlowError(f"{DBT_CLOUD_CONFIG}'s project has no `account-host`")
|
|
623
|
+
raw_id = (account_id or str(entry.get("account-id") or "")).strip()
|
|
624
|
+
# ASCII digits only, as `_project_id`'s.
|
|
625
|
+
if not re.fullmatch(r"[0-9]+", raw_id) or int(raw_id) == 0:
|
|
626
|
+
raise MetricFlowError("dbt Cloud's account id must be a positive integer")
|
|
627
|
+
scheme, separator, _ = host.partition("://")
|
|
628
|
+
api_url = host if separator and scheme in ("http", "https") else f"https://{host}"
|
|
629
|
+
entry_token = str(entry.get("token-value") or "").strip() or None
|
|
630
|
+
token_source = str(DBT_CLOUD_CONFIG)
|
|
631
|
+
if os.environ.get(PUBLISH_TOKEN_ENV, "").strip():
|
|
632
|
+
token_source = PUBLISH_TOKEN_ENV
|
|
633
|
+
api_headers = _token_headers(_PublishTarget(api_url, entry_token))
|
|
634
|
+
if not api_headers:
|
|
635
|
+
raise MetricFlowError(
|
|
636
|
+
f"setting up dbt Cloud needs a token: `{PUBLISH_TOKEN_ENV}`, or {DBT_CLOUD_CONFIG}'s"
|
|
637
|
+
)
|
|
638
|
+
# As a publish finds its server: the var's gets only `PUBLISH_TOKEN_ENV`.
|
|
639
|
+
url = _rendered_var(config, PUBLISH_URL_VAR)
|
|
640
|
+
target = _PublishTarget(url) if url else _PublishTarget(_semantic_layer_url(host), entry_token)
|
|
641
|
+
return DbtCloudAccount(
|
|
642
|
+
api_url=api_url,
|
|
643
|
+
account_id=int(raw_id),
|
|
644
|
+
api_headers=api_headers,
|
|
645
|
+
token_source=token_source,
|
|
646
|
+
semantic_layer_url=target.url,
|
|
647
|
+
semantic_layer_headers=_token_headers(target),
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def publish_environment(config: RuntimeConfig) -> str:
|
|
652
|
+
"""The MetricFlow environment the project publishes to (see `_environment`)."""
|
|
653
|
+
return _environment(config)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def publish_profile(config: RuntimeConfig) -> str | None:
|
|
657
|
+
"""The dbt Cloud profile a publish creates its environment on: the `PROFILE_VAR` var."""
|
|
658
|
+
return _rendered_var(config, PROFILE_VAR)
|
|
569
659
|
|
|
570
660
|
|
|
571
661
|
def local_url(manifest: Manifest, config: RuntimeConfig) -> str:
|
|
@@ -577,9 +667,14 @@ def local_url(manifest: Manifest, config: RuntimeConfig) -> str:
|
|
|
577
667
|
return _manager.url(manifest, config)
|
|
578
668
|
|
|
579
669
|
|
|
580
|
-
def compile_locally(
|
|
581
|
-
|
|
582
|
-
|
|
670
|
+
def compile_locally(
|
|
671
|
+
manifest: Manifest, config: RuntimeConfig, sql: str, input_dialect: str | None = None
|
|
672
|
+
) -> Compiled:
|
|
673
|
+
"""
|
|
674
|
+
Compiles semantic SQL as a MetricFlow model's is, on the server `local_url` runs: read in
|
|
675
|
+
`input_dialect` if given, else the warehouse's.
|
|
676
|
+
"""
|
|
677
|
+
return _manager.compile(manifest, config, sql, input_dialect)
|
|
583
678
|
|
|
584
679
|
|
|
585
680
|
@dataclass(frozen=True)
|