dbt-metricflow-sql 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/PKG-INFO +1 -1
  2. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/README.md +50 -0
  3. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/pyproject.toml +1 -1
  4. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/cli.py +59 -11
  5. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/client.py +19 -5
  6. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/llms-full.txt +63 -2
  7. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/plugin.py +114 -19
  8. dbt_metricflow_sql-0.3.0/src/dbt_metricflow_sql/sl_config.py +1030 -0
  9. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/.gitignore +0 -0
  10. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/LICENSE +0 -0
  11. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/PYPI.md +0 -0
  12. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/__init__.py +0 -0
  13. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/dependencies.py +0 -0
  14. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/docs.py +0 -0
  15. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/fragment.schema.json +0 -0
  16. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/semantics.py +0 -0
  17. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/src/dbt_metricflow_sql/server.py +0 -0
  18. {dbt_metricflow_sql-0.2.0 → dbt_metricflow_sql-0.3.0}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: dbt-metricflow-sql
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: dbt v1 plugin for MetricFlow
5
5
  Project-URL: Documentation, https://congenial-bassoon-7zkjv8w.pages.github.io/
6
6
  License-Expression: LicenseRef-Fivetran
@@ -12,6 +12,9 @@ it, so install the plugin with its `sidecar` extra:
12
12
  pip install 'dbt-metricflow-sql[sidecar]'
13
13
  ```
14
14
 
15
+ An install from source, e.g. `pip install -e`, has no docs for `dbtsl docs` until
16
+ `just bundle-docs` builds them: the wheel's `llms-full.txt` is a build artifact.
17
+
15
18
  For another command, set the `metricflow_server` var, as a list of arguments or a string:
16
19
 
17
20
  ```yaml
@@ -188,6 +191,41 @@ warns of keys it doesn't know as custom ones; the plugin declares these to it.
188
191
  naming the redirect's host. That includes the upload's, e.g. S3's for a bucket in another
189
192
  region.
190
193
 
194
+ ## Setting up dbt Cloud
195
+
196
+ Before the first publish, the dbt Cloud account needs its `experimental-semantic-layer` flag on,
197
+ which dbt Labs sets, and a **profile** for the environment to run as: a warehouse connection,
198
+ with a **login** (its credential), which a dbt Cloud group is **mapped** to, so its users run as
199
+ it, or a service token **granted**, as tokens ignore mappings. `dbtsl config` manages them, one
200
+ API call per command, mostly, on the account, host and token a publish uses:
201
+
202
+ ```sh
203
+ dbtsl config status # everything set up, and every gap, each with the command filling it
204
+ dbtsl config profiles create prod --adapter-type bigquery --connection project_id=my-project
205
+ dbtsl config logins create --profile prod analytics_sa --keyfile ~/keys/sa.json
206
+ dbtsl config mappings create --profile prod --login analytics_sa --group Analysts --priority 100
207
+ dbtsl config grants create --profile prod --login analytics_sa --service-token ci
208
+ dbtsl publish --target prod --vars '{metricflow_publish_profile: prod}' # creates the environment
209
+ ```
210
+
211
+ - `profiles`, `logins`, `mappings` and `grants` `list`, `create` and `delete`; profiles and
212
+ logins also `get` and `update`, keeping the fields not given. `environments` `list`,
213
+ `set-default` and `delete` the Semantic Layer's. Anything is named by its name or its id.
214
+ - `create --if-missing` returns the one with the same name, or group or token and login, instead
215
+ of failing, so a script can run it again.
216
+ - dbt Cloud checks the fields, and its errors are reported per field. Secrets come only from
217
+ files (`--keyfile`, `--private-key-file`) and environment variables (`--password-env`,
218
+ `--private-key-passphrase-env`, `--secret-env KEY=VAR`), go only in a login's create or update,
219
+ and are never printed.
220
+ - `--json` prints one JSON document on stdout, `"schema": "dbtsl.config/v1"`, its `result` or its
221
+ `error`'s stable `code`. Exit codes: 0 done, 2 usage, 3 `status` found gaps, 4 token, 5
222
+ unavailable or 404, 6 not found, 7 rejected.
223
+ - dbt Cloud answers these routes 404 alike whether the flag is off, the token lacks a permission,
224
+ or, for up to an hour after the flag is turned on, a server doesn't see it yet: a 404 is retried
225
+ for `--retry-for` seconds (default 120), which is safe, as nothing ran.
226
+
227
+ `dbtsl docs dbt-platform-setup` has the recipe.
228
+
191
229
  ## Querying
192
230
 
193
231
  `dbtsl` also reads the semantic layer, from the command line:
@@ -209,6 +247,18 @@ server's default. `query` and `explain` use Trino's client protocol, `explain` a
209
247
  `EXPLAIN <query>`, which compiles without running; `search` and `get-objects` use the server's
210
248
  `/v1/search` and `/v1/objects`. SQL of `-` is read from stdin.
211
249
 
250
+ `query` and `explain` read their SQL in the target's dialect, e.g. BigQuery's
251
+ `DATE_TRUNC(date, YEAR)` for a BigQuery target, as a MetricFlow model's is, so a query reads the
252
+ same with `--local` as without. `--dialect` names another, e.g. `--dialect trino` for Trino's
253
+ `DATE_TRUNC('year', date)`: any of sqlglot's dialects, case-insensitively. The server answers an
254
+ unknown one with an error, which the command fails with. Either way, the SQL compiles to the
255
+ warehouse's dialect. The server is told the dialect as `X-Dbt-Dialect`; Trino clients that don't
256
+ send it are read as Trino's.
257
+
258
+ ```sh
259
+ dbtsl query --dialect trino "SELECT DATE_TRUNC('year', date) AS year, AGG(revenue) FROM sales GROUP BY 1"
260
+ ```
261
+
212
262
  With `--local`, they parse the project, and read its own semantics instead, as `dbt build`
213
263
  would compile them, on the server the plugin runs to compile MetricFlow models: `query` compiles the SQL there
214
264
  and runs it on the target's warehouse, with dbt's own adapter, so the tables it reads must be
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  name = "dbt-metricflow-sql"
3
3
  # A placeholder: a release sets its tag's version, with scripts/set_version.py.
4
- version = "0.2.0"
4
+ version = "0.3.0"
5
5
  description = "dbt v1 plugin for MetricFlow"
6
6
  # Its PyPI page: README.md is for this repository.
7
7
  readme = "PYPI.md"
@@ -12,12 +12,16 @@
12
12
  - `dbtsl get-objects ID...`: prints the semantic layer's objects by id, as JSON.
13
13
  - `dbtsl docs [PAGE]`: prints MetricFlow's docs, as Markdown, for AI agents: with no page, the
14
14
  pages it has; with `--all`, every page.
15
+ - `dbtsl config profiles|logins|mappings|grants|environments|status`: sets up the dbt Cloud
16
+ account's Semantic Layer for the project to publish to (see `dbt_metricflow_sql.sl_config`).
15
17
 
16
18
  Each of the last four reads the server and environment the project publishes to, with the same
17
19
  vars and token, the environment naming the Trino catalog: it loads the project's and profile's
18
20
  config, with the dbt arguments it's given, e.g. `--target prod`, without parsing the project, so
19
21
  nothing else in the project, e.g. a model that doesn't parse, can fail it. Queries run on the
20
- server, over Trino's protocol, and so does `explain`, as `EXPLAIN <query>`. With `--local`, it
22
+ server, over Trino's protocol, and so does `explain`, as `EXPLAIN <query>`, their SQL read in the
23
+ target's dialect, e.g. BigQuery's, as a MetricFlow model's is, or in the one `--dialect` names,
24
+ e.g. `--dialect trino`. With `--local`, it
21
25
  parses the project, quietly, and reads its own semantics instead, on the server the plugin runs
22
26
  to compile MetricFlow models; a query runs on the target's warehouse, through dbt's adapter. SQL
23
27
  of `-` is read from stdin.
@@ -47,7 +51,7 @@ from dbt_common.clients.system import get_env
47
51
  from dbt_common.context import set_invocation_context
48
52
  from dbt_common.exceptions import DbtRuntimeError
49
53
 
50
- from dbt_metricflow_sql import client, plugin
54
+ from dbt_metricflow_sql import client, plugin, sl_config
51
55
  from dbt_metricflow_sql.client import MetricFlowError, QueryResult
52
56
  from dbt_metricflow_sql.docs import DocsMissingError, DocsPage, read_docs, split_pages
53
57
  from dbt_metricflow_sql.semantics import SemanticsError
@@ -77,8 +81,12 @@ def main(argv: Sequence[str] | None = None) -> int:
77
81
  parser.error(f"unrecognized arguments: {' '.join(extra)}")
78
82
  if args.command == DOCS:
79
83
  return _docs(args.page, args.all)
84
+ if args.command == sl_config.CONFIG:
85
+ return sl_config.run(args, _load_config)
80
86
  reader: _Reader | None = (
81
- _local_reader(args.dbt_args) if args.local else _remote_reader(args.dbt_args)
87
+ _local_reader(args.dbt_args, args.dialect)
88
+ if args.local
89
+ else _remote_reader(args.dbt_args, args.dialect)
82
90
  )
83
91
  if reader is None:
84
92
  return 1
@@ -121,10 +129,12 @@ def _parser() -> argparse.ArgumentParser:
121
129
  ),
122
130
  )
123
131
  query = _reading_command(commands, QUERY, "run semantic SQL, printing its result")
132
+ _add_dialect(query)
124
133
  query.add_argument("sql", help="the semantic SQL to run, or `-` to read it from stdin")
125
134
  explain = _reading_command(
126
135
  commands, EXPLAIN, "compile semantic SQL, printing the warehouse SQL it compiles to"
127
136
  )
137
+ _add_dialect(explain)
128
138
  explain.add_argument("sql", help="the semantic SQL to compile, or `-` to read it from stdin")
129
139
  search = _reading_command(commands, SEARCH, "search the semantic layer's objects")
130
140
  search.add_argument("query", nargs="+", help="keywords, e.g. `return rate`")
@@ -145,9 +155,22 @@ def _parser() -> argparse.ArgumentParser:
145
155
  )
146
156
  docs.add_argument("page", nargs="?", help="the page to print, e.g. `concepts`")
147
157
  docs.add_argument("--all", action="store_true", help="print every page")
158
+ sl_config.add_parser(commands, _add_dbt_options)
148
159
  return parser
149
160
 
150
161
 
162
+ def _add_dbt_options(command: argparse.ArgumentParser, options: Sequence[str]) -> None:
163
+ """dbt's `options`, e.g. `--target`, collected into `dbt_args` in the order given."""
164
+ command.set_defaults(dbt_args=[])
165
+ for option in options:
166
+ command.add_argument(
167
+ option,
168
+ action=_PassToDbt,
169
+ metavar=option.lstrip("-").upper(),
170
+ help="as dbt's own option",
171
+ )
172
+
173
+
151
174
  def _reading_command(
152
175
  commands: argparse._SubParsersAction[argparse.ArgumentParser], name: str, summary: str
153
176
  ) -> argparse.ArgumentParser:
@@ -168,7 +191,8 @@ def _reading_command(
168
191
  " queries on the target's warehouse with dbt's adapter"
169
192
  ),
170
193
  )
171
- command.set_defaults(dbt_args=[])
194
+ # Only `query` and `explain` take `--dialect`.
195
+ command.set_defaults(dbt_args=[], dialect=None)
172
196
  for option in _DBT_OPTIONS:
173
197
  command.add_argument(
174
198
  option,
@@ -179,6 +203,21 @@ def _reading_command(
179
203
  return command
180
204
 
181
205
 
206
+ def _add_dialect(command: argparse.ArgumentParser) -> None:
207
+ """`--dialect`, the dialect the command's SQL is written in, instead of the target's."""
208
+ command.add_argument(
209
+ "--dialect",
210
+ # Case-insensitive, e.g. `BigQuery`, as the server's statements are, though `/v1/compile`'s
211
+ # aren't. The server checks the name, so it's not checked here.
212
+ type=str.lower,
213
+ metavar="DIALECT",
214
+ help=(
215
+ "the SQL dialect the semantic SQL is written in, e.g. `trino` or `bigquery`"
216
+ " (default: the target's, e.g. `bigquery` for a BigQuery target)"
217
+ ),
218
+ )
219
+
220
+
182
221
  class _PassToDbt(argparse.Action):
183
222
  """Collects an option, e.g. `--target prod`, into `dbt_args`, in the order given."""
184
223
 
@@ -215,9 +254,16 @@ class _RemoteReader:
215
254
  """Reads the environment the project publishes to, on the server it publishes to."""
216
255
 
217
256
  layer: plugin.SemanticLayer
257
+ # The dialect queries are written in, if not the target's (see `SemanticLayer.input_dialect`).
258
+ dialect: str | None = None
218
259
 
219
260
  def query(self, sql: str) -> QueryResult:
220
- return client.run_query(self.layer.url, sql, self.layer.headers)
261
+ return client.run_query(
262
+ self.layer.url,
263
+ sql,
264
+ self.layer.headers,
265
+ input_dialect=self.dialect or self.layer.input_dialect,
266
+ )
221
267
 
222
268
  def explain(self, sql: str) -> str:
223
269
  # The server compiles `EXPLAIN <query>` without running it, in the environment's dialect.
@@ -240,9 +286,11 @@ class _LocalReader:
240
286
  manifest: Manifest
241
287
  # The invocation's, as the plugin's own hooks find it.
242
288
  config: RuntimeConfig
289
+ # The dialect queries are written in, if not the target's.
290
+ dialect: str | None = None
243
291
 
244
292
  def query(self, sql: str) -> QueryResult:
245
- compiled = plugin.compile_locally(self.manifest, self.config, sql)
293
+ compiled = plugin.compile_locally(self.manifest, self.config, sql, self.dialect)
246
294
  if compiled.explain:
247
295
  # As the server answers `EXPLAIN <query>`: its compiled SQL, without running it.
248
296
  return QueryResult(
@@ -262,7 +310,7 @@ class _LocalReader:
262
310
  )
263
311
 
264
312
  def explain(self, sql: str) -> str:
265
- return plugin.compile_locally(self.manifest, self.config, sql).sql
313
+ return plugin.compile_locally(self.manifest, self.config, sql, self.dialect).sql
266
314
 
267
315
  def search(self, query: str, limit: int) -> tuple[client.SearchHit, ...]:
268
316
  return client.search(plugin.local_url(self.manifest, self.config), query, limit)
@@ -271,13 +319,13 @@ class _LocalReader:
271
319
  return client.get_objects(plugin.local_url(self.manifest, self.config), ids)
272
320
 
273
321
 
274
- def _remote_reader(dbt_args: Sequence[str]) -> _RemoteReader | None:
322
+ def _remote_reader(dbt_args: Sequence[str], dialect: str | None = None) -> _RemoteReader | None:
275
323
  """
276
324
  The reader of the server the project publishes to, from its config alone (see
277
325
  `_load_config`), or None, reporting why, if it can't find the server.
278
326
  """
279
327
  try:
280
- return _RemoteReader(plugin.semantic_layer(_load_config(dbt_args)))
328
+ return _RemoteReader(plugin.semantic_layer(_load_config(dbt_args)), dialect)
281
329
  except click.ClickException as error:
282
330
  # dbt's own option's, e.g. `--vars` that isn't YAML.
283
331
  print(error.format_message(), file=sys.stderr)
@@ -306,10 +354,10 @@ def _load_config(dbt_args: Sequence[str]) -> RuntimeConfig:
306
354
  return RuntimeConfig.from_args(flags)
307
355
 
308
356
 
309
- def _local_reader(dbt_args: Sequence[str]) -> _LocalReader | None:
357
+ def _local_reader(dbt_args: Sequence[str], dialect: str | None = None) -> _LocalReader | None:
310
358
  """The reader of the project `dbt parse` parses, quietly, or None if it fails, as dbt logs."""
311
359
  parsed = _parse(dbt_args, quiet=True)
312
- return None if parsed is None else _LocalReader(*parsed)
360
+ return None if parsed is None else _LocalReader(*parsed, dialect)
313
361
 
314
362
 
315
363
  def _parse(dbt_args: Sequence[str], quiet: bool = False) -> tuple[Manifest, RuntimeConfig] | None:
@@ -26,6 +26,9 @@ PROFILE_HEADER = "X-Dbt-Profile"
26
26
  # The header naming the catalog, i.e. the environment, a query, compile, search or lookup reads, as
27
27
  # Trino's protocol names a session's catalog, e.g. `prod`. Without it, the server reads its default.
28
28
  TRINO_CATALOG_HEADER = "X-Trino-Catalog"
29
+ # The header naming the dialect a query's semantic SQL is written in, e.g. `bigquery`. Without it,
30
+ # the server reads it as Trino's; it answers one it doesn't know with an error.
31
+ DIALECT_HEADER = "X-Dbt-Dialect"
29
32
  # Trino's protocol requires a user; the server identifies the caller by its token instead.
30
33
  _TRINO_USER_HEADER = "X-Trino-User"
31
34
  _TRINO_USER = "dbtsl"
@@ -60,13 +63,16 @@ class Compiled:
60
63
  explain: bool = False
61
64
 
62
65
 
63
- def compile_sql(url: str, sql: str, dialect: str) -> Compiled:
64
- """Metricflow SQL, in the warehouse's `dialect`, to warehouse SQL, pretty-printed."""
66
+ def compile_sql(url: str, sql: str, dialect: str, input_dialect: str | None = None) -> Compiled:
67
+ """
68
+ Metricflow SQL, in `input_dialect` if given, else the warehouse's `dialect`, to warehouse SQL,
69
+ pretty-printed.
70
+ """
65
71
  response = _request(
66
72
  url,
67
73
  "POST",
68
74
  "/v1/compile",
69
- {"sql": sql, "input_dialect": dialect, "output_dialect": dialect},
75
+ {"sql": sql, "input_dialect": input_dialect or dialect, "output_dialect": dialect},
70
76
  ).body
71
77
  return Compiled(
72
78
  sql=str(response["sql"]),
@@ -175,14 +181,22 @@ class QueryResult:
175
181
  numeric: tuple[bool, ...]
176
182
 
177
183
 
178
- def run_query(url: str, sql: str, headers: Mapping[str, str] | None = None) -> QueryResult:
184
+ def run_query(
185
+ url: str,
186
+ sql: str,
187
+ headers: Mapping[str, str] | None = None,
188
+ input_dialect: str | None = None,
189
+ ) -> QueryResult:
179
190
  """
180
191
  Runs the semantic SQL `sql` on the server at `url`, over Trino's client protocol: submits it
181
192
  to `/v1/statement`, then follows each response's `nextUri` until there is none, collecting its
182
193
  rows. A `nextUri` is requested from `url`'s own scheme and host, whatever it names, so
183
- `headers`, e.g. credentials, never go elsewhere. Raises `MetricFlowError` if the query fails.
194
+ `headers`, e.g. credentials, never go elsewhere. `sql` is read in `input_dialect`, e.g.
195
+ `bigquery`, if given, else as Trino's. Raises `MetricFlowError` if the query fails.
184
196
  """
185
197
  request_headers = {**(headers or {}), _TRINO_USER_HEADER: _TRINO_USER}
198
+ if input_dialect:
199
+ request_headers[DIALECT_HEADER] = input_dialect
186
200
  results = _request(
187
201
  url,
188
202
  "POST",
@@ -26,9 +26,9 @@
26
26
 
27
27
  1. [Signup](https://www.getdbt.com/signup) if you don’t already have an account.
28
28
 
29
- 2. Configure a data warehouse connection
29
+ 2. Set up [dbt Platform CLI](https://docs.getdbt.com/docs/platform/dbt-cli-installation)
30
30
 
31
- 3. Set up [dbt Platform CLI](https://docs.getdbt.com/docs/platform/dbt-cli-installation)
31
+ 3. [Configure the Semantic Layer in dbt Platform](../dbt-platform-setup/)
32
32
 
33
33
  4. **Model your data**
34
34
 
@@ -63,6 +63,67 @@
63
63
 
64
64
  To compile queries locally, use the `--local` flag on CLI commands:
65
65
 
66
+ # dbt Platform setup
67
+
68
+ > Set up dbt Cloud's Semantic Layer for a project with `dbtsl config`, from the command line.
69
+
70
+ Before a project publishes to dbt Cloud’s Semantic Layer for the first time, its account needs:
71
+
72
+ * a **profile**: a warehouse connection, which an environment runs as;
73
+ * a **login** on the profile: the warehouse credential queries run with;
74
+ * a **mapping** from a dbt Cloud group to the login, so the group’s users run as it. A user in several mapped groups runs as the login of the mapping with the highest priority;
75
+ * or a **grant** of the login to a service token. Service tokens ignore mappings.
76
+
77
+ dbt Labs turns on the account’s `experimental-semantic-layer` flag. Then `dbtsl config` sets up the rest, and the first `dbtsl publish` creates the environment on the profile. The account’s first environment becomes its default.
78
+
79
+ ## The recipe
80
+
81
+ Start with `status`. It is read-only, and it reports every gap, each with the command that fills it. Its UPPERCASE arguments are placeholders to fill in.
82
+
83
+ ```sh
84
+ dbtsl config status
85
+ ```
86
+
87
+ 1. Create the profile. Name it after the target that publishes, e.g. `prod`, or set the `metricflow_publish_profile` var to its name.
88
+
89
+ ```sh
90
+ dbtsl config profiles create prod --adapter-type bigquery \
91
+ --connection project_id=my-project --if-missing
92
+ # Snowflake:
93
+ dbtsl config profiles create prod --adapter-type snowflake --connection account=ab12345 \
94
+ --connection database=ANALYTICS --connection warehouse=REPORTING --if-missing
95
+ ```
96
+
97
+ 2. Create a login with its secret. Secrets come only from files and environment variables, and are never printed.
98
+
99
+ ```sh
100
+ dbtsl config logins create --profile prod analytics_sa --keyfile ~/keys/sa.json --if-missing
101
+ # Snowflake, with a key pair, or `--password-env VAR`:
102
+ dbtsl config logins create --profile prod reporter --login-detail user=REPORTER \
103
+ --login-detail role=REPORTER --private-key-file ~/keys/reporter.p8 \
104
+ --private-key-passphrase-env SNOWFLAKE_PASSPHRASE --if-missing
105
+ ```
106
+
107
+ 3. Map a group to the login, or grant the login to a service token, e.g. CI’s. Groups and service tokens are named by their name or id.
108
+
109
+ ```sh
110
+ dbtsl config mappings create --profile prod --login analytics_sa \
111
+ --group Analysts --priority 100 --if-missing
112
+ dbtsl config grants create --profile prod --login analytics_sa --service-token ci --if-missing
113
+ ```
114
+
115
+ 4. Publish. This creates the environment on the profile.
116
+
117
+ ```sh
118
+ dbtsl publish --target prod --vars '{metricflow_publish_profile: prod}'
119
+ ```
120
+
121
+ Then `dbtsl config status` reports `Ready.`
122
+
123
+ ## Details
124
+
125
+ Each command’s options are in its help: `dbtsl config --help`, and e.g. `dbtsl config logins create --help`.
126
+
66
127
  # Overview
67
128
 
68
129
  MetricFlow is the engine that powers the dbt semantic layer.
@@ -150,9 +150,9 @@ class MetricflowManager:
150
150
  # The server, once something needs it; or what kept it from starting.
151
151
  self._server: ServerProcess | None = None
152
152
  self._start_failure: MetricFlowError | SemanticsError | None = None
153
- # Compilations by dialect and semantic SQL, so a model compiles once, as dbt builds its
153
+ # Compilations by dialect, input dialect and semantic SQL, so a model compiles once, as dbt builds its
154
154
  # graph, however many times dbt compiles it.
155
- self._compiled: dict[tuple[str, str], Compiled] = {}
155
+ self._compiled: dict[tuple[str, str | None, str], Compiled] = {}
156
156
  # The physical tables each model's compiled SQL reads, by unique id.
157
157
  self.tables: dict[str, tuple[tuple[str, ...], ...]] = {}
158
158
 
@@ -199,15 +199,22 @@ class MetricflowManager:
199
199
  raise MetricFlowError(f"starting the server failed: {failure}") from failure
200
200
  return server.url
201
201
 
202
- def compile(self, manifest: Manifest, config: RuntimeConfig, sql: str) -> Compiled:
202
+ def compile(
203
+ self,
204
+ manifest: Manifest,
205
+ config: RuntimeConfig,
206
+ sql: str,
207
+ input_dialect: str | None = None,
208
+ ) -> Compiled:
209
+ """Compiles `sql`, in `input_dialect` if given, else the warehouse's, to the warehouse's."""
203
210
  url = self.url(manifest, config)
204
211
  # dbt's adapter types are sqlglot's dialect names: bigquery, duckdb, snowflake, ...
205
- key = (config.credentials.type, sql)
212
+ key = (config.credentials.type, input_dialect, sql)
206
213
  with self._lock:
207
214
  compiled = self._compiled.get(key)
208
215
  # Outside the lock, so models compile concurrently on dbt's threads.
209
216
  if compiled is None:
210
- compiled = client.compile_sql(url, sql, config.credentials.type)
217
+ compiled = client.compile_sql(url, sql, config.credentials.type, input_dialect)
211
218
  with self._lock:
212
219
  self._compiled[key] = compiled
213
220
  return compiled
@@ -353,10 +360,20 @@ def _publish_target(config: RuntimeConfig, publishing: str) -> _PublishTarget |
353
360
 
354
361
  def _dbt_cloud_target(config: RuntimeConfig) -> _PublishTarget:
355
362
  """
356
- The Semantic Layer of the dbt Cloud account of the project's entry in `DBT_CLOUD_CONFIG`, with
357
- the entry's token. The entry is `dbt_project.yml`'s `dbt-cloud: project-id`, else the file's
358
- active project. Raises `MetricFlowError` naming what's missing. Never echoes the file, which
359
- holds tokens: YAML's errors quote the line they're on.
363
+ The Semantic Layer of the dbt Cloud account of the project's entry in `DBT_CLOUD_CONFIG` (see
364
+ `_dbt_cloud_entry`), with the entry's token.
365
+ """
366
+ entry = _dbt_cloud_entry(config)
367
+ host = str(entry.get("account-host") or "")
368
+ token = str(entry.get("token-value") or "").strip() or None
369
+ return _PublishTarget(_semantic_layer_url(host), token)
370
+
371
+
372
+ def _dbt_cloud_entry(config: RuntimeConfig) -> dict[str, Any]:
373
+ """
374
+ The project's entry in `DBT_CLOUD_CONFIG`: `dbt_project.yml`'s `dbt-cloud: project-id`, else
375
+ the file's active project. Raises `MetricFlowError` naming what's missing. Never echoes the
376
+ file, which holds tokens: YAML's errors quote the line they're on.
360
377
  """
361
378
  path = DBT_CLOUD_CONFIG
362
379
  try:
@@ -387,10 +404,8 @@ def _dbt_cloud_target(config: RuntimeConfig) -> _PublishTarget:
387
404
  ]
388
405
  if not entries:
389
406
  raise MetricFlowError(f"{path} has no project {str(project_id).strip()}")
390
- entry = entries[0]
391
- host = str(entry.get("account-host") or "")
392
- token = str(entry.get("token-value") or "").strip() or None
393
- return _PublishTarget(_semantic_layer_url(host), token)
407
+ entry: dict[str, Any] = entries[0]
408
+ return entry
394
409
 
395
410
 
396
411
  def _semantic_layer_url(account_host: str) -> str:
@@ -548,14 +563,18 @@ class SemanticLayer:
548
563
  url: str
549
564
  # The token, and the catalog naming the environment, if any.
550
565
  headers: Mapping[str, str] = field(repr=False)
566
+ # The dialect queries' semantic SQL is written in: the target warehouse's, e.g. `bigquery`, as
567
+ # a MetricFlow model's is, so a query reads the same here as with `--local`.
568
+ input_dialect: str | None = None
551
569
 
552
570
 
553
571
  def semantic_layer(config: RuntimeConfig) -> SemanticLayer:
554
572
  """
555
573
  The server the project publishes to (see `_publish_target`), whatever the `metricflow_publish`
556
574
  var says, with the headers reading the environment it publishes to there: the token (see
557
- `_token_headers`), and the environment (see `_environment`) as the Trino catalog. Raises
558
- `MetricFlowError` for a server it can't find, or a header it can't send.
575
+ `_token_headers`), and the environment (see `_environment`) as the Trino catalog; queries are
576
+ read in the target's dialect. Raises `MetricFlowError` for a server it can't find, or a header
577
+ it can't send.
559
578
  """
560
579
  target = _publish_target(config, _Publishing.REQUIRED)
561
580
  assert target is not None, "a required publish always has a target"
@@ -565,7 +584,78 @@ def semantic_layer(config: RuntimeConfig) -> SemanticLayer:
565
584
  ((f"the `{ENVIRONMENT_VAR}` var", client.TRINO_CATALOG_HEADER, _environment(config)),),
566
585
  )
567
586
  )
568
- return SemanticLayer(url=target.url, headers=headers)
587
+ # dbt's adapter types are sqlglot's dialect names: bigquery, duckdb, snowflake, ...
588
+ return SemanticLayer(url=target.url, headers=headers, input_dialect=config.credentials.type)
589
+
590
+
591
+ @dataclass(frozen=True)
592
+ class DbtCloudAccount:
593
+ """The dbt Cloud account the project publishes to, for `dbtsl config` to set up."""
594
+
595
+ # dbt Cloud's API, e.g. `https://ab123.us1.dbt.com`.
596
+ api_url: str
597
+ account_id: int
598
+ # The headers authenticating to `api_url`: the bearer token.
599
+ api_headers: Mapping[str, str] = field(repr=False)
600
+ # Where the token came from, e.g. `DBT_CLOUD_TOKEN`; never the token itself.
601
+ token_source: str
602
+ # The server the project publishes to (see `_publish_target`), and the headers authenticating
603
+ # to it, as a publish sends them.
604
+ semantic_layer_url: str
605
+ semantic_layer_headers: Mapping[str, str] = field(repr=False)
606
+
607
+
608
+ def dbt_cloud_account(
609
+ config: RuntimeConfig, account_host: str | None = None, account_id: str | None = None
610
+ ) -> DbtCloudAccount:
611
+ """
612
+ The dbt Cloud account of the project's entry in `DBT_CLOUD_CONFIG` (see `_dbt_cloud_entry`),
613
+ or of `account_host` and `account_id` where they're given, then not needing the file. The
614
+ token is `PUBLISH_TOKEN_ENV`, else the entry's. A host may name its scheme: `http://` only for
615
+ this machine's, e.g. a test's. Raises `MetricFlowError` naming what's missing.
616
+ """
617
+ entry: Mapping[str, Any] = {}
618
+ if account_host is None or account_id is None:
619
+ entry = _dbt_cloud_entry(config)
620
+ host = (account_host or str(entry.get("account-host") or "")).strip().rstrip("/")
621
+ if not host:
622
+ raise MetricFlowError(f"{DBT_CLOUD_CONFIG}'s project has no `account-host`")
623
+ raw_id = (account_id or str(entry.get("account-id") or "")).strip()
624
+ # ASCII digits only, as `_project_id`'s.
625
+ if not re.fullmatch(r"[0-9]+", raw_id) or int(raw_id) == 0:
626
+ raise MetricFlowError("dbt Cloud's account id must be a positive integer")
627
+ scheme, separator, _ = host.partition("://")
628
+ api_url = host if separator and scheme in ("http", "https") else f"https://{host}"
629
+ entry_token = str(entry.get("token-value") or "").strip() or None
630
+ token_source = str(DBT_CLOUD_CONFIG)
631
+ if os.environ.get(PUBLISH_TOKEN_ENV, "").strip():
632
+ token_source = PUBLISH_TOKEN_ENV
633
+ api_headers = _token_headers(_PublishTarget(api_url, entry_token))
634
+ if not api_headers:
635
+ raise MetricFlowError(
636
+ f"setting up dbt Cloud needs a token: `{PUBLISH_TOKEN_ENV}`, or {DBT_CLOUD_CONFIG}'s"
637
+ )
638
+ # As a publish finds its server: the var's gets only `PUBLISH_TOKEN_ENV`.
639
+ url = _rendered_var(config, PUBLISH_URL_VAR)
640
+ target = _PublishTarget(url) if url else _PublishTarget(_semantic_layer_url(host), entry_token)
641
+ return DbtCloudAccount(
642
+ api_url=api_url,
643
+ account_id=int(raw_id),
644
+ api_headers=api_headers,
645
+ token_source=token_source,
646
+ semantic_layer_url=target.url,
647
+ semantic_layer_headers=_token_headers(target),
648
+ )
649
+
650
+
651
+ def publish_environment(config: RuntimeConfig) -> str:
652
+ """The MetricFlow environment the project publishes to (see `_environment`)."""
653
+ return _environment(config)
654
+
655
+
656
+ def publish_profile(config: RuntimeConfig) -> str | None:
657
+ """The dbt Cloud profile a publish creates its environment on: the `PROFILE_VAR` var."""
658
+ return _rendered_var(config, PROFILE_VAR)
569
659
 
570
660
 
571
661
  def local_url(manifest: Manifest, config: RuntimeConfig) -> str:
@@ -577,9 +667,14 @@ def local_url(manifest: Manifest, config: RuntimeConfig) -> str:
577
667
  return _manager.url(manifest, config)
578
668
 
579
669
 
580
- def compile_locally(manifest: Manifest, config: RuntimeConfig, sql: str) -> Compiled:
581
- """Compiles semantic SQL as a MetricFlow model's is, on the server `local_url` runs."""
582
- return _manager.compile(manifest, config, sql)
670
+ def compile_locally(
671
+ manifest: Manifest, config: RuntimeConfig, sql: str, input_dialect: str | None = None
672
+ ) -> Compiled:
673
+ """
674
+ Compiles semantic SQL as a MetricFlow model's is, on the server `local_url` runs: read in
675
+ `input_dialect` if given, else the warehouse's.
676
+ """
677
+ return _manager.compile(manifest, config, sql, input_dialect)
583
678
 
584
679
 
585
680
  @dataclass(frozen=True)