sqlmesh 0.195.1.dev4__py3-none-any.whl → 0.195.1.dev5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sqlmesh/_version.py CHANGED
@@ -17,5 +17,5 @@ __version__: str
17
17
  __version_tuple__: VERSION_TUPLE
18
18
  version_tuple: VERSION_TUPLE
19
19
 
20
- __version__ = version = '0.195.1.dev4'
21
- __version_tuple__ = version_tuple = (0, 195, 1, 'dev4')
20
+ __version__ = version = '0.195.1.dev5'
21
+ __version_tuple__ = version_tuple = (0, 195, 1, 'dev5')
sqlmesh/cli/main.py CHANGED
@@ -6,11 +6,15 @@ import sys
6
6
  import typing as t
7
7
 
8
8
  import click
9
-
10
9
  from sqlmesh import configure_logging, remove_excess_logs
11
10
  from sqlmesh.cli import error_handler
12
11
  from sqlmesh.cli import options as opt
13
- from sqlmesh.cli.example_project import ProjectTemplate, init_example_project
12
+ from sqlmesh.cli.project_init import (
13
+ ProjectTemplate,
14
+ init_example_project,
15
+ InitCliMode,
16
+ interactive_init,
17
+ )
14
18
  from sqlmesh.core.analytics import cli_analytics
15
19
  from sqlmesh.core.console import configure_console, get_console
16
20
  from sqlmesh.utils import Verbosity
@@ -22,6 +26,7 @@ from pathlib import Path
22
26
 
23
27
  logger = logging.getLogger(__name__)
24
28
 
29
+
25
30
  SKIP_LOAD_COMMANDS = (
26
31
  "clean",
27
32
  "create_external_models",
@@ -138,7 +143,7 @@ def cli(
138
143
 
139
144
 
140
145
  @cli.command("init")
141
- @click.argument("sql_dialect", required=False)
146
+ @click.argument("engine", required=False)
142
147
  @click.option(
143
148
  "-t",
144
149
  "--template",
@@ -160,24 +165,83 @@ def cli(
160
165
  @cli_analytics
161
166
  def init(
162
167
  ctx: click.Context,
163
- sql_dialect: t.Optional[str] = None,
168
+ engine: t.Optional[str] = None,
164
169
  template: t.Optional[str] = None,
165
170
  dlt_pipeline: t.Optional[str] = None,
166
171
  dlt_path: t.Optional[str] = None,
167
172
  ) -> None:
168
173
  """Create a new SQLMesh repository."""
169
- try:
170
- project_template = ProjectTemplate(template.lower() if template else "default")
171
- except ValueError:
172
- raise click.ClickException(f"Invalid project template '{template}'")
173
- init_example_project(
174
- ctx.obj,
175
- dialect=sql_dialect,
174
+ project_template = None
175
+ if template:
176
+ try:
177
+ project_template = ProjectTemplate(template.lower())
178
+ except ValueError:
179
+ template_strings = "', '".join([template.value for template in ProjectTemplate])
180
+ raise click.ClickException(
181
+ f"Invalid project template '{template}'. Please specify one of '{template_strings}'."
182
+ )
183
+
184
+ if engine or project_template == ProjectTemplate.DBT:
185
+ init_example_project(
186
+ path=ctx.obj,
187
+ template=project_template or ProjectTemplate.DEFAULT,
188
+ engine_type=engine,
189
+ pipeline=dlt_pipeline,
190
+ dlt_path=dlt_path,
191
+ )
192
+ return
193
+
194
+ import sqlmesh.utils.rich as srich
195
+
196
+ console = srich.console
197
+
198
+ project_template, engine_type, cli_mode = interactive_init(ctx.obj, console, project_template)
199
+
200
+ config_path = init_example_project(
201
+ path=ctx.obj,
176
202
  template=project_template,
203
+ engine_type=engine_type,
204
+ cli_mode=cli_mode or InitCliMode.DEFAULT,
177
205
  pipeline=dlt_pipeline,
178
206
  dlt_path=dlt_path,
179
207
  )
180
208
 
209
+ engine_install_text = ""
210
+ if engine_type and engine_type not in ("duckdb", "motherduck"):
211
+ install_text = (
212
+ "pyspark" if engine_type == "spark" else f"sqlmesh\\[{engine_type.replace('_', '')}]"
213
+ )
214
+ engine_install_text = f'• Run command in CLI to install your SQL engine\'s Python dependencies: pip install "{install_text}"\n'
215
+ # interactive init does not support DLT template
216
+ next_step_text = {
217
+ ProjectTemplate.DEFAULT: f"{engine_install_text}• Update your gateway connection settings (e.g., username/password) in the project configuration file:\n {config_path}",
218
+ ProjectTemplate.DBT: "",
219
+ }
220
+ next_step_text[ProjectTemplate.EMPTY] = next_step_text[ProjectTemplate.DEFAULT]
221
+
222
+ quickstart_text = {
223
+ ProjectTemplate.DEFAULT: "Quickstart guide:\nhttps://sqlmesh.readthedocs.io/en/stable/quickstart/cli/",
224
+ ProjectTemplate.DBT: "dbt guide:\nhttps://sqlmesh.readthedocs.io/en/stable/integrations/dbt/",
225
+ }
226
+ quickstart_text[ProjectTemplate.EMPTY] = quickstart_text[ProjectTemplate.DEFAULT]
227
+
228
+ console.print(f"""──────────────────────────────
229
+
230
+ Your SQLMesh project is ready!
231
+
232
+ Next steps:
233
+ {next_step_text[project_template]}
234
+ • Run command in CLI: sqlmesh plan
235
+ • (Optional) Explain a plan: sqlmesh plan --explain
236
+
237
+ {quickstart_text[project_template]}
238
+
239
+ Need help?
240
+ • Docs: https://sqlmesh.readthedocs.io
241
+ • Slack: https://www.tobikodata.com/slack
242
+ • GitHub: https://github.com/TobikoData/sqlmesh/issues
243
+ """)
244
+
181
245
 
182
246
  @cli.command("render")
183
247
  @click.argument("model")
@@ -0,0 +1,497 @@
1
+ import typing as t
2
+ from enum import Enum
3
+ from pathlib import Path
4
+ from dataclasses import dataclass
5
+ from rich.prompt import Prompt
6
+ from rich.console import Console
7
+ from sqlmesh.integrations.dlt import generate_dlt_models_and_settings
8
+ from sqlmesh.utils.date import yesterday_ds
9
+ from sqlmesh.utils.errors import SQLMeshError
10
+
11
+ from sqlmesh.core.config.connection import (
12
+ CONNECTION_CONFIG_TO_TYPE,
13
+ DIALECT_TO_TYPE,
14
+ INIT_DISPLAY_INFO_TO_TYPE,
15
+ )
16
+
17
+
18
+ PRIMITIVES = (str, int, bool, float)
19
+
20
+
21
+ class ProjectTemplate(Enum):
22
+ DEFAULT = "default"
23
+ DBT = "dbt"
24
+ EMPTY = "empty"
25
+ DLT = "dlt"
26
+
27
+
28
+ class InitCliMode(Enum):
29
+ DEFAULT = "default"
30
+ FLOW = "flow"
31
+
32
+
33
+ def _gen_config(
34
+ engine_type: t.Optional[str],
35
+ settings: t.Optional[str],
36
+ start: t.Optional[str],
37
+ template: ProjectTemplate,
38
+ cli_mode: InitCliMode,
39
+ dialect: t.Optional[str] = None,
40
+ ) -> str:
41
+ project_dialect = dialect or DIALECT_TO_TYPE.get(engine_type)
42
+
43
+ connection_settings = (
44
+ settings
45
+ or """ type: duckdb
46
+ database: db.db"""
47
+ )
48
+
49
+ if not settings and template != ProjectTemplate.DBT:
50
+ doc_link = "https://sqlmesh.readthedocs.io/en/stable/integrations/engines{engine_link}"
51
+ engine_link = ""
52
+
53
+ if engine_type in CONNECTION_CONFIG_TO_TYPE:
54
+ required_fields = []
55
+ non_required_fields = []
56
+
57
+ for name, field in CONNECTION_CONFIG_TO_TYPE[engine_type].model_fields.items():
58
+ field_name = field.alias or name
59
+
60
+ default_value = field.get_default()
61
+
62
+ if isinstance(default_value, Enum):
63
+ default_value = default_value.value
64
+ elif not isinstance(default_value, PRIMITIVES):
65
+ default_value = ""
66
+
67
+ required = field.is_required() or field_name == "type"
68
+ option_str = f" {'# ' if not required else ''}{field_name}: {default_value}\n"
69
+
70
+ # specify the DuckDB database field so quickstart runs out of the box
71
+ if engine_type == "duckdb" and field_name == "database":
72
+ option_str = " database: db.db\n"
73
+ required = True
74
+
75
+ if required:
76
+ required_fields.append(option_str)
77
+ else:
78
+ non_required_fields.append(option_str)
79
+
80
+ connection_settings = "".join(required_fields + non_required_fields)
81
+
82
+ engine_link = f"/{engine_type}/#connection-options"
83
+
84
+ connection_settings = (
85
+ " # For more information on configuring the connection to your execution engine, visit:\n"
86
+ " # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connection\n"
87
+ f" # {doc_link.format(engine_link=engine_link)}\n{connection_settings}"
88
+ )
89
+
90
+ default_configs = {
91
+ ProjectTemplate.DEFAULT: f"""# --- Gateway Connection ---
92
+ gateways:
93
+ {engine_type}:
94
+ connection:
95
+ {connection_settings}
96
+ default_gateway: {engine_type}
97
+
98
+ # --- Model Defaults ---
99
+ # https://sqlmesh.readthedocs.io/en/stable/reference/model_configuration/#model-defaults
100
+
101
+ model_defaults:
102
+ dialect: {project_dialect}
103
+ start: {start or yesterday_ds()} # Start date for backfill history
104
+ cron: '@daily' # Run models daily at 12am UTC (can override per model)
105
+
106
+ # --- Linting Rules ---
107
+ # Enforce standards for your team
108
+ # https://sqlmesh.readthedocs.io/en/stable/guides/linter/
109
+
110
+ linter:
111
+ enabled: true
112
+ rules:
113
+ - ambiguousorinvalidcolumn
114
+ - invalidselectstarexpansion
115
+ """,
116
+ ProjectTemplate.DBT: """from pathlib import Path
117
+
118
+ from sqlmesh.dbt.loader import sqlmesh_config
119
+
120
+ config = sqlmesh_config(Path(__file__).parent)
121
+ """,
122
+ }
123
+
124
+ default_configs[ProjectTemplate.EMPTY] = default_configs[ProjectTemplate.DEFAULT]
125
+ default_configs[ProjectTemplate.DLT] = default_configs[ProjectTemplate.DEFAULT]
126
+
127
+ flow_cli_mode = """
128
+ # FLOW: Minimal prompts, automatic changes, summary output
129
+ # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#plan
130
+
131
+ plan:
132
+ no_diff: true # Hide detailed text differences for changed models
133
+ no_prompts: true # No interactive prompts
134
+ auto_apply: true # Apply changes automatically
135
+
136
+ # --- Optional: Set a default target environment ---
137
+ # This is intended for local development to prevent users from accidentally applying plans to the prod environment.
138
+ # It is a development only config and should NOT be committed to your git repo.
139
+ # https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#default-target-environment
140
+
141
+ # Uncomment the following line to use a default target environment derived from the logged in user's name.
142
+ # default_target_environment: dev_{{ user() }}
143
+
144
+ # Example usage:
145
+ # sqlmesh plan # Automatically resolves to: sqlmesh plan dev_yourname
146
+ # sqlmesh plan prod # Specify `prod` to apply changes to production
147
+ """
148
+
149
+ return default_configs[template] + (flow_cli_mode if cli_mode == InitCliMode.FLOW else "")
150
+
151
+
152
+ @dataclass
153
+ class ExampleObjects:
154
+ sql_models: t.Dict[str, str]
155
+ python_models: t.Dict[str, str]
156
+ seeds: t.Dict[str, str]
157
+ audits: t.Dict[str, str]
158
+ tests: t.Dict[str, str]
159
+ sql_macros: t.Dict[str, str]
160
+ python_macros: t.Dict[str, str]
161
+
162
+
163
+ def _gen_example_objects(schema_name: str) -> ExampleObjects:
164
+ sql_models: t.Dict[str, str] = {}
165
+ python_models: t.Dict[str, str] = {}
166
+ seeds: t.Dict[str, str] = {}
167
+ audits: t.Dict[str, str] = {}
168
+ tests: t.Dict[str, str] = {}
169
+ sql_macros: t.Dict[str, str] = {}
170
+ python_macros: t.Dict[str, str] = {"__init__": ""}
171
+
172
+ full_model_name = f"{schema_name}.full_model"
173
+ incremental_model_name = f"{schema_name}.incremental_model"
174
+ seed_model_name = f"{schema_name}.seed_model"
175
+
176
+ sql_models[full_model_name] = f"""MODEL (
177
+ name {full_model_name},
178
+ kind FULL,
179
+ cron '@daily',
180
+ grain item_id,
181
+ audits (assert_positive_order_ids),
182
+ );
183
+
184
+ SELECT
185
+ item_id,
186
+ COUNT(DISTINCT id) AS num_orders,
187
+ FROM
188
+ {incremental_model_name}
189
+ GROUP BY item_id
190
+ """
191
+
192
+ sql_models[incremental_model_name] = f"""MODEL (
193
+ name {incremental_model_name},
194
+ kind INCREMENTAL_BY_TIME_RANGE (
195
+ time_column event_date
196
+ ),
197
+ start '2020-01-01',
198
+ cron '@daily',
199
+ grain (id, event_date)
200
+ );
201
+
202
+ SELECT
203
+ id,
204
+ item_id,
205
+ event_date,
206
+ FROM
207
+ {seed_model_name}
208
+ WHERE
209
+ event_date BETWEEN @start_date AND @end_date
210
+ """
211
+
212
+ sql_models[seed_model_name] = f"""MODEL (
213
+ name {seed_model_name},
214
+ kind SEED (
215
+ path '../seeds/seed_data.csv'
216
+ ),
217
+ columns (
218
+ id INTEGER,
219
+ item_id INTEGER,
220
+ event_date DATE
221
+ ),
222
+ grain (id, event_date)
223
+ );
224
+ """
225
+
226
+ seeds["seed_data"] = """id,item_id,event_date
227
+ 1,2,2020-01-01
228
+ 2,1,2020-01-01
229
+ 3,3,2020-01-03
230
+ 4,1,2020-01-04
231
+ 5,1,2020-01-05
232
+ 6,1,2020-01-06
233
+ 7,1,2020-01-07
234
+ """
235
+
236
+ audits["assert_positive_order_ids"] = """AUDIT (
237
+ name assert_positive_order_ids,
238
+ );
239
+
240
+ SELECT *
241
+ FROM @this_model
242
+ WHERE
243
+ item_id < 0
244
+ """
245
+
246
+ tests["test_full_model"] = f"""test_example_full_model:
247
+ model: {full_model_name}
248
+ inputs:
249
+ {incremental_model_name}:
250
+ rows:
251
+ - id: 1
252
+ item_id: 1
253
+ - id: 2
254
+ item_id: 1
255
+ - id: 3
256
+ item_id: 2
257
+ outputs:
258
+ query:
259
+ rows:
260
+ - item_id: 1
261
+ num_orders: 2
262
+ - item_id: 2
263
+ num_orders: 1
264
+ """
265
+
266
+ return ExampleObjects(
267
+ sql_models=sql_models,
268
+ python_models=python_models,
269
+ seeds=seeds,
270
+ audits=audits,
271
+ tests=tests,
272
+ python_macros=python_macros,
273
+ sql_macros=sql_macros,
274
+ )
275
+
276
+
277
+ def init_example_project(
278
+ path: t.Union[str, Path],
279
+ engine_type: t.Optional[str],
280
+ dialect: t.Optional[str] = None,
281
+ template: ProjectTemplate = ProjectTemplate.DEFAULT,
282
+ pipeline: t.Optional[str] = None,
283
+ dlt_path: t.Optional[str] = None,
284
+ schema_name: str = "sqlmesh_example",
285
+ cli_mode: InitCliMode = InitCliMode.DEFAULT,
286
+ ) -> Path:
287
+ root_path = Path(path)
288
+ config_extension = "py" if template == ProjectTemplate.DBT else "yaml"
289
+ config_path = root_path / f"config.{config_extension}"
290
+ audits_path = root_path / "audits"
291
+ macros_path = root_path / "macros"
292
+ models_path = root_path / "models"
293
+ seeds_path = root_path / "seeds"
294
+ tests_path = root_path / "tests"
295
+
296
+ if config_path.exists():
297
+ raise SQLMeshError(
298
+ f"Found an existing config file '{config_path}'.\n\nPlease change to another directory or remove the existing file."
299
+ )
300
+
301
+ if template == ProjectTemplate.DBT and not Path(root_path, "dbt_project.yml").exists():
302
+ raise SQLMeshError(
303
+ "Required dbt project file 'dbt_project.yml' not found in the current directory.\n\nPlease add it or change directories before running `sqlmesh init` to set up your project."
304
+ )
305
+
306
+ engine_types = "', '".join(CONNECTION_CONFIG_TO_TYPE)
307
+ if engine_type is None and template != ProjectTemplate.DBT:
308
+ raise SQLMeshError(
309
+ f"Missing `engine` argument to `sqlmesh init` - please specify a SQL engine for your project. Options: '{engine_types}'."
310
+ )
311
+
312
+ if engine_type and engine_type not in CONNECTION_CONFIG_TO_TYPE:
313
+ raise SQLMeshError(
314
+ f"Invalid engine '{engine_type}'. Please specify one of '{engine_types}'."
315
+ )
316
+
317
+ models: t.Set[t.Tuple[str, str]] = set()
318
+ settings = None
319
+ start = None
320
+ if engine_type and template == ProjectTemplate.DLT:
321
+ project_dialect = dialect or DIALECT_TO_TYPE.get(engine_type)
322
+ if pipeline and project_dialect:
323
+ dlt_models, settings, start = generate_dlt_models_and_settings(
324
+ pipeline_name=pipeline, dialect=project_dialect, dlt_path=dlt_path
325
+ )
326
+ else:
327
+ raise SQLMeshError(
328
+ "Please provide a DLT pipeline with the `--dlt-pipeline` flag to generate a SQLMesh project from DLT."
329
+ )
330
+
331
+ _create_config(config_path, engine_type, dialect, settings, start, template, cli_mode)
332
+ if template == ProjectTemplate.DBT:
333
+ return config_path
334
+
335
+ _create_folders([audits_path, macros_path, models_path, seeds_path, tests_path])
336
+
337
+ if template == ProjectTemplate.DLT:
338
+ _create_object_files(
339
+ models_path, {model[0].split(".")[-1]: model[1] for model in dlt_models}, "sql"
340
+ )
341
+ return config_path
342
+
343
+ example_objects = _gen_example_objects(schema_name=schema_name)
344
+
345
+ if template != ProjectTemplate.EMPTY:
346
+ _create_object_files(models_path, example_objects.sql_models, "sql")
347
+ _create_object_files(models_path, example_objects.python_models, "py")
348
+ _create_object_files(seeds_path, example_objects.seeds, "csv")
349
+ _create_object_files(audits_path, example_objects.audits, "sql")
350
+ _create_object_files(tests_path, example_objects.tests, "yaml")
351
+ _create_object_files(macros_path, example_objects.python_macros, "py")
352
+ _create_object_files(macros_path, example_objects.sql_macros, "sql")
353
+
354
+ return config_path
355
+
356
+
357
+ def _create_folders(target_folders: t.Sequence[Path]) -> None:
358
+ for folder_path in target_folders:
359
+ folder_path.mkdir(exist_ok=True)
360
+ (folder_path / ".gitkeep").touch()
361
+
362
+
363
+ def _create_config(
364
+ config_path: Path,
365
+ engine_type: t.Optional[str],
366
+ dialect: t.Optional[str],
367
+ settings: t.Optional[str],
368
+ start: t.Optional[str],
369
+ template: ProjectTemplate,
370
+ cli_mode: InitCliMode,
371
+ ) -> None:
372
+ project_config = _gen_config(engine_type, settings, start, template, cli_mode, dialect)
373
+
374
+ _write_file(
375
+ config_path,
376
+ project_config,
377
+ )
378
+
379
+
380
+ def _create_object_files(path: Path, object_dict: t.Dict[str, str], file_extension: str) -> None:
381
+ for object_name, object_def in object_dict.items():
382
+ # file name is table component of catalog.schema.table
383
+ _write_file(path / f"{object_name.split('.')[-1]}.{file_extension}", object_def)
384
+
385
+
386
+ def _write_file(path: Path, payload: str) -> None:
387
+ with open(path, "w", encoding="utf-8") as fd:
388
+ fd.write(payload)
389
+
390
+
391
+ def interactive_init(
392
+ path: Path,
393
+ console: Console,
394
+ project_template: t.Optional[ProjectTemplate] = None,
395
+ ) -> t.Tuple[ProjectTemplate, t.Optional[str], t.Optional[InitCliMode]]:
396
+ console.print("──────────────────────────────")
397
+ console.print("Welcome to SQLMesh!")
398
+
399
+ project_template = _init_template_prompt(console) if not project_template else project_template
400
+
401
+ if project_template == ProjectTemplate.DBT:
402
+ return (project_template, None, None)
403
+
404
+ engine_type = _init_engine_prompt(console)
405
+ cli_mode = _init_cli_mode_prompt(console)
406
+
407
+ return (project_template, engine_type, cli_mode)
408
+
409
+
410
+ def _init_integer_prompt(
411
+ console: Console, err_msg_entity: str, num_options: int, retry_func: t.Callable[[t.Any], t.Any]
412
+ ) -> int:
413
+ err_msg = "\nERROR: '{option_str}' is not a valid {err_msg_entity} number - please enter a number between 1 and {num_options} or exit with control+c\n"
414
+ while True:
415
+ option_str = Prompt.ask("Enter a number", console=console)
416
+
417
+ value_error = False
418
+ try:
419
+ option_num = int(option_str)
420
+ except ValueError:
421
+ value_error = True
422
+
423
+ if value_error or option_num < 1 or option_num > num_options:
424
+ console.print(
425
+ err_msg.format(
426
+ option_str=option_str, err_msg_entity=err_msg_entity, num_options=num_options
427
+ ),
428
+ style="red",
429
+ )
430
+ continue
431
+ console.print("")
432
+ return option_num
433
+
434
+
435
+ def _init_display_choices(values_dict: t.Dict[str, str], console: Console) -> t.Dict[int, str]:
436
+ display_num_to_value = {}
437
+ for i, value_str in enumerate(values_dict.keys()):
438
+ console.print(f" \[{i + 1}] {' ' if i < 9 else ''}{value_str} {values_dict[value_str]}")
439
+ display_num_to_value[i + 1] = value_str
440
+ console.print("")
441
+ return display_num_to_value
442
+
443
+
444
+ def _init_template_prompt(console: Console) -> ProjectTemplate:
445
+ console.print("──────────────────────────────\n")
446
+ console.print("What type of project do you want to set up?\n")
447
+
448
+ # These are ordered for user display - do not reorder
449
+ template_descriptions = {
450
+ ProjectTemplate.DEFAULT.name: "- Create SQLMesh example project models and files",
451
+ ProjectTemplate.DBT.value: " - You have an existing dbt project and want to run it with SQLMesh",
452
+ ProjectTemplate.EMPTY.name: " - Create a SQLMesh configuration file and project directories only",
453
+ }
454
+
455
+ display_num_to_template = _init_display_choices(template_descriptions, console)
456
+
457
+ template_num = _init_integer_prompt(
458
+ console, "project type", len(template_descriptions), _init_template_prompt
459
+ )
460
+
461
+ return ProjectTemplate(display_num_to_template[template_num].lower())
462
+
463
+
464
+ def _init_engine_prompt(console: Console) -> str:
465
+ console.print("──────────────────────────────\n")
466
+ console.print("Choose your SQL engine:\n")
467
+
468
+ # INIT_DISPLAY_INFO_TO_TYPE is a dict of {engine_type: (display_order, display_name)}
469
+ DISPLAY_NAME_TO_TYPE = {v[1]: k for k, v in INIT_DISPLAY_INFO_TO_TYPE.items()}
470
+ ordered_engine_display_names = {
471
+ info[1]: "" for info in sorted(INIT_DISPLAY_INFO_TO_TYPE.values(), key=lambda x: x[0])
472
+ }
473
+ display_num_to_display_name = _init_display_choices(ordered_engine_display_names, console)
474
+
475
+ engine_num = _init_integer_prompt(
476
+ console, "engine", len(ordered_engine_display_names), _init_engine_prompt
477
+ )
478
+
479
+ return DISPLAY_NAME_TO_TYPE[display_num_to_display_name[engine_num]]
480
+
481
+
482
+ def _init_cli_mode_prompt(console: Console) -> InitCliMode:
483
+ console.print("──────────────────────────────\n")
484
+ console.print("Choose your SQLMesh CLI experience:\n")
485
+
486
+ cli_mode_descriptions = {
487
+ InitCliMode.DEFAULT.name: "- See and control every detail",
488
+ InitCliMode.FLOW.name: " - Automatically run changes and show summary output",
489
+ }
490
+
491
+ display_num_to_cli_mode = _init_display_choices(cli_mode_descriptions, console)
492
+
493
+ cli_mode_num = _init_integer_prompt(
494
+ console, "config", len(cli_mode_descriptions), _init_cli_mode_prompt
495
+ )
496
+
497
+ return InitCliMode(display_num_to_cli_mode[cli_mode_num].lower())
@@ -87,6 +87,9 @@ def _get_engine_import_validator(
87
87
 
88
88
  class ConnectionConfig(abc.ABC, BaseConfig):
89
89
  type_: str
90
+ DIALECT: t.ClassVar[str]
91
+ DISPLAY_NAME: t.ClassVar[str]
92
+ DISPLAY_ORDER: t.ClassVar[int]
90
93
  concurrent_tasks: int
91
94
  register_comments: bool
92
95
  pre_ping: bool
@@ -463,6 +466,9 @@ class MotherDuckConnectionConfig(BaseDuckDBConnectionConfig):
463
466
  """Configuration for the MotherDuck connection."""
464
467
 
465
468
  type_: t.Literal["motherduck"] = Field(alias="type", default="motherduck")
469
+ DIALECT: t.ClassVar[t.Literal["duckdb"]] = "duckdb"
470
+ DISPLAY_NAME: t.ClassVar[t.Literal["MotherDuck"]] = "MotherDuck"
471
+ DISPLAY_ORDER: t.ClassVar[t.Literal[5]] = 5
466
472
 
467
473
  @property
468
474
  def _connection_kwargs_keys(self) -> t.Set[str]:
@@ -487,6 +493,9 @@ class DuckDBConnectionConfig(BaseDuckDBConnectionConfig):
487
493
  """Configuration for the DuckDB connection."""
488
494
 
489
495
  type_: t.Literal["duckdb"] = Field(alias="type", default="duckdb")
496
+ DIALECT: t.ClassVar[t.Literal["duckdb"]] = "duckdb"
497
+ DISPLAY_NAME: t.ClassVar[t.Literal["DuckDB"]] = "DuckDB"
498
+ DISPLAY_ORDER: t.ClassVar[t.Literal[1]] = 1
490
499
 
491
500
 
492
501
  class SnowflakeConnectionConfig(ConnectionConfig):
@@ -537,6 +546,9 @@ class SnowflakeConnectionConfig(ConnectionConfig):
537
546
  session_parameters: t.Optional[dict] = None
538
547
 
539
548
  type_: t.Literal["snowflake"] = Field(alias="type", default="snowflake")
549
+ DIALECT: t.ClassVar[t.Literal["snowflake"]] = "snowflake"
550
+ DISPLAY_NAME: t.ClassVar[t.Literal["Snowflake"]] = "Snowflake"
551
+ DISPLAY_ORDER: t.ClassVar[t.Literal[2]] = 2
540
552
 
541
553
  _concurrent_tasks_validator = concurrent_tasks_validator
542
554
 
@@ -733,6 +745,9 @@ class DatabricksConnectionConfig(ConnectionConfig):
733
745
  pre_ping: t.Literal[False] = False
734
746
 
735
747
  type_: t.Literal["databricks"] = Field(alias="type", default="databricks")
748
+ DIALECT: t.ClassVar[t.Literal["databricks"]] = "databricks"
749
+ DISPLAY_NAME: t.ClassVar[t.Literal["Databricks"]] = "Databricks"
750
+ DISPLAY_ORDER: t.ClassVar[t.Literal[3]] = 3
736
751
 
737
752
  _concurrent_tasks_validator = concurrent_tasks_validator
738
753
  _http_headers_validator = http_headers_validator
@@ -989,6 +1004,9 @@ class BigQueryConnectionConfig(ConnectionConfig):
989
1004
  pre_ping: t.Literal[False] = False
990
1005
 
991
1006
  type_: t.Literal["bigquery"] = Field(alias="type", default="bigquery")
1007
+ DIALECT: t.ClassVar[t.Literal["bigquery"]] = "bigquery"
1008
+ DISPLAY_NAME: t.ClassVar[t.Literal["BigQuery"]] = "BigQuery"
1009
+ DISPLAY_ORDER: t.ClassVar[t.Literal[4]] = 4
992
1010
 
993
1011
  _engine_import_validator = _get_engine_import_validator("google.cloud.bigquery", "bigquery")
994
1012
 
@@ -1129,7 +1147,12 @@ class GCPPostgresConnectionConfig(ConnectionConfig):
1129
1147
  timeout: t.Optional[int] = None
1130
1148
  scopes: t.Tuple[str, ...] = ("https://www.googleapis.com/auth/sqlservice.admin",)
1131
1149
  driver: str = "pg8000"
1150
+
1132
1151
  type_: t.Literal["gcp_postgres"] = Field(alias="type", default="gcp_postgres")
1152
+ DIALECT: t.ClassVar[t.Literal["postgres"]] = "postgres"
1153
+ DISPLAY_NAME: t.ClassVar[t.Literal["GCP Postgres"]] = "GCP Postgres"
1154
+ DISPLAY_ORDER: t.ClassVar[t.Literal[13]] = 13
1155
+
1133
1156
  concurrent_tasks: int = 4
1134
1157
  register_comments: bool = True
1135
1158
  pre_ping: bool = True
@@ -1264,6 +1287,9 @@ class RedshiftConnectionConfig(ConnectionConfig):
1264
1287
  pre_ping: bool = False
1265
1288
 
1266
1289
  type_: t.Literal["redshift"] = Field(alias="type", default="redshift")
1290
+ DIALECT: t.ClassVar[t.Literal["redshift"]] = "redshift"
1291
+ DISPLAY_NAME: t.ClassVar[t.Literal["Redshift"]] = "Redshift"
1292
+ DISPLAY_ORDER: t.ClassVar[t.Literal[7]] = 7
1267
1293
 
1268
1294
  _engine_import_validator = _get_engine_import_validator("redshift_connector", "redshift")
1269
1295
 
@@ -1325,6 +1351,9 @@ class PostgresConnectionConfig(ConnectionConfig):
1325
1351
  pre_ping: bool = True
1326
1352
 
1327
1353
  type_: t.Literal["postgres"] = Field(alias="type", default="postgres")
1354
+ DIALECT: t.ClassVar[t.Literal["postgres"]] = "postgres"
1355
+ DISPLAY_NAME: t.ClassVar[t.Literal["Postgres"]] = "Postgres"
1356
+ DISPLAY_ORDER: t.ClassVar[t.Literal[12]] = 12
1328
1357
 
1329
1358
  _engine_import_validator = _get_engine_import_validator("psycopg2", "postgres")
1330
1359
 
@@ -1378,6 +1407,9 @@ class MySQLConnectionConfig(ConnectionConfig):
1378
1407
  pre_ping: bool = True
1379
1408
 
1380
1409
  type_: t.Literal["mysql"] = Field(alias="type", default="mysql")
1410
+ DIALECT: t.ClassVar[t.Literal["mysql"]] = "mysql"
1411
+ DISPLAY_NAME: t.ClassVar[t.Literal["MySQL"]] = "MySQL"
1412
+ DISPLAY_ORDER: t.ClassVar[t.Literal[14]] = 14
1381
1413
 
1382
1414
  _engine_import_validator = _get_engine_import_validator("pymysql", "mysql")
1383
1415
 
@@ -1440,6 +1472,9 @@ class MSSQLConnectionConfig(ConnectionConfig):
1440
1472
  pre_ping: bool = True
1441
1473
 
1442
1474
  type_: t.Literal["mssql"] = Field(alias="type", default="mssql")
1475
+ DIALECT: t.ClassVar[t.Literal["tsql"]] = "tsql"
1476
+ DISPLAY_NAME: t.ClassVar[t.Literal["MSSQL"]] = "MSSQL"
1477
+ DISPLAY_ORDER: t.ClassVar[t.Literal[11]] = 11
1443
1478
 
1444
1479
  @model_validator(mode="before")
1445
1480
  @classmethod
@@ -1581,6 +1616,8 @@ class MSSQLConnectionConfig(ConnectionConfig):
1581
1616
 
1582
1617
  class AzureSQLConnectionConfig(MSSQLConnectionConfig):
1583
1618
  type_: t.Literal["azuresql"] = Field(alias="type", default="azuresql") # type: ignore
1619
+ DISPLAY_NAME: t.ClassVar[t.Literal["Azure SQL"]] = "Azure SQL" # type: ignore
1620
+ DISPLAY_ORDER: t.ClassVar[t.Literal[10]] = 10 # type: ignore
1584
1621
 
1585
1622
  @property
1586
1623
  def _extra_engine_config(self) -> t.Dict[str, t.Any]:
@@ -1601,6 +1638,9 @@ class SparkConnectionConfig(ConnectionConfig):
1601
1638
  pre_ping: t.Literal[False] = False
1602
1639
 
1603
1640
  type_: t.Literal["spark"] = Field(alias="type", default="spark")
1641
+ DIALECT: t.ClassVar[t.Literal["spark"]] = "spark"
1642
+ DISPLAY_NAME: t.ClassVar[t.Literal["Spark"]] = "Spark"
1643
+ DISPLAY_ORDER: t.ClassVar[t.Literal[8]] = 8
1604
1644
 
1605
1645
  _engine_import_validator = _get_engine_import_validator("pyspark", "spark")
1606
1646
 
@@ -1719,6 +1759,9 @@ class TrinoConnectionConfig(ConnectionConfig):
1719
1759
  pre_ping: t.Literal[False] = False
1720
1760
 
1721
1761
  type_: t.Literal["trino"] = Field(alias="type", default="trino")
1762
+ DIALECT: t.ClassVar[t.Literal["trino"]] = "trino"
1763
+ DISPLAY_NAME: t.ClassVar[t.Literal["Trino"]] = "Trino"
1764
+ DISPLAY_ORDER: t.ClassVar[t.Literal[9]] = 9
1722
1765
 
1723
1766
  _engine_import_validator = _get_engine_import_validator("trino", "trino")
1724
1767
 
@@ -1879,6 +1922,9 @@ class ClickhouseConnectionConfig(ConnectionConfig):
1879
1922
  connection_pool_options: t.Optional[t.Dict[str, t.Any]] = None
1880
1923
 
1881
1924
  type_: t.Literal["clickhouse"] = Field(alias="type", default="clickhouse")
1925
+ DIALECT: t.ClassVar[t.Literal["clickhouse"]] = "clickhouse"
1926
+ DISPLAY_NAME: t.ClassVar[t.Literal["ClickHouse"]] = "ClickHouse"
1927
+ DISPLAY_ORDER: t.ClassVar[t.Literal[6]] = 6
1882
1928
 
1883
1929
  _engine_import_validator = _get_engine_import_validator("clickhouse_connect", "clickhouse")
1884
1930
 
@@ -2003,6 +2049,9 @@ class AthenaConnectionConfig(ConnectionConfig):
2003
2049
  pre_ping: t.Literal[False] = False
2004
2050
 
2005
2051
  type_: t.Literal["athena"] = Field(alias="type", default="athena")
2052
+ DIALECT: t.ClassVar[t.Literal["athena"]] = "athena"
2053
+ DISPLAY_NAME: t.ClassVar[t.Literal["Athena"]] = "Athena"
2054
+ DISPLAY_ORDER: t.ClassVar[t.Literal[15]] = 15
2006
2055
 
2007
2056
  _engine_import_validator = _get_engine_import_validator("pyathena", "athena")
2008
2057
 
@@ -2071,6 +2120,9 @@ class RisingwaveConnectionConfig(ConnectionConfig):
2071
2120
  pre_ping: bool = True
2072
2121
 
2073
2122
  type_: t.Literal["risingwave"] = Field(alias="type", default="risingwave")
2123
+ DIALECT: t.ClassVar[t.Literal["risingwave"]] = "risingwave"
2124
+ DISPLAY_NAME: t.ClassVar[t.Literal["RisingWave"]] = "RisingWave"
2125
+ DISPLAY_ORDER: t.ClassVar[t.Literal[16]] = 16
2074
2126
 
2075
2127
  _engine_import_validator = _get_engine_import_validator("psycopg2", "risingwave")
2076
2128
 
@@ -2115,6 +2167,27 @@ CONNECTION_CONFIG_TO_TYPE = {
2115
2167
  )
2116
2168
  }
2117
2169
 
2170
+ DIALECT_TO_TYPE = {
2171
+ tpe.all_field_infos()["type_"].default: tpe.DIALECT
2172
+ for tpe in subclasses(
2173
+ __name__,
2174
+ ConnectionConfig,
2175
+ exclude=(ConnectionConfig, BaseDuckDBConnectionConfig),
2176
+ )
2177
+ }
2178
+
2179
+ INIT_DISPLAY_INFO_TO_TYPE = {
2180
+ tpe.all_field_infos()["type_"].default: (
2181
+ tpe.DISPLAY_ORDER,
2182
+ tpe.DISPLAY_NAME,
2183
+ )
2184
+ for tpe in subclasses(
2185
+ __name__,
2186
+ ConnectionConfig,
2187
+ exclude=(ConnectionConfig, BaseDuckDBConnectionConfig),
2188
+ )
2189
+ }
2190
+
2118
2191
 
2119
2192
  def parse_connection_config(v: t.Dict[str, t.Any]) -> ConnectionConfig:
2120
2193
  if "type" not in v:
@@ -138,7 +138,7 @@ def generate_dlt_models(
138
138
  force: bool,
139
139
  dlt_path: t.Optional[str] = None,
140
140
  ) -> t.List[str]:
141
- from sqlmesh.cli.example_project import _create_models
141
+ from sqlmesh.cli.project_init import _create_object_files
142
142
 
143
143
  sqlmesh_models, _, _ = generate_dlt_models_and_settings(
144
144
  pipeline_name=pipeline_name,
@@ -152,7 +152,11 @@ def generate_dlt_models(
152
152
  sqlmesh_models = {model for model in sqlmesh_models if model[0] not in existing_models}
153
153
 
154
154
  if sqlmesh_models:
155
- _create_models(models_path=context.path / "models", models=sqlmesh_models)
155
+ _create_object_files(
156
+ context.path / "models",
157
+ {model[0].split(".")[-1]: model[1] for model in sqlmesh_models},
158
+ "sql",
159
+ )
156
160
  return [model[0] for model in sqlmesh_models]
157
161
  return []
158
162
 
sqlmesh/magics.py CHANGED
@@ -26,15 +26,16 @@ from IPython.core.magic import (
26
26
  from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
27
27
  from IPython.utils.process import arg_split
28
28
  from rich.jupyter import JupyterRenderable
29
- from sqlmesh.cli.example_project import ProjectTemplate, init_example_project
29
+ from sqlmesh.cli.project_init import ProjectTemplate, init_example_project
30
30
  from sqlmesh.core import analytics
31
31
  from sqlmesh.core.config import load_configs
32
+ from sqlmesh.core.config.connection import INIT_DISPLAY_INFO_TO_TYPE
32
33
  from sqlmesh.core.console import create_console, set_console, configure_console
33
34
  from sqlmesh.core.context import Context
34
35
  from sqlmesh.core.dialect import format_model_expressions, parse
35
36
  from sqlmesh.core.model import load_sql_based_model
36
37
  from sqlmesh.core.test import ModelTestMetadata
37
- from sqlmesh.utils import sqlglot_dialects, yaml, Verbosity, optional_import
38
+ from sqlmesh.utils import yaml, Verbosity, optional_import
38
39
  from sqlmesh.utils.errors import MagicError, MissingContextException, SQLMeshError
39
40
 
40
41
  logger = logging.getLogger(__name__)
@@ -198,9 +199,9 @@ class SQLMeshMagics(Magics):
198
199
  @magic_arguments()
199
200
  @argument("path", type=str, help="The path where the new SQLMesh project should be created.")
200
201
  @argument(
201
- "sql_dialect",
202
+ "engine",
202
203
  type=str,
203
- help=f"Default model SQL dialect. Supported values: {sqlglot_dialects()}.",
204
+ help=f"Project SQL engine. Supported values: '{', '.join([info[1] for info in sorted(INIT_DISPLAY_INFO_TO_TYPE.values(), key=lambda x: x[0])])}'.", # type: ignore
204
205
  )
205
206
  @argument(
206
207
  "--template",
@@ -229,7 +230,12 @@ class SQLMeshMagics(Magics):
229
230
  except ValueError:
230
231
  raise MagicError(f"Invalid project template '{args.template}'")
231
232
  init_example_project(
232
- args.path, args.sql_dialect, project_template, args.dlt_pipeline, args.dlt_path
233
+ path=args.path,
234
+ engine_type=args.engine,
235
+ dialect=None,
236
+ template=project_template,
237
+ pipeline=args.dlt_pipeline,
238
+ dlt_path=args.dlt_path,
233
239
  )
234
240
  html = str(
235
241
  h(
sqlmesh/utils/yaml.py CHANGED
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import getpass
3
4
  import io
4
5
  import typing as t
5
6
  from decimal import Decimal
@@ -14,6 +15,7 @@ from sqlmesh.utils.jinja import ENVIRONMENT, create_var
14
15
 
15
16
  JINJA_METHODS = {
16
17
  "env_var": lambda key, default=None: getenv(key, default),
18
+ "user": lambda: getpass.getuser(),
17
19
  }
18
20
 
19
21
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sqlmesh
3
- Version: 0.195.1.dev4
3
+ Version: 0.195.1.dev5
4
4
  Summary: Next-generation data transformation framework
5
5
  Author-email: "TobikoData Inc." <engineering@tobikodata.com>
6
6
  License: Apache License
@@ -1,14 +1,14 @@
1
1
  sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
2
- sqlmesh/_version.py,sha256=W2g-e2AVI33a-naNGAiuozPk-wN4pGYR9p2budZemfY,528
3
- sqlmesh/magics.py,sha256=d7udR_sieKWzp_YXNFPA2O173xDCqk6u9CIKB7Skb2c,41206
2
+ sqlmesh/_version.py,sha256=HJO_GXCREVs-EIHbm1D5ew1lkKP8RQu5o4NWac9a8KQ,528
3
+ sqlmesh/magics.py,sha256=nMfwSW5rIkdnyO9MXYm9u24hNl6dqgAWybC-MnRcNnE,41451
4
4
  sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
5
  sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
6
  sqlmesh/cicd/bot.py,sha256=2zlbn-DXkqQzr3lA0__IGU4XaIfXBXBKLWXNI2DRJX8,759
7
7
  sqlmesh/cicd/config.py,sha256=mpgkisSBBovlNrNZir2unT8Wes6b38RW2UsU_o4e8vY,109
8
8
  sqlmesh/cli/__init__.py,sha256=MAXAyO7isbyM-PaSdFjkJNcjn32WjOl0XzxT5gvp_vo,1690
9
- sqlmesh/cli/example_project.py,sha256=QAFQbw_kzqn30GLTthqsuWXbFDBpX8W6RsCgQ-jx65M,9308
10
- sqlmesh/cli/main.py,sha256=wo5_057ofKgB6D32bl6zBCNZDKwBSXrZv5IzGpRnB3o,33179
9
+ sqlmesh/cli/main.py,sha256=5hFnJY_yvzdD-HNtXFE_p6CTaXiEIxzCko5LCrrN7Uo,35533
11
10
  sqlmesh/cli/options.py,sha256=-4ZclMIeIg8tghLowtmQNmpEFlm0QtOVEh40NqRdFHA,2499
11
+ sqlmesh/cli/project_init.py,sha256=oqHZ8wSIfFwQHmcE6OUgqH0srCcXOFsD3_EABNasrx0,16418
12
12
  sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
13
13
  sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
14
14
  sqlmesh/core/console.py,sha256=_B9rmOn0WbgP8DsST8Lg9ODGu9u0WA2D2jRrJRX0VMM,153805
@@ -41,7 +41,7 @@ sqlmesh/core/config/__init__.py,sha256=b4gFb04tpeZpowqJD-7h3U3TLgxrpQhZahVEAo6VW
41
41
  sqlmesh/core/config/base.py,sha256=t8NQmsgQoZSc-k0dlDiCb8t1jj0AMYdGZ-6se9q_Pks,4898
42
42
  sqlmesh/core/config/categorizer.py,sha256=6vzUoNLjR6GOEb_2mYVz2TwmMv2BfldgHX2u-Le5HZs,1975
43
43
  sqlmesh/core/config/common.py,sha256=rsb2uVFqPEX3DMsWG_hApwztCuRJP-FAmaQ_5L3-S8g,4006
44
- sqlmesh/core/config/connection.py,sha256=sxRPhP0q088kInXtZinEK2FbkQHkzJZrGMDF-VnWmtA,82137
44
+ sqlmesh/core/config/connection.py,sha256=-KF0xufXPt8HIYZmRdyQokyq59xOiCS4S2zSgIi0LUg,85462
45
45
  sqlmesh/core/config/feature_flag.py,sha256=JRcD31OG_t2cYc24meKbHDGB0iVID0P07V2oOeB_mgA,203
46
46
  sqlmesh/core/config/format.py,sha256=6CXFbvnor56xbldKE-Vrm9k_ABRoY4v6vgIb3mCihiQ,1355
47
47
  sqlmesh/core/config/gateway.py,sha256=tYngyqwd_4Qr9lhcv2hlvLvb_2pgYYtKu6hdGsTr-4I,1931
@@ -157,7 +157,7 @@ sqlmesh/engines/spark/db_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
157
157
  sqlmesh/engines/spark/db_api/errors.py,sha256=LSXuHHhkzHvKlcdwmhCM9qm51KyKoP8gQH-p_W3Oow4,148
158
158
  sqlmesh/engines/spark/db_api/spark_session.py,sha256=uQP0VFFYtp3N9mK7zKw6SoiGVhJDDYPylqx46yDwGEc,4667
159
159
  sqlmesh/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
160
- sqlmesh/integrations/dlt.py,sha256=Blj_707XadGtg9bQZI8nZN4vKsxILV8182JZbnJa8mc,7651
160
+ sqlmesh/integrations/dlt.py,sha256=mA9ym16gAN2O8RYOzSPeTX5xsXPuXVRPdiX7dkGGZlo,7744
161
161
  sqlmesh/integrations/llm.py,sha256=URgp3JKr06HYv6hGlJ7DQ9m3QCc41xRWK23HzmGpbDI,1662
162
162
  sqlmesh/integrations/slack.py,sha256=nxLxu5WztGbZH3JdqnzyPqmJUMfRdJ_49LQ7zR-u39Q,6801
163
163
  sqlmesh/integrations/github/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -282,8 +282,8 @@ sqlmesh/utils/process.py,sha256=vpXcpJ1eIbivs7gZdwTKSJI7OS6kKBebnaBzz75ykvs,2428
282
282
  sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,11685
283
283
  sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
284
284
  sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
285
- sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
286
- sqlmesh-0.195.1.dev4.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
285
+ sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
286
+ sqlmesh-0.195.1.dev5.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
287
287
  web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
288
288
  web/client/dist/index.html,sha256=drXwzSCEVDTMfaIwzJYjnT8Xk4c-Y4wNJz-CflUa_vY,5472
289
289
  web/client/dist/assets/Audits-DrNQaTQA.js,sha256=E8JF632bsRUm6LGtgXlyHrJOYQe5VW2CgRZPaZNKvgg,1041
@@ -401,8 +401,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
401
401
  web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
402
402
  web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
403
403
  web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
404
- sqlmesh-0.195.1.dev4.dist-info/METADATA,sha256=L5VdwpRstKaJDH3j1MPFjmdhqzgNyDchcLQYXa-59hA,26690
405
- sqlmesh-0.195.1.dev4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
406
- sqlmesh-0.195.1.dev4.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
407
- sqlmesh-0.195.1.dev4.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
408
- sqlmesh-0.195.1.dev4.dist-info/RECORD,,
404
+ sqlmesh-0.195.1.dev5.dist-info/METADATA,sha256=VmS7Njc9KzrGOl9Xm328ltW6gCIcymnckLe4e83fL4I,26690
405
+ sqlmesh-0.195.1.dev5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
406
+ sqlmesh-0.195.1.dev5.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
407
+ sqlmesh-0.195.1.dev5.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
408
+ sqlmesh-0.195.1.dev5.dist-info/RECORD,,
@@ -1,338 +0,0 @@
1
- import typing as t
2
- from enum import Enum
3
- from pathlib import Path
4
- from dataclasses import dataclass
5
-
6
- import click
7
- from sqlglot import Dialect
8
- from sqlmesh.integrations.dlt import generate_dlt_models_and_settings
9
- from sqlmesh.utils.date import yesterday_ds
10
-
11
- from sqlmesh.core.config.connection import CONNECTION_CONFIG_TO_TYPE
12
-
13
-
14
- PRIMITIVES = (str, int, bool, float)
15
-
16
-
17
- class ProjectTemplate(Enum):
18
- DBT = "dbt"
19
- DLT = "dlt"
20
- DEFAULT = "default"
21
- EMPTY = "empty"
22
-
23
-
24
- def _gen_config(
25
- dialect: t.Optional[str],
26
- settings: t.Optional[str],
27
- start: t.Optional[str],
28
- template: ProjectTemplate,
29
- ) -> str:
30
- connection_settings = (
31
- settings
32
- or """ type: duckdb
33
- database: db.db"""
34
- )
35
-
36
- if not settings and template != ProjectTemplate.DBT:
37
- doc_link = "https://sqlmesh.readthedocs.io/en/stable/integrations/engines{engine_link}"
38
- engine_link = ""
39
-
40
- engine = "mssql" if dialect == "tsql" else dialect
41
-
42
- if engine in CONNECTION_CONFIG_TO_TYPE:
43
- required_fields = []
44
- non_required_fields = []
45
-
46
- for name, field in CONNECTION_CONFIG_TO_TYPE[engine].model_fields.items():
47
- field_name = field.alias or name
48
- default_value = field.get_default()
49
-
50
- if isinstance(default_value, Enum):
51
- default_value = default_value.value
52
- elif not isinstance(default_value, PRIMITIVES):
53
- default_value = ""
54
-
55
- required = field.is_required() or field_name == "type"
56
- option_str = f" {'# ' if not required else ''}{field_name}: {default_value}\n"
57
-
58
- # specify the DuckDB database field so quickstart runs out of the box
59
- if engine == "duckdb" and field_name == "database":
60
- option_str = " database: db.db\n"
61
- required = True
62
-
63
- if required:
64
- required_fields.append(option_str)
65
- else:
66
- non_required_fields.append(option_str)
67
-
68
- connection_settings = "".join(required_fields + non_required_fields)
69
-
70
- engine_link = f"/{engine}/#connection-options"
71
-
72
- connection_settings = (
73
- " # For more information on configuring the connection to your execution engine, visit:\n"
74
- " # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connections\n"
75
- f" # {doc_link.format(engine_link=engine_link)}\n{connection_settings}"
76
- )
77
-
78
- default_configs = {
79
- ProjectTemplate.DEFAULT: f"""gateways:
80
- {dialect}:
81
- connection:
82
- {connection_settings}
83
-
84
- default_gateway: {dialect}
85
-
86
- model_defaults:
87
- dialect: {dialect}
88
- start: {start or yesterday_ds()}
89
- """,
90
- ProjectTemplate.DBT: """from pathlib import Path
91
-
92
- from sqlmesh.dbt.loader import sqlmesh_config
93
-
94
- config = sqlmesh_config(Path(__file__).parent)
95
- """,
96
- }
97
-
98
- default_configs[ProjectTemplate.EMPTY] = default_configs[ProjectTemplate.DEFAULT]
99
- default_configs[ProjectTemplate.DLT] = default_configs[ProjectTemplate.DEFAULT]
100
- return default_configs[template]
101
-
102
-
103
- @dataclass
104
- class ExampleObjects:
105
- schema_name: str
106
- full_model_name: str
107
- full_model_def: str
108
- incremental_model_name: str
109
- incremental_model_def: str
110
- seed_model_name: str
111
- seed_model_def: str
112
- seed_data: str
113
- audit_def: str
114
- test_def: str
115
-
116
- def models(self) -> t.Set[t.Tuple[str, str]]:
117
- return {
118
- (self.full_model_name, self.full_model_def),
119
- (self.incremental_model_name, self.incremental_model_def),
120
- (self.seed_model_name, self.seed_model_def),
121
- }
122
-
123
-
124
- def _gen_example_objects(schema_name: str) -> ExampleObjects:
125
- full_model_name = f"{schema_name}.full_model"
126
- incremental_model_name = f"{schema_name}.incremental_model"
127
- seed_model_name = f"{schema_name}.seed_model"
128
-
129
- full_model_def = f"""MODEL (
130
- name {full_model_name},
131
- kind FULL,
132
- cron '@daily',
133
- grain item_id,
134
- audits (assert_positive_order_ids),
135
- );
136
-
137
- SELECT
138
- item_id,
139
- COUNT(DISTINCT id) AS num_orders,
140
- FROM
141
- {incremental_model_name}
142
- GROUP BY item_id
143
- """
144
-
145
- incremental_model_def = f"""MODEL (
146
- name {incremental_model_name},
147
- kind INCREMENTAL_BY_TIME_RANGE (
148
- time_column event_date
149
- ),
150
- start '2020-01-01',
151
- cron '@daily',
152
- grain (id, event_date)
153
- );
154
-
155
- SELECT
156
- id,
157
- item_id,
158
- event_date,
159
- FROM
160
- {seed_model_name}
161
- WHERE
162
- event_date BETWEEN @start_date AND @end_date
163
- """
164
-
165
- seed_model_def = f"""MODEL (
166
- name {seed_model_name},
167
- kind SEED (
168
- path '../seeds/seed_data.csv'
169
- ),
170
- columns (
171
- id INTEGER,
172
- item_id INTEGER,
173
- event_date DATE
174
- ),
175
- grain (id, event_date)
176
- );
177
- """
178
-
179
- audit_def = """AUDIT (
180
- name assert_positive_order_ids,
181
- );
182
-
183
- SELECT *
184
- FROM @this_model
185
- WHERE
186
- item_id < 0
187
- """
188
-
189
- seed_data = """id,item_id,event_date
190
- 1,2,2020-01-01
191
- 2,1,2020-01-01
192
- 3,3,2020-01-03
193
- 4,1,2020-01-04
194
- 5,1,2020-01-05
195
- 6,1,2020-01-06
196
- 7,1,2020-01-07
197
- """
198
-
199
- test_def = f"""test_example_full_model:
200
- model: {full_model_name}
201
- inputs:
202
- {incremental_model_name}:
203
- rows:
204
- - id: 1
205
- item_id: 1
206
- - id: 2
207
- item_id: 1
208
- - id: 3
209
- item_id: 2
210
- outputs:
211
- query:
212
- rows:
213
- - item_id: 1
214
- num_orders: 2
215
- - item_id: 2
216
- num_orders: 1
217
- """
218
-
219
- return ExampleObjects(
220
- schema_name=schema_name,
221
- full_model_name=full_model_name,
222
- full_model_def=full_model_def,
223
- incremental_model_name=incremental_model_name,
224
- incremental_model_def=incremental_model_def,
225
- seed_model_name=seed_model_name,
226
- seed_model_def=seed_model_def,
227
- seed_data=seed_data,
228
- audit_def=audit_def,
229
- test_def=test_def,
230
- )
231
-
232
-
233
- def init_example_project(
234
- path: t.Union[str, Path],
235
- dialect: t.Optional[str],
236
- template: ProjectTemplate = ProjectTemplate.DEFAULT,
237
- pipeline: t.Optional[str] = None,
238
- dlt_path: t.Optional[str] = None,
239
- schema_name: str = "sqlmesh_example",
240
- ) -> None:
241
- root_path = Path(path)
242
- config_extension = "py" if template == ProjectTemplate.DBT else "yaml"
243
- config_path = root_path / f"config.{config_extension}"
244
- audits_path = root_path / "audits"
245
- macros_path = root_path / "macros"
246
- models_path = root_path / "models"
247
- seeds_path = root_path / "seeds"
248
- tests_path = root_path / "tests"
249
-
250
- if config_path.exists():
251
- raise click.ClickException(f"Found an existing config in '{config_path}'")
252
-
253
- if not dialect and template != ProjectTemplate.DBT:
254
- raise click.ClickException(
255
- "Default SQL dialect is a required argument for SQLMesh projects"
256
- )
257
-
258
- models: t.Set[t.Tuple[str, str]] = set()
259
- settings = None
260
- start = None
261
- if template == ProjectTemplate.DLT:
262
- if pipeline and dialect:
263
- models, settings, start = generate_dlt_models_and_settings(
264
- pipeline_name=pipeline, dialect=dialect, dlt_path=dlt_path
265
- )
266
- else:
267
- raise click.ClickException(
268
- "DLT pipeline is a required argument to generate a SQLMesh project from DLT"
269
- )
270
-
271
- _create_config(config_path, dialect, settings, start, template)
272
- if template == ProjectTemplate.DBT:
273
- return
274
-
275
- _create_folders([audits_path, macros_path, models_path, seeds_path, tests_path])
276
-
277
- if template == ProjectTemplate.DLT:
278
- _create_models(models_path, models)
279
- return
280
-
281
- example_objects = _gen_example_objects(schema_name=schema_name)
282
-
283
- if template != ProjectTemplate.EMPTY:
284
- _create_macros(macros_path)
285
- _create_audits(audits_path, example_objects)
286
- _create_models(models_path, example_objects.models())
287
- _create_seeds(seeds_path, example_objects)
288
- _create_tests(tests_path, example_objects)
289
-
290
-
291
- def _create_folders(target_folders: t.Sequence[Path]) -> None:
292
- for folder_path in target_folders:
293
- folder_path.mkdir(exist_ok=True)
294
- (folder_path / ".gitkeep").touch()
295
-
296
-
297
- def _create_config(
298
- config_path: Path,
299
- dialect: t.Optional[str],
300
- settings: t.Optional[str],
301
- start: t.Optional[str],
302
- template: ProjectTemplate,
303
- ) -> None:
304
- if dialect:
305
- Dialect.get_or_raise(dialect)
306
-
307
- project_config = _gen_config(dialect, settings, start, template)
308
-
309
- _write_file(
310
- config_path,
311
- project_config,
312
- )
313
-
314
-
315
- def _create_macros(macros_path: Path) -> None:
316
- (macros_path / "__init__.py").touch()
317
-
318
-
319
- def _create_audits(audits_path: Path, example_objects: ExampleObjects) -> None:
320
- _write_file(audits_path / "assert_positive_order_ids.sql", example_objects.audit_def)
321
-
322
-
323
- def _create_models(models_path: Path, models: t.Set[t.Tuple[str, str]]) -> None:
324
- for model_name, model_def in models:
325
- _write_file(models_path / f"{model_name.split('.')[-1]}.sql", model_def)
326
-
327
-
328
- def _create_seeds(seeds_path: Path, example_objects: ExampleObjects) -> None:
329
- _write_file(seeds_path / "seed_data.csv", example_objects.seed_data)
330
-
331
-
332
- def _create_tests(tests_path: Path, example_objects: ExampleObjects) -> None:
333
- _write_file(tests_path / "test_full_model.yaml", example_objects.test_def)
334
-
335
-
336
- def _write_file(path: Path, payload: str) -> None:
337
- with open(path, "w", encoding="utf-8") as fd:
338
- fd.write(payload)