sqlmesh 0.209.1.dev10__py3-none-any.whl → 0.209.1.dev12__py3-none-any.whl

sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.209.1.dev10'
-__version_tuple__ = version_tuple = (0, 209, 1, 'dev10')
+__version__ = version = '0.209.1.dev12'
+__version_tuple__ = version_tuple = (0, 209, 1, 'dev12')
 
 __commit_id__ = commit_id = None

{sqlmesh-0.209.1.dev10.dist-info → sqlmesh-0.209.1.dev12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.209.1.dev10
+Version: 0.209.1.dev12
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.209.1.dev10.dist-info → sqlmesh-0.209.1.dev12.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=Igshe-vepS3oVl2sTH0hj1iCEG61O35goUT34bEh2t4,723
+sqlmesh/_version.py,sha256=7OqlZXPJ-sO1ZD5h7xeYT3AcKstkLcz3pDByvyZckQs,723
 sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -288,7 +288,11 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.209.1.dev10.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.209.1.dev12.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh_dbt/__init__.py,sha256=awYS5y5mz-1NUmx6i5h5NSTJ7tidRl9NC0FAnFWSF6U,350
+sqlmesh_dbt/cli.py,sha256=Ekn77WLMmyyr31aoDoWrZp5DS0rfiddy7pNohBK8G0s,2115
+sqlmesh_dbt/console.py,sha256=1B0B5_R1DlrEkyWVDl8G78-B6_K-ZtLPsQEhqPCfLEU,188
+sqlmesh_dbt/operations.py,sha256=u8O5mrbX942y6lGBHMg1GPIJLTWVOUiVw35uG3okjKg,5461
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=SXykdSRNgrT1k6I2dPZPEfpxPjIsi8-yVFijt-naSA4,5472
 web/client/dist/assets/Audits-Jm353fhk.js,sha256=ry6b5IKT3hLAfD5C3T2BqxulfauEV0n0YZ2vsP4eQ_U,1041
@@ -406,8 +410,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.209.1.dev10.dist-info/METADATA,sha256=SYcGEzmGQzEPLJIypLUOFqWWZDiBQxYqWUENLU0gp2I,26686
-sqlmesh-0.209.1.dev10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.209.1.dev10.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.209.1.dev10.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.209.1.dev10.dist-info/RECORD,,
+sqlmesh-0.209.1.dev12.dist-info/METADATA,sha256=M3xduNBEn62vHm2RLj8dwbolcTWTz77Rk0Mlu3z_tOY,26686
+sqlmesh-0.209.1.dev12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.209.1.dev12.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
+sqlmesh-0.209.1.dev12.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
+sqlmesh-0.209.1.dev12.dist-info/RECORD,,

{sqlmesh-0.209.1.dev10.dist-info → sqlmesh-0.209.1.dev12.dist-info}/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
 sqlmesh = sqlmesh.cli.main:cli
 sqlmesh_cicd = sqlmesh.cicd.bot:bot
+sqlmesh_dbt = sqlmesh_dbt.cli:dbt
 sqlmesh_lsp = sqlmesh.lsp.main:main

{sqlmesh-0.209.1.dev10.dist-info → sqlmesh-0.209.1.dev12.dist-info}/top_level.txt

@@ -1,2 +1,3 @@
 sqlmesh
+sqlmesh_dbt
 web

sqlmesh_dbt/__init__.py

@@ -0,0 +1,5 @@
+# Note: `sqlmesh_dbt` is deliberately in its own package from `sqlmesh` to avoid the upfront time overhead
+# that comes from `import sqlmesh`
+#
+# Obviously we still have to `import sqlmesh` at some point but this allows us to defer it until needed,
+# which means we can make the CLI feel more responsive by being able to output something immediately

sqlmesh_dbt/cli.py

@@ -0,0 +1,80 @@
+import typing as t
+import sys
+import click
+from sqlmesh_dbt.operations import DbtOperations, create
+
+
+def _get_dbt_operations(ctx: click.Context) -> DbtOperations:
+    if not isinstance(ctx.obj, DbtOperations):
+        raise ValueError(f"Unexpected click context object: {type(ctx.obj)}")
+    return ctx.obj
+
+
+@click.group()
+@click.pass_context
+def dbt(ctx: click.Context) -> None:
+    """
+    An ELT tool for managing your SQL transformations and data models, powered by the SQLMesh engine.
+    """
+
+    if "--help" in sys.argv:
+        # we dont need to import sqlmesh/load the project for CLI help
+        return
+
+    # TODO: conditionally call create() if there are times we dont want/need to import sqlmesh and load a project
+    ctx.obj = create()
+
+
+@dbt.command()
+@click.option("-s", "-m", "--select", "--models", "--model", help="Specify the nodes to include.")
+@click.option(
+    "-f",
+    "--full-refresh",
+    help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.",
+)
+@click.pass_context
+def run(ctx: click.Context, select: t.Optional[str], full_refresh: bool) -> None:
+    """Compile SQL and execute against the current target database."""
+    _get_dbt_operations(ctx).run(select=select, full_refresh=full_refresh)
+
+
+@dbt.command(name="list")
+@click.pass_context
+def list_(ctx: click.Context) -> None:
+    """List the resources in your project"""
+    _get_dbt_operations(ctx).list_()
+
+
+@dbt.command(name="ls", hidden=True)  # hidden alias for list
+@click.pass_context
+def ls(ctx: click.Context) -> None:
+    """List the resources in your project"""
+    ctx.forward(list_)
+
+
+def _not_implemented(name: str) -> None:
+    @dbt.command(name=name)
+    def _not_implemented() -> None:
+        """Not implemented"""
+        click.echo(f"dbt {name} not implemented")
+
+
+for subcommand in (
+    "build",
+    "clean",
+    "clone",
+    "compile",
+    "debug",
+    "deps",
+    "docs",
+    "init",
+    "parse",
+    "retry",
+    "run-operation",
+    "seed",
+    "show",
+    "snapshot",
+    "source",
+    "test",
+):
+    _not_implemented(subcommand)

sqlmesh_dbt/console.py

@@ -0,0 +1,8 @@
+from sqlmesh.core.console import TerminalConsole
+
+
+class DbtCliConsole(TerminalConsole):
+    # TODO: build this out
+
+    def print(self, msg: str) -> None:
+        return self._print(msg)

sqlmesh_dbt/operations.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+import typing as t
+from rich.progress import Progress
+from pathlib import Path
+
+if t.TYPE_CHECKING:
+    # important to gate these to be able to defer importing sqlmesh until we need to
+    from sqlmesh.core.context import Context
+    from sqlmesh.dbt.project import Project
+    from sqlmesh_dbt.console import DbtCliConsole
+
+
+class DbtOperations:
+    def __init__(self, sqlmesh_context: Context, dbt_project: Project):
+        self.context = sqlmesh_context
+        self.project = dbt_project
+
+    def list_(self) -> None:
+        for _, model in self.context.models.items():
+            self.console.print(model.name)
+
+    def run(self, select: t.Optional[str] = None, full_refresh: bool = False) -> None:
+        # A dbt run both updates data and changes schemas and has no way of rolling back so more closely maps to a SQLMesh forward-only plan
+        # TODO: if --full-refresh specified, mark incrementals as breaking instead of forward_only?
+
+        # TODO: we need to either convert DBT selector syntax to SQLMesh selector syntax
+        # or make the model selection engine configurable
+        select_models = None
+        if select:
+            if "," in select:
+                select_models = select.split(",")
+            else:
+                select_models = select.split(" ")
+
+        self.context.plan(
+            select_models=select_models,
+            no_auto_categorization=True,  # everything is breaking / foward-only
+            run=True,
+            no_diff=True,
+            no_prompts=True,
+            auto_apply=True,
+        )
+
+    @property
+    def console(self) -> DbtCliConsole:
+        console = self.context.console
+        from sqlmesh_dbt.console import DbtCliConsole
+
+        if not isinstance(console, DbtCliConsole):
+            raise ValueError(f"Expecting dbt cli console, got: {console}")
+
+        return console
+
+
+def create(
+    project_dir: t.Optional[Path] = None, profiles_dir: t.Optional[Path] = None, debug: bool = False
+) -> DbtOperations:
+    with Progress(transient=True) as progress:
+        # Indeterminate progress bar before SQLMesh import to provide feedback to the user that something is indeed happening
+        load_task_id = progress.add_task("Loading engine", total=None)
+
+        from sqlmesh import configure_logging
+        from sqlmesh.core.context import Context
+        from sqlmesh.dbt.loader import sqlmesh_config, DbtLoader
+        from sqlmesh.core.console import set_console
+        from sqlmesh_dbt.console import DbtCliConsole
+        from sqlmesh.utils.errors import SQLMeshError
+
+        configure_logging(force_debug=debug)
+        set_console(DbtCliConsole())
+
+        progress.update(load_task_id, description="Loading project", total=None)
+
+        # inject default start date if one is not specified to prevent the user from having to do anything
+        _inject_default_start_date(project_dir)
+
+        config = sqlmesh_config(
+            project_root=project_dir,
+            # do we want to use a local duckdb for state?
+            # warehouse state has a bunch of overhead to initialize, is slow for ongoing operations and will create tables that perhaps the user was not expecting
+            # on the other hand, local state is not portable
+            state_connection=None,
+        )
+
+        sqlmesh_context = Context(
+            config=config,
+            load=True,
+        )
+
+        # this helps things which want a default project-level start date, like the "effective from date" for forward-only plans
+        if not sqlmesh_context.config.model_defaults.start:
+            min_start_date = min(
+                (
+                    model.start
+                    for model in sqlmesh_context.models.values()
+                    if model.start is not None
+                ),
+                default=None,
+            )
+            sqlmesh_context.config.model_defaults.start = min_start_date
+
+        dbt_loader = sqlmesh_context._loaders[0]
+        if not isinstance(dbt_loader, DbtLoader):
+            raise SQLMeshError(f"Unexpected loader type: {type(dbt_loader)}")
+
+        # so that DbtOperations can query information from the DBT project files in order to invoke SQLMesh correctly
+        dbt_project = dbt_loader._projects[0]
+
+        return DbtOperations(sqlmesh_context, dbt_project)
+
+
+def _inject_default_start_date(project_dir: t.Optional[Path] = None) -> None:
+    """
+    SQLMesh needs a start date to as the starting point for calculating intervals on incremental models
+
+    Rather than forcing the user to update their config manually or having a default that is not saved between runs,
+    we can inject it automatically to the dbt_project.yml file
+    """
+    from sqlmesh.dbt.project import PROJECT_FILENAME, load_yaml
+    from sqlmesh.utils.yaml import dump
+    from sqlmesh.utils.date import yesterday_ds
+
+    project_yaml_path = (project_dir or Path.cwd()) / PROJECT_FILENAME
+    if project_yaml_path.exists():
+        loaded_project_file = load_yaml(project_yaml_path)
+        start_date_keys = ("start", "+start")
+        if "models" in loaded_project_file and all(
+            k not in loaded_project_file["models"] for k in start_date_keys
+        ):
+            loaded_project_file["models"]["+start"] = yesterday_ds()
+            # todo: this may format the file differently, is that acceptable?
+            with project_yaml_path.open("w") as f:
+                dump(loaded_project_file, f)