sqlmesh 0.225.1.dev15__py3-none-any.whl → 0.225.1.dev17__py3-none-any.whl

sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.225.1.dev15'
-__version_tuple__ = version_tuple = (0, 225, 1, 'dev15')
+__version__ = version = '0.225.1.dev17'
+__version_tuple__ = version_tuple = (0, 225, 1, 'dev17')
 
 __commit_id__ = commit_id = None

sqlmesh/core/config/loader.py

@@ -178,6 +178,7 @@ def load_config_from_paths(
 
             dbt_python_config = sqlmesh_config(
                 project_root=dbt_project_file.parent,
+                profiles_dir=kwargs.pop("profiles_dir", None),
                 dbt_profile_name=kwargs.pop("profile", None),
                 dbt_target_name=kwargs.pop("target", None),
                 variables=variables,

sqlmesh/dbt/column.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import typing as t
+import logging
 
 from sqlglot import exp, parse_one
 from sqlglot.helper import ensure_list
@@ -9,6 +10,8 @@ from sqlmesh.dbt.common import GeneralConfig
 from sqlmesh.utils.conversions import ensure_bool
 from sqlmesh.utils.pydantic import field_validator
 
+logger = logging.getLogger(__name__)
+
 
 def yaml_to_columns(
     yaml: t.Dict[str, ColumnConfig] | t.List[t.Dict[str, ColumnConfig]],
@@ -31,11 +34,20 @@ def column_types_to_sqlmesh(
     Returns:
         A dict of column name to exp.DataType
     """
-    return {
-        name: parse_one(column.data_type, into=exp.DataType, dialect=dialect or "")
-        for name, column in columns.items()
-        if column.enabled and column.data_type
-    }
+    col_types_to_sqlmesh: t.Dict[str, exp.DataType] = {}
+    for name, column in columns.items():
+        if column.enabled and column.data_type:
+            column_def = parse_one(
+                f"{name} {column.data_type}", into=exp.ColumnDef, dialect=dialect or ""
+            )
+            if column_def.args.get("constraints"):
+                logger.warning(
+                    f"Ignoring unsupported constraints for column '{name}' with definition '{column.data_type}'. Please refer to github.com/TobikoData/sqlmesh/issues/4717 for more information."
+                )
+            kind = column_def.kind
+            if kind:
+                col_types_to_sqlmesh[name] = kind
+    return col_types_to_sqlmesh
 
 
 def column_descriptions_to_sqlmesh(columns: t.Dict[str, ColumnConfig]) -> t.Dict[str, str]:

sqlmesh/dbt/context.py

@@ -37,6 +37,8 @@ class DbtContext:
     """Context for DBT environment"""
 
     project_root: Path = Path()
+    profiles_dir: t.Optional[Path] = None
+    """Optional override to specify the directory where profiles.yml is located, if not at the :project_root"""
     target_name: t.Optional[str] = None
     profile_name: t.Optional[str] = None
     project_schema: t.Optional[str] = None

sqlmesh/dbt/loader.py

@@ -53,10 +53,18 @@ def sqlmesh_config(
     threads: t.Optional[int] = None,
     register_comments: t.Optional[bool] = None,
     infer_state_schema_name: bool = False,
+    profiles_dir: t.Optional[Path] = None,
     **kwargs: t.Any,
 ) -> Config:
     project_root = project_root or Path()
-    context = DbtContext(project_root=project_root, profile_name=dbt_profile_name)
+    context = DbtContext(
+        project_root=project_root, profiles_dir=profiles_dir, profile_name=dbt_profile_name
+    )
+
+    # note: Profile.load() is called twice with different DbtContext's:
+    # - once here with the above DbtContext (to determine connnection / gateway config which has to be set up before everything else)
+    # - again on the SQLMesh side via GenericContext.load() -> DbtLoader._load_projects() -> Project.load() which constructs a fresh DbtContext and ignores the above one
+    # it's important to ensure that the DbtContext created within the DbtLoader uses the same project root / profiles dir that we use here
     profile = Profile.load(context, target_name=dbt_target_name)
     model_defaults = kwargs.pop("model_defaults", ModelDefaultsConfig())
     if model_defaults.dialect is None:
@@ -98,6 +106,7 @@ def sqlmesh_config(
 
     return Config(
         loader=loader,
+        loader_kwargs=dict(profiles_dir=profiles_dir),
         model_defaults=model_defaults,
         variables=variables or {},
         dbt=RootDbtConfig(infer_state_schema_name=infer_state_schema_name),
@@ -116,9 +125,12 @@ def sqlmesh_config(
 
 
 class DbtLoader(Loader):
-    def __init__(self, context: GenericContext, path: Path) -> None:
+    def __init__(
+        self, context: GenericContext, path: Path, profiles_dir: t.Optional[Path] = None
+    ) -> None:
         self._projects: t.List[Project] = []
         self._macros_max_mtime: t.Optional[float] = None
+        self._profiles_dir = profiles_dir
         super().__init__(context, path)
 
     def load(self) -> LoadedProject:
@@ -225,6 +237,7 @@ class DbtLoader(Loader):
             project = Project.load(
                 DbtContext(
                     project_root=self.config_path,
+                    profiles_dir=self._profiles_dir,
                     target_name=target_name,
                     sqlmesh_config=self.config,
                 ),

sqlmesh/dbt/profile.py

@@ -60,7 +60,7 @@ class Profile:
             if not context.profile_name:
                 raise ConfigError(f"{project_file.stem} must include project name.")
 
-        profile_filepath = cls._find_profile(context.project_root)
+        profile_filepath = cls._find_profile(context.project_root, context.profiles_dir)
         if not profile_filepath:
             raise ConfigError(f"{cls.PROFILE_FILE} not found.")
 
@@ -68,8 +68,8 @@ class Profile:
         return Profile(profile_filepath, target_name, target)
 
     @classmethod
-    def _find_profile(cls, project_root: Path) -> t.Optional[Path]:
-        dir = os.environ.get("DBT_PROFILES_DIR", "")
+    def _find_profile(cls, project_root: Path, profiles_dir: t.Optional[Path]) -> t.Optional[Path]:
+        dir = os.environ.get("DBT_PROFILES_DIR", profiles_dir or "")
         path = Path(project_root, dir, cls.PROFILE_FILE)
         if path.exists():
             return path

{sqlmesh-0.225.1.dev15.dist-info → sqlmesh-0.225.1.dev17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.225.1.dev15
+Version: 0.225.1.dev17
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License
@@ -235,7 +235,7 @@ Requires-Dist: python-dotenv
 Requires-Dist: requests
 Requires-Dist: rich[jupyter]
 Requires-Dist: ruamel.yaml
-Requires-Dist: sqlglot[rs]~=27.24.2
+Requires-Dist: sqlglot[rs]~=27.27.0
 Requires-Dist: tenacity
 Requires-Dist: time-machine
 Requires-Dist: json-stream

{sqlmesh-0.225.1.dev15.dist-info → sqlmesh-0.225.1.dev17.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=v_spqQEhcnGaahp1yPvMqUIa6mhH3cs3Bc1CznxvCEA,7965
-sqlmesh/_version.py,sha256=XJ_E-an9zSgcdG4frk9FvpyPMKbjrN_5k6vGmWAkTng,723
+sqlmesh/_version.py,sha256=0vDUvywH_ZkmJlZuCZc7Q299r7zD869DTT8Mio64uNY,723
 sqlmesh/magics.py,sha256=xLh3u4eqpVrKRVN5KF3X84RPRqjygAB9AJP1TXwH8hg,42086
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -48,7 +48,7 @@ sqlmesh/core/config/format.py,sha256=6CXFbvnor56xbldKE-Vrm9k_ABRoY4v6vgIb3mCihiQ
 sqlmesh/core/config/gateway.py,sha256=tYngyqwd_4Qr9lhcv2hlvLvb_2pgYYtKu6hdGsTr-4I,1931
 sqlmesh/core/config/janitor.py,sha256=jxZWSNZMhNjJR9lWI00rDe2nr_5AOxkKE52Fo5nbq_U,901
 sqlmesh/core/config/linter.py,sha256=4Fh6UgY5FrRD9tk_R41Y-M_r91Wh7EQW3Pc1IdRN9Cg,1287
-sqlmesh/core/config/loader.py,sha256=h8yfsq0YAmamAW-YGysVuluvLL-YAFvmJ5VcIceFq3U,10104
+sqlmesh/core/config/loader.py,sha256=p26Ux_-ekgXtOHuqSMObDGfnAfb8Ou3Rug95mZ_uiOg,10167
 sqlmesh/core/config/migration.py,sha256=3lZIlmIcQH3_hIEm3Y3jFtvpw0YXzRzAnk4OdiklO6M,408
 sqlmesh/core/config/model.py,sha256=NgqyIq7J5DCGhIo9I_htiWjTycydVRlchG5McZuHSd0,4627
 sqlmesh/core/config/naming.py,sha256=KVGaxUgswoQajXR_cHFJvgkmEFcU1V182GkOH1SJxuE,326
@@ -131,14 +131,14 @@ sqlmesh/dbt/__init__.py,sha256=KUv-lW5sG9D2ceXAIzA4MLcjyhzq3E-7qJP4P_PH2EU,144
 sqlmesh/dbt/adapter.py,sha256=z-tFIj3rpVvdBr3y8l40FU531-TQ5H2ctLmjwzMBxwk,21321
 sqlmesh/dbt/basemodel.py,sha256=oUr_Em-TjQbpYZS5gtvMA65JRTdnZM46NO9MWvLBLzQ,14860
 sqlmesh/dbt/builtin.py,sha256=c7spvtMFMR6hfLkuZj9uqBQ093R_hYRv9b4jd9k26QI,19792
-sqlmesh/dbt/column.py,sha256=Kvj_N3D_QegcoNQG7VqRiqy-FRfupud-6ilhlykBOgE,1873
+sqlmesh/dbt/column.py,sha256=T5xEWNf0n1sZ3REWnc5D9RsXt5VrrZ1YlMWZUUuAUxo,2449
 sqlmesh/dbt/common.py,sha256=AezWUEoGDISQj9eOo1Z5kLsXz3oRFb0LikTMEaAX3J8,8593
-sqlmesh/dbt/context.py,sha256=Ofhto7_TFvbRfUnRNJpeZBy8ELDi7LX5hPQko7SdlN0,10920
-sqlmesh/dbt/loader.py,sha256=XpPCoHJkcAuXfeIhKmTrJsaGQSHbrk4twCptPQGbsWI,18466
+sqlmesh/dbt/context.py,sha256=JDfSkVBBV2Xi4nDOwWipVHJRll3ioEmvh7gBglPVvqM,11074
+sqlmesh/dbt/loader.py,sha256=ZTpPFnXuf4hQ8Z7Z6oMzxqN2wMMxsQqhm2x-8a5R1AA,19269
 sqlmesh/dbt/manifest.py,sha256=aea8FaSnMbnjF1JfJx0hDRvg-cEwLCkS207dlf_NCvg,34613
 sqlmesh/dbt/model.py,sha256=uFeNj-bL68fQjj6L2fRAv00UQMZTJSTlrJNRrfvETKk,35117
 sqlmesh/dbt/package.py,sha256=8MOq_kHP2qjj24bpoC3GPnHlOVLYO4V9oVb9krk1Mdk,4759
-sqlmesh/dbt/profile.py,sha256=hAqL8lfT98pSBY-wCInPg2WhFEjCNUGF2bx5UcWFx-4,3939
+sqlmesh/dbt/profile.py,sha256=ilDiSqBqw6lsJLUu4MfJSrIkvtC3fbxlvawKn44lHjc,4009
 sqlmesh/dbt/project.py,sha256=Dxf9JakvvQofhcRa2bSi5tFCU4eHMlyRt_Dd7vwOWQM,4712
 sqlmesh/dbt/relation.py,sha256=8QhnW_mQ6bjhKNZwTI1p9SLdMBCcAxvHfo0Qs97Mw6w,228
 sqlmesh/dbt/seed.py,sha256=4X2SDKNaJv4r91S2kpwgJKhOL6GoEpb8d0gRchrCKo0,4494
@@ -238,12 +238,12 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=0F9RdpuuCoG5NiEDXvWlAGCiJ-59OjSAmgFF5wW05aY,1133
 sqlmesh/utils/yaml.py,sha256=KFBd7hsKNRTtRudGR7d410qUYffQv0EWRcDM8hVNNZg,3025
-sqlmesh-0.225.1.dev15.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.225.1.dev17.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 sqlmesh_dbt/__init__.py,sha256=awYS5y5mz-1NUmx6i5h5NSTJ7tidRl9NC0FAnFWSF6U,350
-sqlmesh_dbt/cli.py,sha256=eAeOZtHOikSDEZNI6hnZYJ-pja6ijFldRV4PeSzwYwU,5744
+sqlmesh_dbt/cli.py,sha256=p9foHjAW9ni7BTOJ2loynk47M0Sf43QIJZRggOzF5tc,6351
 sqlmesh_dbt/console.py,sha256=RwWLYnEZHzn9Xp-e2gbZvkdKbWbBLN146geI84mJitg,1132
 sqlmesh_dbt/error.py,sha256=1sPNU6Dik30DR9WTCvGp3ED-pzNmAA3LhP95BXb3ndI,1146
-sqlmesh_dbt/operations.py,sha256=QiCsz5NtD9XScc3t2ASjtnkyVg0tjiyWkIjp21d5MG4,14196
+sqlmesh_dbt/operations.py,sha256=-9yM-czQpaA-U42kad5a5OJXIAh63FiIBtuP_672AF0,14331
 sqlmesh_dbt/options.py,sha256=noB_qK4uGGi7Erqk1XkkMaFz5aUc6lp44wwn1Nv_LI4,737
 sqlmesh_dbt/selectors.py,sha256=nmVrFsC7CR2A24FdGTp5Wz7MuWreI-xLQTpOTy0H9K4,6543
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -363,8 +363,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=8XTwgOh6QBbNy_hTM1JuHgRjbnie-pGPrphiW-FNLjQ,6058
-sqlmesh-0.225.1.dev15.dist-info/METADATA,sha256=2bhN5fdwuRPD3RyRmeqx11wNE-FtRYripkZfJeKoRdI,26686
-sqlmesh-0.225.1.dev15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.225.1.dev15.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
-sqlmesh-0.225.1.dev15.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
-sqlmesh-0.225.1.dev15.dist-info/RECORD,,
+sqlmesh-0.225.1.dev17.dist-info/METADATA,sha256=3Z-gHxpytLz_svwdo2EkWmyBvt1IhiXS6GYcVleOS74,26686
+sqlmesh-0.225.1.dev17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.225.1.dev17.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
+sqlmesh-0.225.1.dev17.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
+sqlmesh-0.225.1.dev17.dist-info/RECORD,,

sqlmesh_dbt/cli.py

@@ -84,6 +84,16 @@ resource_type_option = click.option(
     type=click.Choice(["debug", "info", "warn", "error", "none"]),
     help="Specify the minimum severity of events that are logged to the console and the log file.",
 )
+@click.option(
+    "--profiles-dir",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Which directory to look in for the profiles.yml file. If not set, dbt will look in the current working directory first, then HOME/.dbt/",
+)
+@click.option(
+    "--project-dir",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.",
+)
 @click.pass_context
 @cli_global_error_handler
 def dbt(
@@ -92,6 +102,8 @@ def dbt(
     target: t.Optional[str] = None,
     debug: bool = False,
     log_level: t.Optional[str] = None,
+    profiles_dir: t.Optional[Path] = None,
+    project_dir: t.Optional[Path] = None,
 ) -> None:
     """
     An ELT tool for managing your SQL transformations and data models, powered by the SQLMesh engine.
@@ -105,7 +117,8 @@ def dbt(
     # that need to be known before we attempt to load the project
     ctx.obj = functools.partial(
         create,
-        project_dir=Path.cwd(),
+        project_dir=project_dir,
+        profiles_dir=profiles_dir,
         profile=profile,
         target=target,
         debug=debug,

sqlmesh_dbt/operations.py

@@ -232,6 +232,7 @@ class DbtOperations:
 
 def create(
     project_dir: t.Optional[Path] = None,
+    profiles_dir: t.Optional[Path] = None,
     profile: t.Optional[str] = None,
     target: t.Optional[str] = None,
     vars: t.Optional[t.Dict[str, t.Any]] = None,
@@ -268,7 +269,11 @@ def create(
         sqlmesh_context = Context(
             paths=[project_dir],
             config_loader_kwargs=dict(
-                profile=profile, target=target, variables=vars, threads=threads
+                profile=profile,
+                target=target,
+                variables=vars,
+                threads=threads,
+                profiles_dir=profiles_dir,
             ),
             load=True,
             # DbtSelector selects based on dbt model fqn's rather than SQLMesh model names