sqlmesh 0.196.1.dev4__py3-none-any.whl → 0.196.1.dev6__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.196.1.dev4'
-__version_tuple__ = version_tuple = (0, 196, 1, 'dev4')
+__version__ = version = '0.196.1.dev6'
+__version_tuple__ = version_tuple = (0, 196, 1, 'dev6')

sqlmesh/core/config/connection.py

@@ -266,6 +266,7 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
         extensions: A list of autoloadable extensions to load.
         connector_config: A dictionary of configuration to pass into the duckdb connector.
         secrets: A list of dictionaries used to generate DuckDB secrets for authenticating with external services (e.g. S3).
+        filesystems: A list of dictionaries used to register `fsspec` filesystems to the DuckDB cursor.
         concurrent_tasks: The maximum number of tasks that can use this connection concurrently.
         register_comments: Whether or not to register model comments with the SQL engine.
         pre_ping: Whether or not to pre-ping the connection before starting a new transaction to ensure it is still alive.
@@ -277,6 +278,7 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
     extensions: t.List[t.Union[str, t.Dict[str, t.Any]]] = []
     connector_config: t.Dict[str, t.Any] = {}
     secrets: t.List[t.Dict[str, t.Any]] = []
+    filesystems: t.List[t.Dict[str, t.Any]] = []
 
     concurrent_tasks: int = 1
     register_comments: bool = True
@@ -371,6 +373,15 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
                             except Exception as e:
                                 raise ConfigError(f"Failed to create secret: {e}")
 
+            if self.filesystems:
+                from fsspec import filesystem  # type: ignore
+
+                for file_system in self.filesystems:
+                    options = file_system.copy()
+                    fs = options.pop("fs")
+                    fs = filesystem(fs, **options)
+                    cursor.register_filesystem(fs)
+
             for i, (alias, path_options) in enumerate(
                 (getattr(self, "catalogs", None) or {}).items()
             ):

sqlmesh/core/snapshot/evaluator.py

@@ -911,39 +911,40 @@ class SnapshotEvaluator:
         ):
             return
 
+        deployability_index = DeployabilityIndex.all_deployable()
+        render_kwargs: t.Dict[str, t.Any] = dict(
+            engine_adapter=adapter,
+            snapshots=parent_snapshots_by_name(snapshot, snapshots),
+            runtime_stage=RuntimeStage.CREATING,
+            deployability_index=deployability_index,
+        )
         target_table_name = snapshot.table_name()
-        if adapter.table_exists(target_table_name):
-            evaluation_strategy = _evaluation_strategy(snapshot, adapter)
-            tmp_table_name = snapshot.table_name(is_deployable=False)
-            logger.info(
-                "Migrating table schema from '%s' to '%s'",
-                tmp_table_name,
-                target_table_name,
-            )
-            evaluation_strategy.migrate(
-                target_table_name=target_table_name,
-                source_table_name=tmp_table_name,
-                snapshot=snapshot,
-                snapshots=parent_snapshots_by_name(snapshot, snapshots),
-                allow_destructive_snapshots=allow_destructive_snapshots,
-            )
-        else:
-            logger.info(
-                "Creating table '%s' for the snapshot of the forward-only model %s",
-                target_table_name,
-                snapshot.snapshot_id,
-            )
-            deployability_index = DeployabilityIndex.all_deployable()
-            render_kwargs: t.Dict[str, t.Any] = dict(
-                engine_adapter=adapter,
-                snapshots=parent_snapshots_by_name(snapshot, snapshots),
-                runtime_stage=RuntimeStage.CREATING,
-                deployability_index=deployability_index,
-            )
-            with (
-                adapter.transaction(),
-                adapter.session(snapshot.model.render_session_properties(**render_kwargs)),
-            ):
+
+        with (
+            adapter.transaction(),
+            adapter.session(snapshot.model.render_session_properties(**render_kwargs)),
+        ):
+            if adapter.table_exists(target_table_name):
+                evaluation_strategy = _evaluation_strategy(snapshot, adapter)
+                tmp_table_name = snapshot.table_name(is_deployable=False)
+                logger.info(
+                    "Migrating table schema from '%s' to '%s'",
+                    tmp_table_name,
+                    target_table_name,
+                )
+                evaluation_strategy.migrate(
+                    target_table_name=target_table_name,
+                    source_table_name=tmp_table_name,
+                    snapshot=snapshot,
+                    snapshots=parent_snapshots_by_name(snapshot, snapshots),
+                    allow_destructive_snapshots=allow_destructive_snapshots,
+                )
+            else:
+                logger.info(
+                    "Creating table '%s' for the snapshot of the forward-only model %s",
+                    target_table_name,
+                    snapshot.snapshot_id,
+                )
                 self._execute_create(
                     snapshot=snapshot,
                     table_name=target_table_name,

sqlmesh/dbt/target.py

@@ -138,6 +138,7 @@ class DuckDbConfig(TargetConfig):
         extensions: A list of autoloadable extensions to load.
         settings: A dictionary of settings to pass into the duckdb connector.
         secrets: A list of secrets to pass to the secret manager in the duckdb connector.
+        filesystems: A list of `fsspec` filesystems to register in the duckdb connection.
     """
 
     type: t.Literal["duckdb"] = "duckdb"
@@ -147,6 +148,7 @@ class DuckDbConfig(TargetConfig):
     extensions: t.Optional[t.List[str]] = None
     settings: t.Optional[t.Dict[str, t.Any]] = None
     secrets: t.Optional[t.List[t.Dict[str, t.Any]]] = None
+    filesystems: t.Optional[t.List[t.Dict[str, t.Any]]] = None
 
     @model_validator(mode="before")
     def validate_authentication(cls, data: t.Any) -> t.Any:
@@ -182,6 +184,8 @@ class DuckDbConfig(TargetConfig):
             kwargs["connector_config"] = self.settings
         if self.secrets is not None:
             kwargs["secrets"] = self.secrets
+        if self.filesystems is not None:
+            kwargs["filesystems"] = self.filesystems
         return DuckDBConnectionConfig(
             database=self.path,
             concurrent_tasks=1,

{sqlmesh-0.196.1.dev4.dist-info → sqlmesh-0.196.1.dev6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.196.1.dev4
+Version: 0.196.1.dev6
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.196.1.dev4.dist-info → sqlmesh-0.196.1.dev6.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=7t8C0kEKJWqTH-kL0UAU1xOCz2miDlgoxSBKJ0QCvMA,528
+sqlmesh/_version.py,sha256=ss8qzuMXEEN16YeHsRoAwO7vZWIY6zgCs81hS4otaiM,528
 sqlmesh/magics.py,sha256=nMfwSW5rIkdnyO9MXYm9u24hNl6dqgAWybC-MnRcNnE,41451
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,7 +41,7 @@ sqlmesh/core/config/__init__.py,sha256=b4gFb04tpeZpowqJD-7h3U3TLgxrpQhZahVEAo6VW
 sqlmesh/core/config/base.py,sha256=t8NQmsgQoZSc-k0dlDiCb8t1jj0AMYdGZ-6se9q_Pks,4898
 sqlmesh/core/config/categorizer.py,sha256=6vzUoNLjR6GOEb_2mYVz2TwmMv2BfldgHX2u-Le5HZs,1975
 sqlmesh/core/config/common.py,sha256=rsb2uVFqPEX3DMsWG_hApwztCuRJP-FAmaQ_5L3-S8g,4006
-sqlmesh/core/config/connection.py,sha256=-KF0xufXPt8HIYZmRdyQokyq59xOiCS4S2zSgIi0LUg,85462
+sqlmesh/core/config/connection.py,sha256=7-9WN1Rq2j9g_TilRlhxuzO4ugWeN3eCtMkU0Um6lmw,85959
 sqlmesh/core/config/feature_flag.py,sha256=JRcD31OG_t2cYc24meKbHDGB0iVID0P07V2oOeB_mgA,203
 sqlmesh/core/config/format.py,sha256=6CXFbvnor56xbldKE-Vrm9k_ABRoY4v6vgIb3mCihiQ,1355
 sqlmesh/core/config/gateway.py,sha256=tYngyqwd_4Qr9lhcv2hlvLvb_2pgYYtKu6hdGsTr-4I,1931
@@ -103,7 +103,7 @@ sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4Tto
 sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
 sqlmesh/core/snapshot/definition.py,sha256=mjGZBpzhKI9gYLmuYhSAs0IBKEo_JdfX66EcR9yRcig,84359
-sqlmesh/core/snapshot/evaluator.py,sha256=zuxNnEKCyImFIT_hWLNc7xtKv4uflkKXOLprkD_hCJE,90132
+sqlmesh/core/snapshot/evaluator.py,sha256=RA-ZFwG_BtkXSmls41gHRaCqa-h-evxXsDLE5jtgxvU,90173
 sqlmesh/core/state_sync/__init__.py,sha256=B60xj-_Xk-4jas3UphLvwkuf04G-9AyGckcblXE0aRg,869
 sqlmesh/core/state_sync/base.py,sha256=Drq4MDWWxO_J-sJIxsMG1-YSgj2L42n5WM2vh1sr6bc,18334
 sqlmesh/core/state_sync/cache.py,sha256=0AsYdQyBqbLdJvkBQ-hDAipsvnn_jSps-nya-Q4XeL0,5337
@@ -139,7 +139,7 @@ sqlmesh/dbt/project.py,sha256=4PJHPVWIKDd71J5__iICqEp21nFiijKjJkiE5kZIH5Y,3980
 sqlmesh/dbt/relation.py,sha256=rgDJQUuENjqq7kpMJJ6AdlgwYmcYjA1oFj6LCYCS-Hg,227
 sqlmesh/dbt/seed.py,sha256=E7pFhtGwRrC9U_VZJNABwLTa-Nk4OeD-gOtjSpvYmDA,4251
 sqlmesh/dbt/source.py,sha256=IZK-aOV-KdWIOPuXqRjJZAZ7phW1HUXoHtE-3clgkx4,4125
-sqlmesh/dbt/target.py,sha256=BOBZbpz2EYRT6A8l1Yomta60qllbwMfTP7oP19FPXqY,42584
+sqlmesh/dbt/target.py,sha256=aHPt0jSb-Rsqg-uGYZY_rwIXUCmJ8YmsaGh2HiczWbI,42831
 sqlmesh/dbt/test.py,sha256=aOTKqYpVafff-TU4Xhbc9eZr6QAuFLMxIu_sV-18JLQ,8282
 sqlmesh/dbt/util.py,sha256=BdtfQ7sxnZ8y8CFJStkePpmRE-KntHCXTefRnXR5Pag,905
 sqlmesh/dbt/converter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -283,7 +283,7 @@ sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,116
 sqlmesh/utils/rich.py,sha256=BpB47G0-JcU41LkGkeGKroexj4NOV5fox4-GOIXPgIc,3545
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.196.1.dev4.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.196.1.dev6.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=drXwzSCEVDTMfaIwzJYjnT8Xk4c-Y4wNJz-CflUa_vY,5472
 web/client/dist/assets/Audits-DrNQaTQA.js,sha256=E8JF632bsRUm6LGtgXlyHrJOYQe5VW2CgRZPaZNKvgg,1041
@@ -401,8 +401,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.196.1.dev4.dist-info/METADATA,sha256=uv6fZGChbG2btFPPUf4yrluOqmTMDoLNEHUdyEYEszs,26690
-sqlmesh-0.196.1.dev4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.196.1.dev4.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.196.1.dev4.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.196.1.dev4.dist-info/RECORD,,
+sqlmesh-0.196.1.dev6.dist-info/METADATA,sha256=MUshvxIdlIMCB-pCyTJkcOz4TP1tnWXG4WNsCy9YmjM,26690
+sqlmesh-0.196.1.dev6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.196.1.dev6.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.196.1.dev6.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.196.1.dev6.dist-info/RECORD,,