sqlmesh 0.184.4.dev2__py3-none-any.whl → 0.184.4.dev4__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.184.4.dev2'
-__version_tuple__ = version_tuple = (0, 184, 4, 'dev2')
+__version__ = version = '0.184.4.dev4'
+__version_tuple__ = version_tuple = (0, 184, 4, 'dev4')

sqlmesh/core/config/connection.py

@@ -13,6 +13,7 @@ from functools import partial
 
 import pydantic
 from pydantic import Field
+from packaging import version
 from sqlglot import exp
 from sqlglot.helper import subclasses
 
@@ -201,6 +202,7 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
         catalogs: Key is the name of the catalog and value is the path.
         extensions: A list of autoloadable extensions to load.
         connector_config: A dictionary of configuration to pass into the duckdb connector.
+        secrets: A list of dictionaries used to generate DuckDB secrets for authenticating with external services (e.g. S3).
         concurrent_tasks: The maximum number of tasks that can use this connection concurrently.
         register_comments: Whether or not to register model comments with the SQL engine.
         pre_ping: Whether or not to pre-ping the connection before starting a new transaction to ensure it is still alive.
@@ -211,6 +213,7 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
     catalogs: t.Optional[t.Dict[str, t.Union[str, DuckDBAttachOptions]]] = None
     extensions: t.List[t.Union[str, t.Dict[str, t.Any]]] = []
     connector_config: t.Dict[str, t.Any] = {}
+    secrets: t.List[t.Dict[str, t.Any]] = []
 
     concurrent_tasks: int = 1
     register_comments: bool = True
@@ -283,6 +286,28 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
                 except Exception as e:
                     raise ConfigError(f"Failed to set connector config {field} to {setting}: {e}")
 
+            if self.secrets:
+                duckdb_version = duckdb.__version__
+                if version.parse(duckdb_version) < version.parse("0.10.0"):
+                    from sqlmesh.core.console import get_console
+
+                    get_console().log_warning(
+                        f"DuckDB version {duckdb_version} does not support secrets-based authentication (requires 0.10.0 or later).\n"
+                        "To use secrets, please upgrade DuckDB. For older versions, configure legacy authentication via `connector_config`.\n"
+                        "More info: https://duckdb.org/docs/stable/extensions/httpfs/s3api_legacy_authentication.html"
+                    )
+                else:
+                    for secrets in self.secrets:
+                        secret_settings: t.List[str] = []
+                        for field, setting in secrets.items():
+                            secret_settings.append(f"{field} '{setting}'")
+                        if secret_settings:
+                            secret_clause = ", ".join(secret_settings)
+                            try:
+                                cursor.execute(f"CREATE SECRET ({secret_clause});")
+                            except Exception as e:
+                                raise ConfigError(f"Failed to create secret: {e}")
+
             for i, (alias, path_options) in enumerate(
                 (getattr(self, "catalogs", None) or {}).items()
             ):

sqlmesh/core/engine_adapter/base.py

@@ -106,6 +106,7 @@ class EngineAdapter:
     SUPPORTS_REPLACE_TABLE = True
     DEFAULT_CATALOG_TYPE = DIALECT
     QUOTE_IDENTIFIERS_IN_VIEWS = True
+    MAX_IDENTIFIER_LENGTH: t.Optional[int] = None
 
     def __init__(
         self,
@@ -2138,14 +2139,12 @@ class EngineAdapter:
         )
         with self.transaction():
             for e in ensure_list(expressions):
-                sql = t.cast(
-                    str,
-                    (
-                        self._to_sql(e, quote=quote_identifiers, **to_sql_kwargs)
-                        if isinstance(e, exp.Expression)
-                        else e
-                    ),
-                )
+                if isinstance(e, exp.Expression):
+                    self._check_identifier_length(e)
+                    sql = self._to_sql(e, quote=quote_identifiers, **to_sql_kwargs)
+                else:
+                    sql = t.cast(str, e)
+
                 self._log_sql(
                     sql,
                     expression=e if isinstance(e, exp.Expression) else None,
@@ -2516,6 +2515,18 @@ class EngineAdapter:
     def _select_columns(cls, columns: t.Iterable[str]) -> exp.Select:
         return exp.select(*(exp.column(c, quoted=True) for c in columns))
 
+    def _check_identifier_length(self, expression: exp.Expression) -> None:
+        if self.MAX_IDENTIFIER_LENGTH is None or not isinstance(expression, exp.DDL):
+            return
+
+        for identifier in expression.find_all(exp.Identifier):
+            name = identifier.name
+            name_length = len(name)
+            if name_length > self.MAX_IDENTIFIER_LENGTH:
+                raise SQLMeshError(
+                    f"Identifier name '{name}' (length {name_length}) exceeds {self.dialect.capitalize()}'s max identifier limit of {self.MAX_IDENTIFIER_LENGTH} characters"
+                )
+
 
 class EngineAdapterWithIndexSupport(EngineAdapter):
     SUPPORTS_INDEXES = True

sqlmesh/core/engine_adapter/mysql.py

@@ -39,6 +39,7 @@ class MySQLEngineAdapter(
     MAX_TABLE_COMMENT_LENGTH = 2048
     MAX_COLUMN_COMMENT_LENGTH = 1024
     SUPPORTS_REPLACE_TABLE = False
+    MAX_IDENTIFIER_LENGTH = 64
     SCHEMA_DIFFER = SchemaDiffer(
         parameterized_type_defaults={
             exp.DataType.build("BIT", dialect=DIALECT).this: [(1,)],

sqlmesh/core/engine_adapter/postgres.py

@@ -34,6 +34,7 @@ class PostgresEngineAdapter(
     HAS_VIEW_BINDING = True
     CURRENT_CATALOG_EXPRESSION = exp.column("current_catalog")
     SUPPORTS_REPLACE_TABLE = False
+    MAX_IDENTIFIER_LENGTH = 63
     SCHEMA_DIFFER = SchemaDiffer(
         parameterized_type_defaults={
             # DECIMAL without precision is "up to 131072 digits before the decimal point; up to 16383 digits after the decimal point"

sqlmesh/core/engine_adapter/risingwave.py

@@ -30,6 +30,7 @@ class RisingwaveEngineAdapter(PostgresEngineAdapter):
     COMMENT_CREATION_VIEW = CommentCreationView.UNSUPPORTED
     SUPPORTS_MATERIALIZED_VIEWS = True
     SUPPORTS_TRANSACTIONS = False
+    MAX_IDENTIFIER_LENGTH = None
 
     def _truncate_table(self, table_name: TableName) -> None:
         return self.execute(exp.Delete(this=exp.to_table(table_name)))

sqlmesh/dbt/target.py

@@ -151,6 +151,7 @@ class DuckDbConfig(TargetConfig):
         path: Location of the database file. If not specified, an in memory database is used.
         extensions: A list of autoloadable extensions to load.
         settings: A dictionary of settings to pass into the duckdb connector.
+        secrets: A list of secrets to pass to the secret manager in the duckdb connector.
     """
 
     type: t.Literal["duckdb"] = "duckdb"
@@ -159,6 +160,7 @@ class DuckDbConfig(TargetConfig):
     path: str = DUCKDB_IN_MEMORY
     extensions: t.Optional[t.List[str]] = None
     settings: t.Optional[t.Dict[str, t.Any]] = None
+    secrets: t.Optional[t.List[t.Dict[str, t.Any]]] = None
 
     @model_validator(mode="before")
     def validate_authentication(cls, data: t.Any) -> t.Any:
@@ -192,6 +194,8 @@ class DuckDbConfig(TargetConfig):
             kwargs["extensions"] = self.extensions
         if self.settings is not None:
             kwargs["connector_config"] = self.settings
+        if self.secrets is not None:
+            kwargs["secrets"] = self.secrets
         return DuckDBConnectionConfig(
             database=self.path,
             concurrent_tasks=1,

{sqlmesh-0.184.4.dev2.dist-info → sqlmesh-0.184.4.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.184.4.dev2
+Version: 0.184.4.dev4
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.184.4.dev2.dist-info → sqlmesh-0.184.4.dev4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=DRovDXI-9D9xeYDP7t8Fq7mI31S2btlN-LHPOK3y7PA,7240
-sqlmesh/_version.py,sha256=wQuD5_QObpaHiyQWgdojLHW571NIDF9meJc6Ge5ITBI,528
+sqlmesh/_version.py,sha256=ifVqs-FEUDrnjUcfvVbgyQx7o0Vun52nDVXWHD0E3wI,528
 sqlmesh/magics.py,sha256=aeOyTdeNnyGzGtJXECDAbqJ8OAplrV4UAYl0I2hhtig,39670
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,7 +41,7 @@ sqlmesh/core/config/__init__.py,sha256=b4gFb04tpeZpowqJD-7h3U3TLgxrpQhZahVEAo6VW
 sqlmesh/core/config/base.py,sha256=t8NQmsgQoZSc-k0dlDiCb8t1jj0AMYdGZ-6se9q_Pks,4898
 sqlmesh/core/config/categorizer.py,sha256=6vzUoNLjR6GOEb_2mYVz2TwmMv2BfldgHX2u-Le5HZs,1975
 sqlmesh/core/config/common.py,sha256=-o_HoRFtR4u84frg_TLno7z7vS9II6ASiOayb59MKHo,2806
-sqlmesh/core/config/connection.py,sha256=kMhHzIKAIz2uiM5XhRgn-yMUOVS1B_aJEmIejGnUClo,71976
+sqlmesh/core/config/connection.py,sha256=2xEBZFjaHgdTiSj0MjDIfCPnOoe5dT4gRQ2RYm0bQ80,73499
 sqlmesh/core/config/feature_flag.py,sha256=JRcD31OG_t2cYc24meKbHDGB0iVID0P07V2oOeB_mgA,203
 sqlmesh/core/config/format.py,sha256=6CXFbvnor56xbldKE-Vrm9k_ABRoY4v6vgIb3mCihiQ,1355
 sqlmesh/core/config/gateway.py,sha256=tYngyqwd_4Qr9lhcv2hlvLvb_2pgYYtKu6hdGsTr-4I,1931
@@ -59,7 +59,7 @@ sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
 sqlmesh/core/engine_adapter/_typing.py,sha256=BAM_U7Vsi0k37XHu_nNnu3whXxuTVDey_uZe3aelKYY,1033
 sqlmesh/core/engine_adapter/athena.py,sha256=WBVAB_YzBJXdiqq51LTd2jmYoMtGrLmtsA8IrIFVnQ0,26199
-sqlmesh/core/engine_adapter/base.py,sha256=XQeJi4Rr5VllnQKFZBYXNs1G142T16ZOVU_l9UYvzoE,100539
+sqlmesh/core/engine_adapter/base.py,sha256=WDyZhieFq9iBQSDsLdf-ae6aPu25d-FpwZ4Lt3FPf6o,101153
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=DOsOdCJyQAVfA9S0GznuH0J1bonMqYVP_yJLDYcxFYM,6772
 sqlmesh/core/engine_adapter/bigquery.py,sha256=gnd2cFc7v95tT_E9XMHHgccS-QwSF2sJHsL8xfcSguk,49249
 sqlmesh/core/engine_adapter/clickhouse.py,sha256=4ACsYsZ8-gvyCozwoEMEayTTUdMjmNea4xUkxtXw6R4,34907
@@ -67,10 +67,10 @@ sqlmesh/core/engine_adapter/databricks.py,sha256=AbNbnt9nQdOd7lTov2zsKROZlzgV2fL
 sqlmesh/core/engine_adapter/duckdb.py,sha256=FfNmBzLySDDi--nTlsSavTwdperCzhpYzDoroCilPcA,4900
 sqlmesh/core/engine_adapter/mixins.py,sha256=HMB0Z9O3p5jaTZMolKiWKqpVDZokf5GoI99YXwEJ8bc,22470
 sqlmesh/core/engine_adapter/mssql.py,sha256=N21kjLrvloxLPMCGloQx5Jew5SIddrCVq3lTrnl8coc,11788
-sqlmesh/core/engine_adapter/mysql.py,sha256=HnMtNjxnCVz3X3-uK8Im-BNgLo_viX6QH3Uo8rMfPeY,6935
-sqlmesh/core/engine_adapter/postgres.py,sha256=XWnV7MDndkCRokb7ih2yxXodw4W8E61yHbuWtRTFE3k,4570
+sqlmesh/core/engine_adapter/mysql.py,sha256=lVC-siXZK3iSfqCmBjZNw8uy09Z8la8v8_oUZ-E2OwI,6966
+sqlmesh/core/engine_adapter/postgres.py,sha256=fdQrsBodfGlNf8lZYkS99U_gjoDyaYKEVKLgScduv3I,4601
 sqlmesh/core/engine_adapter/redshift.py,sha256=_rZWv4rchDAzB6m6idoV1NLHFBKK_gM4nlfeErqXEcE,16691
-sqlmesh/core/engine_adapter/risingwave.py,sha256=NXXuiPfbB5ZSp7igMqZpaX2c-5TYreOlIouPU7iCVJ8,936
+sqlmesh/core/engine_adapter/risingwave.py,sha256=YYCQ0EJaSrR9_P1_2_LmElQ6zSxqkZzhvhdm-2WtWWg,969
 sqlmesh/core/engine_adapter/shared.py,sha256=NyF2osSJDWPp7KlJUBR4GZcS_cJFr4daUPvhGyp4XcQ,12909
 sqlmesh/core/engine_adapter/snowflake.py,sha256=gxZWDOY12itwVWF6anqyZI6bXnaK2wY3Uca0qnH5ffk,27142
 sqlmesh/core/engine_adapter/spark.py,sha256=nDJXH1su08wv3m9X0Mz-x98xhZblq1BvM3EtRYjZhrI,21012
@@ -136,7 +136,7 @@ sqlmesh/dbt/project.py,sha256=4PJHPVWIKDd71J5__iICqEp21nFiijKjJkiE5kZIH5Y,3980
 sqlmesh/dbt/relation.py,sha256=3w_4_Dj-oZXUEdBqOl9U6twbt8S2gDNMaFRb3unqkE8,224
 sqlmesh/dbt/seed.py,sha256=E7pFhtGwRrC9U_VZJNABwLTa-Nk4OeD-gOtjSpvYmDA,4251
 sqlmesh/dbt/source.py,sha256=IZK-aOV-KdWIOPuXqRjJZAZ7phW1HUXoHtE-3clgkx4,4125
-sqlmesh/dbt/target.py,sha256=18jn-BmRO4oR9w-R1_y1RGhQyvzQD9HPu0r8gwJaivI,40298
+sqlmesh/dbt/target.py,sha256=YFCqQPhPcMa_7chMoKTTwIYEqaERlDvNipEGPw2RmM4,40529
 sqlmesh/dbt/test.py,sha256=aOTKqYpVafff-TU4Xhbc9eZr6QAuFLMxIu_sV-18JLQ,8282
 sqlmesh/dbt/util.py,sha256=uDFgAXBRFTHNVLJdNhMASqOURe_1OH132AD97sMjj1E,774
 sqlmesh/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -268,7 +268,7 @@ sqlmesh/utils/pydantic.py,sha256=-u69PDY-4FLR7ODNKBoN7PsPNmZ0wdhsJcCAG2P6vyY,110
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.184.4.dev2.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.184.4.dev4.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=x9qfdHkVkXOwWRFRCZGaixiLj5e9tOQsCDRR92vW9QI,5472
 web/client/dist/assets/Audits-cGDtiiKB.js,sha256=jrS4x1nQ8Klx67744_LjNI2oBOI9XnnygDR-0wmlme8,1041
@@ -386,8 +386,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.184.4.dev2.dist-info/METADATA,sha256=s_ub4vftL-EKckwWIHPNKy72z0qsjlz97GRBoa-yhz8,25498
-sqlmesh-0.184.4.dev2.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-sqlmesh-0.184.4.dev2.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.184.4.dev2.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.184.4.dev2.dist-info/RECORD,,
+sqlmesh-0.184.4.dev4.dist-info/METADATA,sha256=XyejHBbOyFdij3c1e0wMllTRrHkSJVfCJGVXPYMEh9A,25498
+sqlmesh-0.184.4.dev4.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+sqlmesh-0.184.4.dev4.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.184.4.dev4.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.184.4.dev4.dist-info/RECORD,,