sqlmesh 0.204.1.dev4__tar.gz → 0.204.1.dev5__tar.gz

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.204.1.dev4
+Version: 0.204.1.dev5
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.204.1.dev4'
-__version_tuple__ = version_tuple = (0, 204, 1, 'dev4')
+__version__ = version = '0.204.1.dev5'
+__version_tuple__ = version_tuple = (0, 204, 1, 'dev5')

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh/core/config/__init__.py

@@ -2,7 +2,10 @@ from sqlmesh.core.config.categorizer import (
     AutoCategorizationMode as AutoCategorizationMode,
     CategorizerConfig as CategorizerConfig,
 )
-from sqlmesh.core.config.common import EnvironmentSuffixTarget as EnvironmentSuffixTarget
+from sqlmesh.core.config.common import (
+    EnvironmentSuffixTarget as EnvironmentSuffixTarget,
+    TableNamingConvention as TableNamingConvention,
+)
 from sqlmesh.core.config.connection import (
     AthenaConnectionConfig as AthenaConnectionConfig,
     BaseDuckDBConnectionConfig as BaseDuckDBConnectionConfig,

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh/core/config/common.py

@@ -49,6 +49,34 @@ class EnvironmentSuffixTarget(str, Enum):
         return str(self)
 
 
+class TableNamingConvention(str, Enum):
+    # Causes table names at the physical layer to follow the convention:
+    # <schema-name>__<table-name>__<fingerprint>
+    SCHEMA_AND_TABLE = "schema_and_table"
+
+    # Causes table names at the physical layer to follow the convention:
+    # <table-name>__<fingerprint>
+    TABLE_ONLY = "table_only"
+
+    # Takes the table name that would be returned from SCHEMA_AND_TABLE and wraps it in md5()
+    # to generate a hash and prefixes the has with `sqlmesh_md5__`, for the following reasons:
+    # - at a glance, you can still see it's managed by sqlmesh and that md5 was used to generate the hash
+    # - unquoted identifiers that start with numbers can trip up DB engine parsers, so having a text prefix prevents this
+    # This causes table names at the physical layer to follow the convention:
+    # sqlmesh_md5__3b07384d113edec49eaa6238ad5ff00d
+    HASH_MD5 = "hash_md5"
+
+    @classproperty
+    def default(cls) -> TableNamingConvention:
+        return TableNamingConvention.SCHEMA_AND_TABLE
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
 def _concurrent_tasks_validator(v: t.Any) -> int:
     if isinstance(v, str):
         v = int(v)

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh/core/config/root.py

@@ -14,7 +14,7 @@ from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 from sqlmesh.cicd.config import CICDBotConfig
 from sqlmesh.core import constants as c
 from sqlmesh.core.console import get_console
-from sqlmesh.core.config import EnvironmentSuffixTarget
+from sqlmesh.core.config import EnvironmentSuffixTarget, TableNamingConvention
 from sqlmesh.core.config.base import BaseConfig, UpdateStrategy
 from sqlmesh.core.config.common import variables_validator, compile_regex_mapping
 from sqlmesh.core.config.connection import (
@@ -106,6 +106,7 @@ class Config(BaseConfig):
         model_defaults: Default values for model definitions.
         physical_schema_mapping: A mapping from regular expressions to names of schemas in which physical tables for corresponding models will be placed.
         environment_suffix_target: Indicates whether to append the environment name to the schema or table name.
+        physical_table_naming_convention: Indicates how tables should be named at the physical layer
         gateway_managed_virtual_layer: Whether the models' views in the virtual layer are created by the model-specific gateway rather than the default gateway.
         infer_python_dependencies: Whether to statically analyze Python code to automatically infer Python package requirements.
         environment_catalog_mapping: A mapping from regular expressions to catalog names. The catalog name is used to determine the target catalog for a given environment.
@@ -147,6 +148,9 @@ class Config(BaseConfig):
     environment_suffix_target: EnvironmentSuffixTarget = Field(
         default=EnvironmentSuffixTarget.default
     )
+    physical_table_naming_convention: TableNamingConvention = Field(
+        default=TableNamingConvention.default
+    )
     gateway_managed_virtual_layer: bool = False
     infer_python_dependencies: bool = True
     environment_catalog_mapping: RegexKeyDict = {}

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh/core/context.py

@@ -2904,9 +2904,11 @@ class GenericContext(BaseContext, t.Generic[C]):
         fingerprint_cache: t.Dict[str, SnapshotFingerprint] = {}
 
         for node in nodes.values():
-            kwargs = {}
+            kwargs: t.Dict[str, t.Any] = {}
             if node.project in self._projects:
-                kwargs["ttl"] = self.config_for_node(node).snapshot_ttl
+                config = self.config_for_node(node)
+                kwargs["ttl"] = config.snapshot_ttl
+                kwargs["table_naming_convention"] = config.physical_table_naming_convention
 
             snapshot = Snapshot.from_node(
                 node,

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh/core/snapshot/definition.py

@@ -13,6 +13,7 @@ from pydantic import Field
 from sqlglot import exp
 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 
+from sqlmesh.core.config import TableNamingConvention
 from sqlmesh.core import constants as c
 from sqlmesh.core.audit import StandaloneAudit
 from sqlmesh.core.environment import EnvironmentSuffixTarget
@@ -44,7 +45,7 @@ from sqlmesh.utils.metaprogramming import (
     format_evaluated_code_exception,
     Executable,
 )
-from sqlmesh.utils.hashing import hash_data
+from sqlmesh.utils.hashing import hash_data, md5
 from sqlmesh.utils.pydantic import PydanticModel, field_validator
 
 if t.TYPE_CHECKING:
@@ -227,6 +228,7 @@ class SnapshotDataVersion(PydanticModel, frozen=True):
     change_category: t.Optional[SnapshotChangeCategory] = None
     physical_schema_: t.Optional[str] = Field(default=None, alias="physical_schema")
     dev_table_suffix: str
+    table_naming_convention: TableNamingConvention = Field(default=TableNamingConvention.default)
 
     def snapshot_id(self, name: str) -> SnapshotId:
         return SnapshotId(name=name, identifier=self.fingerprint.to_identifier())
@@ -333,6 +335,7 @@ class SnapshotInfoMixin(ModelKindMixin):
     # This can be removed from this model once Pydantic 1 support is dropped (must remain in `Snapshot` though)
     base_table_name_override: t.Optional[str]
     dev_table_suffix: str
+    table_naming_convention: TableNamingConvention = Field(default=TableNamingConvention.default)
 
     @cached_property
     def identifier(self) -> str:
@@ -451,6 +454,7 @@ class SnapshotInfoMixin(ModelKindMixin):
             version,
             catalog=self.fully_qualified_table.catalog,
             suffix=self.dev_table_suffix if is_dev_table else None,
+            naming_convention=self.table_naming_convention,
         )
 
     @property
@@ -580,6 +584,7 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
         migrated: Whether or not this snapshot has been created as a result of migration.
         unrestorable: Whether or not this snapshot can be used to revert its model to a previous version.
         next_auto_restatement_ts: The timestamp which indicates when is the next time this snapshot should be restated.
+        table_naming_convention: Convention to follow when generating the physical table name
     """
 
     name: str
@@ -605,6 +610,9 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
     base_table_name_override: t.Optional[str] = None
     next_auto_restatement_ts: t.Optional[int] = None
     dev_table_suffix: str = "dev"
+    table_naming_convention_: TableNamingConvention = Field(
+        default=TableNamingConvention.default, alias="table_naming_convention"
+    )
 
     @field_validator("ttl")
     @classmethod
@@ -656,6 +664,7 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
         ttl: str = c.DEFAULT_SNAPSHOT_TTL,
         version: t.Optional[str] = None,
         cache: t.Optional[t.Dict[str, SnapshotFingerprint]] = None,
+        table_naming_convention: TableNamingConvention = TableNamingConvention.default,
     ) -> Snapshot:
         """Creates a new snapshot for a node.
 
@@ -666,6 +675,7 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
             ttl: A TTL to determine how long orphaned (snapshots that are not promoted anywhere) should live.
             version: The version that a snapshot is associated with. Usually set during the planning phase.
             cache: Cache of node name to fingerprints.
+            table_naming_convention: Convention to follow when generating the physical table name
 
         Returns:
             The newly created snapshot.
@@ -697,6 +707,7 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
             updated_ts=created_ts,
             ttl=ttl,
             version=version,
+            table_naming_convention=table_naming_convention,
         )
 
     def __eq__(self, other: t.Any) -> bool:
@@ -1013,6 +1024,7 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
             previous_version = self.previous_version
             self.version = previous_version.data_version.version
             self.physical_schema_ = previous_version.physical_schema
+            self.table_naming_convention = previous_version.table_naming_convention
             if self.is_materialized and (category.is_indirect_non_breaking or category.is_metadata):
                 # Reuse the dev table for indirect non-breaking changes.
                 self.dev_version_ = (
@@ -1206,6 +1218,7 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
             custom_materialization=custom_materialization,
             dev_table_suffix=self.dev_table_suffix,
             model_gateway=self.model_gateway,
+            table_naming_convention=self.table_naming_convention,  # type: ignore
         )
 
     @property
@@ -1218,6 +1231,7 @@ class Snapshot(PydanticModel, SnapshotInfoMixin):
             change_category=self.change_category,
             physical_schema=self.physical_schema,
             dev_table_suffix=self.dev_table_suffix,
+            table_naming_convention=self.table_naming_convention,
         )
 
     @property
@@ -1568,14 +1582,41 @@ def table_name(
     version: str,
     catalog: t.Optional[str] = None,
     suffix: t.Optional[str] = None,
+    naming_convention: t.Optional[TableNamingConvention] = None,
 ) -> str:
     table = exp.to_table(name)
 
-    # bigquery projects usually have "-" in them which is illegal in the table name, so we aggressively prune
-    name = "__".join(sanitize_name(part.name) for part in table.parts)
+    naming_convention = naming_convention or TableNamingConvention.default
+
+    if naming_convention == TableNamingConvention.HASH_MD5:
+        # just take a MD5 hash of what we would have generated anyway using SCHEMA_AND_TABLE
+        value_to_hash = table_name(
+            physical_schema=physical_schema,
+            name=name,
+            version=version,
+            catalog=catalog,
+            suffix=suffix,
+            naming_convention=TableNamingConvention.SCHEMA_AND_TABLE,
+        )
+        full_name = f"{c.SQLMESH}_md5__{md5(value_to_hash)}"
+    else:
+        # note: Snapshot._table_name() already strips the catalog from the model name before calling this function
+        # Therefore, a model with 3-part naming like "foo.bar.baz" gets passed as (name="bar.baz", catalog="foo") to this function
+        # This is why there is no TableNamingConvention.CATALOG_AND_SCHEMA_AND_TABLE
+        table_parts = table.parts
+        parts_to_consider = 2 if naming_convention == TableNamingConvention.SCHEMA_AND_TABLE else 1
+
+        # in case the parsed table name has less parts than what the naming convention says we should be considering
+        parts_to_consider = min(len(table_parts), parts_to_consider)
+
+        # bigquery projects usually have "-" in them which is illegal in the table name, so we aggressively prune
+        name = "__".join(sanitize_name(part.name) for part in table_parts[-parts_to_consider:])
+
+        full_name = f"{name}__{version}"
+
     suffix = f"__{suffix}" if suffix else ""
 
-    table.set("this", exp.to_identifier(f"{name}__{version}{suffix}"))
+    table.set("this", exp.to_identifier(f"{full_name}{suffix}"))
     table.set("db", exp.to_identifier(physical_schema))
     if not table.catalog and catalog:
         table.set("catalog", exp.to_identifier(catalog))

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh/utils/hashing.py

@@ -9,7 +9,9 @@ def crc32(data: t.Iterable[t.Optional[str]]) -> str:
     return str(zlib.crc32(_safe_concat(data)))
 
 
-def md5(data: t.Iterable[t.Optional[str]]) -> str:
+def md5(data: t.Union[str, t.Iterable[t.Optional[str]]]) -> str:
+    if isinstance(data, str):
+        data = [data]
     return hashlib.md5(_safe_concat(data)).hexdigest()
 
 

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/sqlmesh.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.204.1.dev4
+Version: 0.204.1.dev5
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/tests/core/test_config.py

@@ -3,6 +3,7 @@ import pathlib
 import re
 from pathlib import Path
 from unittest import mock
+import typing as t
 
 import pytest
 from pytest_mock import MockerFixture
@@ -17,6 +18,7 @@ from sqlmesh.core.config import (
     MotherDuckConnectionConfig,
     BuiltInSchedulerConfig,
     EnvironmentSuffixTarget,
+    TableNamingConvention,
 )
 from sqlmesh.core.config.connection import DuckDBAttachOptions, RedshiftConnectionConfig
 from sqlmesh.core.config.feature_flag import DbtFeatureFlag, FeatureFlag
@@ -1412,3 +1414,30 @@ SQLMESH__MODEL_DEFAULTS__DIALECT="postgres"
         default_gateway="test_gateway",
         model_defaults=ModelDefaultsConfig(dialect="postgres"),
     )
+
+
+@pytest.mark.parametrize(
+    "convention_str, expected",
+    [
+        (None, TableNamingConvention.SCHEMA_AND_TABLE),
+        ("schema_and_table", TableNamingConvention.SCHEMA_AND_TABLE),
+        ("table_only", TableNamingConvention.TABLE_ONLY),
+        ("hash_md5", TableNamingConvention.HASH_MD5),
+    ],
+)
+def test_physical_table_naming_convention(
+    convention_str: t.Optional[str], expected: t.Optional[TableNamingConvention], tmp_path: Path
+):
+    config_part = f"physical_table_naming_convention: {convention_str}" if convention_str else ""
+    (tmp_path / "config.yaml").write_text(f"""
+gateways:
+  test_gateway:
+    connection:
+      type: duckdb      
+model_defaults:
+  dialect: duckdb
+{config_part}
+    """)
+
+    config = load_config_from_paths(Config, project_paths=[tmp_path / "config.yaml"])
+    assert config.physical_table_naming_convention == expected

{sqlmesh-0.204.1.dev4 → sqlmesh-0.204.1.dev5}/tests/core/test_integration.py

@@ -35,6 +35,7 @@ from sqlmesh.core.config import (
     GatewayConfig,
     ModelDefaultsConfig,
     DuckDBConnectionConfig,
+    TableNamingConvention,
 )
 from sqlmesh.core.config.common import EnvironmentSuffixTarget
 from sqlmesh.core.console import Console, get_console
@@ -7164,3 +7165,74 @@ def test_engine_adapters_multi_repo_all_gateways_gathered(copy_to_temp_path):
     gathered_gateways = context.engine_adapters.keys()
     expected_gateways = {"local", "memory", "extra"}
     assert gathered_gateways == expected_gateways
+
+
+def test_physical_table_naming_strategy_table_only(copy_to_temp_path: t.Callable):
+    sushi_context = Context(
+        paths=copy_to_temp_path("examples/sushi"),
+        config=Config(
+            model_defaults=ModelDefaultsConfig(dialect="duckdb"),
+            default_connection=DuckDBConnectionConfig(),
+            physical_table_naming_convention=TableNamingConvention.TABLE_ONLY,
+        ),
+    )
+
+    assert sushi_context.config.physical_table_naming_convention == TableNamingConvention.TABLE_ONLY
+    sushi_context.plan(auto_apply=True)
+
+    adapter = sushi_context.engine_adapter
+
+    snapshot_tables = [
+        dict(catalog=str(r[0]), schema=str(r[1]), table=str(r[2]))
+        for r in adapter.fetchall(
+            "select table_catalog, table_schema, table_name from information_schema.tables where table_type='BASE TABLE'"
+        )
+    ]
+
+    assert all([not t["table"].startswith("sushi") for t in snapshot_tables])
+
+    prod_env = sushi_context.state_reader.get_environment("prod")
+    assert prod_env
+
+    prod_env_snapshots = sushi_context.state_reader.get_snapshots(prod_env.snapshots)
+
+    assert all(
+        s.table_naming_convention == TableNamingConvention.TABLE_ONLY
+        for s in prod_env_snapshots.values()
+    )
+
+
+def test_physical_table_naming_strategy_hash_md5(copy_to_temp_path: t.Callable):
+    sushi_context = Context(
+        paths=copy_to_temp_path("examples/sushi"),
+        config=Config(
+            model_defaults=ModelDefaultsConfig(dialect="duckdb"),
+            default_connection=DuckDBConnectionConfig(),
+            physical_table_naming_convention=TableNamingConvention.HASH_MD5,
+        ),
+    )
+
+    assert sushi_context.config.physical_table_naming_convention == TableNamingConvention.HASH_MD5
+    sushi_context.plan(auto_apply=True)
+
+    adapter = sushi_context.engine_adapter
+
+    snapshot_tables = [
+        dict(catalog=str(r[0]), schema=str(r[1]), table=str(r[2]))
+        for r in adapter.fetchall(
+            "select table_catalog, table_schema, table_name from information_schema.tables where table_type='BASE TABLE'"
+        )
+    ]
+
+    assert all([not t["table"].startswith("sushi") for t in snapshot_tables])
+    assert all([t["table"].startswith("sqlmesh_md5") for t in snapshot_tables])
+
+    prod_env = sushi_context.state_reader.get_environment("prod")
+    assert prod_env
+
+    prod_env_snapshots = sushi_context.state_reader.get_snapshots(prod_env.snapshots)
+
+    assert all(
+        s.table_naming_convention == TableNamingConvention.HASH_MD5
+        for s in prod_env_snapshots.values()
+    )