fabricks 3.0.4__tar.gz → 3.0.5__tar.gz

{fabricks-3.0.4 → fabricks-3.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.4
+Version: 3.0.5
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0
@@ -8,6 +8,7 @@ Requires-Dist: azure-identity>=1.10.0
 Requires-Dist: azure-storage-blob>=12.14.1
 Requires-Dist: azure-storage-queue<13,>=12.10.0
 Requires-Dist: databricks-sdk>=0.20.0
+Requires-Dist: importlib-metadata>=8.6.1
 Requires-Dist: jinja2>=2.11.3
 Requires-Dist: pydantic-settings
 Requires-Dist: pydantic-yaml>=1.4.0

fabricks-3.0.5/fabricks/api/version.py

@@ -0,0 +1,3 @@
+import importlib.metadata
+
+FABRICKS_VERSION = importlib.metadata.version("fabricks")

{fabricks-3.0.4 → fabricks-3.0.5}/fabricks/cdc/base/generator.py

@@ -34,10 +34,8 @@ class Generator(Configurator):
 
         df = self.get_data(src, **kwargs)
 
-        if liquid_clustering:
-            assert cluster_by, "clustering column not found"
-        elif partitioning:
-            assert partition_by, "partitioning column not found"
+        if partitioning is True:
+            assert partition_by, "partitioning column(s) not found"
 
         df = self.reorder_columns(df)
 
@@ -84,22 +82,14 @@ class Generator(Configurator):
             DEFAULT_LOGGER.exception("could not execute sql query", extra={"job": self, "sql": sql})
 
     def optimize_table(self):
-        liquid_clustering = self.table.get_property("delta.feature.liquid") == "supported"
+        columns = None
 
-        if liquid_clustering:
-            self.table.optimize()
-        else:
-            columns = None
-
-            if self.change_data_capture == "scd1":
-                columns = ["__key"]
-            elif self.change_data_capture == "scd2":
-                columns = ["__key", "__valid_from"]
-
-            vorder = self.table.get_property("delta.parquet.vorder.enabled") or "false"
-            vorder = vorder.lower() == "true"
+        if self.change_data_capture == "scd1":
+            columns = ["__key"]
+        elif self.change_data_capture == "scd2":
+            columns = ["__key", "__valid_from"]
 
-            self.table.optimize(columns=columns, vorder=vorder)
+        self.table.optimize(columns=columns)
 
     def get_differences_with_deltatable(self, src: Union[DataFrame, Table, str], **kwargs) -> Optional[DataFrame]:
         if self.is_view:

{fabricks-3.0.4 → fabricks-3.0.5}/fabricks/core/jobs/base/configurator.py

@@ -54,7 +54,6 @@ class Configurator(ABC):
     _cdc: Optional[Union[NoCDC, SCD1, SCD2]] = None
     _change_data_capture: Optional[ChangeDataCaptures] = None
     _mode: Optional[Modes] = None
-    _liquid_clustering: Optional[bool] = False
 
     @property
     @abstractmethod

{fabricks-3.0.4 → fabricks-3.0.5}/fabricks/core/jobs/base/generator.py

@@ -209,33 +209,38 @@ class Generator(Configurator):
                 identity = self.options.table.get_boolean("identity", False)
 
             # first take from job options, then from step options
-            liquid_clustering_job = self.options.table.get_boolean("liquid_clustering", None)
+            liquid_clustering_job = self.options.table.get("liquid_clustering", None)
             liquid_clustering_step = self.step_conf.get("table_options", {}).get("liquid_clustering", None)
             if liquid_clustering_job is not None:
                 liquid_clustering = liquid_clustering_job
             elif liquid_clustering_step:
                 liquid_clustering = liquid_clustering_step
 
-            if liquid_clustering:
-                cluster_by = self.options.table.get_list("cluster_by") or []
-                if not cluster_by:
-                    if "__source" in df.columns:
-                        cluster_by.append("__source")
-                    if "__is_current" in df.columns:
-                        cluster_by.append("__is_current")
-                    if "__key" in df.columns:
-                        cluster_by.append("__key")
-                    elif "__hash" in df.columns:
-                        cluster_by.append("__hash")
-
-                if not cluster_by:
-                    DEFAULT_LOGGER.warning(
-                        "liquid clustering disabled (no clustering columns found)", extra={"job": self}
-                    )
-                    liquid_clustering = False
-                    cluster_by = None
+            if liquid_clustering is not None:
+                if liquid_clustering == "auto":
+                    liquid_clustering = True
+                    cluster_by = []
 
-            if not liquid_clustering:
+                else:
+                    cluster_by = self.options.table.get_list("cluster_by") or []
+                    if not cluster_by:
+                        if "__source" in df.columns:
+                            cluster_by.append("__source")
+                        if "__is_current" in df.columns:
+                            cluster_by.append("__is_current")
+                        if "__key" in df.columns:
+                            cluster_by.append("__key")
+                        elif "__hash" in df.columns:
+                            cluster_by.append("__hash")
+
+                    if not cluster_by:
+                        DEFAULT_LOGGER.warning(
+                            "liquid clustering disabled (no clustering columns found)", extra={"job": self}
+                        )
+                        liquid_clustering = False
+                        cluster_by = None
+
+            if liquid_clustering is None:
                 cluster_by = None
                 partition_by = self.options.table.get_list("partition_by")
                 if partition_by:
@@ -404,11 +409,8 @@ class Generator(Configurator):
                     cluster_by.append("__hash")
 
                 if len(cluster_by) > 0:
-                    self.table.enable_liquid_clustering(cluster_by)
+                    self.table.enable_liquid_clustering(cluster_by, auto=False)
                 else:
-                    DEFAULT_LOGGER.warning(
-                        "liquid clustering not enabled (no clustering column found)", extra={"job": self}
-                    )
-
+                    self.table.enable_liquid_clustering(auto=True)
         else:
             DEFAULT_LOGGER.debug("liquid clustering not enabled", extra={"job": self})

{fabricks-3.0.4 → fabricks-3.0.5}/fabricks/metastore/table.py

@@ -65,15 +65,28 @@ class Table(DbObject):
     @property
     def identity_enabled(self) -> bool:
         assert self.is_registered, f"{self} not registered"
-
         return self.get_property("delta.feature.identityColumns") == "supported"
 
     @property
     def type_widening_enabled(self) -> bool:
         assert self.is_registered, f"{self} not registered"
-
         return self.get_property("delta.enableTypeWidening") == "true"
 
+    @property
+    def liquid_clustering_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.feature.clustering") == "supported"
+
+    @property
+    def auto_liquid_clustering_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.clusterByAuto") == "true"
+
+    @property
+    def vorder_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.parquet.vorder.enabled") == "true"
+
     def drop(self):
         super().drop()
         if self.delta_path.exists():
@@ -160,11 +173,14 @@ class Table(DbObject):
         ddl_tblproperties = "-- not tblproperties"
 
         if liquid_clustering:
-            assert cluster_by
-            if isinstance(cluster_by, str):
-                cluster_by = [cluster_by]
-            cluster_by = [f"`{c}`" for c in cluster_by]
-            ddl_cluster_by = "cluster by (" + ", ".join(cluster_by) + ")"
+            if cluster_by:
+                if isinstance(cluster_by, str):
+                    cluster_by = [cluster_by]
+                cluster_by = [f"`{c}`" for c in cluster_by]
+                ddl_cluster_by = "cluster by (" + ", ".join(cluster_by) + ")"
+
+            else:
+                ddl_cluster_by = "cluster by auto"
 
         if partitioning:
             assert partition_by
@@ -388,37 +404,38 @@ class Table(DbObject):
             pass
         self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
 
-    def optimize(
-        self,
-        columns: Optional[Union[str, List[str]]] = None,
-        vorder: Optional[bool] = False,
-    ):
+    def optimize(self, columns: Optional[Union[str, List[str]]] = None):
         assert self.is_registered, f"{self} not registered"
 
         DEFAULT_LOGGER.info("optimize", extra={"job": self})
 
-        zorder_by = columns is not None
-        if zorder_by:
+        if self.liquid_clustering_enabled:
+            self.spark.sql(f"optimize {self.qualified_name}")
+
+        elif self.auto_liquid_clustering_enabled:
+            self.spark.sql(f"optimize {self.qualified_name}")
+
+        elif columns is None:
+            if self.vorder_enabled:
+                DEFAULT_LOGGER.debug("vorder", extra={"job": self})
+                self.spark.sql(f"optimize {self.qualified_name} vorder")
+            else:
+                self.spark.sql(f"optimize {self.qualified_name}")
+
+        else:
             if isinstance(columns, str):
                 columns = [columns]
             columns = [f"`{c}`" for c in columns]
             cols = ", ".join(columns)
 
-            if vorder:
+            if self.vorder_enabled:
                 DEFAULT_LOGGER.debug(f"zorder by {cols} vorder", extra={"job": self})
                 self.spark.sql(f"optimize {self.qualified_name} zorder by ({cols}) vorder")
+
             else:
                 DEFAULT_LOGGER.debug(f"zorder by {cols}", extra={"job": self})
                 self.spark.sql(f"optimize {self.qualified_name} zorder by ({cols})")
 
-        elif vorder:
-            DEFAULT_LOGGER.debug("vorder", extra={"job": self})
-            self.spark.sql(f"optimize {self.qualified_name} vorder")
-
-        else:
-            DEFAULT_LOGGER.debug("optimize", extra={"job": self})
-            self.spark.sql(f"optimize {self.qualified_name}")
-
     def analyze(self):
         assert self.is_registered, f"{self} not registered"
 
@@ -658,18 +675,25 @@ class Table(DbObject):
         df = self.spark.sql(f"describe history {self.qualified_name}")
         return df
 
-    def enable_liquid_clustering(self, columns: Union[str, List[str]]):
+    def enable_liquid_clustering(self, columns: Optional[Union[str, List[str]]] = None, auto: Optional[bool] = False):
         assert self.is_registered, f"{self} not registered"
 
-        if isinstance(columns, str):
-            columns = [columns]
-        columns = [f"`{c}`" for c in columns]
-        cols = ", ".join(columns)
-        DEFAULT_LOGGER.info(f"cluster by {cols}", extra={"job": self})
+        if auto:
+            DEFAULT_LOGGER.info("cluster by auto", extra={"job": self})
+            self.spark.sql(f"alter table {self.qualified_name} cluster by automatic")
 
-        self.spark.sql(
-            f"""
-            alter table {self.qualified_name}
-            cluster by ({cols})
-            """
-        )
+        else:
+            assert columns, "at least one clustering column must be specified"
+
+            if isinstance(columns, str):
+                columns = [columns]
+            columns = [f"`{c}`" for c in columns]
+            cols = ", ".join(columns)
+
+            DEFAULT_LOGGER.info(f"cluster by {cols}", extra={"job": self})
+            self.spark.sql(
+                f"""
+                alter table {self.qualified_name}
+                cluster by ({cols})
+                """
+            )

{fabricks-3.0.4 → fabricks-3.0.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.4"
+version = "3.0.5"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },
@@ -13,6 +13,7 @@ dependencies = [
     "azure-storage-blob >=12.14.1",
     "azure-storage-queue >=12.10.0,<13",
     "databricks-sdk >=0.20.0",
+    "importlib-metadata>=8.6.1",
     "jinja2 >=2.11.3",
     "pydantic>=1.10.21",
     "pydantic-settings",

fabricks-3.0.5/tests/integration/runtime/gold/transf/fact/memory.sql

@@ -0,0 +1 @@
+select `id` as id, `name` as monarch, `doubleField` as value from silver.monarch_scd1__current