fabricks 3.0.3__tar.gz → 3.0.5__tar.gz

{fabricks-3.0.3 → fabricks-3.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.3
+Version: 3.0.5
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0
@@ -8,6 +8,7 @@ Requires-Dist: azure-identity>=1.10.0
 Requires-Dist: azure-storage-blob>=12.14.1
 Requires-Dist: azure-storage-queue<13,>=12.10.0
 Requires-Dist: databricks-sdk>=0.20.0
+Requires-Dist: importlib-metadata>=8.6.1
 Requires-Dist: jinja2>=2.11.3
 Requires-Dist: pydantic-settings
 Requires-Dist: pydantic-yaml>=1.4.0

fabricks-3.0.5/fabricks/api/version.py

@@ -0,0 +1,3 @@
+import importlib.metadata
+
+FABRICKS_VERSION = importlib.metadata.version("fabricks")

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/cdc/base/configurator.py

@@ -37,6 +37,10 @@ class Configurator(ABC):
     def is_registered(self):
         return self.table.is_registered
 
+    @property
+    def qualified_name(self):
+        return f"{self.database}_{'_'.join(self.levels)}"
+
     @abstractmethod
     def get_query(self, src: Union[DataFrame, Table, str], **kwargs):
         raise NotImplementedError()

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/cdc/base/generator.py

@@ -34,10 +34,8 @@ class Generator(Configurator):
 
         df = self.get_data(src, **kwargs)
 
-        if liquid_clustering:
-            assert cluster_by, "clustering column not found"
-        elif partitioning:
-            assert partition_by, "partitioning column not found"
+        if partitioning is True:
+            assert partition_by, "partitioning column(s) not found"
 
         df = self.reorder_columns(df)
 
@@ -84,22 +82,14 @@ class Generator(Configurator):
             DEFAULT_LOGGER.exception("could not execute sql query", extra={"job": self, "sql": sql})
 
     def optimize_table(self):
-        liquid_clustering = self.table.get_property("delta.feature.liquid") == "supported"
+        columns = None
 
-        if liquid_clustering:
-            self.table.optimize()
-        else:
-            columns = None
-
-            if self.change_data_capture == "scd1":
-                columns = ["__key"]
-            elif self.change_data_capture == "scd2":
-                columns = ["__key", "__valid_from"]
-
-            vorder = self.table.get_property("delta.parquet.vorder.enabled") or "false"
-            vorder = vorder.lower() == "true"
+        if self.change_data_capture == "scd1":
+            columns = ["__key"]
+        elif self.change_data_capture == "scd2":
+            columns = ["__key", "__valid_from"]
 
-            self.table.optimize(columns=columns, vorder=vorder)
+        self.table.optimize(columns=columns)
 
     def get_differences_with_deltatable(self, src: Union[DataFrame, Table, str], **kwargs) -> Optional[DataFrame]:
         if self.is_view:

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/cdc/base/merger.py

@@ -101,7 +101,7 @@ class Merger(Processor):
             self.create_table(src, **kwargs)
 
         df = self.get_data(src, **kwargs)
-        global_temp_view = f"{self.database}_{'_'.join(self.levels)}__merge"
+        global_temp_view = f"{self.qualified_name}__merge"
         view = create_or_replace_global_temp_view(global_temp_view, df, uuid=kwargs.get("uuid", False))
 
         merge = self.get_merge_query(view, **kwargs)

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/cdc/base/processor.py

@@ -16,7 +16,7 @@ from fabricks.utils.sqlglot import fix as fix_sql
 class Processor(Generator):
     def get_data(self, src: Union[DataFrame, Table, str], **kwargs) -> DataFrame:
         if isinstance(src, (DataFrame, CDataFrame)):
-            name = f"{self.database}_{'_'.join(self.levels)}__data"
+            name = f"{self.qualified_name}__data"
             global_temp_view = create_or_replace_global_temp_view(name, src, uuid=kwargs.get("uuid", False))
             src = f"select * from {global_temp_view}"
 
@@ -347,8 +347,11 @@ class Processor(Generator):
         df = self.get_data(src, **kwargs)
         df = self.reorder_columns(df)
 
+        name = f"{self.qualified_name}__append"
+        create_or_replace_global_temp_view(name, df, uuid=kwargs.get("uuid", False))
+
         DEFAULT_LOGGER.debug("append", extra={"job": self})
-        df.write.format("delta").mode("append").save(self.table.delta_path.string)
+        self.spark.sql(f"insert into table {self.table} by name select * from global_temp.{name}")
 
     def overwrite(
         self,
@@ -366,9 +369,11 @@ class Processor(Generator):
             if kwargs.get("update_where"):
                 dynamic = True
 
-        writer = df.write.format("delta").mode("overwrite")
         if dynamic:
-            writer.option("partitionOverwriteMode", "dynamic")
+            self.spark.sql("set spark.sql.sources.partitionOverwriteMode = dynamic")
+
+        name = f"{self.qualified_name}__overwrite"
+        create_or_replace_global_temp_view(name, df, uuid=kwargs.get("uuid", False))
 
-        DEFAULT_LOGGER.info("overwrite", extra={"job": self})
-        writer.save(self.table.delta_path.string)
+        DEFAULT_LOGGER.debug("overwrite", extra={"job": self})
+        self.spark.sql(f"insert overwrite table {self.table} by name select * from global_temp.{name}")

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/core/jobs/base/_types.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, List, Literal, Optional, TypedDict, Union
+from typing import List, Literal, Optional, TypedDict, Union
 
 from pydantic import BaseModel, ConfigDict, model_validator
 from pyspark.sql.types import StringType, StructField, StructType
@@ -33,8 +33,8 @@ Origins = Literal["parser", "job"]
 
 
 class SparkOptions(TypedDict):
-    sql: Optional[dict[Any, Any]]
-    conf: Optional[dict[Any, Any]]
+    sql: Optional[dict[str, str]]
+    conf: Optional[dict[str, str]]
 
 
 class TableOptions(TypedDict):
@@ -45,17 +45,17 @@ class TableOptions(TypedDict):
     cluster_by: Optional[List[str]]
     powerbi: Optional[bool]
     bloomfilter_by: Optional[List[str]]
-    constraints: Optional[dict[Any, Any]]
-    properties: Optional[dict[Any, Any]]
+    constraints: Optional[dict[str, str]]
+    properties: Optional[dict[str, str]]
     comment: Optional[str]
-    calculated_columns: Optional[dict[Any, Any]]
+    calculated_columns: Optional[dict[str, str]]
     retention_days: Optional[int]
 
 
 class _InvokeOptions(TypedDict):
     notebook: str
     timeout: int
-    arguments: Optional[dict[Any, Any]]
+    arguments: Optional[dict[str, str]]
 
 
 class InvokerOptions(TypedDict):
@@ -66,7 +66,7 @@ class InvokerOptions(TypedDict):
 
 class ExtenderOptions(TypedDict):
     extender: str
-    arguments: Optional[dict[Any, Any]]
+    arguments: Optional[dict[str, str]]
 
 
 class CheckOptions(TypedDict):
@@ -90,7 +90,7 @@ class BronzeOptions(TypedDict):
     filter_where: Optional[str]
     # extra
     encrypted_columns: Optional[List[str]]
-    calculated_columns: Optional[dict[Any, Any]]
+    calculated_columns: Optional[dict[str, str]]
     operation: Optional[Operations]
     timeout: Optional[int]
 
@@ -106,7 +106,7 @@ class SilverOptions(TypedDict):
     deduplicate: Optional[bool]
     stream: Optional[bool]
     # else
-    order_duplicate_by: Optional[dict[Any, Any]]
+    order_duplicate_by: Optional[dict[str, str]]
     timeout: Optional[int]
 
 
@@ -141,7 +141,7 @@ class BaseJobConf:
 
 @dataclass
 class JobConfBronze(BaseJobConf):
-    step: str
+    step: TBronze
     options: BronzeOptions
     table_options: Optional[TableOptions] = None
     parser_options: Optional[ParserOptions] = None
@@ -155,7 +155,7 @@ class JobConfBronze(BaseJobConf):
 
 @dataclass
 class JobConfSilver(BaseJobConf):
-    step: str
+    step: TSilver
     options: SilverOptions
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None
@@ -168,7 +168,7 @@ class JobConfSilver(BaseJobConf):
 
 @dataclass
 class JobConfGold(BaseJobConf):
-    step: str
+    step: TGold
     options: Optional[GoldOptions]
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/core/jobs/base/configurator.py

@@ -54,7 +54,6 @@ class Configurator(ABC):
     _cdc: Optional[Union[NoCDC, SCD1, SCD2]] = None
     _change_data_capture: Optional[ChangeDataCaptures] = None
     _mode: Optional[Modes] = None
-    _liquid_clustering: Optional[bool] = False
 
     @property
     @abstractmethod

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/core/jobs/base/generator.py

@@ -209,33 +209,38 @@ class Generator(Configurator):
                 identity = self.options.table.get_boolean("identity", False)
 
             # first take from job options, then from step options
-            liquid_clustering_job = self.options.table.get_boolean("liquid_clustering", None)
+            liquid_clustering_job = self.options.table.get("liquid_clustering", None)
             liquid_clustering_step = self.step_conf.get("table_options", {}).get("liquid_clustering", None)
             if liquid_clustering_job is not None:
                 liquid_clustering = liquid_clustering_job
             elif liquid_clustering_step:
                 liquid_clustering = liquid_clustering_step
 
-            if liquid_clustering:
-                cluster_by = self.options.table.get_list("cluster_by") or []
-                if not cluster_by:
-                    if "__source" in df.columns:
-                        cluster_by.append("__source")
-                    if "__is_current" in df.columns:
-                        cluster_by.append("__is_current")
-                    if "__key" in df.columns:
-                        cluster_by.append("__key")
-                    elif "__hash" in df.columns:
-                        cluster_by.append("__hash")
-
-                if not cluster_by:
-                    DEFAULT_LOGGER.warning(
-                        "liquid clustering disabled (no clustering columns found)", extra={"job": self}
-                    )
-                    liquid_clustering = False
-                    cluster_by = None
+            if liquid_clustering is not None:
+                if liquid_clustering == "auto":
+                    liquid_clustering = True
+                    cluster_by = []
 
-            if not liquid_clustering:
+                else:
+                    cluster_by = self.options.table.get_list("cluster_by") or []
+                    if not cluster_by:
+                        if "__source" in df.columns:
+                            cluster_by.append("__source")
+                        if "__is_current" in df.columns:
+                            cluster_by.append("__is_current")
+                        if "__key" in df.columns:
+                            cluster_by.append("__key")
+                        elif "__hash" in df.columns:
+                            cluster_by.append("__hash")
+
+                    if not cluster_by:
+                        DEFAULT_LOGGER.warning(
+                            "liquid clustering disabled (no clustering columns found)", extra={"job": self}
+                        )
+                        liquid_clustering = False
+                        cluster_by = None
+
+            if liquid_clustering is None:
                 cluster_by = None
                 partition_by = self.options.table.get_list("partition_by")
                 if partition_by:
@@ -404,11 +409,8 @@ class Generator(Configurator):
                     cluster_by.append("__hash")
 
                 if len(cluster_by) > 0:
-                    self.table.enable_liquid_clustering(cluster_by)
+                    self.table.enable_liquid_clustering(cluster_by, auto=False)
                 else:
-                    DEFAULT_LOGGER.warning(
-                        "liquid clustering not enabled (no clustering column found)", extra={"job": self}
-                    )
-
+                    self.table.enable_liquid_clustering(auto=True)
         else:
             DEFAULT_LOGGER.debug("liquid clustering not enabled", extra={"job": self})

{fabricks-3.0.3 → fabricks-3.0.5}/fabricks/metastore/table.py

@@ -65,15 +65,28 @@ class Table(DbObject):
     @property
     def identity_enabled(self) -> bool:
         assert self.is_registered, f"{self} not registered"
-
         return self.get_property("delta.feature.identityColumns") == "supported"
 
     @property
     def type_widening_enabled(self) -> bool:
         assert self.is_registered, f"{self} not registered"
-
         return self.get_property("delta.enableTypeWidening") == "true"
 
+    @property
+    def liquid_clustering_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.feature.clustering") == "supported"
+
+    @property
+    def auto_liquid_clustering_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.clusterByAuto") == "true"
+
+    @property
+    def vorder_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.parquet.vorder.enabled") == "true"
+
     def drop(self):
         super().drop()
         if self.delta_path.exists():
@@ -160,11 +173,14 @@ class Table(DbObject):
         ddl_tblproperties = "-- not tblproperties"
 
         if liquid_clustering:
-            assert cluster_by
-            if isinstance(cluster_by, str):
-                cluster_by = [cluster_by]
-            cluster_by = [f"`{c}`" for c in cluster_by]
-            ddl_cluster_by = "cluster by (" + ", ".join(cluster_by) + ")"
+            if cluster_by:
+                if isinstance(cluster_by, str):
+                    cluster_by = [cluster_by]
+                cluster_by = [f"`{c}`" for c in cluster_by]
+                ddl_cluster_by = "cluster by (" + ", ".join(cluster_by) + ")"
+
+            else:
+                ddl_cluster_by = "cluster by auto"
 
         if partitioning:
             assert partition_by
@@ -388,37 +404,38 @@ class Table(DbObject):
             pass
         self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
 
-    def optimize(
-        self,
-        columns: Optional[Union[str, List[str]]] = None,
-        vorder: Optional[bool] = False,
-    ):
+    def optimize(self, columns: Optional[Union[str, List[str]]] = None):
         assert self.is_registered, f"{self} not registered"
 
         DEFAULT_LOGGER.info("optimize", extra={"job": self})
 
-        zorder_by = columns is not None
-        if zorder_by:
+        if self.liquid_clustering_enabled:
+            self.spark.sql(f"optimize {self.qualified_name}")
+
+        elif self.auto_liquid_clustering_enabled:
+            self.spark.sql(f"optimize {self.qualified_name}")
+
+        elif columns is None:
+            if self.vorder_enabled:
+                DEFAULT_LOGGER.debug("vorder", extra={"job": self})
+                self.spark.sql(f"optimize {self.qualified_name} vorder")
+            else:
+                self.spark.sql(f"optimize {self.qualified_name}")
+
+        else:
             if isinstance(columns, str):
                 columns = [columns]
             columns = [f"`{c}`" for c in columns]
             cols = ", ".join(columns)
 
-            if vorder:
+            if self.vorder_enabled:
                 DEFAULT_LOGGER.debug(f"zorder by {cols} vorder", extra={"job": self})
                 self.spark.sql(f"optimize {self.qualified_name} zorder by ({cols}) vorder")
+
             else:
                 DEFAULT_LOGGER.debug(f"zorder by {cols}", extra={"job": self})
                 self.spark.sql(f"optimize {self.qualified_name} zorder by ({cols})")
 
-        elif vorder:
-            DEFAULT_LOGGER.debug("vorder", extra={"job": self})
-            self.spark.sql(f"optimize {self.qualified_name} vorder")
-
-        else:
-            DEFAULT_LOGGER.debug("optimize", extra={"job": self})
-            self.spark.sql(f"optimize {self.qualified_name}")
-
     def analyze(self):
         assert self.is_registered, f"{self} not registered"
 
@@ -658,18 +675,25 @@ class Table(DbObject):
         df = self.spark.sql(f"describe history {self.qualified_name}")
         return df
 
-    def enable_liquid_clustering(self, columns: Union[str, List[str]]):
+    def enable_liquid_clustering(self, columns: Optional[Union[str, List[str]]] = None, auto: Optional[bool] = False):
         assert self.is_registered, f"{self} not registered"
 
-        if isinstance(columns, str):
-            columns = [columns]
-        columns = [f"`{c}`" for c in columns]
-        cols = ", ".join(columns)
-        DEFAULT_LOGGER.info(f"cluster by {cols}", extra={"job": self})
+        if auto:
+            DEFAULT_LOGGER.info("cluster by auto", extra={"job": self})
+            self.spark.sql(f"alter table {self.qualified_name} cluster by automatic")
 
-        self.spark.sql(
-            f"""
-            alter table {self.qualified_name}
-            cluster by ({cols})
-            """
-        )
+        else:
+            assert columns, "at least one clustering column must be specified"
+
+            if isinstance(columns, str):
+                columns = [columns]
+            columns = [f"`{c}`" for c in columns]
+            cols = ", ".join(columns)
+
+            DEFAULT_LOGGER.info(f"cluster by {cols}", extra={"job": self})
+            self.spark.sql(
+                f"""
+                alter table {self.qualified_name}
+                cluster by ({cols})
+                """
+            )

{fabricks-3.0.3 → fabricks-3.0.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.3"
+version = "3.0.5"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },
@@ -13,6 +13,7 @@ dependencies = [
     "azure-storage-blob >=12.14.1",
     "azure-storage-queue >=12.10.0,<13",
     "databricks-sdk >=0.20.0",
+    "importlib-metadata>=8.6.1",
     "jinja2 >=2.11.3",
     "pydantic>=1.10.21",
     "pydantic-settings",

fabricks-3.0.5/tests/integration/runtime/gold/transf/fact/memory.sql

@@ -0,0 +1 @@
+select `id` as id, `name` as monarch, `doubleField` as value from silver.monarch_scd1__current