fabricks 3.0.4__py3-none-any.whl → 3.0.5.1__py3-none-any.whl

fabricks/api/version.py

@@ -0,0 +1,3 @@
+import importlib.metadata
+
+FABRICKS_VERSION = importlib.metadata.version("fabricks")

fabricks/cdc/base/configurator.py

@@ -126,7 +126,7 @@ class Configurator(ABC):
 
     def has_data(self, src: Union[DataFrame, Table, str], **kwargs) -> bool:
         df = self.get_src(src=src)
-        return df.count() > 0
+        return not df.isEmpty()
 
     def get_columns(self, src: Union[DataFrame, Table, str], backtick: Optional[bool] = True) -> List[str]:
         if backtick:

fabricks/cdc/base/generator.py

@@ -34,10 +34,8 @@ class Generator(Configurator):
 
         df = self.get_data(src, **kwargs)
 
-        if liquid_clustering:
-            assert cluster_by, "clustering column not found"
-        elif partitioning:
-            assert partition_by, "partitioning column not found"
+        if partitioning is True:
+            assert partition_by, "partitioning column(s) not found"
 
         df = self.reorder_columns(df)
 
@@ -84,22 +82,14 @@ class Generator(Configurator):
             DEFAULT_LOGGER.exception("could not execute sql query", extra={"job": self, "sql": sql})
 
     def optimize_table(self):
-        liquid_clustering = self.table.get_property("delta.feature.liquid") == "supported"
+        columns = None
 
-        if liquid_clustering:
-            self.table.optimize()
-        else:
-            columns = None
-
-            if self.change_data_capture == "scd1":
-                columns = ["__key"]
-            elif self.change_data_capture == "scd2":
-                columns = ["__key", "__valid_from"]
-
-            vorder = self.table.get_property("delta.parquet.vorder.enabled") or "false"
-            vorder = vorder.lower() == "true"
+        if self.change_data_capture == "scd1":
+            columns = ["__key"]
+        elif self.change_data_capture == "scd2":
+            columns = ["__key", "__valid_from"]
 
-            self.table.optimize(columns=columns, vorder=vorder)
+        self.table.optimize(columns=columns)
 
     def get_differences_with_deltatable(self, src: Union[DataFrame, Table, str], **kwargs) -> Optional[DataFrame]:
         if self.is_view:

fabricks/cdc/base/processor.py

@@ -36,8 +36,6 @@ class Processor(Generator):
         columns = self.get_columns(src, backtick=False)
         fields = [c for c in columns if not c.startswith("__")]
 
-        has_data = self.has_data(src)
-
         keys = kwargs.get("keys", None)
         mode = kwargs.get("mode", "complete")
 
@@ -80,6 +78,12 @@ class Processor(Generator):
         deduplicate_hash = kwargs.get("deduplicate_hash", None)
         soft_delete = kwargs.get("soft_delete", None)
         correct_valid_from = kwargs.get("correct_valid_from", None)
+        delete_missing = kwargs.get("delete_missing", None)
+
+        if mode == "update" and delete_missing:
+            has_data = self.has_data(src)
+        else:
+            has_data = True
 
         if slice is None:
             if mode == "update" and has_timestamp and has_rows:

fabricks/cdc/scd.py

@@ -9,6 +9,7 @@ from fabricks.metastore.table import Table
 class SCD(BaseCDC):
     def delete_missing(self, src: Union[DataFrame, Table, str], **kwargs):
         kwargs["add_operation"] = "reload"
+        kwargs["delete_missing"] = True
         kwargs["mode"] = "update"
         self.merge(src, **kwargs)
 

fabricks/core/jobs/base/configurator.py

@@ -54,7 +54,6 @@ class Configurator(ABC):
     _cdc: Optional[Union[NoCDC, SCD1, SCD2]] = None
     _change_data_capture: Optional[ChangeDataCaptures] = None
     _mode: Optional[Modes] = None
-    _liquid_clustering: Optional[bool] = False
 
     @property
     @abstractmethod

fabricks/core/jobs/base/generator.py

@@ -209,33 +209,38 @@ class Generator(Configurator):
                 identity = self.options.table.get_boolean("identity", False)
 
             # first take from job options, then from step options
-            liquid_clustering_job = self.options.table.get_boolean("liquid_clustering", None)
+            liquid_clustering_job = self.options.table.get("liquid_clustering", None)
             liquid_clustering_step = self.step_conf.get("table_options", {}).get("liquid_clustering", None)
             if liquid_clustering_job is not None:
                 liquid_clustering = liquid_clustering_job
             elif liquid_clustering_step:
                 liquid_clustering = liquid_clustering_step
 
-            if liquid_clustering:
-                cluster_by = self.options.table.get_list("cluster_by") or []
-                if not cluster_by:
-                    if "__source" in df.columns:
-                        cluster_by.append("__source")
-                    if "__is_current" in df.columns:
-                        cluster_by.append("__is_current")
-                    if "__key" in df.columns:
-                        cluster_by.append("__key")
-                    elif "__hash" in df.columns:
-                        cluster_by.append("__hash")
-
-                if not cluster_by:
-                    DEFAULT_LOGGER.warning(
-                        "liquid clustering disabled (no clustering columns found)", extra={"job": self}
-                    )
-                    liquid_clustering = False
-                    cluster_by = None
+            if liquid_clustering is not None:
+                if liquid_clustering == "auto":
+                    liquid_clustering = True
+                    cluster_by = []
 
-            if not liquid_clustering:
+                else:
+                    cluster_by = self.options.table.get_list("cluster_by") or []
+                    if not cluster_by:
+                        if "__source" in df.columns:
+                            cluster_by.append("__source")
+                        if "__is_current" in df.columns:
+                            cluster_by.append("__is_current")
+                        if "__key" in df.columns:
+                            cluster_by.append("__key")
+                        elif "__hash" in df.columns:
+                            cluster_by.append("__hash")
+
+                    if not cluster_by:
+                        DEFAULT_LOGGER.warning(
+                            "liquid clustering disabled (no clustering columns found)", extra={"job": self}
+                        )
+                        liquid_clustering = False
+                        cluster_by = None
+
+            if liquid_clustering is None:
                 cluster_by = None
                 partition_by = self.options.table.get_list("partition_by")
                 if partition_by:
@@ -404,11 +409,8 @@ class Generator(Configurator):
                     cluster_by.append("__hash")
 
                 if len(cluster_by) > 0:
-                    self.table.enable_liquid_clustering(cluster_by)
+                    self.table.enable_liquid_clustering(cluster_by, auto=False)
                 else:
-                    DEFAULT_LOGGER.warning(
-                        "liquid clustering not enabled (no clustering column found)", extra={"job": self}
-                    )
-
+                    self.table.enable_liquid_clustering(auto=True)
         else:
             DEFAULT_LOGGER.debug("liquid clustering not enabled", extra={"job": self})

fabricks/metastore/table.py

@@ -65,15 +65,28 @@ class Table(DbObject):
     @property
     def identity_enabled(self) -> bool:
         assert self.is_registered, f"{self} not registered"
-
         return self.get_property("delta.feature.identityColumns") == "supported"
 
     @property
     def type_widening_enabled(self) -> bool:
         assert self.is_registered, f"{self} not registered"
-
         return self.get_property("delta.enableTypeWidening") == "true"
 
+    @property
+    def liquid_clustering_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.feature.clustering") == "supported"
+
+    @property
+    def auto_liquid_clustering_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.clusterByAuto") == "true"
+
+    @property
+    def vorder_enabled(self) -> bool:
+        assert self.is_registered, f"{self} not registered"
+        return self.get_property("delta.parquet.vorder.enabled") == "true"
+
     def drop(self):
         super().drop()
         if self.delta_path.exists():
@@ -160,11 +173,14 @@ class Table(DbObject):
         ddl_tblproperties = "-- not tblproperties"
 
         if liquid_clustering:
-            assert cluster_by
-            if isinstance(cluster_by, str):
-                cluster_by = [cluster_by]
-            cluster_by = [f"`{c}`" for c in cluster_by]
-            ddl_cluster_by = "cluster by (" + ", ".join(cluster_by) + ")"
+            if cluster_by:
+                if isinstance(cluster_by, str):
+                    cluster_by = [cluster_by]
+                cluster_by = [f"`{c}`" for c in cluster_by]
+                ddl_cluster_by = "cluster by (" + ", ".join(cluster_by) + ")"
+
+            else:
+                ddl_cluster_by = "cluster by auto"
 
         if partitioning:
             assert partition_by
@@ -388,37 +404,38 @@ class Table(DbObject):
             pass
         self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
 
-    def optimize(
-        self,
-        columns: Optional[Union[str, List[str]]] = None,
-        vorder: Optional[bool] = False,
-    ):
+    def optimize(self, columns: Optional[Union[str, List[str]]] = None):
         assert self.is_registered, f"{self} not registered"
 
         DEFAULT_LOGGER.info("optimize", extra={"job": self})
 
-        zorder_by = columns is not None
-        if zorder_by:
+        if self.liquid_clustering_enabled:
+            self.spark.sql(f"optimize {self.qualified_name}")
+
+        elif self.auto_liquid_clustering_enabled:
+            self.spark.sql(f"optimize {self.qualified_name}")
+
+        elif columns is None:
+            if self.vorder_enabled:
+                DEFAULT_LOGGER.debug("vorder", extra={"job": self})
+                self.spark.sql(f"optimize {self.qualified_name} vorder")
+            else:
+                self.spark.sql(f"optimize {self.qualified_name}")
+
+        else:
             if isinstance(columns, str):
                 columns = [columns]
             columns = [f"`{c}`" for c in columns]
             cols = ", ".join(columns)
 
-            if vorder:
+            if self.vorder_enabled:
                 DEFAULT_LOGGER.debug(f"zorder by {cols} vorder", extra={"job": self})
                 self.spark.sql(f"optimize {self.qualified_name} zorder by ({cols}) vorder")
+
             else:
                 DEFAULT_LOGGER.debug(f"zorder by {cols}", extra={"job": self})
                 self.spark.sql(f"optimize {self.qualified_name} zorder by ({cols})")
 
-        elif vorder:
-            DEFAULT_LOGGER.debug("vorder", extra={"job": self})
-            self.spark.sql(f"optimize {self.qualified_name} vorder")
-
-        else:
-            DEFAULT_LOGGER.debug("optimize", extra={"job": self})
-            self.spark.sql(f"optimize {self.qualified_name}")
-
     def analyze(self):
         assert self.is_registered, f"{self} not registered"
 
@@ -658,18 +675,25 @@ class Table(DbObject):
         df = self.spark.sql(f"describe history {self.qualified_name}")
         return df
 
-    def enable_liquid_clustering(self, columns: Union[str, List[str]]):
+    def enable_liquid_clustering(self, columns: Optional[Union[str, List[str]]] = None, auto: Optional[bool] = False):
         assert self.is_registered, f"{self} not registered"
 
-        if isinstance(columns, str):
-            columns = [columns]
-        columns = [f"`{c}`" for c in columns]
-        cols = ", ".join(columns)
-        DEFAULT_LOGGER.info(f"cluster by {cols}", extra={"job": self})
+        if auto:
+            DEFAULT_LOGGER.info("cluster by auto", extra={"job": self})
+            self.spark.sql(f"alter table {self.qualified_name} cluster by automatic")
 
-        self.spark.sql(
-            f"""
-            alter table {self.qualified_name}
-            cluster by ({cols})
-            """
-        )
+        else:
+            assert columns, "at least one clustering column must be specified"
+
+            if isinstance(columns, str):
+                columns = [columns]
+            columns = [f"`{c}`" for c in columns]
+            cols = ", ".join(columns)
+
+            DEFAULT_LOGGER.info(f"cluster by {cols}", extra={"job": self})
+            self.spark.sql(
+                f"""
+                alter table {self.qualified_name}
+                cluster by ({cols})
+                """
+            )

{fabricks-3.0.4.dist-info → fabricks-3.0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.4
+Version: 3.0.5.1
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0
@@ -8,6 +8,7 @@ Requires-Dist: azure-identity>=1.10.0
 Requires-Dist: azure-storage-blob>=12.14.1
 Requires-Dist: azure-storage-queue<13,>=12.10.0
 Requires-Dist: databricks-sdk>=0.20.0
+Requires-Dist: importlib-metadata>=8.6.1
 Requires-Dist: jinja2>=2.11.3
 Requires-Dist: pydantic-settings
 Requires-Dist: pydantic-yaml>=1.4.0

{fabricks-3.0.4.dist-info → fabricks-3.0.5.1.dist-info}/RECORD

@@ -10,6 +10,7 @@ fabricks/api/parsers.py,sha256=nPUDzQ_Hz0fVmnBfGCqqHo7X7R6M-oGsXWDYSikjB54,121
 fabricks/api/schedules.py,sha256=omxxRU5xC_ee5pA5v1ZXpz9pv0INqEdBIlhs1RYkhIk,349
 fabricks/api/udfs.py,sha256=3JTX4OWkoW7_AP9pUKHVS0C6zIBVdOJoAn8MpmB6R48,124
 fabricks/api/utils.py,sha256=a-YrCXkDFzMmcNN8QOSDs_-YQtSePaDP4C4WYMX2AEg,196
+fabricks/api/version.py,sha256=FukX94EbtmJMeajxyOwka8fMfFeaVc0cuM3I5CVIuK8,85
 fabricks/api/views.py,sha256=dPqsGgDs9QOYc-5_QG_i4F_VoaFO5hGZQnIPZ31h5Ps,156
 fabricks/api/cdc/__init__.py,sha256=Cl3LhLbQrA42IvNLqoV7CCbjQEYQMJfO6cAZv1l1aas,196
 fabricks/api/cdc/nocdc.py,sha256=3E1Cn6cPHfEszGMaHEknrLqEvVKS-5-hk8s_GRu6TYY,58
@@ -31,16 +32,16 @@ fabricks/api/notebooks/vacuum.py,sha256=F88-alJyR4rh1ZB4CbvMqyJvAC-6l73GHtq6eBkk
 fabricks/cdc/__init__.py,sha256=_ncE8b8xuT2HqWC3JiCa4JCb_na2xQnVz3M6tLkAXD8,302
 fabricks/cdc/cdc.py,sha256=2CjPUtogWjnvyLjwiyVllcyDV1gpJ0QoRP0yUsiHXuc,69
 fabricks/cdc/nocdc.py,sha256=Nwj0pE3NjSVyLxKs9PUimHzWcKN5ehHt1trrlq69qE4,518
-fabricks/cdc/scd.py,sha256=r1NVK9QAKJG4tRSpEAksvOO3nAuNwRLZoGmNG2TsypE,630
+fabricks/cdc/scd.py,sha256=HzC9ifEu45B4P2aOSgi97AGB-C56l6sKTLqdVinnHKo,670
 fabricks/cdc/scd1.py,sha256=WsOVRsp55WEw4-7nEtb3dfv310icExrj-zEJSEehyz8,334
 fabricks/cdc/scd2.py,sha256=4vZkhc8pJAUlgiBmIw9j_2RsWuAFMcgCkU3WMVt0A-A,334
 fabricks/cdc/base/__init__.py,sha256=1uec9NHg3J5TWPMR09EsCMO1g8_3Dt6ZhC_b61Sg7JY,143
 fabricks/cdc/base/_types.py,sha256=IMI5bT4IFfqSnjTVrPBHsJkRXNdaRcMVUYW8qpfsTs0,82
 fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
-fabricks/cdc/base/configurator.py,sha256=fbQg4C1AH0BIis_Pdrv3BLkcjGnYOaCrxbjNc-95bj8,5269
-fabricks/cdc/base/generator.py,sha256=r_6S556wuNvl4eqo1L6-AfyPNj3mHQl8lTspWiFGFYU,6161
+fabricks/cdc/base/configurator.py,sha256=lInLgLUm_h2VN43vUFzOsxwk4yOQxnDplR6F9fC0rEE,5271
+fabricks/cdc/base/generator.py,sha256=OQuNGjblFeCP0JLJpJKbaXQmzREISw7PLiN04Nt9nu4,5735
 fabricks/cdc/base/merger.py,sha256=suule_MRyI-qXwBaUpKiBLmduZpvI01nwn6MBarkI24,3991
-fabricks/cdc/base/processor.py,sha256=e0JKpz6sGGXvc6U6IFagxy8iZVtE3YAhfLP2mWXMiBs,14239
+fabricks/cdc/base/processor.py,sha256=d52pvaHI9KTvztT2NCb8Vl-tIGc66ZPyuxXR4p3Vq1U,14393
 fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/cdc/templates/filter.sql.jinja,sha256=H0-nAN7HzxDa3p_Qu2U_LeJnBCcBKR6xzIM0VnckeuQ,234
 fabricks/cdc/templates/merge.sql.jinja,sha256=iNpgqGiuI2QABmyTkHCibRr_5r7SASb3yqojhNP3e20,144
@@ -115,9 +116,9 @@ fabricks/core/jobs/silver.py,sha256=wn6c6hoeppjlWf6EutB_8qE5Sxu2PIWk5iQecRUuJ5o,
 fabricks/core/jobs/base/__init__.py,sha256=_AdWtyL7yZG2TOZ9e8WyNPrOjmm6EDkI_TNym5cLDws,208
 fabricks/core/jobs/base/_types.py,sha256=xNKHpzof_mPd97ytvk3wrPnXd1_VdLilh1yQgFF3Ois,6769
 fabricks/core/jobs/base/checker.py,sha256=LPK5f3ucT7T4Z7LjlOyHPXFfb94J_DdYVp6X85wIvDk,5324
-fabricks/core/jobs/base/configurator.py,sha256=ARj920yJJdNtD0Iz8IjbhSyJqlVrXzSz29n0mXofo-k,11569
+fabricks/core/jobs/base/configurator.py,sha256=Dwx7B09PDXefmrY5MtkD3NMuiqE66RtvNJomnzaMjfE,11522
 fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
-fabricks/core/jobs/base/generator.py,sha256=LdI3PDrwee5rjwlFlduA4_s-7rE1AsnFrYdgQJL5_tE,15527
+fabricks/core/jobs/base/generator.py,sha256=3WgL8JweaK31WSIxKGdTsc32dAHAxLOxJT5PQdp_jO4,15657
 fabricks/core/jobs/base/invoker.py,sha256=xJV9fLtY36qfnclqKqNBsjryyR8x39wfhbYJtzOPRyM,6342
 fabricks/core/jobs/base/job.py,sha256=dWmk2PpQH2NETaaDS6KoiefRnDHfDMdCyhmogkdcSFI,93
 fabricks/core/jobs/base/processor.py,sha256=QmyUM11drJ9o8vF5he4rdztcfO7HjiXNhbk_AwJakUM,8324
@@ -145,7 +146,7 @@ fabricks/metastore/_types.py,sha256=NXYxwQHP0sCllM0N6QBbaK4CdtM_m_rHFDxRNRfBcLU,
 fabricks/metastore/database.py,sha256=1EjbRh2b6xEdHJyc4C4xee6FXDiKuPgm-8Q3Gqt7eds,1942
 fabricks/metastore/dbobject.py,sha256=EdxofFMCx6XdqFkm9Z5x4ywW4sstvdpc1d_EhYsE0KY,1883
 fabricks/metastore/pyproject.toml,sha256=6RZM9RMKMDF_EAequhORZ7TD0BQNk7aBCTWAv-sRcp0,519
-fabricks/metastore/table.py,sha256=YywtGc6z_zMwvK9wORvvys50AB-gBd9ZGAdtQq614yc,24312
+fabricks/metastore/table.py,sha256=luKm_kMHBBPHn_J5Tx5Aw6k8yOttgODa4FvzpD-4on0,25453
 fabricks/metastore/utils.py,sha256=8SxhjDkz_aSH4IGUusel7hqOQxP9U8PNBCY0M7GH00Y,1355
 fabricks/metastore/view.py,sha256=Va7xdFtOW9GcDSlyoZNgcF07qty9abtex41au6OSz6c,1381
 fabricks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -171,6 +172,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=u9FFYvWyq9VQdNJNu79-SCN9iGUB
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=mpaSxBNcl6N0QheGLx8rjeyWUvy1Yvvj4raGRv7GL5M,1229
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.4.dist-info/METADATA,sha256=nFCQ4-fewJvmjPgINnMmf07MFDMOyMp8dVOFMd6Vbwo,682
-fabricks-3.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.4.dist-info/RECORD,,
+fabricks-3.0.5.1.dist-info/METADATA,sha256=6gsR4UJcnoz4x2dPwdNT3jYIyEZlpdBqCee0-LsTJJ0,725
+fabricks-3.0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fabricks-3.0.5.1.dist-info/RECORD,,