fabricks 3.0.16__py3-none-any.whl → 3.0.17__py3-none-any.whl

fabricks/cdc/base/configurator.py

@@ -90,6 +90,7 @@ class Configurator(ABC):
             "__identity",
             "__source",
             "__key",
+            "__hash",
             "__timestamp",
             "__valid_from",
             "__valid_to",
@@ -110,7 +111,6 @@ class Configurator(ABC):
         cols = [
             "__operation",
             "__metadata",
-            "__hash",
             "__rescued_data",
         ]
 
@@ -126,6 +126,7 @@ class Configurator(ABC):
             "__identity",
             "__source",
             "__key",
+            "__hash",
             "__timestamp",
             "__valid_from",
             "__valid_to",
@@ -134,7 +135,6 @@ class Configurator(ABC):
             # Trailing
             "__operation",
             "__metadata",
-            "__hash",
             "__rescued_data",
         ]
 
@@ -192,11 +192,6 @@ class Configurator(ABC):
         leading = self.allowed_ouput_leading__columns
         trailing = self.allowed_output_trailing__columns
 
-        # move __hash to the front of the table to ensure statistics are present
-        if "__key" not in columns and "__hash" in columns:
-            leading = ["__hash" if c == "__key" else c for c in leading]
-            trailing = [c for c in trailing if c != "__hash"]
-
         __leading = [c for c in leading if c in columns]
         __trailing = [c for c in trailing if c in columns]
 

fabricks/cdc/base/processor.py

@@ -51,6 +51,7 @@ class Processor(Generator):
 
         overwrite = []
         exclude = kwargs.get("exclude", [])  # used by silver to exclude __operation from output if not update
+        cast = kwargs.get("cast", {})  # used by silver to cast columns to target types
 
         order_duplicate_by = kwargs.get("order_duplicate_by", None)
         if order_duplicate_by:
@@ -140,6 +141,8 @@ class Processor(Generator):
         # override timestamp if added and found in df
         if add_timestamp and "__timestamp" in inputs:
             overwrite.append("__timestamp")
+        elif "__timestamp" in inputs:
+            cast["__timestamp"] = "timestamp"
 
         # override key if added and found in df (key needed for merge)
         if add_key and "__key" in inputs:
@@ -356,6 +359,8 @@ class Processor(Generator):
             "correct_valid_from": correct_valid_from,
             # overwrite
             "overwrite": overwrite,
+            # cast
+            "cast": cast,
             # filter
             "slices": None,
             "sources": None,

fabricks/cdc/templates/ctes/base.sql.jinja

@@ -5,10 +5,14 @@ with
     __base as (
         select
             *
-            {% if overwrite %}
+            {% if overwrite or cast %}
                 -- will be overwritten below
-                except ({% for o in overwrite %}{{ o }}, {% endfor %})
+                except (
+                    {% for o in overwrite %}{{ o }}, {% endfor %}
+                    {% for key, value in cast.items() %} {{ key }}, {% endfor %}
+                )
             {% endif %},
+            {% for key, value in cast.items() %} cast({{ key }} as {{ value }}) as {{ key }}, {% endfor %}
             {% if add_calculated_columns %} {% for c in add_calculated_columns %} {{ c }}, {% endfor %} {% endif %}
             {% if add_timestamp %} cast(current_date() as timestamp) as __timestamp, {% endif %}
             {% if add_operation %} cast('{{ add_operation }}' as string) as __operation, {% endif %}

fabricks/cdc/templates/queries/context.sql.jinja

@@ -141,6 +141,9 @@
 {%- if overwrite %}
 ☐ overwrite: {{ overwrite | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
+{%- if cast %}
+☐ cast: {{ cast | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
 
 👨‍👩‍👧 PARENTS
 {%- if parent_slice %}

fabricks/core/jobs/base/generator.py

@@ -1,5 +1,5 @@
 from abc import abstractmethod
-from typing import Optional, Sequence, Union, cast
+from typing import List, Optional, Sequence, Union, cast
 
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import lit
@@ -166,6 +166,31 @@ class Generator(Configurator):
         """
         ...
 
+    def _get_clustering_columns(self, df: DataFrame) -> Optional[List[str]]:
+        columns = self.options.table.get_list("cluster_by")
+        if columns:
+            return columns
+
+        columns = []
+
+        if "__source" in df.columns:
+            columns = ["__source"]
+        if "__is_current" in df.columns:
+            columns.append("__is_current")
+
+        if "__key" in df.columns:
+            columns.append("__key")
+        elif "__hash" in df.columns:
+            columns.append("__hash")
+
+        if columns:
+            DEFAULT_LOGGER.debug(f"found clustering columns ({', '.join(columns)})", extra={"label": self})
+            return columns
+
+        else:
+            DEFAULT_LOGGER.debug("could not determine any clustering column", extra={"label": self})
+            return None
+
     def create_table(self):
         def _create_table(df: DataFrame, batch: Optional[int] = 0):
             df = self.base_transform(df)
@@ -242,20 +267,13 @@ class Generator(Configurator):
                     cluster_by = []
 
                 else:
-                    cluster_by = self.options.table.get_list("cluster_by") or []
-                    if not cluster_by:
-                        if "__source" in df.columns:
-                            cluster_by.append("__source")
-                        if "__is_current" in df.columns:
-                            cluster_by.append("__is_current")
-                        if "__key" in df.columns:
-                            cluster_by.append("__key")
-                        elif "__hash" in df.columns:
-                            cluster_by.append("__hash")
-
-                    if not cluster_by:
-                        DEFAULT_LOGGER.debug("could not determine clustering column", extra={"label": self})
-                        liquid_clustering = False
+                    cluster_by = self._get_clustering_columns(df)
+
+                    if cluster_by:
+                        liquid_clustering = True
+
+                    else:
+                        liquid_clustering = None
                         cluster_by = None
 
             if liquid_clustering is None:
@@ -446,20 +464,12 @@ class Generator(Configurator):
             enable = enable_step
 
         if enable:
-            cluster_by = self.options.table.get_list("cluster_by") or []
-            if not cluster_by:
-                if "__source" in df.columns:
-                    cluster_by.append("__source")
-                if "__is_current" in df.columns:
-                    cluster_by.append("__is_current")
-                if "__key" in df.columns:
-                    cluster_by.append("__key")
-                elif "__hash" in df.columns:
-                    cluster_by.append("__hash")
-
-                if len(cluster_by) > 0:
-                    self.table.enable_liquid_clustering(cluster_by, auto=False)
-                else:
-                    self.table.enable_liquid_clustering(auto=True)
+            cluster_by = self._get_clustering_columns(df)
+
+            if cluster_by and len(cluster_by) > 0:
+                self.table.enable_liquid_clustering(cluster_by, auto=False)
+            else:
+                self.table.enable_liquid_clustering(auto=True)
+
         else:
-            DEFAULT_LOGGER.debug("could not enable liquid clustering", extra={"label": self})
+            DEFAULT_LOGGER.debug("liquid clustering disabled", extra={"label": self})

fabricks/core/jobs/bronze.py

@@ -2,7 +2,7 @@ from typing import Optional, Sequence, Union, cast
 
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import expr, lit, md5
-from pyspark.sql.types import Row
+from pyspark.sql.types import Row, TimestampType
 
 from fabricks.cdc.nocdc import NoCDC
 from fabricks.context import VARIABLES
@@ -91,6 +91,11 @@ class Bronze(BaseJob):
         try:
             df = self.spark.sql(f"select * from {file_format}.`{self.data_path}`")
             assert len(df.columns) > 1, "external table must have at least one column"
+            if "__timestamp" in df.columns:
+                assert isinstance(df.schema["__timestamp"].dataType, TimestampType), (
+                    "__timestamp must be of type timestamp"
+                )
+
         except Exception as e:
             DEFAULT_LOGGER.exception("read external table failed", extra={"label": self})
             raise e

fabricks/metastore/table.py

@@ -579,6 +579,14 @@ class Table(DbObject):
 
         return self.spark.sql(f"describe detail {self.qualified_name}")
 
+    def get_partitions(self) -> List[str]:
+        assert self.registered, f"{self} not registered"
+
+        try:
+            return self.spark.sql(f"show partitions {self.qualified_name}").columns
+        except AnalysisException:
+            return []
+
     def get_properties(self) -> DataFrame:
         assert self.registered, f"{self} not registered"
 

{fabricks-3.0.16.dist-info → fabricks-3.0.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.16
+Version: 3.0.17
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.16.dist-info → fabricks-3.0.17.dist-info}/RECORD

@@ -38,15 +38,15 @@ fabricks/cdc/scd2.py,sha256=4vZkhc8pJAUlgiBmIw9j_2RsWuAFMcgCkU3WMVt0A-A,334
 fabricks/cdc/base/__init__.py,sha256=kU4LmQ7x1rekCt8T3r83MmAQac6n2ov-Gh8mBbxIC48,157
 fabricks/cdc/base/_types.py,sha256=WloCDC3ATrn0aZJ6E8BRYKZx19N3EE56r6qlBYhcuvQ,257
 fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
-fabricks/cdc/base/configurator.py,sha256=w6Ywif87iv1WG-5OM3XkzIRrsns-_QQ6XlADpk0YLlw,6434
+fabricks/cdc/base/configurator.py,sha256=DzODL0wO0FqfmIMQKEofqQKYeOCR6vaO0Am92ctcAk4,6158
 fabricks/cdc/base/generator.py,sha256=pa_GJn7Pdi5vMnXN8zExmOPMpCqdZ3QoxHEB0wv0lsk,5933
 fabricks/cdc/base/merger.py,sha256=3qUUs0uqmwOMdXc50kV3Zo9omuQuUUFgtMLBrg4E-wk,4159
-fabricks/cdc/base/processor.py,sha256=gL3pWMaBRsc0oB93ISnH2x07WbmtM_QEIx8qrUcUoZ0,17704
+fabricks/cdc/base/processor.py,sha256=hExUGGVOX1Px0IWw65MOaDeeUbSMtjRekLdqEQ0U6Mw,17921
 fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/cdc/templates/filter.sql.jinja,sha256=AQcOj4KRwKscVG3zepnEAN9Yxb03AM7068hqW7dtVI8,236
 fabricks/cdc/templates/merge.sql.jinja,sha256=YS9wWckCVsUI1pUYiRSFBIuQ16WU3EPWSkhZVy2niBA,221
 fabricks/cdc/templates/query.sql.jinja,sha256=Z0kSm9sRKJTLQ2Lb3NS7yu93GBxNls9HL7uAjTdirjk,868
-fabricks/cdc/templates/ctes/base.sql.jinja,sha256=gO0dEQ00_NXOeZKRxfWvoUjpks0nHK-Is5H7xdI_J6s,1637
+fabricks/cdc/templates/ctes/base.sql.jinja,sha256=rtp5hRMxcoD3NupzlAFvH8BREKHUnLEirHvJCoOmQk8,1871
 fabricks/cdc/templates/ctes/current.sql.jinja,sha256=bb72XdaJVce5k57dx_N6T4OfyQDUrFLulr3VM6irdn0,1278
 fabricks/cdc/templates/ctes/deduplicate_hash.sql.jinja,sha256=DHcSNhAvoFZY0gSv3Zylat9ysGXKR22OGXs49KTNCjA,1095
 fabricks/cdc/templates/ctes/deduplicate_key.sql.jinja,sha256=4-K4nk2Bfr_8c_x3YDYBP7JUx4c-b3Ef-7HFx0s2jrc,1122
@@ -61,7 +61,7 @@ fabricks/cdc/templates/merges/nocdc.sql.jinja,sha256=lA4-PXogC_u6RqummOQX_WBFCNq
 fabricks/cdc/templates/merges/scd1.sql.jinja,sha256=GimfwgEobGgCzPce_FJdvQY9jmRJXFUZ4_CVhRgTTqY,1600
 fabricks/cdc/templates/merges/scd2.sql.jinja,sha256=5qBO_1lr7xa-Ep8RqvJGCRISOv4uo_tiAtVOybp4tUU,1210
 fabricks/cdc/templates/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fabricks/cdc/templates/queries/context.sql.jinja,sha256=19mAAAM9Gh2PAzuupCzc68zxEHH3spTpHZHGU9vke8o,3883
+fabricks/cdc/templates/queries/context.sql.jinja,sha256=x-t3CzdRaSXi8Xi7aYaNdIVyRwVOxmxw8xBZyfnFE40,3977
 fabricks/cdc/templates/queries/final.sql.jinja,sha256=vxH434CO5k8Ia7tugaH8LC1co7Epaj7Z1M7Y9BdqzaI,111
 fabricks/cdc/templates/queries/scd1.sql.jinja,sha256=siHULgKE3uRBGQYZFUR_eHNqFuGgO9xUCRVV2jnAXAI,3019
 fabricks/cdc/templates/queries/scd2.sql.jinja,sha256=Nn0wUs9N-_QviZqUKRWAFdD17RR3EFBTMs9BpBu6z7E,3877
@@ -91,7 +91,7 @@ fabricks/core/dags/run.py,sha256=RIDULb9WakObSyYzmkglh8RwFRwC8-NFC-1yPDMkBC0,107
 fabricks/core/dags/terminator.py,sha256=Y6pV2UnSyrCIx2AQWJXoHk5Roq12gZqpennHx_Lbnzs,793
 fabricks/core/dags/utils.py,sha256=4kyobLGl4tO0Flo6LxNzYjCU_G42vns1LrkxTO5_KLY,1585
 fabricks/core/jobs/__init__.py,sha256=W_1m6LoGiXBml_8cucedju_hllSSnZGKsZjyFh-2oJw,531
-fabricks/core/jobs/bronze.py,sha256=eDH2YLHbOgBoJoGZTFLJS9igqnqkeJtM56nahQK6zJ4,13815
+fabricks/core/jobs/bronze.py,sha256=mz3YDm4ft9sP51k4lUR6gmT1rkQBLvdaePAD3uo3m1c,14040
 fabricks/core/jobs/get_job.py,sha256=35zay3Z_WoJIylzEQlGle6UvrE1EClfRbFEVGvszof0,3675
 fabricks/core/jobs/get_job_conf.py,sha256=3vAj_usCbNqejMUKOF85LPaHBYAwxgrDG7LYgY-vBUw,4812
 fabricks/core/jobs/get_job_id.py,sha256=6dLyzxGHlRvJZVJSwZkCk3iXzWkIhePC_6FhoP0gEN4,607
@@ -105,7 +105,7 @@ fabricks/core/jobs/base/_types.py,sha256=y66BtJlJskq7wGzn7te5XYjO-NEqeQGUC11kkbe
 fabricks/core/jobs/base/checker.py,sha256=Cdfh8rQYy4MvMFl0HyC3alGUWm8zrFXk08m2t2JMu6Y,5477
 fabricks/core/jobs/base/configurator.py,sha256=9G5F7Qg5FWHPbHgdh8Qxc85OoSX0rnjD4c9itwU5KKc,10415
 fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
-fabricks/core/jobs/base/generator.py,sha256=Dk82tj21NhR9wwgXzMp8JlKQ6D9HnjVlK9fvDmoYLbk,17646
+fabricks/core/jobs/base/generator.py,sha256=TWiJtQvEH0uH8YMFlOxxi6CpWnrDNua61PCbdamOtHA,17487
 fabricks/core/jobs/base/invoker.py,sha256=FvjfpNqi542slxC2yLu1BIu5EklNUWySxDF8cD_SqKQ,7602
 fabricks/core/jobs/base/job.py,sha256=dWmk2PpQH2NETaaDS6KoiefRnDHfDMdCyhmogkdcSFI,93
 fabricks/core/jobs/base/processor.py,sha256=qkNiJSSLaEnivKGBcd9UZyIVFexnv-n1p_5mCZIy1rA,9076
@@ -143,7 +143,7 @@ fabricks/metastore/_types.py,sha256=NXYxwQHP0sCllM0N6QBbaK4CdtM_m_rHFDxRNRfBcLU,
 fabricks/metastore/database.py,sha256=23VAKKzjrwlEaj28DNNmiOhcfdKRzYk8eEfq-PzINbg,1924
 fabricks/metastore/dbobject.py,sha256=ve8p48OqEpJYsqWNhgesGSE0emM--uY8QrvBRoR3j3g,1881
 fabricks/metastore/pyproject.toml,sha256=6RZM9RMKMDF_EAequhORZ7TD0BQNk7aBCTWAv-sRcp0,519
-fabricks/metastore/table.py,sha256=AaoNL-1mz4A0CCb3tH_0BUurYPjA1oL5pioCYlEMtu4,29113
+fabricks/metastore/table.py,sha256=lAxOky3BgpPudcIEZl2UbNgFiTHYfdtvcIHy9D5D78s,29368
 fabricks/metastore/utils.py,sha256=8SxhjDkz_aSH4IGUusel7hqOQxP9U8PNBCY0M7GH00Y,1355
 fabricks/metastore/view.py,sha256=f7hKJWtnH1KmZym8dkoucKOTndntzai_f2YqferxHLs,1431
 fabricks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -171,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=5k-R6zCgUAtapQgxT4turcx1IQ-b
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=lTQ0CfUhcvn3xTCcT_Ns6PMDBsO5UEfa2S9XpJiLJ9c,1250
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.16.dist-info/METADATA,sha256=EUzxCf5zxbG8Dmst3_r4jyjK4Xb78n3pY0gajpZwqrg,798
-fabricks-3.0.16.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-fabricks-3.0.16.dist-info/RECORD,,
+fabricks-3.0.17.dist-info/METADATA,sha256=JTkqiUkvRKtEQKIWDsQhsNcmvMJ81vCx9kQhyxfznwM,798
+fabricks-3.0.17.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+fabricks-3.0.17.dist-info/RECORD,,