fabricks 3.0.15__tar.gz → 3.0.17__tar.gz

{fabricks-3.0.15 → fabricks-3.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.15
+Version: 3.0.17
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/cdc/base/configurator.py

@@ -90,6 +90,7 @@ class Configurator(ABC):
             "__identity",
             "__source",
             "__key",
+            "__hash",
             "__timestamp",
             "__valid_from",
             "__valid_to",
@@ -110,7 +111,6 @@ class Configurator(ABC):
         cols = [
             "__operation",
             "__metadata",
-            "__hash",
             "__rescued_data",
         ]
 
@@ -126,6 +126,7 @@ class Configurator(ABC):
             "__identity",
             "__source",
             "__key",
+            "__hash",
             "__timestamp",
             "__valid_from",
             "__valid_to",
@@ -134,7 +135,6 @@ class Configurator(ABC):
             # Trailing
             "__operation",
             "__metadata",
-            "__hash",
             "__rescued_data",
         ]
 
@@ -192,11 +192,6 @@ class Configurator(ABC):
         leading = self.allowed_ouput_leading__columns
         trailing = self.allowed_output_trailing__columns
 
-        # move __hash to the front of the table to ensure statistics are present
-        if "__key" not in columns and "__hash" in columns:
-            leading = ["__hash" if c == "__key" else c for c in leading]
-            trailing = [c for c in trailing if c != "__hash"]
-
         __leading = [c for c in leading if c in columns]
         __trailing = [c for c in trailing if c in columns]
 

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/cdc/base/processor.py

@@ -51,6 +51,7 @@ class Processor(Generator):
 
         overwrite = []
         exclude = kwargs.get("exclude", [])  # used by silver to exclude __operation from output if not update
+        cast = kwargs.get("cast", {})  # used by silver to cast columns to target types
 
         order_duplicate_by = kwargs.get("order_duplicate_by", None)
         if order_duplicate_by:
@@ -140,6 +141,8 @@ class Processor(Generator):
         # override timestamp if added and found in df
         if add_timestamp and "__timestamp" in inputs:
             overwrite.append("__timestamp")
+        elif "__timestamp" in inputs:
+            cast["__timestamp"] = "timestamp"
 
         # override key if added and found in df (key needed for merge)
         if add_key and "__key" in inputs:
@@ -356,6 +359,8 @@ class Processor(Generator):
             "correct_valid_from": correct_valid_from,
             # overwrite
             "overwrite": overwrite,
+            # cast
+            "cast": cast,
             # filter
             "slices": None,
             "sources": None,

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/cdc/templates/ctes/base.sql.jinja

@@ -5,10 +5,14 @@ with
     __base as (
         select
             *
-            {% if overwrite %}
+            {% if overwrite or cast %}
                 -- will be overwritten below
-                except ({% for o in overwrite %}{{ o }}, {% endfor %})
+                except (
+                    {% for o in overwrite %}{{ o }}, {% endfor %}
+                    {% for key, value in cast.items() %} {{ key }}, {% endfor %}
+                )
             {% endif %},
+            {% for key, value in cast.items() %} cast({{ key }} as {{ value }}) as {{ key }}, {% endfor %}
             {% if add_calculated_columns %} {% for c in add_calculated_columns %} {{ c }}, {% endfor %} {% endif %}
             {% if add_timestamp %} cast(current_date() as timestamp) as __timestamp, {% endif %}
             {% if add_operation %} cast('{{ add_operation }}' as string) as __operation, {% endif %}

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/cdc/templates/queries/context.sql.jinja

@@ -141,6 +141,9 @@
 {%- if overwrite %}
 ☐ overwrite: {{ overwrite | join(", ") | truncate(100, killwords=True) }}
 {%- endif %}
+{%- if cast %}
+☐ cast: {{ cast | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
 
 👨‍👩‍👧 PARENTS
 {%- if parent_slice %}

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/core/jobs/base/generator.py

@@ -1,5 +1,5 @@
 from abc import abstractmethod
-from typing import Optional, Sequence, Union, cast
+from typing import List, Optional, Sequence, Union, cast
 
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import lit
@@ -166,6 +166,31 @@ class Generator(Configurator):
         """
         ...
 
+    def _get_clustering_columns(self, df: DataFrame) -> Optional[List[str]]:
+        columns = self.options.table.get_list("cluster_by")
+        if columns:
+            return columns
+
+        columns = []
+
+        if "__source" in df.columns:
+            columns = ["__source"]
+        if "__is_current" in df.columns:
+            columns.append("__is_current")
+
+        if "__key" in df.columns:
+            columns.append("__key")
+        elif "__hash" in df.columns:
+            columns.append("__hash")
+
+        if columns:
+            DEFAULT_LOGGER.debug(f"found clustering columns ({', '.join(columns)})", extra={"label": self})
+            return columns
+
+        else:
+            DEFAULT_LOGGER.debug("could not determine any clustering column", extra={"label": self})
+            return None
+
     def create_table(self):
         def _create_table(df: DataFrame, batch: Optional[int] = 0):
             df = self.base_transform(df)
@@ -242,20 +267,13 @@ class Generator(Configurator):
                     cluster_by = []
 
                 else:
-                    cluster_by = self.options.table.get_list("cluster_by") or []
-                    if not cluster_by:
-                        if "__source" in df.columns:
-                            cluster_by.append("__source")
-                        if "__is_current" in df.columns:
-                            cluster_by.append("__is_current")
-                        if "__key" in df.columns:
-                            cluster_by.append("__key")
-                        elif "__hash" in df.columns:
-                            cluster_by.append("__hash")
-
-                    if not cluster_by:
-                        DEFAULT_LOGGER.debug("could not determine clustering column", extra={"label": self})
-                        liquid_clustering = False
+                    cluster_by = self._get_clustering_columns(df)
+
+                    if cluster_by:
+                        liquid_clustering = True
+
+                    else:
+                        liquid_clustering = None
                         cluster_by = None
 
             if liquid_clustering is None:
@@ -446,20 +464,12 @@ class Generator(Configurator):
             enable = enable_step
 
         if enable:
-            cluster_by = self.options.table.get_list("cluster_by") or []
-            if not cluster_by:
-                if "__source" in df.columns:
-                    cluster_by.append("__source")
-                if "__is_current" in df.columns:
-                    cluster_by.append("__is_current")
-                if "__key" in df.columns:
-                    cluster_by.append("__key")
-                elif "__hash" in df.columns:
-                    cluster_by.append("__hash")
-
-                if len(cluster_by) > 0:
-                    self.table.enable_liquid_clustering(cluster_by, auto=False)
-                else:
-                    self.table.enable_liquid_clustering(auto=True)
+            cluster_by = self._get_clustering_columns(df)
+
+            if cluster_by and len(cluster_by) > 0:
+                self.table.enable_liquid_clustering(cluster_by, auto=False)
+            else:
+                self.table.enable_liquid_clustering(auto=True)
+
         else:
-            DEFAULT_LOGGER.debug("could not enable liquid clustering", extra={"label": self})
+            DEFAULT_LOGGER.debug("liquid clustering disabled", extra={"label": self})

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/core/jobs/bronze.py

@@ -2,7 +2,7 @@ from typing import Optional, Sequence, Union, cast
 
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import expr, lit, md5
-from pyspark.sql.types import Row
+from pyspark.sql.types import Row, TimestampType
 
 from fabricks.cdc.nocdc import NoCDC
 from fabricks.context import VARIABLES
@@ -91,6 +91,11 @@ class Bronze(BaseJob):
         try:
             df = self.spark.sql(f"select * from {file_format}.`{self.data_path}`")
             assert len(df.columns) > 1, "external table must have at least one column"
+            if "__timestamp" in df.columns:
+                assert isinstance(df.schema["__timestamp"].dataType, TimestampType), (
+                    "__timestamp must be of type timestamp"
+                )
+
         except Exception as e:
             DEFAULT_LOGGER.exception("read external table failed", extra={"label": self})
             raise e

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/core/masks.py

@@ -7,7 +7,7 @@ from fabricks.context import CATALOG, PATH_MASKS, SPARK
 from fabricks.context.log import DEFAULT_LOGGER
 
 
-def register_all_masks():
+def register_all_masks(override: bool = False):
     """
     Register all masks.
     """
@@ -16,7 +16,7 @@ def register_all_masks():
     for mask in get_masks():
         split = mask.split(".")
         try:
-            register_mask(mask=split[0])
+            register_mask(mask=split[0], override=override)
         except Exception as e:
             DEFAULT_LOGGER.exception(f"could not register mask {mask}", exc_info=e)
 
@@ -40,13 +40,16 @@ def is_registered(mask: str, spark: Optional[SparkSession] = None) -> bool:
     return not df.isEmpty()
 
 
-def register_mask(mask: str, spark: Optional[SparkSession] = None):
+def register_mask(mask: str, override: Optional[bool] = False, spark: Optional[SparkSession] = None):
     if spark is None:
         spark = SPARK
     assert spark is not None
 
-    if not is_registered(mask, spark):
-        DEFAULT_LOGGER.debug(f"register mask {mask}")
+    if not is_registered(mask, spark) or override:
+        if override:
+            DEFAULT_LOGGER.debug(f"override mask {mask}")
+        else:
+            DEFAULT_LOGGER.debug(f"register mask {mask}")
 
         path = PATH_MASKS.joinpath(f"{mask}.sql")
         spark.sql(path.get_sql())

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/core/udfs.py

@@ -60,8 +60,8 @@ def is_registered(udf: str, spark: Optional[SparkSession] = None) -> bool:
 def register_udf(
     udf: str,
     extension: Optional[str] = None,
+    override: Optional[bool] = False,
     spark: Optional[SparkSession] = None,
-    override: bool = False,
 ):
     """
     Register a user-defined function (UDF).

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/deploy/__init__.py

@@ -25,12 +25,12 @@ class Deploy:
         deploy_views()
 
     @staticmethod
-    def udfs():
-        deploy_udfs()
+    def udfs(override: bool = True):
+        deploy_udfs(override=override)
 
     @staticmethod
-    def masks():
-        deploy_masks()
+    def masks(override: bool = True):
+        deploy_masks(override=override)
 
     @staticmethod
     def notebooks():

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/deploy/masks.py

@@ -2,7 +2,7 @@ from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.core.masks import register_all_masks
 
 
-def deploy_masks():
+def deploy_masks(override: bool = True):
     DEFAULT_LOGGER.info("create or replace masks")
 
-    register_all_masks()
+    register_all_masks(override=override)

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/deploy/tables.py

@@ -9,9 +9,9 @@ from fabricks.metastore.table import Table
 def deploy_tables(drop: bool = False):
     DEFAULT_LOGGER.info("create or replace fabricks (default) tables")
 
-    create_table_log(drop)
-    create_table_dummy(drop)
-    create_table_step(drop)
+    create_table_log(drop=drop)
+    create_table_dummy(drop=drop)
+    create_table_step(drop=drop)
 
 
 def create_table_step(drop: bool = False):

{fabricks-3.0.15 → fabricks-3.0.17}/fabricks/metastore/table.py

@@ -579,6 +579,14 @@ class Table(DbObject):
 
         return self.spark.sql(f"describe detail {self.qualified_name}")
 
+    def get_partitions(self) -> List[str]:
+        assert self.registered, f"{self} not registered"
+
+        try:
+            return self.spark.sql(f"show partitions {self.qualified_name}").columns
+        except AnalysisException:
+            return []
+
     def get_properties(self) -> DataFrame:
         assert self.registered, f"{self} not registered"
 

{fabricks-3.0.15 → fabricks-3.0.17}/format.sh

@@ -45,7 +45,7 @@ fi
 
 if [ -z "$build" ]
 then
-    build="true"
+    build="false"
 fi
 
 if [ "$all" = "true" ] || [ "$python" = "true" ]

{fabricks-3.0.15 → fabricks-3.0.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.15"
+version = "3.0.17"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

{fabricks-3.0.15 → fabricks-3.0.17}/tests/integration/data/job1/prince/2022/01/01/0001/prince_202201010001.json

@@ -7,5 +7,14 @@
         "BEL_RestoredDateUtc": null,
         "BEL_IsFullLoad": true,
         "BEL_UpdateDateUtc": "2022-01-01 01:00:00"
+    },
+    {
+        "@Id": 6004,
+        "Näàme": "Gabriel",
+        "double Field!": 0.19000102,
+        "BEL_DeleteDateUtc": null,
+        "BEL_RestoredDateUtc": null,
+        "BEL_IsFullLoad": true,
+        "BEL_UpdateDateUtc": "2022-01-01 01:00:00"
     }
 ]

{fabricks-3.0.15 → fabricks-3.0.17}/tests/integration/fabricksconfig.json

@@ -2,7 +2,7 @@
     "runtime": "runtime",
     "notebooks": "runtime/fabricks/notebooks",
     "job_config_from_yaml": true,
-    "loglevel": "info",
+    "loglevel": "debug",
     "debugmode": true,
     "config": "runtime/fabricks/conf.uc.fabricks.yml",
     "devmode": true

{fabricks-3.0.15 → fabricks-3.0.17}/tests/integration/jobs/job1/test_semantic.py

@@ -65,6 +65,7 @@ def test_semantic_fact_zstd():
 @pytest.mark.order(139)
 def test_semantic_fact_powerbi():
     j = get_job(step="semantic", topic="fact", item="powerbi")
+
     props_df = j.table.show_properties()
     assert props_df.where("key == 'delta.minReaderVersion'").select("value").collect()[0][0] == "2", (
         "minReaderVersion <> 2"
@@ -72,3 +73,7 @@ def test_semantic_fact_powerbi():
     assert props_df.where("key == 'delta.minWriterVersion'").select("value").collect()[0][0] == "5", (
         "minWriterVersion <> 5"
     )
+
+    partitions = j.table.get_partitions()
+    assert len(partitions) == 1, "partition not found"
+    assert partitions[0].lower() == "monarch", "partition <> monarch"

fabricks-3.0.17/tests/integration/landing_to_raw.py

@@ -0,0 +1,46 @@
+# Databricks notebook source
+# MAGIC %run ./add_missing_modules
+
+# COMMAND ----------
+
+from logging import DEBUG
+
+from databricks.sdk.runtime import dbutils, spark
+
+from fabricks.context import CATALOG
+from fabricks.context.log import DEFAULT_LOGGER
+from tests.integration._types import paths
+from tests.integration.utils import git_to_landing, landing_to_raw
+
+# COMMAND ----------
+
+DEFAULT_LOGGER.setLevel(DEBUG)
+
+# COMMAND ----------
+
+if CATALOG:
+    try:
+        spark.sql(f"use catalog {CATALOG}")
+        spark.sql("drop schema if exists bronze cascade")
+        spark.sql("create schema bronze")
+
+    except Exception:
+        pass
+
+# COMMAND ----------
+
+paths.landing.rm()
+git_to_landing()
+
+# COMMAND ----------
+
+paths.raw.rm()
+landing_to_raw(1)
+
+# COMMAND ----------
+
+paths.out.rm()
+
+# COMMAND ----------
+
+dbutils.notebook.exit(value="exit (0)")  # type: ignore

{fabricks-3.0.15 → fabricks-3.0.17}/tests/integration/runtime/semantic/fact/_config.semantic.yml

@@ -42,6 +42,7 @@
       mode: complete
     table_options:
       powerbi: true
+      partition_by: [Monarch]
 - job:
     step: semantic
     topic: fact

{fabricks-3.0.15 → fabricks-3.0.17}/tests/integration/runtime/silver/_config.princes.yml

@@ -16,3 +16,11 @@
     options:
       mode: update
       change_data_capture: scd1
+- job:
+    step: silver
+    topic: prince
+    item: scd2
+    tags: [test]
+    options:
+      mode: update
+      change_data_capture: scd2

{fabricks-3.0.15 → fabricks-3.0.17}/tests/integration/runtime/uc/bronze/_config.princes.yml

@@ -18,3 +18,12 @@
       uri: abfss://test@$datahub/raw/2/prince
       parser: monarch
       keys: ['@Id']
+- job:
+    step: bronze
+    topic: prince
+    item: scd2
+    options:
+      mode: register
+      uri: abfss://test@$datahub/raw/2/delta/prince
+      parser: monarch
+      keys: ['@Id']

{fabricks-3.0.15 → fabricks-3.0.17}/tests/integration/utils.py

@@ -1,4 +1,5 @@
 import os
+import re
 from typing import Any, List, Union, cast
 
 import pandas as pd
@@ -22,7 +23,7 @@ def create_random_tables():
     spark.sql(f"create table if not exists bronze.princess_no_column using delta location '{uri}'")
 
 
-def convert_parquet_to_delta(topic: str):
+def convert_parquet_to_delta(topic: str, deletelog: bool = True):
     for i in range(1, 4):
         dfs = []
 
@@ -30,7 +31,11 @@ def convert_parquet_to_delta(topic: str):
         if i > 1:
             root = root.joinpath(str(i))
 
-        for p in [f"{topic}__deletelog", topic]:
+        _paths = [topic]
+        if deletelog:
+            _paths.append(f"{topic}__deletelog")
+
+        for p in _paths:
             df = (
                 spark.read.option("pathGlobFilter", "*.parquet")
                 .option("recursiveFileLookup", "True")
@@ -59,7 +64,13 @@ def convert_parquet_to_delta(topic: str):
         )
         df = df.withColumn("__timestamp", expr("to_timestamp(__timestamp, 'yyyyMMddHHmmss')"))
         df = df.drop("__split", "__split_size", "__file_path", "__file_name")
-        df.write.mode("append").option("mergeSchema", "True").format("delta").save(f"{root}/delta/{topic}")
+
+        writer = df.write.mode("append").option("mergeSchema", "True").format("delta")
+
+        if any(not re.match(r"^[a-zA-Z0-9_]+$", c) for c in df.columns):
+            writer = writer.option("delta.columnMapping.mode", "name")
+
+        writer.save(f"{root}/delta/{topic}")
 
 
 def convert_json_to_parquet(from_dir: Path, to_dir: Path):
@@ -128,6 +139,7 @@ def landing_to_raw(iter: Union[int, List[int]]):
 
     convert_parquet_to_delta("regent")
     convert_parquet_to_delta("monarch")
+    convert_parquet_to_delta("prince", deletelog=False)
 
 
 def create_expected_views():

{fabricks-3.0.15 → fabricks-3.0.17}/uv.lock

@@ -645,7 +645,7 @@ wheels = [
 
 [[package]]
 name = "fabricks"
-version = "3.0.15"
+version = "3.0.17"
 source = { editable = "." }
 dependencies = [
     { name = "azure-data-tables" },