fabricks 3.0.2__tar.gz → 3.0.4__tar.gz

{fabricks-3.0.2 → fabricks-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.2
+Version: 3.0.4
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.2 → fabricks-3.0.4}/fabricks/cdc/base/configurator.py

@@ -37,6 +37,10 @@ class Configurator(ABC):
     def is_registered(self):
         return self.table.is_registered
 
+    @property
+    def qualified_name(self):
+        return f"{self.database}_{'_'.join(self.levels)}"
+
     @abstractmethod
     def get_query(self, src: Union[DataFrame, Table, str], **kwargs):
         raise NotImplementedError()

{fabricks-3.0.2 → fabricks-3.0.4}/fabricks/cdc/base/merger.py

@@ -101,7 +101,7 @@ class Merger(Processor):
             self.create_table(src, **kwargs)
 
         df = self.get_data(src, **kwargs)
-        global_temp_view = f"{self.database}_{'_'.join(self.levels)}__merge"
+        global_temp_view = f"{self.qualified_name}__merge"
         view = create_or_replace_global_temp_view(global_temp_view, df, uuid=kwargs.get("uuid", False))
 
         merge = self.get_merge_query(view, **kwargs)

{fabricks-3.0.2 → fabricks-3.0.4}/fabricks/cdc/base/processor.py

@@ -16,7 +16,7 @@ from fabricks.utils.sqlglot import fix as fix_sql
 class Processor(Generator):
     def get_data(self, src: Union[DataFrame, Table, str], **kwargs) -> DataFrame:
         if isinstance(src, (DataFrame, CDataFrame)):
-            name = f"{self.database}_{'_'.join(self.levels)}__data"
+            name = f"{self.qualified_name}__data"
             global_temp_view = create_or_replace_global_temp_view(name, src, uuid=kwargs.get("uuid", False))
             src = f"select * from {global_temp_view}"
 
@@ -347,8 +347,11 @@ class Processor(Generator):
         df = self.get_data(src, **kwargs)
         df = self.reorder_columns(df)
 
+        name = f"{self.qualified_name}__append"
+        create_or_replace_global_temp_view(name, df, uuid=kwargs.get("uuid", False))
+
         DEFAULT_LOGGER.debug("append", extra={"job": self})
-        df.write.format("delta").mode("append").save(self.table.delta_path.string)
+        self.spark.sql(f"insert into table {self.table} by name select * from global_temp.{name}")
 
     def overwrite(
         self,
@@ -366,9 +369,11 @@ class Processor(Generator):
             if kwargs.get("update_where"):
                 dynamic = True
 
-        writer = df.write.format("delta").mode("overwrite")
         if dynamic:
-            writer.option("partitionOverwriteMode", "dynamic")
+            self.spark.sql("set spark.sql.sources.partitionOverwriteMode = dynamic")
+
+        name = f"{self.qualified_name}__overwrite"
+        create_or_replace_global_temp_view(name, df, uuid=kwargs.get("uuid", False))
 
-        DEFAULT_LOGGER.info("overwrite", extra={"job": self})
-        writer.save(self.table.delta_path.string)
+        DEFAULT_LOGGER.debug("overwrite", extra={"job": self})
+        self.spark.sql(f"insert overwrite table {self.table} by name select * from global_temp.{name}")

{fabricks-3.0.2 → fabricks-3.0.4}/fabricks/core/jobs/base/_types.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, List, Literal, Optional, TypedDict, Union
+from typing import List, Literal, Optional, TypedDict, Union
 
 from pydantic import BaseModel, ConfigDict, model_validator
 from pyspark.sql.types import StringType, StructField, StructType
@@ -33,8 +33,8 @@ Origins = Literal["parser", "job"]
 
 
 class SparkOptions(TypedDict):
-    sql: Optional[dict[Any, Any]]
-    conf: Optional[dict[Any, Any]]
+    sql: Optional[dict[str, str]]
+    conf: Optional[dict[str, str]]
 
 
 class TableOptions(TypedDict):
@@ -45,17 +45,17 @@ class TableOptions(TypedDict):
     cluster_by: Optional[List[str]]
     powerbi: Optional[bool]
     bloomfilter_by: Optional[List[str]]
-    constraints: Optional[dict[Any, Any]]
-    properties: Optional[dict[Any, Any]]
+    constraints: Optional[dict[str, str]]
+    properties: Optional[dict[str, str]]
     comment: Optional[str]
-    calculated_columns: Optional[dict[Any, Any]]
+    calculated_columns: Optional[dict[str, str]]
     retention_days: Optional[int]
 
 
 class _InvokeOptions(TypedDict):
     notebook: str
     timeout: int
-    arguments: Optional[dict[Any, Any]]
+    arguments: Optional[dict[str, str]]
 
 
 class InvokerOptions(TypedDict):
@@ -66,7 +66,7 @@ class InvokerOptions(TypedDict):
 
 class ExtenderOptions(TypedDict):
     extender: str
-    arguments: Optional[dict[Any, Any]]
+    arguments: Optional[dict[str, str]]
 
 
 class CheckOptions(TypedDict):
@@ -90,7 +90,7 @@ class BronzeOptions(TypedDict):
     filter_where: Optional[str]
     # extra
     encrypted_columns: Optional[List[str]]
-    calculated_columns: Optional[dict[Any, Any]]
+    calculated_columns: Optional[dict[str, str]]
     operation: Optional[Operations]
     timeout: Optional[int]
 
@@ -106,7 +106,7 @@ class SilverOptions(TypedDict):
     deduplicate: Optional[bool]
     stream: Optional[bool]
     # else
-    order_duplicate_by: Optional[dict[Any, Any]]
+    order_duplicate_by: Optional[dict[str, str]]
     timeout: Optional[int]
 
 
@@ -141,7 +141,7 @@ class BaseJobConf:
 
 @dataclass
 class JobConfBronze(BaseJobConf):
-    step: str
+    step: TBronze
     options: BronzeOptions
     table_options: Optional[TableOptions] = None
     parser_options: Optional[ParserOptions] = None
@@ -155,7 +155,7 @@ class JobConfBronze(BaseJobConf):
 
 @dataclass
 class JobConfSilver(BaseJobConf):
-    step: str
+    step: TSilver
     options: SilverOptions
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None
@@ -168,7 +168,7 @@ class JobConfSilver(BaseJobConf):
 
 @dataclass
 class JobConfGold(BaseJobConf):
-    step: str
+    step: TGold
     options: Optional[GoldOptions]
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None

{fabricks-3.0.2 → fabricks-3.0.4}/fabricks/core/jobs/base/configurator.py

@@ -301,9 +301,8 @@ class Configurator(ABC):
         """
         if self.mode == "memory":
             DEFAULT_LOGGER.debug("memory (no optimize)", extra={"job": self})
-        else:
-            assert self.table.exists()
 
+        else:
             if vacuum:
                 self.vacuum()
             if optimize:
@@ -312,19 +311,23 @@ class Configurator(ABC):
                 self.table.compute_statistics()
 
     def vacuum(self):
-        job = self.options.table.get("retention_days")
-        step = self.step_conf.get("table_options", {}).get("retention_days", None)
-        runtime = CONF_RUNTIME.get("options", {}).get("retention_days")
-
-        if job is not None:
-            retention_days = job
-        elif step:
-            retention_days = step
+        if self.mode == "memory":
+            DEFAULT_LOGGER.debug("memory (no vacuum)", extra={"job": self})
+
         else:
-            assert runtime
-            retention_days = runtime
+            job = self.options.table.get("retention_days")
+            step = self.step_conf.get("table_options", {}).get("retention_days", None)
+            runtime = CONF_RUNTIME.get("options", {}).get("retention_days")
+
+            if job is not None:
+                retention_days = job
+            elif step:
+                retention_days = step
+            else:
+                assert runtime
+                retention_days = runtime
 
-        self.table.vacuum(retention_days=retention_days)
+            self.table.vacuum(retention_days=retention_days)
 
     def __str__(self):
         return f"{self.step}.{self.topic}_{self.item}"

{fabricks-3.0.2 → fabricks-3.0.4}/fabricks/core/jobs/bronze.py

@@ -103,6 +103,21 @@ class Bronze(BaseJob):
         DEFAULT_LOGGER.debug("drop external table", extra={"job": self})
         self.spark.sql(f"drop table if exists {self.qualified_name}")
 
+    def analyze_external_table(self):
+        DEFAULT_LOGGER.debug("analyze external table", extra={"job": self})
+        self.spark.sql(f"analyze table {self.qualified_name} compute statistics")
+
+    def vacuum_external_table(self, retention_hours: Optional[int] = 168):
+        from delta import DeltaTable
+
+        DEFAULT_LOGGER.debug("vacuum external table", extra={"job": self})
+        try:
+            dt = DeltaTable.forPath(self.spark, self.data_path.string)
+            self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = False")
+            dt.vacuum(retention_hours)
+        finally:
+            self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
+
     def optimize_external_table(
         self,
         vacuum: Optional[bool] = True,
@@ -110,20 +125,10 @@ class Bronze(BaseJob):
     ):
         DEFAULT_LOGGER.debug("optimize external table", extra={"job": self})
         if vacuum:
-            from delta import DeltaTable
-
-            dt = DeltaTable.forPath(self.spark, self.data_path.string)
-            retention_days = 7
-            DEFAULT_LOGGER.debug(f"{self.data_path} - vacuum table (removing files older than {retention_days} days)")
-            try:
-                self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = False")
-                dt.vacuum(retention_days * 24)
-            finally:
-                self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
+            self.vacuum_external_table()
 
         if analyze:
-            DEFAULT_LOGGER.debug(f"{self.data_path} - compute delta statistics")
-            self.spark.sql(f"analyze table delta.`{self.data_path}` compute delta statistics")
+            self.analyze_external_table()
 
     @property
     def parser(self) -> BaseParser:
@@ -370,6 +375,14 @@ class Bronze(BaseJob):
         else:
             super().optimize(vacuum=vacuum, optimize=optimize, analyze=analyze)
 
+    def vacuum(self):
+        if self.mode == "memory":
+            DEFAULT_LOGGER.info("memory (no vacuum)", extra={"job": self})
+        elif self.mode == "register":
+            self.vacuum_external_table()
+        else:
+            super().vacuum()
+
     def overwrite(self):
         self.truncate()
         self.run()

{fabricks-3.0.2 → fabricks-3.0.4}/fabricks/metastore/table.py

@@ -433,7 +433,11 @@ class Table(DbObject):
         cols = [
             f"`{name}`"
             for name, dtype in self.dataframe.dtypes
-            if not dtype.startswith("struct") and not dtype.startswith("array") and name not in ["__metadata"]
+            if not dtype.startswith("struct")
+            and not dtype.startswith("array")
+            and not dtype.startswith("variant")
+            and not dtype.startswith("map")
+            and name not in ["__metadata"]
         ]
         cols = ", ".join(sorted(cols))
         self.spark.sql(f"analyze table {self.qualified_name} compute statistics for columns {cols}")

{fabricks-3.0.2 → fabricks-3.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.2"
+version = "3.0.4"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

{fabricks-3.0.2 → fabricks-3.0.4}/uv.lock

@@ -627,7 +627,7 @@ wheels = [
 
 [[package]]
 name = "fabricks"
-version = "3.0.2"
+version = "3.0.4"
 source = { editable = "." }
 dependencies = [
     { name = "azure-data-tables" },