fabricks 3.0.1__py3-none-any.whl → 3.0.3__py3-none-any.whl

fabricks/core/dags/base.py

@@ -1,7 +1,7 @@
 import re
 from typing import Optional
 
-from azure.core.exceptions import ServiceRequestError
+from azure.core.exceptions import AzureError
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import expr
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
@@ -31,7 +31,7 @@ class BaseDags:
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=1, max=10),
-        retry=retry_if_exception_type((Exception, ServiceRequestError)),
+        retry=retry_if_exception_type((Exception, AzureError)),
         reraise=True,
     )
     def get_table(self) -> AzureTable:

fabricks/core/dags/processor.py

@@ -4,7 +4,7 @@ import time
 from multiprocessing import Process
 from typing import Any, List, Union
 
-from azure.core.exceptions import ServiceRequestError
+from azure.core.exceptions import AzureError
 from databricks.sdk.runtime import dbutils, spark
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 
@@ -49,7 +49,7 @@ class DagProcessor(BaseDags):
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=1, max=10),
-        retry=retry_if_exception_type((Exception, ServiceRequestError)),
+        retry=retry_if_exception_type((Exception, AzureError)),
         reraise=True,
     )
     def query(self, data: Any) -> List[dict]:
@@ -58,7 +58,7 @@ class DagProcessor(BaseDags):
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=1, max=10),
-        retry=retry_if_exception_type((Exception, ServiceRequestError)),
+        retry=retry_if_exception_type((Exception, AzureError)),
         reraise=True,
     )
     def upsert(self, data: Any) -> None:
@@ -67,7 +67,7 @@ class DagProcessor(BaseDags):
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=1, max=10),
-        retry=retry_if_exception_type((Exception, ServiceRequestError)),
+        retry=retry_if_exception_type((Exception, AzureError)),
         reraise=True,
     )
     def delete(self, data: Any) -> None:

fabricks/core/jobs/base/_types.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import List, Literal, Optional, TypedDict, Union
+from typing import Any, List, Literal, Optional, TypedDict, Union
 
 from pydantic import BaseModel, ConfigDict, model_validator
 from pyspark.sql.types import StringType, StructField, StructType
@@ -33,8 +33,8 @@ Origins = Literal["parser", "job"]
 
 
 class SparkOptions(TypedDict):
-    sql: Optional[dict[str, str]]
-    conf: Optional[dict[str, str]]
+    sql: Optional[dict[Any, Any]]
+    conf: Optional[dict[Any, Any]]
 
 
 class TableOptions(TypedDict):
@@ -45,17 +45,17 @@ class TableOptions(TypedDict):
     cluster_by: Optional[List[str]]
     powerbi: Optional[bool]
     bloomfilter_by: Optional[List[str]]
-    constraints: Optional[dict[str, str]]
-    properties: Optional[dict[str, str]]
+    constraints: Optional[dict[Any, Any]]
+    properties: Optional[dict[Any, Any]]
     comment: Optional[str]
-    calculated_columns: Optional[dict[str, str]]
+    calculated_columns: Optional[dict[Any, Any]]
     retention_days: Optional[int]
 
 
 class _InvokeOptions(TypedDict):
     notebook: str
     timeout: int
-    arguments: Optional[dict[str, str]]
+    arguments: Optional[dict[Any, Any]]
 
 
 class InvokerOptions(TypedDict):
@@ -66,7 +66,7 @@ class InvokerOptions(TypedDict):
 
 class ExtenderOptions(TypedDict):
     extender: str
-    arguments: Optional[dict[str, str]]
+    arguments: Optional[dict[Any, Any]]
 
 
 class CheckOptions(TypedDict):
@@ -90,7 +90,7 @@ class BronzeOptions(TypedDict):
     filter_where: Optional[str]
     # extra
     encrypted_columns: Optional[List[str]]
-    calculated_columns: Optional[dict[str, str]]
+    calculated_columns: Optional[dict[Any, Any]]
     operation: Optional[Operations]
     timeout: Optional[int]
 
@@ -106,7 +106,7 @@ class SilverOptions(TypedDict):
     deduplicate: Optional[bool]
     stream: Optional[bool]
     # else
-    order_duplicate_by: Optional[dict[str, str]]
+    order_duplicate_by: Optional[dict[Any, Any]]
     timeout: Optional[int]
 
 
@@ -141,7 +141,7 @@ class BaseJobConf:
 
 @dataclass
 class JobConfBronze(BaseJobConf):
-    step: TBronze
+    step: str
     options: BronzeOptions
     table_options: Optional[TableOptions] = None
     parser_options: Optional[ParserOptions] = None
@@ -155,7 +155,7 @@ class JobConfBronze(BaseJobConf):
 
 @dataclass
 class JobConfSilver(BaseJobConf):
-    step: TSilver
+    step: str
     options: SilverOptions
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None
@@ -168,7 +168,7 @@ class JobConfSilver(BaseJobConf):
 
 @dataclass
 class JobConfGold(BaseJobConf):
-    step: TGold
+    step: str
     options: Optional[GoldOptions]
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None

fabricks/core/jobs/base/configurator.py

@@ -301,9 +301,8 @@ class Configurator(ABC):
         """
         if self.mode == "memory":
             DEFAULT_LOGGER.debug("memory (no optimize)", extra={"job": self})
-        else:
-            assert self.table.exists()
 
+        else:
             if vacuum:
                 self.vacuum()
             if optimize:
@@ -312,19 +311,23 @@ class Configurator(ABC):
                 self.table.compute_statistics()
 
     def vacuum(self):
-        job = self.options.table.get("retention_days")
-        step = self.step_conf.get("table_options", {}).get("retention_days", None)
-        runtime = CONF_RUNTIME.get("options", {}).get("retention_days")
-
-        if job is not None:
-            retention_days = job
-        elif step:
-            retention_days = step
+        if self.mode == "memory":
+            DEFAULT_LOGGER.debug("memory (no vacuum)", extra={"job": self})
+
         else:
-            assert runtime
-            retention_days = runtime
+            job = self.options.table.get("retention_days")
+            step = self.step_conf.get("table_options", {}).get("retention_days", None)
+            runtime = CONF_RUNTIME.get("options", {}).get("retention_days")
+
+            if job is not None:
+                retention_days = job
+            elif step:
+                retention_days = step
+            else:
+                assert runtime
+                retention_days = runtime
 
-        self.table.vacuum(retention_days=retention_days)
+            self.table.vacuum(retention_days=retention_days)
 
     def __str__(self):
         return f"{self.step}.{self.topic}_{self.item}"

fabricks/core/jobs/bronze.py

@@ -103,6 +103,21 @@ class Bronze(BaseJob):
         DEFAULT_LOGGER.debug("drop external table", extra={"job": self})
         self.spark.sql(f"drop table if exists {self.qualified_name}")
 
+    def analyze_external_table(self):
+        DEFAULT_LOGGER.debug("analyze external table", extra={"job": self})
+        self.spark.sql(f"analyze table {self.qualified_name} compute statistics")
+
+    def vacuum_external_table(self, retention_hours: Optional[int] = 168):
+        from delta import DeltaTable
+
+        DEFAULT_LOGGER.debug("vacuum external table", extra={"job": self})
+        try:
+            dt = DeltaTable.forPath(self.spark, self.data_path.string)
+            self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = False")
+            dt.vacuum(retention_hours)
+        finally:
+            self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
+
     def optimize_external_table(
         self,
         vacuum: Optional[bool] = True,
@@ -110,20 +125,10 @@ class Bronze(BaseJob):
     ):
         DEFAULT_LOGGER.debug("optimize external table", extra={"job": self})
         if vacuum:
-            from delta import DeltaTable
-
-            dt = DeltaTable.forPath(self.spark, self.data_path.string)
-            retention_days = 7
-            DEFAULT_LOGGER.debug(f"{self.data_path} - vacuum table (removing files older than {retention_days} days)")
-            try:
-                self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = False")
-                dt.vacuum(retention_days * 24)
-            finally:
-                self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
+            self.vacuum_external_table()
 
         if analyze:
-            DEFAULT_LOGGER.debug(f"{self.data_path} - compute delta statistics")
-            self.spark.sql(f"analyze table delta.`{self.data_path}` compute delta statistics")
+            self.analyze_external_table()
 
     @property
     def parser(self) -> BaseParser:
@@ -318,9 +323,9 @@ class Bronze(BaseJob):
 
     def for_each_run(self, **kwargs):
         if self.mode == "register":
-            DEFAULT_LOGGER.info("register (no run)", extra={"job": self})
+            DEFAULT_LOGGER.debug("register (no run)", extra={"job": self})
         elif self.mode == "memory":
-            DEFAULT_LOGGER.info("memory (no run)", extra={"job": self})
+            DEFAULT_LOGGER.debug("memory (no run)", extra={"job": self})
         else:
             super().for_each_run(**kwargs)
 
@@ -370,6 +375,14 @@ class Bronze(BaseJob):
         else:
             super().optimize(vacuum=vacuum, optimize=optimize, analyze=analyze)
 
+    def vacuum(self):
+        if self.mode == "memory":
+            DEFAULT_LOGGER.info("memory (no vacuum)", extra={"job": self})
+        elif self.mode == "register":
+            self.vacuum_external_table()
+        else:
+            super().vacuum()
+
     def overwrite(self):
         self.truncate()
         self.run()

fabricks/metastore/table.py

@@ -433,7 +433,11 @@ class Table(DbObject):
         cols = [
             f"`{name}`"
             for name, dtype in self.dataframe.dtypes
-            if not dtype.startswith("struct") and not dtype.startswith("array") and name not in ["__metadata"]
+            if not dtype.startswith("struct")
+            and not dtype.startswith("array")
+            and not dtype.startswith("variant")
+            and not dtype.startswith("map")
+            and name not in ["__metadata"]
         ]
         cols = ", ".join(sorted(cols))
         self.spark.sql(f"analyze table {self.qualified_name} compute statistics for columns {cols}")

{fabricks-3.0.1.dist-info → fabricks-3.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.1
+Version: 3.0.3
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.1.dist-info → fabricks-3.0.3.dist-info}/RECORD

@@ -93,10 +93,10 @@ fabricks/core/udfs.py,sha256=sxahq2WZSy427i8_HLV1y4P_RjdvCRrm3muXGMufr1s,3440
 fabricks/core/utils.py,sha256=qdn2ElpqBgDsW55-tACWZaFOT0ebrBYg2fenqSgd6YI,2456
 fabricks/core/views.py,sha256=6fmQgxXIB3NDhabti0vs5DSlq1kHJ-forNHJgvBxslU,1013
 fabricks/core/dags/__init__.py,sha256=0DUKzVcXcROvxkN19P_kaOJ7da5BAM7Vt8EGQbp2KSY,240
-fabricks/core/dags/base.py,sha256=31rAck3eAYGXDPUbFGIIhnw3cAOpbZK3yjpMkRFUxro,3157
+fabricks/core/dags/base.py,sha256=tFj27SqeZUZ7pB_LOWkpdowZz5gj30JUANI4gWK3Pl8,3139
 fabricks/core/dags/generator.py,sha256=WoPsXLA89O8y75YpjvEo_NhLoXdZh3oSDrOU-lnAj94,4826
 fabricks/core/dags/log.py,sha256=pszqun3lSFzfyKbahsPWDgP0MOH58H2dpK42prc6sk0,782
-fabricks/core/dags/processor.py,sha256=NwMuZHWency81Pq7rTPI5cXb_ACQ2flHTiixZ5eTitw,7734
+fabricks/core/dags/processor.py,sha256=_8Htq7rP_6hZx0htxISrxRIA9gjar4c44meVQ5spX7A,7698
 fabricks/core/dags/run.py,sha256=RIDULb9WakObSyYzmkglh8RwFRwC8-NFC-1yPDMkBC0,1074
 fabricks/core/dags/terminator.py,sha256=Y6pV2UnSyrCIx2AQWJXoHk5Roq12gZqpennHx_Lbnzs,793
 fabricks/core/dags/utils.py,sha256=17IIza8zaaVkhqCudXUr4GSYXZZCobGB3FLgbS4eAjs,1229
@@ -105,7 +105,7 @@ fabricks/core/deploy/tables.py,sha256=HPQxdLfggSzL3PTqHnBaJnScXuROK_oko-vqjiXWrn
 fabricks/core/deploy/udfs.py,sha256=TBTwMJAuZpsShu_z-Z-LENo3P_eWX69qUkmFuRqv3kU,516
 fabricks/core/deploy/views.py,sha256=DAPCF1gijNGVioKUhRH_PS0koAYhoeDgMvBy-UJ6GQc,13756
 fabricks/core/jobs/__init__.py,sha256=W_1m6LoGiXBml_8cucedju_hllSSnZGKsZjyFh-2oJw,531
-fabricks/core/jobs/bronze.py,sha256=PRZ5aJX-XIX3hal_mgav43pcTDVaO5KeeJrUYOu0drM,13181
+fabricks/core/jobs/bronze.py,sha256=89nsucGyTGAYPd6pO-MZUmi031P62gp095PTYjSftjg,13514
 fabricks/core/jobs/get_job.py,sha256=35zay3Z_WoJIylzEQlGle6UvrE1EClfRbFEVGvszof0,3675
 fabricks/core/jobs/get_job_conf.py,sha256=3vAj_usCbNqejMUKOF85LPaHBYAwxgrDG7LYgY-vBUw,4812
 fabricks/core/jobs/get_job_id.py,sha256=6dLyzxGHlRvJZVJSwZkCk3iXzWkIhePC_6FhoP0gEN4,607
@@ -113,9 +113,9 @@ fabricks/core/jobs/get_jobs.py,sha256=5E1J95vFYDqa2n9DKpJn4ujD0MW-P38pNc6T6V8LDm
 fabricks/core/jobs/gold.py,sha256=EQ6nCNdvWTBt194tmXvMWZItw53o7x49nwJ4UiCSDH0,13996
 fabricks/core/jobs/silver.py,sha256=wn6c6hoeppjlWf6EutB_8qE5Sxu2PIWk5iQecRUuJ5o,13163
 fabricks/core/jobs/base/__init__.py,sha256=_AdWtyL7yZG2TOZ9e8WyNPrOjmm6EDkI_TNym5cLDws,208
-fabricks/core/jobs/base/_types.py,sha256=xNKHpzof_mPd97ytvk3wrPnXd1_VdLilh1yQgFF3Ois,6769
+fabricks/core/jobs/base/_types.py,sha256=r_s1mG-kHErvC9tMX5ndNEgA_3ASD2ouEkYylbhe-18,6764
 fabricks/core/jobs/base/checker.py,sha256=LPK5f3ucT7T4Z7LjlOyHPXFfb94J_DdYVp6X85wIvDk,5324
-fabricks/core/jobs/base/configurator.py,sha256=671-ksuf5FDIUtCLmt_HhLYhUl34jUdKb7S1wiotBzk,11439
+fabricks/core/jobs/base/configurator.py,sha256=ARj920yJJdNtD0Iz8IjbhSyJqlVrXzSz29n0mXofo-k,11569
 fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
 fabricks/core/jobs/base/generator.py,sha256=LdI3PDrwee5rjwlFlduA4_s-7rE1AsnFrYdgQJL5_tE,15527
 fabricks/core/jobs/base/invoker.py,sha256=xJV9fLtY36qfnclqKqNBsjryyR8x39wfhbYJtzOPRyM,6342
@@ -145,7 +145,7 @@ fabricks/metastore/_types.py,sha256=NXYxwQHP0sCllM0N6QBbaK4CdtM_m_rHFDxRNRfBcLU,
 fabricks/metastore/database.py,sha256=1EjbRh2b6xEdHJyc4C4xee6FXDiKuPgm-8Q3Gqt7eds,1942
 fabricks/metastore/dbobject.py,sha256=EdxofFMCx6XdqFkm9Z5x4ywW4sstvdpc1d_EhYsE0KY,1883
 fabricks/metastore/pyproject.toml,sha256=6RZM9RMKMDF_EAequhORZ7TD0BQNk7aBCTWAv-sRcp0,519
-fabricks/metastore/table.py,sha256=3rQZDOJZ2GchpVQ4N9Cu-SgEZI2TlUGqO9cWchDsitE,24196
+fabricks/metastore/table.py,sha256=YywtGc6z_zMwvK9wORvvys50AB-gBd9ZGAdtQq614yc,24312
 fabricks/metastore/utils.py,sha256=8SxhjDkz_aSH4IGUusel7hqOQxP9U8PNBCY0M7GH00Y,1355
 fabricks/metastore/view.py,sha256=Va7xdFtOW9GcDSlyoZNgcF07qty9abtex41au6OSz6c,1381
 fabricks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -171,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=u9FFYvWyq9VQdNJNu79-SCN9iGUB
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=mpaSxBNcl6N0QheGLx8rjeyWUvy1Yvvj4raGRv7GL5M,1229
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.1.dist-info/METADATA,sha256=5jGdhyCSur762goGfzWn5wtH_emHDD2MIDTogOazLcE,682
-fabricks-3.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.1.dist-info/RECORD,,
+fabricks-3.0.3.dist-info/METADATA,sha256=dztXMj5CrebPZZmMtr-LdpcS2Zg6sO0x0OCUzwLik3s,682
+fabricks-3.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fabricks-3.0.3.dist-info/RECORD,,