fabricks 3.0.3__py3-none-any.whl → 3.0.4__py3-none-any.whl

fabricks/cdc/base/configurator.py

@@ -37,6 +37,10 @@ class Configurator(ABC):
     def is_registered(self):
         return self.table.is_registered
 
+    @property
+    def qualified_name(self):
+        return f"{self.database}_{'_'.join(self.levels)}"
+
     @abstractmethod
     def get_query(self, src: Union[DataFrame, Table, str], **kwargs):
         raise NotImplementedError()

fabricks/cdc/base/merger.py

@@ -101,7 +101,7 @@ class Merger(Processor):
             self.create_table(src, **kwargs)
 
         df = self.get_data(src, **kwargs)
-        global_temp_view = f"{self.database}_{'_'.join(self.levels)}__merge"
+        global_temp_view = f"{self.qualified_name}__merge"
         view = create_or_replace_global_temp_view(global_temp_view, df, uuid=kwargs.get("uuid", False))
 
         merge = self.get_merge_query(view, **kwargs)

fabricks/cdc/base/processor.py

@@ -16,7 +16,7 @@ from fabricks.utils.sqlglot import fix as fix_sql
 class Processor(Generator):
     def get_data(self, src: Union[DataFrame, Table, str], **kwargs) -> DataFrame:
         if isinstance(src, (DataFrame, CDataFrame)):
-            name = f"{self.database}_{'_'.join(self.levels)}__data"
+            name = f"{self.qualified_name}__data"
             global_temp_view = create_or_replace_global_temp_view(name, src, uuid=kwargs.get("uuid", False))
             src = f"select * from {global_temp_view}"
 
@@ -347,8 +347,11 @@ class Processor(Generator):
         df = self.get_data(src, **kwargs)
         df = self.reorder_columns(df)
 
+        name = f"{self.qualified_name}__append"
+        create_or_replace_global_temp_view(name, df, uuid=kwargs.get("uuid", False))
+
         DEFAULT_LOGGER.debug("append", extra={"job": self})
-        df.write.format("delta").mode("append").save(self.table.delta_path.string)
+        self.spark.sql(f"insert into table {self.table} by name select * from global_temp.{name}")
 
     def overwrite(
         self,
@@ -366,9 +369,11 @@ class Processor(Generator):
             if kwargs.get("update_where"):
                 dynamic = True
 
-        writer = df.write.format("delta").mode("overwrite")
         if dynamic:
-            writer.option("partitionOverwriteMode", "dynamic")
+            self.spark.sql("set spark.sql.sources.partitionOverwriteMode = dynamic")
+
+        name = f"{self.qualified_name}__overwrite"
+        create_or_replace_global_temp_view(name, df, uuid=kwargs.get("uuid", False))
 
-        DEFAULT_LOGGER.info("overwrite", extra={"job": self})
-        writer.save(self.table.delta_path.string)
+        DEFAULT_LOGGER.debug("overwrite", extra={"job": self})
+        self.spark.sql(f"insert overwrite table {self.table} by name select * from global_temp.{name}")

fabricks/core/jobs/base/_types.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, List, Literal, Optional, TypedDict, Union
+from typing import List, Literal, Optional, TypedDict, Union
 
 from pydantic import BaseModel, ConfigDict, model_validator
 from pyspark.sql.types import StringType, StructField, StructType
@@ -33,8 +33,8 @@ Origins = Literal["parser", "job"]
 
 
 class SparkOptions(TypedDict):
-    sql: Optional[dict[Any, Any]]
-    conf: Optional[dict[Any, Any]]
+    sql: Optional[dict[str, str]]
+    conf: Optional[dict[str, str]]
 
 
 class TableOptions(TypedDict):
@@ -45,17 +45,17 @@ class TableOptions(TypedDict):
     cluster_by: Optional[List[str]]
     powerbi: Optional[bool]
     bloomfilter_by: Optional[List[str]]
-    constraints: Optional[dict[Any, Any]]
-    properties: Optional[dict[Any, Any]]
+    constraints: Optional[dict[str, str]]
+    properties: Optional[dict[str, str]]
     comment: Optional[str]
-    calculated_columns: Optional[dict[Any, Any]]
+    calculated_columns: Optional[dict[str, str]]
     retention_days: Optional[int]
 
 
 class _InvokeOptions(TypedDict):
     notebook: str
     timeout: int
-    arguments: Optional[dict[Any, Any]]
+    arguments: Optional[dict[str, str]]
 
 
 class InvokerOptions(TypedDict):
@@ -66,7 +66,7 @@ class InvokerOptions(TypedDict):
 
 class ExtenderOptions(TypedDict):
     extender: str
-    arguments: Optional[dict[Any, Any]]
+    arguments: Optional[dict[str, str]]
 
 
 class CheckOptions(TypedDict):
@@ -90,7 +90,7 @@ class BronzeOptions(TypedDict):
     filter_where: Optional[str]
     # extra
     encrypted_columns: Optional[List[str]]
-    calculated_columns: Optional[dict[Any, Any]]
+    calculated_columns: Optional[dict[str, str]]
     operation: Optional[Operations]
     timeout: Optional[int]
 
@@ -106,7 +106,7 @@ class SilverOptions(TypedDict):
     deduplicate: Optional[bool]
     stream: Optional[bool]
     # else
-    order_duplicate_by: Optional[dict[Any, Any]]
+    order_duplicate_by: Optional[dict[str, str]]
     timeout: Optional[int]
 
 
@@ -141,7 +141,7 @@ class BaseJobConf:
 
 @dataclass
 class JobConfBronze(BaseJobConf):
-    step: str
+    step: TBronze
     options: BronzeOptions
     table_options: Optional[TableOptions] = None
     parser_options: Optional[ParserOptions] = None
@@ -155,7 +155,7 @@ class JobConfBronze(BaseJobConf):
 
 @dataclass
 class JobConfSilver(BaseJobConf):
-    step: str
+    step: TSilver
     options: SilverOptions
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None
@@ -168,7 +168,7 @@ class JobConfSilver(BaseJobConf):
 
 @dataclass
 class JobConfGold(BaseJobConf):
-    step: str
+    step: TGold
     options: Optional[GoldOptions]
     table_options: Optional[TableOptions] = None
     check_options: Optional[CheckOptions] = None

{fabricks-3.0.3.dist-info → fabricks-3.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.3
+Version: 3.0.4
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.3.dist-info → fabricks-3.0.4.dist-info}/RECORD

@@ -37,10 +37,10 @@ fabricks/cdc/scd2.py,sha256=4vZkhc8pJAUlgiBmIw9j_2RsWuAFMcgCkU3WMVt0A-A,334
 fabricks/cdc/base/__init__.py,sha256=1uec9NHg3J5TWPMR09EsCMO1g8_3Dt6ZhC_b61Sg7JY,143
 fabricks/cdc/base/_types.py,sha256=IMI5bT4IFfqSnjTVrPBHsJkRXNdaRcMVUYW8qpfsTs0,82
 fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
-fabricks/cdc/base/configurator.py,sha256=aUHc1wSc738gEsYkTHP_d_PTxdZHKxw7hpH26d4Zu1A,5166
+fabricks/cdc/base/configurator.py,sha256=fbQg4C1AH0BIis_Pdrv3BLkcjGnYOaCrxbjNc-95bj8,5269
 fabricks/cdc/base/generator.py,sha256=r_6S556wuNvl4eqo1L6-AfyPNj3mHQl8lTspWiFGFYU,6161
-fabricks/cdc/base/merger.py,sha256=p0VBVfE4i6xnxqL811Jp38vk45GAR5ip_Mlu8KU5188,4009
-fabricks/cdc/base/processor.py,sha256=8sFw2pOlHsDPhhZ0VIsYZjGrMK9HzVnLBOaNw0o6u7Y,13953
+fabricks/cdc/base/merger.py,sha256=suule_MRyI-qXwBaUpKiBLmduZpvI01nwn6MBarkI24,3991
+fabricks/cdc/base/processor.py,sha256=e0JKpz6sGGXvc6U6IFagxy8iZVtE3YAhfLP2mWXMiBs,14239
 fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/cdc/templates/filter.sql.jinja,sha256=H0-nAN7HzxDa3p_Qu2U_LeJnBCcBKR6xzIM0VnckeuQ,234
 fabricks/cdc/templates/merge.sql.jinja,sha256=iNpgqGiuI2QABmyTkHCibRr_5r7SASb3yqojhNP3e20,144
@@ -113,7 +113,7 @@ fabricks/core/jobs/get_jobs.py,sha256=5E1J95vFYDqa2n9DKpJn4ujD0MW-P38pNc6T6V8LDm
 fabricks/core/jobs/gold.py,sha256=EQ6nCNdvWTBt194tmXvMWZItw53o7x49nwJ4UiCSDH0,13996
 fabricks/core/jobs/silver.py,sha256=wn6c6hoeppjlWf6EutB_8qE5Sxu2PIWk5iQecRUuJ5o,13163
 fabricks/core/jobs/base/__init__.py,sha256=_AdWtyL7yZG2TOZ9e8WyNPrOjmm6EDkI_TNym5cLDws,208
-fabricks/core/jobs/base/_types.py,sha256=r_s1mG-kHErvC9tMX5ndNEgA_3ASD2ouEkYylbhe-18,6764
+fabricks/core/jobs/base/_types.py,sha256=xNKHpzof_mPd97ytvk3wrPnXd1_VdLilh1yQgFF3Ois,6769
 fabricks/core/jobs/base/checker.py,sha256=LPK5f3ucT7T4Z7LjlOyHPXFfb94J_DdYVp6X85wIvDk,5324
 fabricks/core/jobs/base/configurator.py,sha256=ARj920yJJdNtD0Iz8IjbhSyJqlVrXzSz29n0mXofo-k,11569
 fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
@@ -171,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=u9FFYvWyq9VQdNJNu79-SCN9iGUB
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=mpaSxBNcl6N0QheGLx8rjeyWUvy1Yvvj4raGRv7GL5M,1229
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.3.dist-info/METADATA,sha256=dztXMj5CrebPZZmMtr-LdpcS2Zg6sO0x0OCUzwLik3s,682
-fabricks-3.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.3.dist-info/RECORD,,
+fabricks-3.0.4.dist-info/METADATA,sha256=nFCQ4-fewJvmjPgINnMmf07MFDMOyMp8dVOFMd6Vbwo,682
+fabricks-3.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fabricks-3.0.4.dist-info/RECORD,,