fabricks 3.0.5__py3-none-any.whl → 3.0.5.2__py3-none-any.whl

fabricks/cdc/base/configurator.py

@@ -126,7 +126,7 @@ class Configurator(ABC):
 
     def has_data(self, src: Union[DataFrame, Table, str], **kwargs) -> bool:
         df = self.get_src(src=src)
-        return df.count() > 0
+        return not df.isEmpty()
 
     def get_columns(self, src: Union[DataFrame, Table, str], backtick: Optional[bool] = True) -> List[str]:
         if backtick:

fabricks/cdc/base/processor.py

@@ -36,8 +36,6 @@ class Processor(Generator):
         columns = self.get_columns(src, backtick=False)
         fields = [c for c in columns if not c.startswith("__")]
 
-        has_data = self.has_data(src)
-
         keys = kwargs.get("keys", None)
         mode = kwargs.get("mode", "complete")
 
@@ -80,6 +78,12 @@ class Processor(Generator):
         deduplicate_hash = kwargs.get("deduplicate_hash", None)
         soft_delete = kwargs.get("soft_delete", None)
         correct_valid_from = kwargs.get("correct_valid_from", None)
+        delete_missing = kwargs.get("delete_missing", None)
+
+        if mode == "update" and delete_missing:
+            has_data = self.has_data(src)
+        else:
+            has_data = True
 
         if slice is None:
             if mode == "update" and has_timestamp and has_rows:

fabricks/cdc/scd.py

@@ -9,6 +9,7 @@ from fabricks.metastore.table import Table
 class SCD(BaseCDC):
     def delete_missing(self, src: Union[DataFrame, Table, str], **kwargs):
         kwargs["add_operation"] = "reload"
+        kwargs["delete_missing"] = True
         kwargs["mode"] = "update"
         self.merge(src, **kwargs)
 

fabricks/core/deploy/views.py

@@ -288,8 +288,14 @@ def create_or_replace_logs_pivot_view():
         not done and not failed and not skipped and array_contains(statuses, 'running') as timed_out,
         not array_contains(statuses, 'running') as cancelled,
         max(l.notebook_id) as notebook_id,
-        max(l.timestamp) filter(where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status = 'scheduled' ) as scheduled_time,
+        max(l.timestamp) filter (where l.status = 'waiting' ) as waiting_time,
+        max(l.timestamp) filter (where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status = 'running' ) as running_time,
+        max(l.timestamp) filter (where l.status = 'done' ) as done_time,
+        max(l.timestamp) filter (where l.status = 'failed' ) as failed_time,
         max(l.timestamp) filter(where l.status = 'ok') as end_time,
+        max(l.timestamp) filter(where l.status = 'ok') as ok_time,
         max(l.exception) as exception
       from
         fabricks.logs l
@@ -313,6 +319,12 @@ def create_or_replace_logs_pivot_view():
       g.notebook_id,
       g.start_time,
       g.end_time,
+      g.scheduled_time,
+      g.waiting_time,
+      g.running_time,
+      g.done_time,
+      g.failed_time,
+      g.ok_time,
       if(g.timed_out, null, date_diff(SECOND, start_time, end_time)) as duration,
       g.exception
     from

{fabricks-3.0.5.dist-info → fabricks-3.0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.5
+Version: 3.0.5.2
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.5.dist-info → fabricks-3.0.5.2.dist-info}/RECORD

@@ -32,16 +32,16 @@ fabricks/api/notebooks/vacuum.py,sha256=F88-alJyR4rh1ZB4CbvMqyJvAC-6l73GHtq6eBkk
 fabricks/cdc/__init__.py,sha256=_ncE8b8xuT2HqWC3JiCa4JCb_na2xQnVz3M6tLkAXD8,302
 fabricks/cdc/cdc.py,sha256=2CjPUtogWjnvyLjwiyVllcyDV1gpJ0QoRP0yUsiHXuc,69
 fabricks/cdc/nocdc.py,sha256=Nwj0pE3NjSVyLxKs9PUimHzWcKN5ehHt1trrlq69qE4,518
-fabricks/cdc/scd.py,sha256=r1NVK9QAKJG4tRSpEAksvOO3nAuNwRLZoGmNG2TsypE,630
+fabricks/cdc/scd.py,sha256=HzC9ifEu45B4P2aOSgi97AGB-C56l6sKTLqdVinnHKo,670
 fabricks/cdc/scd1.py,sha256=WsOVRsp55WEw4-7nEtb3dfv310icExrj-zEJSEehyz8,334
 fabricks/cdc/scd2.py,sha256=4vZkhc8pJAUlgiBmIw9j_2RsWuAFMcgCkU3WMVt0A-A,334
 fabricks/cdc/base/__init__.py,sha256=1uec9NHg3J5TWPMR09EsCMO1g8_3Dt6ZhC_b61Sg7JY,143
 fabricks/cdc/base/_types.py,sha256=IMI5bT4IFfqSnjTVrPBHsJkRXNdaRcMVUYW8qpfsTs0,82
 fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
-fabricks/cdc/base/configurator.py,sha256=fbQg4C1AH0BIis_Pdrv3BLkcjGnYOaCrxbjNc-95bj8,5269
+fabricks/cdc/base/configurator.py,sha256=lInLgLUm_h2VN43vUFzOsxwk4yOQxnDplR6F9fC0rEE,5271
 fabricks/cdc/base/generator.py,sha256=OQuNGjblFeCP0JLJpJKbaXQmzREISw7PLiN04Nt9nu4,5735
 fabricks/cdc/base/merger.py,sha256=suule_MRyI-qXwBaUpKiBLmduZpvI01nwn6MBarkI24,3991
-fabricks/cdc/base/processor.py,sha256=e0JKpz6sGGXvc6U6IFagxy8iZVtE3YAhfLP2mWXMiBs,14239
+fabricks/cdc/base/processor.py,sha256=d52pvaHI9KTvztT2NCb8Vl-tIGc66ZPyuxXR4p3Vq1U,14393
 fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/cdc/templates/filter.sql.jinja,sha256=H0-nAN7HzxDa3p_Qu2U_LeJnBCcBKR6xzIM0VnckeuQ,234
 fabricks/cdc/templates/merge.sql.jinja,sha256=iNpgqGiuI2QABmyTkHCibRr_5r7SASb3yqojhNP3e20,144
@@ -104,7 +104,7 @@ fabricks/core/dags/utils.py,sha256=17IIza8zaaVkhqCudXUr4GSYXZZCobGB3FLgbS4eAjs,1
 fabricks/core/deploy/__init__.py,sha256=x-uY3oz82MsyiScuykGf8oDPWYA0b0LTdegB4q219Eo,375
 fabricks/core/deploy/tables.py,sha256=HPQxdLfggSzL3PTqHnBaJnScXuROK_oko-vqjiXWrn0,2552
 fabricks/core/deploy/udfs.py,sha256=TBTwMJAuZpsShu_z-Z-LENo3P_eWX69qUkmFuRqv3kU,516
-fabricks/core/deploy/views.py,sha256=DAPCF1gijNGVioKUhRH_PS0koAYhoeDgMvBy-UJ6GQc,13756
+fabricks/core/deploy/views.py,sha256=nY2L-yg5fwK40ONjQ_VrOGMrh4eJ6326UDSq3AnGrxA,14340
 fabricks/core/jobs/__init__.py,sha256=W_1m6LoGiXBml_8cucedju_hllSSnZGKsZjyFh-2oJw,531
 fabricks/core/jobs/bronze.py,sha256=89nsucGyTGAYPd6pO-MZUmi031P62gp095PTYjSftjg,13514
 fabricks/core/jobs/get_job.py,sha256=35zay3Z_WoJIylzEQlGle6UvrE1EClfRbFEVGvszof0,3675
@@ -172,6 +172,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=u9FFYvWyq9VQdNJNu79-SCN9iGUB
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=mpaSxBNcl6N0QheGLx8rjeyWUvy1Yvvj4raGRv7GL5M,1229
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.5.dist-info/METADATA,sha256=OmTWAfGA3KTwew_b_Es26qhPLZYt0J3Dlz31XFAYlQs,723
-fabricks-3.0.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.5.dist-info/RECORD,,
+fabricks-3.0.5.2.dist-info/METADATA,sha256=MNN4cMvghY7Xedj3FhM8i4UvBmUyAia3-AaXPzLKqQw,725
+fabricks-3.0.5.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fabricks-3.0.5.2.dist-info/RECORD,,