fabricks 3.0.5__tar.gz → 3.0.5.2__tar.gz

{fabricks-3.0.5 → fabricks-3.0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.5
+Version: 3.0.5.2
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.5 → fabricks-3.0.5.2}/fabricks/cdc/base/configurator.py

@@ -126,7 +126,7 @@ class Configurator(ABC):
 
     def has_data(self, src: Union[DataFrame, Table, str], **kwargs) -> bool:
         df = self.get_src(src=src)
-        return df.count() > 0
+        return not df.isEmpty()
 
     def get_columns(self, src: Union[DataFrame, Table, str], backtick: Optional[bool] = True) -> List[str]:
         if backtick:

{fabricks-3.0.5 → fabricks-3.0.5.2}/fabricks/cdc/base/processor.py

@@ -36,8 +36,6 @@ class Processor(Generator):
         columns = self.get_columns(src, backtick=False)
         fields = [c for c in columns if not c.startswith("__")]
 
-        has_data = self.has_data(src)
-
         keys = kwargs.get("keys", None)
         mode = kwargs.get("mode", "complete")
 
@@ -80,6 +78,12 @@ class Processor(Generator):
         deduplicate_hash = kwargs.get("deduplicate_hash", None)
         soft_delete = kwargs.get("soft_delete", None)
         correct_valid_from = kwargs.get("correct_valid_from", None)
+        delete_missing = kwargs.get("delete_missing", None)
+
+        if mode == "update" and delete_missing:
+            has_data = self.has_data(src)
+        else:
+            has_data = True
 
         if slice is None:
             if mode == "update" and has_timestamp and has_rows:

{fabricks-3.0.5 → fabricks-3.0.5.2}/fabricks/cdc/scd.py

@@ -9,6 +9,7 @@ from fabricks.metastore.table import Table
 class SCD(BaseCDC):
     def delete_missing(self, src: Union[DataFrame, Table, str], **kwargs):
         kwargs["add_operation"] = "reload"
+        kwargs["delete_missing"] = True
         kwargs["mode"] = "update"
         self.merge(src, **kwargs)
 

{fabricks-3.0.5 → fabricks-3.0.5.2}/fabricks/core/deploy/views.py

@@ -288,8 +288,14 @@ def create_or_replace_logs_pivot_view():
         not done and not failed and not skipped and array_contains(statuses, 'running') as timed_out,
         not array_contains(statuses, 'running') as cancelled,
         max(l.notebook_id) as notebook_id,
-        max(l.timestamp) filter(where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status = 'scheduled' ) as scheduled_time,
+        max(l.timestamp) filter (where l.status = 'waiting' ) as waiting_time,
+        max(l.timestamp) filter (where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status = 'running' ) as running_time,
+        max(l.timestamp) filter (where l.status = 'done' ) as done_time,
+        max(l.timestamp) filter (where l.status = 'failed' ) as failed_time,
         max(l.timestamp) filter(where l.status = 'ok') as end_time,
+        max(l.timestamp) filter(where l.status = 'ok') as ok_time,
         max(l.exception) as exception
       from
         fabricks.logs l
@@ -313,6 +319,12 @@ def create_or_replace_logs_pivot_view():
       g.notebook_id,
       g.start_time,
       g.end_time,
+      g.scheduled_time,
+      g.waiting_time,
+      g.running_time,
+      g.done_time,
+      g.failed_time,
+      g.ok_time,
       if(g.timed_out, null, date_diff(SECOND, start_time, end_time)) as duration,
       g.exception
     from

{fabricks-3.0.5 → fabricks-3.0.5.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.5"
+version = "3.0.5.2"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },