fabricks 3.0.5__py3-none-any.whl → 3.0.5.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. fabricks/cdc/base/configurator.py +1 -1
  2. fabricks/cdc/base/processor.py +6 -2
  3. fabricks/cdc/scd.py +1 -0
  4. {fabricks-3.0.5.dist-info → fabricks-3.0.5.1.dist-info}/METADATA +1 -1
  5. {fabricks-3.0.5.dist-info → fabricks-3.0.5.1.dist-info}/RECORD +6 -6
  6. {fabricks-3.0.5.dist-info → fabricks-3.0.5.1.dist-info}/WHEEL +0 -0
fabricks/cdc/base/configurator.py CHANGED
@@ -126,7 +126,7 @@ class Configurator(ABC):
126
126
 
127
127
  def has_data(self, src: Union[DataFrame, Table, str], **kwargs) -> bool:
128
128
  df = self.get_src(src=src)
129
- return df.count() > 0
129
+ return not df.isEmpty()
130
130
 
131
131
  def get_columns(self, src: Union[DataFrame, Table, str], backtick: Optional[bool] = True) -> List[str]:
132
132
  if backtick:
fabricks/cdc/base/processor.py CHANGED
@@ -36,8 +36,6 @@ class Processor(Generator):
36
36
  columns = self.get_columns(src, backtick=False)
37
37
  fields = [c for c in columns if not c.startswith("__")]
38
38
 
39
- has_data = self.has_data(src)
40
-
41
39
  keys = kwargs.get("keys", None)
42
40
  mode = kwargs.get("mode", "complete")
43
41
 
@@ -80,6 +78,12 @@ class Processor(Generator):
80
78
  deduplicate_hash = kwargs.get("deduplicate_hash", None)
81
79
  soft_delete = kwargs.get("soft_delete", None)
82
80
  correct_valid_from = kwargs.get("correct_valid_from", None)
81
+ delete_missing = kwargs.get("delete_missing", None)
82
+
83
+ if mode == "update" and delete_missing:
84
+ has_data = self.has_data(src)
85
+ else:
86
+ has_data = True
83
87
 
84
88
  if slice is None:
85
89
  if mode == "update" and has_timestamp and has_rows:
fabricks/cdc/scd.py CHANGED
@@ -9,6 +9,7 @@ from fabricks.metastore.table import Table
9
9
  class SCD(BaseCDC):
10
10
  def delete_missing(self, src: Union[DataFrame, Table, str], **kwargs):
11
11
  kwargs["add_operation"] = "reload"
12
+ kwargs["delete_missing"] = True
12
13
  kwargs["mode"] = "update"
13
14
  self.merge(src, **kwargs)
14
15
 
{fabricks-3.0.5.dist-info → fabricks-3.0.5.1.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fabricks
3
- Version: 3.0.5
3
+ Version: 3.0.5.1
4
4
  Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
5
5
  Requires-Python: <4,>=3.9
6
6
  Requires-Dist: azure-data-tables<13,>=12.5.0
{fabricks-3.0.5.dist-info → fabricks-3.0.5.1.dist-info}/RECORD RENAMED
@@ -32,16 +32,16 @@ fabricks/api/notebooks/vacuum.py,sha256=F88-alJyR4rh1ZB4CbvMqyJvAC-6l73GHtq6eBkk
32
32
  fabricks/cdc/__init__.py,sha256=_ncE8b8xuT2HqWC3JiCa4JCb_na2xQnVz3M6tLkAXD8,302
33
33
  fabricks/cdc/cdc.py,sha256=2CjPUtogWjnvyLjwiyVllcyDV1gpJ0QoRP0yUsiHXuc,69
34
34
  fabricks/cdc/nocdc.py,sha256=Nwj0pE3NjSVyLxKs9PUimHzWcKN5ehHt1trrlq69qE4,518
35
- fabricks/cdc/scd.py,sha256=r1NVK9QAKJG4tRSpEAksvOO3nAuNwRLZoGmNG2TsypE,630
35
+ fabricks/cdc/scd.py,sha256=HzC9ifEu45B4P2aOSgi97AGB-C56l6sKTLqdVinnHKo,670
36
36
  fabricks/cdc/scd1.py,sha256=WsOVRsp55WEw4-7nEtb3dfv310icExrj-zEJSEehyz8,334
37
37
  fabricks/cdc/scd2.py,sha256=4vZkhc8pJAUlgiBmIw9j_2RsWuAFMcgCkU3WMVt0A-A,334
38
38
  fabricks/cdc/base/__init__.py,sha256=1uec9NHg3J5TWPMR09EsCMO1g8_3Dt6ZhC_b61Sg7JY,143
39
39
  fabricks/cdc/base/_types.py,sha256=IMI5bT4IFfqSnjTVrPBHsJkRXNdaRcMVUYW8qpfsTs0,82
40
40
  fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
41
- fabricks/cdc/base/configurator.py,sha256=fbQg4C1AH0BIis_Pdrv3BLkcjGnYOaCrxbjNc-95bj8,5269
41
+ fabricks/cdc/base/configurator.py,sha256=lInLgLUm_h2VN43vUFzOsxwk4yOQxnDplR6F9fC0rEE,5271
42
42
  fabricks/cdc/base/generator.py,sha256=OQuNGjblFeCP0JLJpJKbaXQmzREISw7PLiN04Nt9nu4,5735
43
43
  fabricks/cdc/base/merger.py,sha256=suule_MRyI-qXwBaUpKiBLmduZpvI01nwn6MBarkI24,3991
44
- fabricks/cdc/base/processor.py,sha256=e0JKpz6sGGXvc6U6IFagxy8iZVtE3YAhfLP2mWXMiBs,14239
44
+ fabricks/cdc/base/processor.py,sha256=d52pvaHI9KTvztT2NCb8Vl-tIGc66ZPyuxXR4p3Vq1U,14393
45
45
  fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
46
46
  fabricks/cdc/templates/filter.sql.jinja,sha256=H0-nAN7HzxDa3p_Qu2U_LeJnBCcBKR6xzIM0VnckeuQ,234
47
47
  fabricks/cdc/templates/merge.sql.jinja,sha256=iNpgqGiuI2QABmyTkHCibRr_5r7SASb3yqojhNP3e20,144
@@ -172,6 +172,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=u9FFYvWyq9VQdNJNu79-SCN9iGUB
172
172
  fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
173
173
  fabricks/utils/write/delta.py,sha256=mpaSxBNcl6N0QheGLx8rjeyWUvy1Yvvj4raGRv7GL5M,1229
174
174
  fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
175
- fabricks-3.0.5.dist-info/METADATA,sha256=OmTWAfGA3KTwew_b_Es26qhPLZYt0J3Dlz31XFAYlQs,723
176
- fabricks-3.0.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
177
- fabricks-3.0.5.dist-info/RECORD,,
175
+ fabricks-3.0.5.1.dist-info/METADATA,sha256=6gsR4UJcnoz4x2dPwdNT3jYIyEZlpdBqCee0-LsTJJ0,725
176
+ fabricks-3.0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
177
+ fabricks-3.0.5.1.dist-info/RECORD,,