fabricks 3.0.12__py3-none-any.whl → 3.0.13__py3-none-any.whl

fabricks/context/helpers.py

@@ -1,59 +1,36 @@
+import os
+import pathlib
 from typing import List
 
 from fabricks.utils.path import Path
 
 
-def get_config_from_toml():
-    import os
-    import pathlib
-    import sys
-
-    if sys.version_info >= (3, 11):
-        import tomllib
-    else:
-        import tomli as tomllib  # type: ignore
-
+def get_config_from_file():
     path = pathlib.Path(os.getcwd())
-    while path is not None and not (path / "pyproject.toml").exists():
-        if path == path.parent:
-            break
-        path = path.parent
-
-    if (path / "pyproject.toml").exists():
-        with open((path / "pyproject.toml"), "rb") as f:
-            config = tomllib.load(f)
-            return path, config.get("tool", {}).get("fabricks", {})
 
-    return None, {}
-
-
-def get_config_from_json():
-    import json
-    import os
-    import pathlib
-
-    path = pathlib.Path(os.getcwd())
-    while path is not None and not (path / "fabricksconfig.json").exists():
+    while path is not None and (not (path / "pyproject.toml").exists() or (path / "fabricksconfig.json").exists()):
         if path == path.parent:
             break
         path = path.parent
 
     if (path / "fabricksconfig.json").exists():
+        import json
+
         with open((path / "fabricksconfig.json"), "r") as f:
             config = json.load(f)
-            return path, config
-
-    return None, {}
+            return path, config, "json"
 
+    if (path / "pyproject.toml").exists():
+        import sys
 
-def get_config_from_file():
-    json_path, json_config = get_config_from_json()
-    if json_config:
-        return json_path, json_config, "json"
+        if sys.version_info >= (3, 11):
+            import tomllib
+        else:
+            import tomli as tomllib  # type: ignore
 
-    pyproject_path, pyproject_config = get_config_from_toml()
-    if pyproject_config:
-        return pyproject_path, pyproject_config, "pyproject"
+        with open((path / "pyproject.toml"), "rb") as f:
+            config = tomllib.load(f)
+            return path, config.get("tool", {}).get("fabricks", {}), "pyproject"
 
     return None, {}, None
 

fabricks/core/extenders.py

@@ -1,8 +1,8 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Callable
 
 from fabricks.context import IS_UNITY_CATALOG, PATH_EXTENDERS
 from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.utils.helpers import load_module_from_path
 
 EXTENDERS: dict[str, Callable] = {}
 
@@ -14,12 +14,7 @@ def get_extender(name: str) -> Callable:
     else:
         DEFAULT_LOGGER.debug(f"could not check if extender exists ({path.string})")
 
-    spec = spec_from_file_location(name, path.string)
-    assert spec, "no valid extender found in {path.string}"
-    assert spec.loader is not None
-
-    mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
+    load_module_from_path(name, path)
     e = EXTENDERS[name]
 
     return e

fabricks/core/jobs/base/generator.py

@@ -339,7 +339,7 @@ class Generator(Configurator):
                     self.table.add_table_comment(comment=comment)
 
         else:
-            DEFAULT_LOGGER.debug("table exists, skip creation", extra={"label": self})
+            DEFAULT_LOGGER.debug("table already exists, skipped creation", extra={"label": self})
 
     def _update_schema(
         self,

fabricks/core/parsers/get_parser.py

@@ -1,9 +1,9 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Optional
 
 from fabricks.context import PATH_PARSERS
 from fabricks.core.parsers._types import ParserOptions
 from fabricks.core.parsers.base import PARSERS, BaseParser
+from fabricks.utils.helpers import load_module_from_path
 
 
 def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> BaseParser:
@@ -11,12 +11,7 @@ def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> Bas
         path = PATH_PARSERS.joinpath(name).append(".py")
         assert path.exists(), f"parser not found ({path})"
 
-        spec = spec_from_file_location(name, path.string)
-        assert spec, f"parser not found ({path})"
-        assert spec.loader is not None
-
-        mod = module_from_spec(spec)
-        spec.loader.exec_module(mod)
+        load_module_from_path(name, path)
         parser = PARSERS[name](parser_options)
 
     else:

fabricks/core/steps/base.py

@@ -228,15 +228,22 @@ class BaseStep:
             DEFAULT_LOGGER.exception("fail to get jobs", extra={"label": self})
             raise e
 
-    def create_db_objects(self, retry: Optional[bool] = True) -> List[Dict]:
+    def create_db_objects(
+        self,
+        retry: Optional[bool] = True,
+        update_lists: Optional[bool] = True,
+        incremental: Optional[bool] = False,
+    ) -> List[Dict]:
         DEFAULT_LOGGER.info("create db objects", extra={"label": self})
 
         df = self.get_jobs()
-        table_df = self.database.get_tables()
-        view_df = self.database.get_views()
 
-        df = df.join(table_df, "job_id", how="left_anti")
-        df = df.join(view_df, "job_id", how="left_anti")
+        if incremental:
+            table_df = self.database.get_tables()
+            view_df = self.database.get_views()
+
+            df = df.join(table_df, "job_id", how="left_anti")
+            df = df.join(view_df, "job_id", how="left_anti")
 
         if df:
             results = run_in_parallel(
@@ -248,15 +255,16 @@ class BaseStep:
                 loglevel=logging.CRITICAL,
             )
 
-        self.update_tables_list()
-        self.update_views_list()
+        if update_lists:
+            self.update_tables_list()
+            self.update_views_list()
 
         errors = [res for res in results if res.get("error")]
 
         if errors:
             if retry:
                 DEFAULT_LOGGER.warning("retry to create jobs", extra={"label": self})
-                return self.create_db_objects(retry=False)
+                return self.create_db_objects(retry=False, update_lists=update_lists, incremental=incremental)
 
         return errors
 

fabricks/utils/azure_table.py

@@ -1,5 +1,4 @@
-import time
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 from azure.data.tables import TableClient, TableServiceClient
 from pyspark.sql import DataFrame
@@ -99,27 +98,29 @@ class AzureTable:
         if self._table_client is not None:
             self._table_client.close()
 
-    def submit(self, operations: List, retry: Optional[bool] = True):
-        try:
-            partitions = set()
-            for d in operations:
-                partitions.add(d[1]["PartitionKey"])
-
-            for p in partitions:
-                _operations = [d for d in operations if d[1].get("PartitionKey") == p]
-                t = 50
-                if len(_operations) < t:
-                    self.table.submit_transaction(_operations)
-                else:
-                    transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
-                    for transaction in transactions:
-                        self.table.submit_transaction(transaction)
-        except Exception as e:
-            if retry:
-                time.sleep(10)
-                self.submit(operations, retry=False)
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        retry=retry_if_exception_type((Exception)),
+        reraise=True,
+    )
+    def _submit_with_retry(self, data: Any):
+        self.table.submit_transaction(data)
+
+    def submit(self, operations: List):
+        partitions = set()
+        for d in operations:
+            partitions.add(d[1]["PartitionKey"])
+
+        for p in partitions:
+            _operations = [d for d in operations if d[1].get("PartitionKey") == p]
+            t = 50
+            if len(_operations) < t:
+                self._submit_with_retry(_operations)
             else:
-                raise e
+                transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
+                for transaction in transactions:
+                    self._submit_with_retry(transaction)
 
     def delete(self, data: Union[List, DataFrame, dict]):
         if isinstance(data, DataFrameLike):

fabricks/utils/helpers.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 from functools import reduce
 from queue import Queue
 from typing import Any, Callable, Iterable, List, Literal, Optional, Union
@@ -216,13 +217,28 @@ def run_notebook(path: Path, timeout: Optional[int] = None, **kwargs):
     dbutils.notebook.run(path.get_notebook_path(), timeout, {**kwargs})  # type: ignore
 
 
-def xxhash64(s: Any):
+def xxhash64(s: Any) -> int:
     df = spark.sql(f"select xxhash64(cast('{s}' as string)) as xxhash64")
     return df.collect()[0][0]
 
 
-def md5(s: Any):
+def md5(s: Any) -> str:
     from hashlib import md5
 
     md5 = md5(str(s).encode())
     return md5.hexdigest()
+
+
+def load_module_from_path(name: str, path: Path):
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    sys.path.append(str(path.parent))
+
+    spec = spec_from_file_location(name, path.string)
+    assert spec, f"no valid module found in {path.string}"
+    assert spec.loader is not None
+
+    textwrap_module = module_from_spec(spec)
+    spec.loader.exec_module(textwrap_module)
+
+    return textwrap_module

fabricks/utils/path.py

@@ -1,4 +1,5 @@
 import os
+import posixpath
 from pathlib import Path as PathlibPath
 from typing import List, Optional, Union
 
@@ -120,8 +121,13 @@ class Path:
             return False
 
     def joinpath(self, *other):
-        new_path = self.pathlibpath.joinpath(*other)
-        return Path(path=new_path, assume_git=self.assume_git)
+        parts = [str(o) for o in other]
+        base = self.string
+
+        joined = posixpath.join(base, *parts)
+        new = posixpath.normpath(joined)
+
+        return Path(path=new, assume_git=self.assume_git)
 
     def append(self, other: str):
         new_path = self.string + other

{fabricks-3.0.12.dist-info → fabricks-3.0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.12
+Version: 3.0.13
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.12.dist-info → fabricks-3.0.13.dist-info}/RECORD

@@ -70,14 +70,14 @@ fabricks/cdc/templates/queries/nocdc/update.sql.jinja,sha256=mjNUwGVhZ08yUkdv9sC
 fabricks/context/__init__.py,sha256=qfntJ9O6omzY_t6AhDP6Ndu9C5LMiVdWbo6ikhtoe7o,1446
 fabricks/context/_types.py,sha256=FzQJ35vp0uc6pAq18bc-VHwMVEWtd0VDdm8xQmNr2Sg,2681
 fabricks/context/config.py,sha256=EmLUnswuWfrncaNJMDjvdMg-1lD8aneKAY8IDna7VPE,4814
-fabricks/context/helpers.py,sha256=GV9MscE8p6CsHbjVC-Qnqtv9VBf893DoxfLa057hFT8,2061
+fabricks/context/helpers.py,sha256=nyHF0yhTJ_cDQT5Q47z0CYh4QHOJyzyRNRBfrUYDIbU,1552
 fabricks/context/log.py,sha256=CadrRf8iL6iXlGIGIhEIswa7wGqC-E-oLwWcGTyJ10s,2074
 fabricks/context/runtime.py,sha256=87PtX6SqLoFd0PGxgisF6dLlxtCHaHxkMMIt34UyB2w,3479
 fabricks/context/secret.py,sha256=iRM-KU-JcJAEOLoGJ8S4Oh65-yt674W6CDTSkOE7SXw,3192
 fabricks/context/spark_session.py,sha256=BPaxKJXHZDI5oQiOPhmua_xjXnrVgluh--AVpvUgbck,2553
 fabricks/context/utils.py,sha256=EQRscdUhdjwk2htZu8gCgNZ9PfRzzrR6e1kRrIbVlBM,2786
 fabricks/core/__init__.py,sha256=LaqDi4xuyHAoLOvS44PQdZdRfq9SmVr7mB6BDHyxYpc,209
-fabricks/core/extenders.py,sha256=39bSm9QiW4vBAyT659joE-5p_EZiNM4gi8KA3-OgX3E,917
+fabricks/core/extenders.py,sha256=oJzfv0hWxusnGmrjMwbrGyKfot8xzA4XtNquPWfFgPo,727
 fabricks/core/job_schema.py,sha256=6-70oy0ZJd3V9AiXfc0Q8b8NVEynxQza_h7mB13uB-s,853
 fabricks/core/masks.py,sha256=3UCxcCi-TgFHB7xT5ZvmEa8RMKev23X_JLE70Pr3rpY,1347
 fabricks/core/udfs.py,sha256=gu7K-ohxcO0TdgA7IjzVMOZatZQYhFTklHo60a6k_Yc,2960
@@ -105,7 +105,7 @@ fabricks/core/jobs/base/_types.py,sha256=y66BtJlJskq7wGzn7te5XYjO-NEqeQGUC11kkbe
 fabricks/core/jobs/base/checker.py,sha256=Cdfh8rQYy4MvMFl0HyC3alGUWm8zrFXk08m2t2JMu6Y,5477
 fabricks/core/jobs/base/configurator.py,sha256=9G5F7Qg5FWHPbHgdh8Qxc85OoSX0rnjD4c9itwU5KKc,10415
 fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
-fabricks/core/jobs/base/generator.py,sha256=TI4Wy8tlVRIKEczx6qmrIf2ppGwLvhdaatb9pnCvzlI,17635
+fabricks/core/jobs/base/generator.py,sha256=Dk82tj21NhR9wwgXzMp8JlKQ6D9HnjVlK9fvDmoYLbk,17646
 fabricks/core/jobs/base/invoker.py,sha256=FvjfpNqi542slxC2yLu1BIu5EklNUWySxDF8cD_SqKQ,7602
 fabricks/core/jobs/base/job.py,sha256=dWmk2PpQH2NETaaDS6KoiefRnDHfDMdCyhmogkdcSFI,93
 fabricks/core/jobs/base/processor.py,sha256=qkNiJSSLaEnivKGBcd9UZyIVFexnv-n1p_5mCZIy1rA,9076
@@ -113,7 +113,7 @@ fabricks/core/parsers/__init__.py,sha256=TGjyUeiiTkJrAxIpu2D_c2rQcbe5YRpmBW9oh0F
 fabricks/core/parsers/_types.py,sha256=JC2Oh-wUvaX8SBzeuf5owPgRaj-Q3-7MXxyIYPQ7QwA,147
 fabricks/core/parsers/base.py,sha256=P8IrLQKGakwaAQ-4gf4vElVwWoSpkixYd9kNthu1VDM,3292
 fabricks/core/parsers/decorator.py,sha256=kn_Mj-JLWTFaRiciZ3KavmSUcWFPY3ve-buMruHrX_Q,307
-fabricks/core/parsers/get_parser.py,sha256=TTnVPwKqKpFu6jJJnXEuiEctWGtimk8w2p1jF2U7ibg,909
+fabricks/core/parsers/get_parser.py,sha256=mauofS626h9wpPZtlZFqIb1jcKM3Jz4D_36uvd-Lv4k,717
 fabricks/core/parsers/utils.py,sha256=qdn2ElpqBgDsW55-tACWZaFOT0ebrBYg2fenqSgd6YI,2456
 fabricks/core/schedules/__init__.py,sha256=bDjNMcm7itimAo4gun0W4W9bZKwZmWUjkMqAQIcqI2Y,431
 fabricks/core/schedules/diagrams.py,sha256=YA4T7Etl_UPfW-3IGFq5Xj9OlXZGQ27Aot6RVa3ZUgg,578
@@ -126,7 +126,7 @@ fabricks/core/schedules/terminate.py,sha256=-RvtOrxTOZl2sZQ6KfNHJL6H2LCAEMSVRyyl
 fabricks/core/schedules/views.py,sha256=8hYwPLCvvN-nem2lNAKvUY5hC71v88z4-y8j0poUApM,1949
 fabricks/core/steps/__init__.py,sha256=JP-kaDa890-9XqBSPp6YdssAexdxv-MqQ__WfVYdgeg,132
 fabricks/core/steps/_types.py,sha256=VxIrH3nFwmPlwG-UI8sDDP0AwK_9jlsy6yQp6YfgtqE,90
-fabricks/core/steps/base.py,sha256=UTzVqdWaho9zgMrloq8ndqcWQ9R5Z-O1SInIrm9byB4,14403
+fabricks/core/steps/base.py,sha256=MJe2q9s1siM89YkpHDqldtbtKQgkhDB_cFa2-e_irvs,14642
 fabricks/core/steps/get_step.py,sha256=8q4rEDdTTZNJsXB2l5XY-Ktoow8ZHsON_tx5yKMUIzg,284
 fabricks/core/steps/get_step_conf.py,sha256=UPT3gB1Sh5yzawZ9qiVQlvVAKaxPX82gaWBDzxx75EM,633
 fabricks/deploy/__init__.py,sha256=ntxtFnzeho_WneVoL5CCqbI4rKApKgdmi9j0HKb0LJc,2375
@@ -149,13 +149,13 @@ fabricks/metastore/view.py,sha256=f7hKJWtnH1KmZym8dkoucKOTndntzai_f2YqferxHLs,14
 fabricks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/utils/_types.py,sha256=AuOhknlozqx5QdAdvZSA6xAWhU8k4nxG1vxIkOVgHeY,184
 fabricks/utils/azure_queue.py,sha256=wtKAq_MD5QLxelerDO475dzL-SySIrxt9d5KGi-8vvw,3102
-fabricks/utils/azure_table.py,sha256=7tCD1iM7UWREaSQVVmtgHCAebLtWPP9ZmuU5zDALyo0,5305
+fabricks/utils/azure_table.py,sha256=J_UAPiCN89rL5FNmwIGg43Z6FSH8evVIWijDEgYJxk4,5294
 fabricks/utils/console.py,sha256=X4lLgL_UxCjoFRx-ZRCwzdBveRGPKlFYZDi6vl7uevQ,1017
 fabricks/utils/fdict.py,sha256=cdnvNBSXKJIDKSdhQGJA4CGv0qLn5IVYKQ111l7nM9I,7978
-fabricks/utils/helpers.py,sha256=h7SuOVpBP5qcgX1nM1suvkXG9BhiK5-257EBepCvrO8,7452
+fabricks/utils/helpers.py,sha256=fKv6mpT-428xTSjdLfm7TnN1Xo9FadrSIY1qzYgWCzs,7909
 fabricks/utils/log.py,sha256=LCQEM81PhdojiyLrtEzv1QM__bWbaEhGddyd0IqyGXM,7985
 fabricks/utils/mermaid.py,sha256=XoiVxPaUJS4TC_ybA-e78qFzQkQ46uPf055JiiNDdSg,986
-fabricks/utils/path.py,sha256=Bs3PayWtg62-mrsDbvu8kh0VLZZhX7tU9YiyHFiYNhs,6698
+fabricks/utils/path.py,sha256=ToTTS8QKGsWq8cR8SDE2ocWKx2GsuZ5psPTuAMq9R4s,6813
 fabricks/utils/pip.py,sha256=UHo7NTjFGJNghWBuuDow28xUkZYg2YrlbAP49IxZyXY,1522
 fabricks/utils/pydantic.py,sha256=W0fiDLVMFrrInfQw2s5YPeSEvkN-4k864u3UyPoHaz4,2516
 fabricks/utils/spark.py,sha256=QWVpbGwOvURIVBlR7ygt6NQ9MHUsIDvlquJ65iI8UBI,2007
@@ -171,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=5k-R6zCgUAtapQgxT4turcx1IQ-b
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=lTQ0CfUhcvn3xTCcT_Ns6PMDBsO5UEfa2S9XpJiLJ9c,1250
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.12.dist-info/METADATA,sha256=LYFVTMlw8ueiyViUZC5ZZHk2LZkpNnVGO2B1ZhYrdTg,798
-fabricks-3.0.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.12.dist-info/RECORD,,
+fabricks-3.0.13.dist-info/METADATA,sha256=H7Yv6dwfPTscFWMQWayOB7t_eURWWYCGxPGzWsV71KQ,798
+fabricks-3.0.13.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+fabricks-3.0.13.dist-info/RECORD,,

{fabricks-3.0.12.dist-info → fabricks-3.0.13.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any