fabricks 3.0.11__tar.gz → 3.0.13__tar.gz

{fabricks-3.0.11 → fabricks-3.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.11
+Version: 3.0.13
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.11 → fabricks-3.0.13}/README.md

@@ -271,12 +271,8 @@ with
     from dates d
     left join newkey r on d.__timestamp between r.__valid_from and r.__valid_to and d.only_offer_id = r.only_offer_id
     group by d.only_offer_id, __timestamp, __operation
-    
 ```
 
-
-
-
 ## 🧾 Conclusion
 
 Fabricks helps you define pipelines declaratively in YAML and implement logic in SQL, while handling CDC, dependencies, logging, and table lifecycle. Start from the examples near each step, then evolve to advanced features like checks, UDFs, extenders, and semantic properties.

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/cdc/base/merger.py

@@ -89,6 +89,7 @@ class Merger(Processor):
                 sql = sql.replace("{src}", "src")
                 sql = fix_sql(sql)
                 sql = sql.replace("`src`", "{src}")
+
                 DEFAULT_LOGGER.debug("merge", extra={"label": self, "sql": sql})
 
             except Exception as e:

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/cdc/base/processor.py

@@ -390,11 +390,12 @@ class Processor(Generator):
         try:
             sql = template.render(**context)
             if fix:
-                DEFAULT_LOGGER.debug("fix context", extra={"label": self, "sql": sql})
                 sql = self.fix_sql(sql)
+            else:
+                DEFAULT_LOGGER.debug("print query", extra={"label": self, "sql": sql})
 
         except (Exception, TypeError) as e:
-            DEFAULT_LOGGER.exception("fail to execute sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         row = self.spark.sql(sql).collect()[0]
@@ -425,7 +426,7 @@ class Processor(Generator):
 
         except (Exception, TypeError) as e:
             DEFAULT_LOGGER.debug("context", extra={"label": self, "context": context})
-            DEFAULT_LOGGER.exception("fail to generate sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         return sql

fabricks-3.0.13/fabricks/context/config.py

@@ -0,0 +1,127 @@
+import logging
+import os
+from typing import Final
+
+from fabricks.context.helpers import get_config_from_file
+from fabricks.utils.log import get_logger
+from fabricks.utils.path import Path
+from fabricks.utils.spark import spark
+
+logger, _ = get_logger("logs", level=logging.DEBUG)
+file_path, file_config, origin = get_config_from_file()
+
+if file_path:
+    logger.debug(f"found {origin} config ({file_path})", extra={"label": "config"})
+
+# path to runtime
+runtime = os.environ.get("FABRICKS_RUNTIME", "none")
+runtime = None if runtime.lower() == "none" else runtime
+if runtime is None:
+    if runtime := file_config.get("runtime"):
+        assert file_path is not None
+        runtime = file_path.joinpath(runtime)
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve runtime from env", extra={"label": "config"})
+
+if runtime is None:
+    if file_path is not None:
+        runtime = file_path
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+    else:
+        raise ValueError(
+            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
+        )
+
+path_runtime = Path(runtime, assume_git=True)
+
+# path to config
+config = os.environ.get("FABRICKS_CONFIG")
+if config is None:
+    if config := file_config.get("config"):
+        assert file_path is not None
+        config = file_path.joinpath(config)
+        logger.debug(f"resolve config from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve config from env", extra={"label": "config"})
+
+if config is None:
+    logger.debug("resolve config from default path", extra={"label": "config"})
+    config = path_runtime.joinpath(
+        "fabricks",
+        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
+    ).string
+
+path_config = Path(config, assume_git=True)
+
+# path to notebooks
+notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
+notebooks = None if notebooks.lower() == "none" else notebooks
+if notebooks is None:
+    if notebooks := file_config.get("notebooks"):
+        assert file_path is not None
+        notebooks = file_path.joinpath(notebooks)
+        logger.debug(f"resolve notebooks from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve notebooks from env", extra={"label": "config"})
+
+if notebooks is None:
+    logger.debug("resolve notebooks from default path", extra={"label": "config"})
+    notebooks = path_runtime.joinpath("notebooks")
+
+path_notebooks = Path(str(notebooks), assume_git=True)
+
+# job config from yaml
+is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
+if is_job_config_from_yaml is None:
+    if is_job_config_from_yaml := file_config.get("job_config_from_yaml"):
+        logger.debug(f"resolve job_config_from_yaml from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve job_config_from_yaml from env", extra={"label": "config"})
+
+# debug mode
+is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
+if is_debugmode is None:
+    if is_debugmode := file_config.get("debugmode"):
+        logger.debug(f"resolve debugmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve debugmode from env", extra={"label": "config"})
+
+# dev mode
+is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
+if is_devmode is None:
+    if is_devmode := file_config.get("devmode"):
+        logger.debug(f"resolve devmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve devmode from env", extra={"label": "config"})
+
+# log level
+loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
+if loglevel is None:
+    if loglevel := file_config.get("loglevel"):
+        logger.debug(f"resolve loglevel from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve loglevel from env", extra={"label": "config"})
+
+loglevel = loglevel.upper() if loglevel else "INFO"
+if loglevel == "DEBUG":
+    _loglevel = logging.DEBUG
+elif loglevel == "INFO":
+    _loglevel = logging.INFO
+elif loglevel == "WARNING":
+    _loglevel = logging.WARNING
+elif loglevel == "ERROR":
+    _loglevel = logging.ERROR
+elif loglevel == "CRITICAL":
+    _loglevel = logging.CRITICAL
+else:
+    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
+
+# Constants
+PATH_CONFIG: Final[Path] = path_config
+PATH_RUNTIME: Final[Path] = path_runtime
+PATH_NOTEBOOKS: Final[Path] = path_notebooks
+IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
+IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
+IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
+LOGLEVEL: Final[int] = _loglevel

fabricks-3.0.13/fabricks/context/helpers.py

@@ -0,0 +1,58 @@
+import os
+import pathlib
+from typing import List
+
+from fabricks.utils.path import Path
+
+
+def get_config_from_file():
+    path = pathlib.Path(os.getcwd())
+
+    while path is not None and (not (path / "pyproject.toml").exists() or (path / "fabricksconfig.json").exists()):
+        if path == path.parent:
+            break
+        path = path.parent
+
+    if (path / "fabricksconfig.json").exists():
+        import json
+
+        with open((path / "fabricksconfig.json"), "r") as f:
+            config = json.load(f)
+            return path, config, "json"
+
+    if (path / "pyproject.toml").exists():
+        import sys
+
+        if sys.version_info >= (3, 11):
+            import tomllib
+        else:
+            import tomli as tomllib  # type: ignore
+
+        with open((path / "pyproject.toml"), "rb") as f:
+            config = tomllib.load(f)
+            return path, config.get("tool", {}).get("fabricks", {}), "pyproject"
+
+    return None, {}, None
+
+
+def get_storage_paths(objects: List[dict], variables: dict) -> dict:
+    d = {}
+    for o in objects:
+        if o:
+            name = o.get("name")
+            assert name
+            uri = o.get("path_options", {}).get("storage")
+            assert uri
+            d[name] = Path.from_uri(uri, regex=variables)
+    return d
+
+
+def get_runtime_path(objects: List[dict], root: Path) -> dict:
+    d = {}
+    for o in objects:
+        name = o.get("name")
+        assert name
+        uri = o.get("path_options", {}).get("runtime")
+        assert uri
+        d[name] = root.joinpath(uri)
+    return d

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/context/runtime.py

@@ -1,11 +1,12 @@
-from typing import Final, List, Optional
+from typing import Final, Optional
 
 import yaml
 
-from fabricks.context.config import path_config, path_runtime
+from fabricks.context.config import PATH_CONFIG, PATH_RUNTIME
+from fabricks.context.helpers import get_runtime_path, get_storage_paths
 from fabricks.utils.path import Path
 
-with open(str(path_config)) as f:
+with open(str(PATH_CONFIG)) as f:
     data = yaml.safe_load(f)
 
 conf: dict = [d["conf"] for d in data][0]
@@ -51,67 +52,42 @@ FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_cr
 
 path_udfs = path_options.get("udfs", "fabricks/udfs")
 assert path_udfs, "path to udfs mandatory"
-PATH_UDFS: Final[Path] = path_runtime.joinpath(path_udfs)
+PATH_UDFS: Final[Path] = PATH_RUNTIME.joinpath(path_udfs)
 
 path_parsers = path_options.get("parsers", "fabricks/parsers")
 assert path_parsers, "path to parsers mandatory"
-PATH_PARSERS: Final[Path] = path_runtime.joinpath(path_parsers)
+PATH_PARSERS: Final[Path] = PATH_RUNTIME.joinpath(path_parsers)
 
 path_extenders = path_options.get("extenders", "fabricks/extenders")
 assert path_extenders, "path to extenders mandatory"
-PATH_EXTENDERS: Final[Path] = path_runtime.joinpath(path_extenders)
+PATH_EXTENDERS: Final[Path] = PATH_RUNTIME.joinpath(path_extenders)
 
 path_views = path_options.get("views", "fabricks/views")
 assert path_views, "path to views mandatory"
-PATH_VIEWS: Final[Path] = path_runtime.joinpath(path_views)
+PATH_VIEWS: Final[Path] = PATH_RUNTIME.joinpath(path_views)
 
 path_schedules = path_options.get("schedules", "fabricks/schedules")
 assert path_schedules, "path to schedules mandatory"
-PATH_SCHEDULES: Final[Path] = path_runtime.joinpath(path_schedules)
+PATH_SCHEDULES: Final[Path] = PATH_RUNTIME.joinpath(path_schedules)
 
 path_requirements = path_options.get("requirements", "fabricks/requirements")
 assert path_requirements, "path to requirements mandatory"
-PATH_REQUIREMENTS: Final[Path] = path_runtime.joinpath(path_requirements)
+PATH_REQUIREMENTS: Final[Path] = PATH_RUNTIME.joinpath(path_requirements)
 
 path_masks = path_options.get("masks", "fabricks/masks")
 assert path_masks, "path to masks mandatory"
-PATH_MASKS: Final[Path] = path_runtime.joinpath(path_masks)
-
-
-def _get_storage_paths(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        if o:
-            name = o.get("name")
-            assert name
-            uri = o.get("path_options", {}).get("storage")
-            assert uri
-            d[name] = Path.from_uri(uri, regex=variables)
-    return d
-
+PATH_MASKS: Final[Path] = PATH_RUNTIME.joinpath(path_masks)
 
 PATHS_STORAGE: Final[dict[str, Path]] = {
     "fabricks": FABRICKS_STORAGE,
-    **_get_storage_paths(BRONZE),
-    **_get_storage_paths(SILVER),
-    **_get_storage_paths(GOLD),
-    **_get_storage_paths(databases),
+    **get_storage_paths(BRONZE, variables),
+    **get_storage_paths(SILVER, variables),
+    **get_storage_paths(GOLD, variables),
+    **get_storage_paths(databases, variables),
 }
 
-
-def _get_runtime_path(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        name = o.get("name")
-        assert name
-        uri = o.get("path_options", {}).get("runtime")
-        assert uri
-        d[name] = path_runtime.joinpath(uri)
-    return d
-
-
 PATHS_RUNTIME: Final[dict[str, Path]] = {
-    **_get_runtime_path(BRONZE),
-    **_get_runtime_path(SILVER),
-    **_get_runtime_path(GOLD),
+    **get_runtime_path(BRONZE, PATH_RUNTIME),
+    **get_runtime_path(SILVER, PATH_RUNTIME),
+    **get_runtime_path(GOLD, PATH_RUNTIME),
 }

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/core/dags/processor.py

@@ -10,7 +10,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_ex
 
 from fabricks.context import PATH_NOTEBOOKS
 from fabricks.core.dags.base import BaseDags
-from fabricks.core.dags.log import LOGGER
+from fabricks.core.dags.log import LOGGER, TABLE_LOG_HANDLER
 from fabricks.core.dags.run import run
 from fabricks.core.jobs.base._types import TStep
 from fabricks.core.steps.get_step import get_step
@@ -148,7 +148,9 @@ class DagProcessor(BaseDags):
                 finally:
                     j["Status"] = "ok"
                     self.table.upsert(j)
+
                     LOGGER.info("end", extra=self.extra(j))
+                    TABLE_LOG_HANDLER.flush()
 
                 dependencies = self.table.query(f"PartitionKey eq 'dependencies' and ParentId eq '{j.get('JobId')}'")
                 self.table.delete(dependencies)

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/core/extenders.py

@@ -1,8 +1,8 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Callable
 
 from fabricks.context import IS_UNITY_CATALOG, PATH_EXTENDERS
 from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.utils.helpers import load_module_from_path
 
 EXTENDERS: dict[str, Callable] = {}
 
@@ -14,12 +14,7 @@ def get_extender(name: str) -> Callable:
     else:
         DEFAULT_LOGGER.debug(f"could not check if extender exists ({path.string})")
 
-    spec = spec_from_file_location(name, path.string)
-    assert spec, "no valid extender found in {path.string}"
-    assert spec.loader is not None
-
-    mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
+    load_module_from_path(name, path)
     e = EXTENDERS[name]
 
     return e

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/core/jobs/base/generator.py

@@ -336,10 +336,10 @@ class Generator(Configurator):
 
                 comment = self.options.table.get("comment")
                 if comment:
-                    self.table.add_comment(comment=comment)
+                    self.table.add_table_comment(comment=comment)
 
         else:
-            DEFAULT_LOGGER.debug("table exists, skip creation", extra={"label": self})
+            DEFAULT_LOGGER.debug("table already exists, skipped creation", extra={"label": self})
 
     def _update_schema(
         self,
@@ -389,6 +389,24 @@ class Generator(Configurator):
     def overwrite_schema(self, df: Optional[DataFrame] = None):
         self._update_schema(df=df, overwrite=True)
 
+    def update_comments(self, table: Optional[bool] = True, columns: Optional[bool] = True):
+        if self.virtual:
+            return
+
+        if self.persist:
+            self.table.drop_comments()
+
+            if table:
+                comment = self.options.table.get("comment")
+                if comment:
+                    self.table.add_table_comment(comment=comment)
+
+            if columns:
+                comments = self.options.table.get_dict("comments")
+                if comments:
+                    for col, comment in comments.items():
+                        self.table.add_column_comment(column=col, comment=comment)
+
     def get_differences_with_deltatable(self, df: Optional[DataFrame] = None):
         if df is None:
             df = self.get_data(stream=self.stream)

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/core/parsers/get_parser.py

@@ -1,9 +1,9 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Optional
 
 from fabricks.context import PATH_PARSERS
 from fabricks.core.parsers._types import ParserOptions
 from fabricks.core.parsers.base import PARSERS, BaseParser
+from fabricks.utils.helpers import load_module_from_path
 
 
 def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> BaseParser:
@@ -11,12 +11,7 @@ def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> Bas
         path = PATH_PARSERS.joinpath(name).append(".py")
         assert path.exists(), f"parser not found ({path})"
 
-        spec = spec_from_file_location(name, path.string)
-        assert spec, f"parser not found ({path})"
-        assert spec.loader is not None
-
-        mod = module_from_spec(spec)
-        spec.loader.exec_module(mod)
+        load_module_from_path(name, path)
         parser = PARSERS[name](parser_options)
 
     else:

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/core/steps/base.py

@@ -228,15 +228,22 @@ class BaseStep:
             DEFAULT_LOGGER.exception("fail to get jobs", extra={"label": self})
             raise e
 
-    def create_db_objects(self, retry: Optional[bool] = True) -> List[Dict]:
+    def create_db_objects(
+        self,
+        retry: Optional[bool] = True,
+        update_lists: Optional[bool] = True,
+        incremental: Optional[bool] = False,
+    ) -> List[Dict]:
         DEFAULT_LOGGER.info("create db objects", extra={"label": self})
 
         df = self.get_jobs()
-        table_df = self.database.get_tables()
-        view_df = self.database.get_views()
 
-        df = df.join(table_df, "job_id", how="left_anti")
-        df = df.join(view_df, "job_id", how="left_anti")
+        if incremental:
+            table_df = self.database.get_tables()
+            view_df = self.database.get_views()
+
+            df = df.join(table_df, "job_id", how="left_anti")
+            df = df.join(view_df, "job_id", how="left_anti")
 
         if df:
             results = run_in_parallel(
@@ -248,15 +255,16 @@ class BaseStep:
                 loglevel=logging.CRITICAL,
             )
 
-        self.update_tables_list()
-        self.update_views_list()
+        if update_lists:
+            self.update_tables_list()
+            self.update_views_list()
 
         errors = [res for res in results if res.get("error")]
 
         if errors:
             if retry:
                 DEFAULT_LOGGER.warning("retry to create jobs", extra={"label": self})
-                return self.create_db_objects(retry=False)
+                return self.create_db_objects(retry=False, update_lists=update_lists, incremental=incremental)
 
         return errors
 

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/deploy/views.py

@@ -276,6 +276,7 @@ def create_or_replace_logs_pivot_view():
         l.step,
         l.job,
         l.job_id,
+        -- flags
         collect_set(l.status) as statuses,
         array_contains(statuses, 'skipped') as skipped,
         array_contains(statuses, 'warned') as warned,
@@ -283,15 +284,19 @@ def create_or_replace_logs_pivot_view():
         array_contains(statuses, 'failed') or (not done and not skipped) as failed,
         not done and not failed and not skipped and array_contains(statuses, 'running') as timed_out,
         not array_contains(statuses, 'running') as cancelled,
+        --
         max(l.notebook_id) as notebook_id,
+        --
+        max(l.timestamp) filter (where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status in ('done', 'ok')) as end_time,
+        --
         max(l.timestamp) filter (where l.status = 'scheduled' ) as scheduled_time,
         max(l.timestamp) filter (where l.status = 'waiting' ) as waiting_time,
-        max(l.timestamp) filter (where l.status = 'running') as start_time,
         max(l.timestamp) filter (where l.status = 'running' ) as running_time,
         max(l.timestamp) filter (where l.status = 'done' ) as done_time,
         max(l.timestamp) filter (where l.status = 'failed' ) as failed_time,
-        max(l.timestamp) filter(where l.status = 'ok') as end_time,
-        max(l.timestamp) filter(where l.status = 'ok') as ok_time,
+        max(l.timestamp) filter (where l.status = 'ok') as ok_time,
+        --
         max(l.exception) as exception
       from
         fabricks.logs l

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/metastore/table.py

@@ -666,7 +666,45 @@ class Table(DbObject):
             """
         )
 
-    def add_comment(self, comment: str):
+    def drop_comments(self):
+        self.drop_table_comment()
+        for col in self.columns:
+            self.drop_column_comment(col)
+
+    def drop_table_comment(self):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug("drop table comment", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on table {self.qualified_name}
+            is null;
+            """
+        )
+
+    def drop_column_comment(self, column: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"drop comment from column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is null;
+            """
+        )
+
+    def add_column_comment(self, column: str, comment: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"add comment '{comment}' to column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is '{comment}';
+            """
+        )
+
+    def add_table_comment(self, comment: str):
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.debug(f"add comment '{comment}'", extra={"label": self})

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/utils/azure_table.py

@@ -1,5 +1,4 @@
-import time
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 from azure.data.tables import TableClient, TableServiceClient
 from pyspark.sql import DataFrame
@@ -99,27 +98,29 @@ class AzureTable:
         if self._table_client is not None:
             self._table_client.close()
 
-    def submit(self, operations: List, retry: Optional[bool] = True):
-        try:
-            partitions = set()
-            for d in operations:
-                partitions.add(d[1]["PartitionKey"])
-
-            for p in partitions:
-                _operations = [d for d in operations if d[1].get("PartitionKey") == p]
-                t = 50
-                if len(_operations) < t:
-                    self.table.submit_transaction(_operations)
-                else:
-                    transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
-                    for transaction in transactions:
-                        self.table.submit_transaction(transaction)
-        except Exception as e:
-            if retry:
-                time.sleep(10)
-                self.submit(operations, retry=False)
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        retry=retry_if_exception_type((Exception)),
+        reraise=True,
+    )
+    def _submit_with_retry(self, data: Any):
+        self.table.submit_transaction(data)
+
+    def submit(self, operations: List):
+        partitions = set()
+        for d in operations:
+            partitions.add(d[1]["PartitionKey"])
+
+        for p in partitions:
+            _operations = [d for d in operations if d[1].get("PartitionKey") == p]
+            t = 50
+            if len(_operations) < t:
+                self._submit_with_retry(_operations)
             else:
-                raise e
+                transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
+                for transaction in transactions:
+                    self._submit_with_retry(transaction)
 
     def delete(self, data: Union[List, DataFrame, dict]):
         if isinstance(data, DataFrameLike):

{fabricks-3.0.11 → fabricks-3.0.13}/fabricks/utils/helpers.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 from functools import reduce
 from queue import Queue
 from typing import Any, Callable, Iterable, List, Literal, Optional, Union
@@ -216,13 +217,28 @@ def run_notebook(path: Path, timeout: Optional[int] = None, **kwargs):
     dbutils.notebook.run(path.get_notebook_path(), timeout, {**kwargs})  # type: ignore
 
 
-def xxhash64(s: Any):
+def xxhash64(s: Any) -> int:
     df = spark.sql(f"select xxhash64(cast('{s}' as string)) as xxhash64")
     return df.collect()[0][0]
 
 
-def md5(s: Any):
+def md5(s: Any) -> str:
     from hashlib import md5
 
     md5 = md5(str(s).encode())
     return md5.hexdigest()
+
+
+def load_module_from_path(name: str, path: Path):
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    sys.path.append(str(path.parent))
+
+    spec = spec_from_file_location(name, path.string)
+    assert spec, f"no valid module found in {path.string}"
+    assert spec.loader is not None
+
+    textwrap_module = module_from_spec(spec)
+    spec.loader.exec_module(textwrap_module)
+
+    return textwrap_module