fabricks 3.0.12__tar.gz → 3.0.13__tar.gz

{fabricks-3.0.12 → fabricks-3.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.12
+Version: 3.0.13
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/context/helpers.py

@@ -1,59 +1,36 @@
+import os
+import pathlib
 from typing import List
 
 from fabricks.utils.path import Path
 
 
-def get_config_from_toml():
-    import os
-    import pathlib
-    import sys
-
-    if sys.version_info >= (3, 11):
-        import tomllib
-    else:
-        import tomli as tomllib  # type: ignore
-
+def get_config_from_file():
     path = pathlib.Path(os.getcwd())
-    while path is not None and not (path / "pyproject.toml").exists():
-        if path == path.parent:
-            break
-        path = path.parent
-
-    if (path / "pyproject.toml").exists():
-        with open((path / "pyproject.toml"), "rb") as f:
-            config = tomllib.load(f)
-            return path, config.get("tool", {}).get("fabricks", {})
 
-    return None, {}
-
-
-def get_config_from_json():
-    import json
-    import os
-    import pathlib
-
-    path = pathlib.Path(os.getcwd())
-    while path is not None and not (path / "fabricksconfig.json").exists():
+    while path is not None and (not (path / "pyproject.toml").exists() or (path / "fabricksconfig.json").exists()):
         if path == path.parent:
             break
         path = path.parent
 
     if (path / "fabricksconfig.json").exists():
+        import json
+
         with open((path / "fabricksconfig.json"), "r") as f:
             config = json.load(f)
-            return path, config
-
-    return None, {}
+            return path, config, "json"
 
+    if (path / "pyproject.toml").exists():
+        import sys
 
-def get_config_from_file():
-    json_path, json_config = get_config_from_json()
-    if json_config:
-        return json_path, json_config, "json"
+        if sys.version_info >= (3, 11):
+            import tomllib
+        else:
+            import tomli as tomllib  # type: ignore
 
-    pyproject_path, pyproject_config = get_config_from_toml()
-    if pyproject_config:
-        return pyproject_path, pyproject_config, "pyproject"
+        with open((path / "pyproject.toml"), "rb") as f:
+            config = tomllib.load(f)
+            return path, config.get("tool", {}).get("fabricks", {}), "pyproject"
 
     return None, {}, None
 

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/core/extenders.py

@@ -1,8 +1,8 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Callable
 
 from fabricks.context import IS_UNITY_CATALOG, PATH_EXTENDERS
 from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.utils.helpers import load_module_from_path
 
 EXTENDERS: dict[str, Callable] = {}
 
@@ -14,12 +14,7 @@ def get_extender(name: str) -> Callable:
     else:
         DEFAULT_LOGGER.debug(f"could not check if extender exists ({path.string})")
 
-    spec = spec_from_file_location(name, path.string)
-    assert spec, "no valid extender found in {path.string}"
-    assert spec.loader is not None
-
-    mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
+    load_module_from_path(name, path)
     e = EXTENDERS[name]
 
     return e

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/core/jobs/base/generator.py

@@ -339,7 +339,7 @@ class Generator(Configurator):
                     self.table.add_table_comment(comment=comment)
 
         else:
-            DEFAULT_LOGGER.debug("table exists, skip creation", extra={"label": self})
+            DEFAULT_LOGGER.debug("table already exists, skipped creation", extra={"label": self})
 
     def _update_schema(
         self,

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/core/parsers/get_parser.py

@@ -1,9 +1,9 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Optional
 
 from fabricks.context import PATH_PARSERS
 from fabricks.core.parsers._types import ParserOptions
 from fabricks.core.parsers.base import PARSERS, BaseParser
+from fabricks.utils.helpers import load_module_from_path
 
 
 def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> BaseParser:
@@ -11,12 +11,7 @@ def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> Bas
         path = PATH_PARSERS.joinpath(name).append(".py")
         assert path.exists(), f"parser not found ({path})"
 
-        spec = spec_from_file_location(name, path.string)
-        assert spec, f"parser not found ({path})"
-        assert spec.loader is not None
-
-        mod = module_from_spec(spec)
-        spec.loader.exec_module(mod)
+        load_module_from_path(name, path)
         parser = PARSERS[name](parser_options)
 
     else:

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/core/steps/base.py

@@ -228,15 +228,22 @@ class BaseStep:
             DEFAULT_LOGGER.exception("fail to get jobs", extra={"label": self})
             raise e
 
-    def create_db_objects(self, retry: Optional[bool] = True) -> List[Dict]:
+    def create_db_objects(
+        self,
+        retry: Optional[bool] = True,
+        update_lists: Optional[bool] = True,
+        incremental: Optional[bool] = False,
+    ) -> List[Dict]:
         DEFAULT_LOGGER.info("create db objects", extra={"label": self})
 
         df = self.get_jobs()
-        table_df = self.database.get_tables()
-        view_df = self.database.get_views()
 
-        df = df.join(table_df, "job_id", how="left_anti")
-        df = df.join(view_df, "job_id", how="left_anti")
+        if incremental:
+            table_df = self.database.get_tables()
+            view_df = self.database.get_views()
+
+            df = df.join(table_df, "job_id", how="left_anti")
+            df = df.join(view_df, "job_id", how="left_anti")
 
         if df:
             results = run_in_parallel(
@@ -248,15 +255,16 @@ class BaseStep:
                 loglevel=logging.CRITICAL,
             )
 
-        self.update_tables_list()
-        self.update_views_list()
+        if update_lists:
+            self.update_tables_list()
+            self.update_views_list()
 
         errors = [res for res in results if res.get("error")]
 
         if errors:
             if retry:
                 DEFAULT_LOGGER.warning("retry to create jobs", extra={"label": self})
-                return self.create_db_objects(retry=False)
+                return self.create_db_objects(retry=False, update_lists=update_lists, incremental=incremental)
 
         return errors
 

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/utils/azure_table.py

@@ -1,5 +1,4 @@
-import time
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 from azure.data.tables import TableClient, TableServiceClient
 from pyspark.sql import DataFrame
@@ -99,27 +98,29 @@ class AzureTable:
         if self._table_client is not None:
             self._table_client.close()
 
-    def submit(self, operations: List, retry: Optional[bool] = True):
-        try:
-            partitions = set()
-            for d in operations:
-                partitions.add(d[1]["PartitionKey"])
-
-            for p in partitions:
-                _operations = [d for d in operations if d[1].get("PartitionKey") == p]
-                t = 50
-                if len(_operations) < t:
-                    self.table.submit_transaction(_operations)
-                else:
-                    transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
-                    for transaction in transactions:
-                        self.table.submit_transaction(transaction)
-        except Exception as e:
-            if retry:
-                time.sleep(10)
-                self.submit(operations, retry=False)
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        retry=retry_if_exception_type((Exception)),
+        reraise=True,
+    )
+    def _submit_with_retry(self, data: Any):
+        self.table.submit_transaction(data)
+
+    def submit(self, operations: List):
+        partitions = set()
+        for d in operations:
+            partitions.add(d[1]["PartitionKey"])
+
+        for p in partitions:
+            _operations = [d for d in operations if d[1].get("PartitionKey") == p]
+            t = 50
+            if len(_operations) < t:
+                self._submit_with_retry(_operations)
             else:
-                raise e
+                transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
+                for transaction in transactions:
+                    self._submit_with_retry(transaction)
 
     def delete(self, data: Union[List, DataFrame, dict]):
         if isinstance(data, DataFrameLike):

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/utils/helpers.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 from functools import reduce
 from queue import Queue
 from typing import Any, Callable, Iterable, List, Literal, Optional, Union
@@ -216,13 +217,28 @@ def run_notebook(path: Path, timeout: Optional[int] = None, **kwargs):
     dbutils.notebook.run(path.get_notebook_path(), timeout, {**kwargs})  # type: ignore
 
 
-def xxhash64(s: Any):
+def xxhash64(s: Any) -> int:
     df = spark.sql(f"select xxhash64(cast('{s}' as string)) as xxhash64")
     return df.collect()[0][0]
 
 
-def md5(s: Any):
+def md5(s: Any) -> str:
     from hashlib import md5
 
     md5 = md5(str(s).encode())
     return md5.hexdigest()
+
+
+def load_module_from_path(name: str, path: Path):
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    sys.path.append(str(path.parent))
+
+    spec = spec_from_file_location(name, path.string)
+    assert spec, f"no valid module found in {path.string}"
+    assert spec.loader is not None
+
+    textwrap_module = module_from_spec(spec)
+    spec.loader.exec_module(textwrap_module)
+
+    return textwrap_module

{fabricks-3.0.12 → fabricks-3.0.13}/fabricks/utils/path.py

@@ -1,4 +1,5 @@
 import os
+import posixpath
 from pathlib import Path as PathlibPath
 from typing import List, Optional, Union
 
@@ -120,8 +121,13 @@ class Path:
             return False
 
     def joinpath(self, *other):
-        new_path = self.pathlibpath.joinpath(*other)
-        return Path(path=new_path, assume_git=self.assume_git)
+        parts = [str(o) for o in other]
+        base = self.string
+
+        joined = posixpath.join(base, *parts)
+        new = posixpath.normpath(joined)
+
+        return Path(path=new, assume_git=self.assume_git)
 
     def append(self, other: str):
         new_path = self.string + other

{fabricks-3.0.12 → fabricks-3.0.13}/format.sh

@@ -1,14 +1,6 @@
-if [[ "$OSTYPE" == "linux-gnu"* ]] 
-then
-    source ./.venv/bin/activate
-elif [[ "$OSTYPE" == "darwin23" ]]
-then
-    source ./.venv/bin/activate
-else
-    source ./.venv/Scripts/activate
-fi
+#!/bin/bash
 
-while getopts :f:p:s:y:g:c:a option
+while getopts :f:p:s:y:g:c:a:b option
 do
     case $option in
         a) all="true";;
@@ -17,6 +9,7 @@ do
         s) sql="true";;
         y) yaml="true";;
         g) git="true";;
+        b) build="true";;
     esac
 done
 
@@ -50,13 +43,18 @@ then
     git="true"
 fi
 
+if [ -z "$build" ]
+then
+    build="true"
+fi
+
 if [ "$all" = "true" ] || [ "$python" = "true" ]
 then
-    echo "🌟 autoflake(ing) 🌟"
+    echo "🖌️ autoflake(ing) 🖌️"
     uv run autoflake -r -i .
-    echo "🌟 isort(ing) 🌟"
+    echo "🖌️ isort(ing) 🖌️"
     uv run isort .
-    echo "🌟 pycln(ing) 🌟"
+    echo "🖌️ pycln(ing) 🖌️"
     uv run pycln .
 fi
 
@@ -69,24 +67,32 @@ fi
 
 if [ "$all" = "true" ] || [ "$python" = "true" ]
 then
-    echo "🌟 pyright(ing) 🌟"
+    echo "🔎 pyright(ing) 🔎"
     uv run pyright .
 fi
 
 if [ "$all" = "true" ] || [ "$sql" = "true" ]
 then
-    echo "🌟 sqlfmt(ing) 🌟"
+    echo "🧼 sqlfmt(ing) 🧼"
     uv run sqlfmt .
 fi
 
 if [ "$all" = "true" ] || [ "$yaml" = "true" ]
 then
-    echo "🌟 yamlfix(ing) 🌟"
+    echo "🧼 yamlfix(ing) 🧼"
     uv run yamlfix . --exclude .venv --exclude .dev --exclude .idea --include *.yml
 fi
 
 if [ "$all" = "true" ] || [ "$git" = "true" ]
 then
-    echo "🌟 git add(ing) 🌟"
+    echo "✅ git add(ing) ✅"
     git add .
 fi
+
+if [ "$all" = "true" ] || [ "$build" = "true" ]
+then
+    echo "🏗️ build wheel 🏗️"
+    uv build
+fi
+
+echo "🎉 All done 🎉"

{fabricks-3.0.12 → fabricks-3.0.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.12"
+version = "3.0.13"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

fabricks-3.0.13/tests/integration/README.md

@@ -0,0 +1,16 @@
+# Integration tests — Databricks runbook
+
+This document describes how to build the `fabricks` wheel from the `framework` package, upload it to Databricks (DBFS), install it on a cluster, and run notebooks or integration tests on Databricks.
+
+**Prerequisites**
+- Python 3.9–3.10 (matches `pyproject.toml`).
+- `uv` installed (for building): `pip install uv`.
+
+**How to proceed ?**
+1) Build the wheel
+- From the repository root (where `framework/pyproject.toml` lives) run `uv build`.
+- The wheel will be written to `dist/` (e.g. `dist/fabricks-3.0.12-py3-none-any.whl`).
+
+2) Upload wheel to DBFS
+
+3) Install the wheel on a cluster

{fabricks-3.0.12 → fabricks-3.0.13}/uv.lock

@@ -645,7 +645,7 @@ wheels = [
 
 [[package]]
 name = "fabricks"
-version = "3.0.12"
+version = "3.0.13"
 source = { editable = "." }
 dependencies = [
     { name = "azure-data-tables" },