fabricks 3.0.11__py3-none-any.whl → 3.0.13__py3-none-any.whl

fabricks/cdc/base/merger.py

@@ -89,6 +89,7 @@ class Merger(Processor):
                 sql = sql.replace("{src}", "src")
                 sql = fix_sql(sql)
                 sql = sql.replace("`src`", "{src}")
+
                 DEFAULT_LOGGER.debug("merge", extra={"label": self, "sql": sql})
 
             except Exception as e:

fabricks/cdc/base/processor.py

@@ -390,11 +390,12 @@ class Processor(Generator):
         try:
             sql = template.render(**context)
             if fix:
-                DEFAULT_LOGGER.debug("fix context", extra={"label": self, "sql": sql})
                 sql = self.fix_sql(sql)
+            else:
+                DEFAULT_LOGGER.debug("print query", extra={"label": self, "sql": sql})
 
         except (Exception, TypeError) as e:
-            DEFAULT_LOGGER.exception("fail to execute sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         row = self.spark.sql(sql).collect()[0]
@@ -425,7 +426,7 @@ class Processor(Generator):
 
         except (Exception, TypeError) as e:
             DEFAULT_LOGGER.debug("context", extra={"label": self, "context": context})
-            DEFAULT_LOGGER.exception("fail to generate sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         return sql

fabricks/context/config.py

@@ -0,0 +1,127 @@
+import logging
+import os
+from typing import Final
+
+from fabricks.context.helpers import get_config_from_file
+from fabricks.utils.log import get_logger
+from fabricks.utils.path import Path
+from fabricks.utils.spark import spark
+
+logger, _ = get_logger("logs", level=logging.DEBUG)
+file_path, file_config, origin = get_config_from_file()
+
+if file_path:
+    logger.debug(f"found {origin} config ({file_path})", extra={"label": "config"})
+
+# path to runtime
+runtime = os.environ.get("FABRICKS_RUNTIME", "none")
+runtime = None if runtime.lower() == "none" else runtime
+if runtime is None:
+    if runtime := file_config.get("runtime"):
+        assert file_path is not None
+        runtime = file_path.joinpath(runtime)
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve runtime from env", extra={"label": "config"})
+
+if runtime is None:
+    if file_path is not None:
+        runtime = file_path
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+    else:
+        raise ValueError(
+            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
+        )
+
+path_runtime = Path(runtime, assume_git=True)
+
+# path to config
+config = os.environ.get("FABRICKS_CONFIG")
+if config is None:
+    if config := file_config.get("config"):
+        assert file_path is not None
+        config = file_path.joinpath(config)
+        logger.debug(f"resolve config from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve config from env", extra={"label": "config"})
+
+if config is None:
+    logger.debug("resolve config from default path", extra={"label": "config"})
+    config = path_runtime.joinpath(
+        "fabricks",
+        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
+    ).string
+
+path_config = Path(config, assume_git=True)
+
+# path to notebooks
+notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
+notebooks = None if notebooks.lower() == "none" else notebooks
+if notebooks is None:
+    if notebooks := file_config.get("notebooks"):
+        assert file_path is not None
+        notebooks = file_path.joinpath(notebooks)
+        logger.debug(f"resolve notebooks from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve notebooks from env", extra={"label": "config"})
+
+if notebooks is None:
+    logger.debug("resolve notebooks from default path", extra={"label": "config"})
+    notebooks = path_runtime.joinpath("notebooks")
+
+path_notebooks = Path(str(notebooks), assume_git=True)
+
+# job config from yaml
+is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
+if is_job_config_from_yaml is None:
+    if is_job_config_from_yaml := file_config.get("job_config_from_yaml"):
+        logger.debug(f"resolve job_config_from_yaml from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve job_config_from_yaml from env", extra={"label": "config"})
+
+# debug mode
+is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
+if is_debugmode is None:
+    if is_debugmode := file_config.get("debugmode"):
+        logger.debug(f"resolve debugmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve debugmode from env", extra={"label": "config"})
+
+# dev mode
+is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
+if is_devmode is None:
+    if is_devmode := file_config.get("devmode"):
+        logger.debug(f"resolve devmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve devmode from env", extra={"label": "config"})
+
+# log level
+loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
+if loglevel is None:
+    if loglevel := file_config.get("loglevel"):
+        logger.debug(f"resolve loglevel from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve loglevel from env", extra={"label": "config"})
+
+loglevel = loglevel.upper() if loglevel else "INFO"
+if loglevel == "DEBUG":
+    _loglevel = logging.DEBUG
+elif loglevel == "INFO":
+    _loglevel = logging.INFO
+elif loglevel == "WARNING":
+    _loglevel = logging.WARNING
+elif loglevel == "ERROR":
+    _loglevel = logging.ERROR
+elif loglevel == "CRITICAL":
+    _loglevel = logging.CRITICAL
+else:
+    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
+
+# Constants
+PATH_CONFIG: Final[Path] = path_config
+PATH_RUNTIME: Final[Path] = path_runtime
+PATH_NOTEBOOKS: Final[Path] = path_notebooks
+IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
+IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
+IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
+LOGLEVEL: Final[int] = _loglevel

fabricks/context/helpers.py

@@ -0,0 +1,58 @@
+import os
+import pathlib
+from typing import List
+
+from fabricks.utils.path import Path
+
+
+def get_config_from_file():
+    path = pathlib.Path(os.getcwd())
+
+    while path is not None and (not (path / "pyproject.toml").exists() or (path / "fabricksconfig.json").exists()):
+        if path == path.parent:
+            break
+        path = path.parent
+
+    if (path / "fabricksconfig.json").exists():
+        import json
+
+        with open((path / "fabricksconfig.json"), "r") as f:
+            config = json.load(f)
+            return path, config, "json"
+
+    if (path / "pyproject.toml").exists():
+        import sys
+
+        if sys.version_info >= (3, 11):
+            import tomllib
+        else:
+            import tomli as tomllib  # type: ignore
+
+        with open((path / "pyproject.toml"), "rb") as f:
+            config = tomllib.load(f)
+            return path, config.get("tool", {}).get("fabricks", {}), "pyproject"
+
+    return None, {}, None
+
+
+def get_storage_paths(objects: List[dict], variables: dict) -> dict:
+    d = {}
+    for o in objects:
+        if o:
+            name = o.get("name")
+            assert name
+            uri = o.get("path_options", {}).get("storage")
+            assert uri
+            d[name] = Path.from_uri(uri, regex=variables)
+    return d
+
+
+def get_runtime_path(objects: List[dict], root: Path) -> dict:
+    d = {}
+    for o in objects:
+        name = o.get("name")
+        assert name
+        uri = o.get("path_options", {}).get("runtime")
+        assert uri
+        d[name] = root.joinpath(uri)
+    return d

fabricks/context/runtime.py

@@ -1,11 +1,12 @@
-from typing import Final, List, Optional
+from typing import Final, Optional
 
 import yaml
 
-from fabricks.context.config import path_config, path_runtime
+from fabricks.context.config import PATH_CONFIG, PATH_RUNTIME
+from fabricks.context.helpers import get_runtime_path, get_storage_paths
 from fabricks.utils.path import Path
 
-with open(str(path_config)) as f:
+with open(str(PATH_CONFIG)) as f:
     data = yaml.safe_load(f)
 
 conf: dict = [d["conf"] for d in data][0]
@@ -51,67 +52,42 @@ FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_cr
 
 path_udfs = path_options.get("udfs", "fabricks/udfs")
 assert path_udfs, "path to udfs mandatory"
-PATH_UDFS: Final[Path] = path_runtime.joinpath(path_udfs)
+PATH_UDFS: Final[Path] = PATH_RUNTIME.joinpath(path_udfs)
 
 path_parsers = path_options.get("parsers", "fabricks/parsers")
 assert path_parsers, "path to parsers mandatory"
-PATH_PARSERS: Final[Path] = path_runtime.joinpath(path_parsers)
+PATH_PARSERS: Final[Path] = PATH_RUNTIME.joinpath(path_parsers)
 
 path_extenders = path_options.get("extenders", "fabricks/extenders")
 assert path_extenders, "path to extenders mandatory"
-PATH_EXTENDERS: Final[Path] = path_runtime.joinpath(path_extenders)
+PATH_EXTENDERS: Final[Path] = PATH_RUNTIME.joinpath(path_extenders)
 
 path_views = path_options.get("views", "fabricks/views")
 assert path_views, "path to views mandatory"
-PATH_VIEWS: Final[Path] = path_runtime.joinpath(path_views)
+PATH_VIEWS: Final[Path] = PATH_RUNTIME.joinpath(path_views)
 
 path_schedules = path_options.get("schedules", "fabricks/schedules")
 assert path_schedules, "path to schedules mandatory"
-PATH_SCHEDULES: Final[Path] = path_runtime.joinpath(path_schedules)
+PATH_SCHEDULES: Final[Path] = PATH_RUNTIME.joinpath(path_schedules)
 
 path_requirements = path_options.get("requirements", "fabricks/requirements")
 assert path_requirements, "path to requirements mandatory"
-PATH_REQUIREMENTS: Final[Path] = path_runtime.joinpath(path_requirements)
+PATH_REQUIREMENTS: Final[Path] = PATH_RUNTIME.joinpath(path_requirements)
 
 path_masks = path_options.get("masks", "fabricks/masks")
 assert path_masks, "path to masks mandatory"
-PATH_MASKS: Final[Path] = path_runtime.joinpath(path_masks)
-
-
-def _get_storage_paths(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        if o:
-            name = o.get("name")
-            assert name
-            uri = o.get("path_options", {}).get("storage")
-            assert uri
-            d[name] = Path.from_uri(uri, regex=variables)
-    return d
-
+PATH_MASKS: Final[Path] = PATH_RUNTIME.joinpath(path_masks)
 
 PATHS_STORAGE: Final[dict[str, Path]] = {
     "fabricks": FABRICKS_STORAGE,
-    **_get_storage_paths(BRONZE),
-    **_get_storage_paths(SILVER),
-    **_get_storage_paths(GOLD),
-    **_get_storage_paths(databases),
+    **get_storage_paths(BRONZE, variables),
+    **get_storage_paths(SILVER, variables),
+    **get_storage_paths(GOLD, variables),
+    **get_storage_paths(databases, variables),
 }
 
-
-def _get_runtime_path(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        name = o.get("name")
-        assert name
-        uri = o.get("path_options", {}).get("runtime")
-        assert uri
-        d[name] = path_runtime.joinpath(uri)
-    return d
-
-
 PATHS_RUNTIME: Final[dict[str, Path]] = {
-    **_get_runtime_path(BRONZE),
-    **_get_runtime_path(SILVER),
-    **_get_runtime_path(GOLD),
+    **get_runtime_path(BRONZE, PATH_RUNTIME),
+    **get_runtime_path(SILVER, PATH_RUNTIME),
+    **get_runtime_path(GOLD, PATH_RUNTIME),
 }

fabricks/core/dags/processor.py

@@ -10,7 +10,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_ex
 
 from fabricks.context import PATH_NOTEBOOKS
 from fabricks.core.dags.base import BaseDags
-from fabricks.core.dags.log import LOGGER
+from fabricks.core.dags.log import LOGGER, TABLE_LOG_HANDLER
 from fabricks.core.dags.run import run
 from fabricks.core.jobs.base._types import TStep
 from fabricks.core.steps.get_step import get_step
@@ -148,7 +148,9 @@ class DagProcessor(BaseDags):
                 finally:
                     j["Status"] = "ok"
                     self.table.upsert(j)
+
                     LOGGER.info("end", extra=self.extra(j))
+                    TABLE_LOG_HANDLER.flush()
 
                 dependencies = self.table.query(f"PartitionKey eq 'dependencies' and ParentId eq '{j.get('JobId')}'")
                 self.table.delete(dependencies)

fabricks/core/extenders.py

@@ -1,8 +1,8 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Callable
 
 from fabricks.context import IS_UNITY_CATALOG, PATH_EXTENDERS
 from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.utils.helpers import load_module_from_path
 
 EXTENDERS: dict[str, Callable] = {}
 
@@ -14,12 +14,7 @@ def get_extender(name: str) -> Callable:
     else:
         DEFAULT_LOGGER.debug(f"could not check if extender exists ({path.string})")
 
-    spec = spec_from_file_location(name, path.string)
-    assert spec, "no valid extender found in {path.string}"
-    assert spec.loader is not None
-
-    mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
+    load_module_from_path(name, path)
     e = EXTENDERS[name]
 
     return e

fabricks/core/jobs/base/generator.py

@@ -336,10 +336,10 @@ class Generator(Configurator):
 
                 comment = self.options.table.get("comment")
                 if comment:
-                    self.table.add_comment(comment=comment)
+                    self.table.add_table_comment(comment=comment)
 
         else:
-            DEFAULT_LOGGER.debug("table exists, skip creation", extra={"label": self})
+            DEFAULT_LOGGER.debug("table already exists, skipped creation", extra={"label": self})
 
     def _update_schema(
         self,
@@ -389,6 +389,24 @@ class Generator(Configurator):
     def overwrite_schema(self, df: Optional[DataFrame] = None):
         self._update_schema(df=df, overwrite=True)
 
+    def update_comments(self, table: Optional[bool] = True, columns: Optional[bool] = True):
+        if self.virtual:
+            return
+
+        if self.persist:
+            self.table.drop_comments()
+
+            if table:
+                comment = self.options.table.get("comment")
+                if comment:
+                    self.table.add_table_comment(comment=comment)
+
+            if columns:
+                comments = self.options.table.get_dict("comments")
+                if comments:
+                    for col, comment in comments.items():
+                        self.table.add_column_comment(column=col, comment=comment)
+
     def get_differences_with_deltatable(self, df: Optional[DataFrame] = None):
         if df is None:
             df = self.get_data(stream=self.stream)

fabricks/core/parsers/get_parser.py

@@ -1,9 +1,9 @@
-from importlib.util import module_from_spec, spec_from_file_location
 from typing import Optional
 
 from fabricks.context import PATH_PARSERS
 from fabricks.core.parsers._types import ParserOptions
 from fabricks.core.parsers.base import PARSERS, BaseParser
+from fabricks.utils.helpers import load_module_from_path
 
 
 def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> BaseParser:
@@ -11,12 +11,7 @@ def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> Bas
         path = PATH_PARSERS.joinpath(name).append(".py")
         assert path.exists(), f"parser not found ({path})"
 
-        spec = spec_from_file_location(name, path.string)
-        assert spec, f"parser not found ({path})"
-        assert spec.loader is not None
-
-        mod = module_from_spec(spec)
-        spec.loader.exec_module(mod)
+        load_module_from_path(name, path)
         parser = PARSERS[name](parser_options)
 
     else:

fabricks/core/steps/base.py

@@ -228,15 +228,22 @@ class BaseStep:
             DEFAULT_LOGGER.exception("fail to get jobs", extra={"label": self})
             raise e
 
-    def create_db_objects(self, retry: Optional[bool] = True) -> List[Dict]:
+    def create_db_objects(
+        self,
+        retry: Optional[bool] = True,
+        update_lists: Optional[bool] = True,
+        incremental: Optional[bool] = False,
+    ) -> List[Dict]:
         DEFAULT_LOGGER.info("create db objects", extra={"label": self})
 
         df = self.get_jobs()
-        table_df = self.database.get_tables()
-        view_df = self.database.get_views()
 
-        df = df.join(table_df, "job_id", how="left_anti")
-        df = df.join(view_df, "job_id", how="left_anti")
+        if incremental:
+            table_df = self.database.get_tables()
+            view_df = self.database.get_views()
+
+            df = df.join(table_df, "job_id", how="left_anti")
+            df = df.join(view_df, "job_id", how="left_anti")
 
         if df:
             results = run_in_parallel(
@@ -248,15 +255,16 @@ class BaseStep:
                 loglevel=logging.CRITICAL,
             )
 
-        self.update_tables_list()
-        self.update_views_list()
+        if update_lists:
+            self.update_tables_list()
+            self.update_views_list()
 
         errors = [res for res in results if res.get("error")]
 
         if errors:
             if retry:
                 DEFAULT_LOGGER.warning("retry to create jobs", extra={"label": self})
-                return self.create_db_objects(retry=False)
+                return self.create_db_objects(retry=False, update_lists=update_lists, incremental=incremental)
 
         return errors
 

fabricks/deploy/views.py

@@ -276,6 +276,7 @@ def create_or_replace_logs_pivot_view():
         l.step,
         l.job,
         l.job_id,
+        -- flags
         collect_set(l.status) as statuses,
         array_contains(statuses, 'skipped') as skipped,
         array_contains(statuses, 'warned') as warned,
@@ -283,15 +284,19 @@ def create_or_replace_logs_pivot_view():
         array_contains(statuses, 'failed') or (not done and not skipped) as failed,
         not done and not failed and not skipped and array_contains(statuses, 'running') as timed_out,
         not array_contains(statuses, 'running') as cancelled,
+        --
         max(l.notebook_id) as notebook_id,
+        --
+        max(l.timestamp) filter (where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status in ('done', 'ok')) as end_time,
+        --
         max(l.timestamp) filter (where l.status = 'scheduled' ) as scheduled_time,
         max(l.timestamp) filter (where l.status = 'waiting' ) as waiting_time,
-        max(l.timestamp) filter (where l.status = 'running') as start_time,
         max(l.timestamp) filter (where l.status = 'running' ) as running_time,
         max(l.timestamp) filter (where l.status = 'done' ) as done_time,
         max(l.timestamp) filter (where l.status = 'failed' ) as failed_time,
-        max(l.timestamp) filter(where l.status = 'ok') as end_time,
-        max(l.timestamp) filter(where l.status = 'ok') as ok_time,
+        max(l.timestamp) filter (where l.status = 'ok') as ok_time,
+        --
         max(l.exception) as exception
       from
         fabricks.logs l

fabricks/metastore/table.py

@@ -666,7 +666,45 @@ class Table(DbObject):
             """
         )
 
-    def add_comment(self, comment: str):
+    def drop_comments(self):
+        self.drop_table_comment()
+        for col in self.columns:
+            self.drop_column_comment(col)
+
+    def drop_table_comment(self):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug("drop table comment", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on table {self.qualified_name}
+            is null;
+            """
+        )
+
+    def drop_column_comment(self, column: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"drop comment from column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is null;
+            """
+        )
+
+    def add_column_comment(self, column: str, comment: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"add comment '{comment}' to column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is '{comment}';
+            """
+        )
+
+    def add_table_comment(self, comment: str):
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.debug(f"add comment '{comment}'", extra={"label": self})

fabricks/utils/azure_table.py

@@ -1,5 +1,4 @@
-import time
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 from azure.data.tables import TableClient, TableServiceClient
 from pyspark.sql import DataFrame
@@ -99,27 +98,29 @@ class AzureTable:
         if self._table_client is not None:
             self._table_client.close()
 
-    def submit(self, operations: List, retry: Optional[bool] = True):
-        try:
-            partitions = set()
-            for d in operations:
-                partitions.add(d[1]["PartitionKey"])
-
-            for p in partitions:
-                _operations = [d for d in operations if d[1].get("PartitionKey") == p]
-                t = 50
-                if len(_operations) < t:
-                    self.table.submit_transaction(_operations)
-                else:
-                    transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
-                    for transaction in transactions:
-                        self.table.submit_transaction(transaction)
-        except Exception as e:
-            if retry:
-                time.sleep(10)
-                self.submit(operations, retry=False)
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        retry=retry_if_exception_type((Exception)),
+        reraise=True,
+    )
+    def _submit_with_retry(self, data: Any):
+        self.table.submit_transaction(data)
+
+    def submit(self, operations: List):
+        partitions = set()
+        for d in operations:
+            partitions.add(d[1]["PartitionKey"])
+
+        for p in partitions:
+            _operations = [d for d in operations if d[1].get("PartitionKey") == p]
+            t = 50
+            if len(_operations) < t:
+                self._submit_with_retry(_operations)
             else:
-                raise e
+                transactions = [_operations[i : i + t] for i in range(0, len(_operations), t)]
+                for transaction in transactions:
+                    self._submit_with_retry(transaction)
 
     def delete(self, data: Union[List, DataFrame, dict]):
         if isinstance(data, DataFrameLike):

fabricks/utils/helpers.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 from functools import reduce
 from queue import Queue
 from typing import Any, Callable, Iterable, List, Literal, Optional, Union
@@ -216,13 +217,28 @@ def run_notebook(path: Path, timeout: Optional[int] = None, **kwargs):
     dbutils.notebook.run(path.get_notebook_path(), timeout, {**kwargs})  # type: ignore
 
 
-def xxhash64(s: Any):
+def xxhash64(s: Any) -> int:
     df = spark.sql(f"select xxhash64(cast('{s}' as string)) as xxhash64")
     return df.collect()[0][0]
 
 
-def md5(s: Any):
+def md5(s: Any) -> str:
     from hashlib import md5
 
     md5 = md5(str(s).encode())
     return md5.hexdigest()
+
+
+def load_module_from_path(name: str, path: Path):
+    from importlib.util import module_from_spec, spec_from_file_location
+
+    sys.path.append(str(path.parent))
+
+    spec = spec_from_file_location(name, path.string)
+    assert spec, f"no valid module found in {path.string}"
+    assert spec.loader is not None
+
+    textwrap_module = module_from_spec(spec)
+    spec.loader.exec_module(textwrap_module)
+
+    return textwrap_module

fabricks/utils/path.py

@@ -1,4 +1,5 @@
 import os
+import posixpath
 from pathlib import Path as PathlibPath
 from typing import List, Optional, Union
 
@@ -120,8 +121,13 @@ class Path:
             return False
 
     def joinpath(self, *other):
-        new_path = self.pathlibpath.joinpath(*other)
-        return Path(path=new_path, assume_git=self.assume_git)
+        parts = [str(o) for o in other]
+        base = self.string
+
+        joined = posixpath.join(base, *parts)
+        new = posixpath.normpath(joined)
+
+        return Path(path=new, assume_git=self.assume_git)
 
     def append(self, other: str):
         new_path = self.string + other

{fabricks-3.0.11.dist-info → fabricks-3.0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.11
+Version: 3.0.13
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.11.dist-info → fabricks-3.0.13.dist-info}/RECORD

@@ -40,8 +40,8 @@ fabricks/cdc/base/_types.py,sha256=WloCDC3ATrn0aZJ6E8BRYKZx19N3EE56r6qlBYhcuvQ,2
 fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
 fabricks/cdc/base/configurator.py,sha256=w6Ywif87iv1WG-5OM3XkzIRrsns-_QQ6XlADpk0YLlw,6434
 fabricks/cdc/base/generator.py,sha256=pa_GJn7Pdi5vMnXN8zExmOPMpCqdZ3QoxHEB0wv0lsk,5933
-fabricks/cdc/base/merger.py,sha256=1_j-hKnKKEcbogyXX0Cm2IdyB-tpDJAIMIz1MwWnXX0,4158
-fabricks/cdc/base/processor.py,sha256=b8ATjVX-dW8JCrt4n3v8HlteTi1498jrItEla0BqynU,17689
+fabricks/cdc/base/merger.py,sha256=3qUUs0uqmwOMdXc50kV3Zo9omuQuUUFgtMLBrg4E-wk,4159
+fabricks/cdc/base/processor.py,sha256=gL3pWMaBRsc0oB93ISnH2x07WbmtM_QEIx8qrUcUoZ0,17704
 fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/cdc/templates/filter.sql.jinja,sha256=AQcOj4KRwKscVG3zepnEAN9Yxb03AM7068hqW7dtVI8,236
 fabricks/cdc/templates/merge.sql.jinja,sha256=YS9wWckCVsUI1pUYiRSFBIuQ16WU3EPWSkhZVy2niBA,221
@@ -69,15 +69,15 @@ fabricks/cdc/templates/queries/nocdc/complete.sql.jinja,sha256=cVKKCSbiuuw1K7BOz
 fabricks/cdc/templates/queries/nocdc/update.sql.jinja,sha256=mjNUwGVhZ08yUkdv9sCTkqyW60p0YavtWTqvSUVrwjA,1283
 fabricks/context/__init__.py,sha256=qfntJ9O6omzY_t6AhDP6Ndu9C5LMiVdWbo6ikhtoe7o,1446
 fabricks/context/_types.py,sha256=FzQJ35vp0uc6pAq18bc-VHwMVEWtd0VDdm8xQmNr2Sg,2681
+fabricks/context/config.py,sha256=EmLUnswuWfrncaNJMDjvdMg-1lD8aneKAY8IDna7VPE,4814
+fabricks/context/helpers.py,sha256=nyHF0yhTJ_cDQT5Q47z0CYh4QHOJyzyRNRBfrUYDIbU,1552
 fabricks/context/log.py,sha256=CadrRf8iL6iXlGIGIhEIswa7wGqC-E-oLwWcGTyJ10s,2074
-fabricks/context/runtime.py,sha256=7pXU_5gPyvChwiyxP4ch59eAgyoNOH9jMUlGWxwXWn8,3915
+fabricks/context/runtime.py,sha256=87PtX6SqLoFd0PGxgisF6dLlxtCHaHxkMMIt34UyB2w,3479
 fabricks/context/secret.py,sha256=iRM-KU-JcJAEOLoGJ8S4Oh65-yt674W6CDTSkOE7SXw,3192
 fabricks/context/spark_session.py,sha256=BPaxKJXHZDI5oQiOPhmua_xjXnrVgluh--AVpvUgbck,2553
 fabricks/context/utils.py,sha256=EQRscdUhdjwk2htZu8gCgNZ9PfRzzrR6e1kRrIbVlBM,2786
-fabricks/context/config/__init__.py,sha256=pFEsGXBQkX5_FP0cwQMX427j6dQuTx81NR9snMxc8cU,3127
-fabricks/context/config/utils.py,sha256=7KCTUiSbqQnDD5mbCO9_o1KbUgD-Xbei_UGgpMQi9nE,1371
 fabricks/core/__init__.py,sha256=LaqDi4xuyHAoLOvS44PQdZdRfq9SmVr7mB6BDHyxYpc,209
-fabricks/core/extenders.py,sha256=39bSm9QiW4vBAyT659joE-5p_EZiNM4gi8KA3-OgX3E,917
+fabricks/core/extenders.py,sha256=oJzfv0hWxusnGmrjMwbrGyKfot8xzA4XtNquPWfFgPo,727
 fabricks/core/job_schema.py,sha256=6-70oy0ZJd3V9AiXfc0Q8b8NVEynxQza_h7mB13uB-s,853
 fabricks/core/masks.py,sha256=3UCxcCi-TgFHB7xT5ZvmEa8RMKev23X_JLE70Pr3rpY,1347
 fabricks/core/udfs.py,sha256=gu7K-ohxcO0TdgA7IjzVMOZatZQYhFTklHo60a6k_Yc,2960
@@ -86,7 +86,7 @@ fabricks/core/dags/__init__.py,sha256=0DUKzVcXcROvxkN19P_kaOJ7da5BAM7Vt8EGQbp2KS
 fabricks/core/dags/base.py,sha256=tFj27SqeZUZ7pB_LOWkpdowZz5gj30JUANI4gWK3Pl8,3139
 fabricks/core/dags/generator.py,sha256=4fp_CRsWnl_UauM9Jx-E4UCaxnm2_Q5103J58fRws2U,4832
 fabricks/core/dags/log.py,sha256=v1xfpQGfddHDz9lflvXOWTXMde3CdERo9jzeSmNDRhY,402
-fabricks/core/dags/processor.py,sha256=dcEWk0y2fuNt7RfxkeCny5axdg5aE4UpD7QwEijcQWM,7864
+fabricks/core/dags/processor.py,sha256=IzjqrpNu6lTYp-Rl2T_8Sb5N0pSk9BhdP9vuL4sBRMg,7930
 fabricks/core/dags/run.py,sha256=RIDULb9WakObSyYzmkglh8RwFRwC8-NFC-1yPDMkBC0,1074
 fabricks/core/dags/terminator.py,sha256=Y6pV2UnSyrCIx2AQWJXoHk5Roq12gZqpennHx_Lbnzs,793
 fabricks/core/dags/utils.py,sha256=4kyobLGl4tO0Flo6LxNzYjCU_G42vns1LrkxTO5_KLY,1585
@@ -105,7 +105,7 @@ fabricks/core/jobs/base/_types.py,sha256=y66BtJlJskq7wGzn7te5XYjO-NEqeQGUC11kkbe
 fabricks/core/jobs/base/checker.py,sha256=Cdfh8rQYy4MvMFl0HyC3alGUWm8zrFXk08m2t2JMu6Y,5477
 fabricks/core/jobs/base/configurator.py,sha256=9G5F7Qg5FWHPbHgdh8Qxc85OoSX0rnjD4c9itwU5KKc,10415
 fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
-fabricks/core/jobs/base/generator.py,sha256=NlJgR1461rx4bOBfvpmBrS6PvNeHD6QH6FOqAZiXDvM,16987
+fabricks/core/jobs/base/generator.py,sha256=Dk82tj21NhR9wwgXzMp8JlKQ6D9HnjVlK9fvDmoYLbk,17646
 fabricks/core/jobs/base/invoker.py,sha256=FvjfpNqi542slxC2yLu1BIu5EklNUWySxDF8cD_SqKQ,7602
 fabricks/core/jobs/base/job.py,sha256=dWmk2PpQH2NETaaDS6KoiefRnDHfDMdCyhmogkdcSFI,93
 fabricks/core/jobs/base/processor.py,sha256=qkNiJSSLaEnivKGBcd9UZyIVFexnv-n1p_5mCZIy1rA,9076
@@ -113,7 +113,7 @@ fabricks/core/parsers/__init__.py,sha256=TGjyUeiiTkJrAxIpu2D_c2rQcbe5YRpmBW9oh0F
 fabricks/core/parsers/_types.py,sha256=JC2Oh-wUvaX8SBzeuf5owPgRaj-Q3-7MXxyIYPQ7QwA,147
 fabricks/core/parsers/base.py,sha256=P8IrLQKGakwaAQ-4gf4vElVwWoSpkixYd9kNthu1VDM,3292
 fabricks/core/parsers/decorator.py,sha256=kn_Mj-JLWTFaRiciZ3KavmSUcWFPY3ve-buMruHrX_Q,307
-fabricks/core/parsers/get_parser.py,sha256=TTnVPwKqKpFu6jJJnXEuiEctWGtimk8w2p1jF2U7ibg,909
+fabricks/core/parsers/get_parser.py,sha256=mauofS626h9wpPZtlZFqIb1jcKM3Jz4D_36uvd-Lv4k,717
 fabricks/core/parsers/utils.py,sha256=qdn2ElpqBgDsW55-tACWZaFOT0ebrBYg2fenqSgd6YI,2456
 fabricks/core/schedules/__init__.py,sha256=bDjNMcm7itimAo4gun0W4W9bZKwZmWUjkMqAQIcqI2Y,431
 fabricks/core/schedules/diagrams.py,sha256=YA4T7Etl_UPfW-3IGFq5Xj9OlXZGQ27Aot6RVa3ZUgg,578
@@ -126,7 +126,7 @@ fabricks/core/schedules/terminate.py,sha256=-RvtOrxTOZl2sZQ6KfNHJL6H2LCAEMSVRyyl
 fabricks/core/schedules/views.py,sha256=8hYwPLCvvN-nem2lNAKvUY5hC71v88z4-y8j0poUApM,1949
 fabricks/core/steps/__init__.py,sha256=JP-kaDa890-9XqBSPp6YdssAexdxv-MqQ__WfVYdgeg,132
 fabricks/core/steps/_types.py,sha256=VxIrH3nFwmPlwG-UI8sDDP0AwK_9jlsy6yQp6YfgtqE,90
-fabricks/core/steps/base.py,sha256=UTzVqdWaho9zgMrloq8ndqcWQ9R5Z-O1SInIrm9byB4,14403
+fabricks/core/steps/base.py,sha256=MJe2q9s1siM89YkpHDqldtbtKQgkhDB_cFa2-e_irvs,14642
 fabricks/core/steps/get_step.py,sha256=8q4rEDdTTZNJsXB2l5XY-Ktoow8ZHsON_tx5yKMUIzg,284
 fabricks/core/steps/get_step_conf.py,sha256=UPT3gB1Sh5yzawZ9qiVQlvVAKaxPX82gaWBDzxx75EM,633
 fabricks/deploy/__init__.py,sha256=ntxtFnzeho_WneVoL5CCqbI4rKApKgdmi9j0HKb0LJc,2375
@@ -136,26 +136,26 @@ fabricks/deploy/schedules.py,sha256=0a5dU1rW6fg8aAp7TTt-l0DgR-4kmzsX2xxV2C30yaw,
 fabricks/deploy/tables.py,sha256=IF822oxOCy12r08Dz54YUK5luud6dtTPxJ4TUIHE-No,2621
 fabricks/deploy/udfs.py,sha256=7fw3O5LgOOxDEhuS3s1yFdqybgFh65r_1IdfZUYeejs,597
 fabricks/deploy/utils.py,sha256=V41r1zVT9KcsICqTLAzpb4ixRk2q2ybJMrGhkPOtG6k,5099
-fabricks/deploy/views.py,sha256=Ket511q0v8QHgkNArb4qVPFDuuQnQn3v-fvr7KO6oxM,14360
+fabricks/deploy/views.py,sha256=8cSt6IzZy-JHHkyqd91NT2hi3LTNTOolimlfSBXMCvU,14434
 fabricks/metastore/README.md,sha256=utPUGAxmjyNMGe43GfL0Gup4MjeTKKwyiUoNVSfMquI,51
 fabricks/metastore/__init__.py,sha256=RhjY2CuqtZBg8fEizzzvW8qszqCM-vSCL1tQGuzoato,174
 fabricks/metastore/_types.py,sha256=NXYxwQHP0sCllM0N6QBbaK4CdtM_m_rHFDxRNRfBcLU,1919
 fabricks/metastore/database.py,sha256=23VAKKzjrwlEaj28DNNmiOhcfdKRzYk8eEfq-PzINbg,1924
 fabricks/metastore/dbobject.py,sha256=ve8p48OqEpJYsqWNhgesGSE0emM--uY8QrvBRoR3j3g,1881
 fabricks/metastore/pyproject.toml,sha256=6RZM9RMKMDF_EAequhORZ7TD0BQNk7aBCTWAv-sRcp0,519
-fabricks/metastore/table.py,sha256=mos701lEU_EwjdwwGb2ey-RSebkg8Ubms2VcOJvIARA,27937
+fabricks/metastore/table.py,sha256=AaoNL-1mz4A0CCb3tH_0BUurYPjA1oL5pioCYlEMtu4,29113
 fabricks/metastore/utils.py,sha256=8SxhjDkz_aSH4IGUusel7hqOQxP9U8PNBCY0M7GH00Y,1355
 fabricks/metastore/view.py,sha256=f7hKJWtnH1KmZym8dkoucKOTndntzai_f2YqferxHLs,1431
 fabricks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/utils/_types.py,sha256=AuOhknlozqx5QdAdvZSA6xAWhU8k4nxG1vxIkOVgHeY,184
 fabricks/utils/azure_queue.py,sha256=wtKAq_MD5QLxelerDO475dzL-SySIrxt9d5KGi-8vvw,3102
-fabricks/utils/azure_table.py,sha256=7tCD1iM7UWREaSQVVmtgHCAebLtWPP9ZmuU5zDALyo0,5305
+fabricks/utils/azure_table.py,sha256=J_UAPiCN89rL5FNmwIGg43Z6FSH8evVIWijDEgYJxk4,5294
 fabricks/utils/console.py,sha256=X4lLgL_UxCjoFRx-ZRCwzdBveRGPKlFYZDi6vl7uevQ,1017
 fabricks/utils/fdict.py,sha256=cdnvNBSXKJIDKSdhQGJA4CGv0qLn5IVYKQ111l7nM9I,7978
-fabricks/utils/helpers.py,sha256=h7SuOVpBP5qcgX1nM1suvkXG9BhiK5-257EBepCvrO8,7452
+fabricks/utils/helpers.py,sha256=fKv6mpT-428xTSjdLfm7TnN1Xo9FadrSIY1qzYgWCzs,7909
 fabricks/utils/log.py,sha256=LCQEM81PhdojiyLrtEzv1QM__bWbaEhGddyd0IqyGXM,7985
 fabricks/utils/mermaid.py,sha256=XoiVxPaUJS4TC_ybA-e78qFzQkQ46uPf055JiiNDdSg,986
-fabricks/utils/path.py,sha256=Bs3PayWtg62-mrsDbvu8kh0VLZZhX7tU9YiyHFiYNhs,6698
+fabricks/utils/path.py,sha256=ToTTS8QKGsWq8cR8SDE2ocWKx2GsuZ5psPTuAMq9R4s,6813
 fabricks/utils/pip.py,sha256=UHo7NTjFGJNghWBuuDow28xUkZYg2YrlbAP49IxZyXY,1522
 fabricks/utils/pydantic.py,sha256=W0fiDLVMFrrInfQw2s5YPeSEvkN-4k864u3UyPoHaz4,2516
 fabricks/utils/spark.py,sha256=QWVpbGwOvURIVBlR7ygt6NQ9MHUsIDvlquJ65iI8UBI,2007
@@ -171,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=5k-R6zCgUAtapQgxT4turcx1IQ-b
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=lTQ0CfUhcvn3xTCcT_Ns6PMDBsO5UEfa2S9XpJiLJ9c,1250
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.11.dist-info/METADATA,sha256=lDGJheQcvE7Psjm6fnUdvH9vMwHuag9sEWlcNNUFzf4,798
-fabricks-3.0.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.11.dist-info/RECORD,,
+fabricks-3.0.13.dist-info/METADATA,sha256=H7Yv6dwfPTscFWMQWayOB7t_eURWWYCGxPGzWsV71KQ,798
+fabricks-3.0.13.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+fabricks-3.0.13.dist-info/RECORD,,

{fabricks-3.0.11.dist-info → fabricks-3.0.13.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

fabricks/context/config/__init__.py

@@ -1,92 +0,0 @@
-import logging
-import os
-from typing import Final
-
-from fabricks.context.config.utils import get_config_from_file
-from fabricks.utils.path import Path
-from fabricks.utils.spark import spark
-
-file_path, file_config = get_config_from_file()
-
-runtime = os.environ.get("FABRICKS_RUNTIME", "none")
-runtime = None if runtime.lower() == "none" else runtime
-if runtime is None:
-    if runtime := file_config.get("runtime"):
-        assert file_path is not None
-        runtime = file_path.joinpath(runtime)
-
-if runtime is None:
-    if file_path is not None:
-        runtime = file_path
-    else:
-        raise ValueError(
-            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
-        )
-
-path_runtime = Path(runtime, assume_git=True)
-PATH_RUNTIME: Final[Path] = path_runtime
-
-notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
-notebooks = None if notebooks.lower() == "none" else notebooks
-if notebooks is None:
-    if notebooks := file_config.get("notebooks"):
-        assert file_path is not None
-        notebooks = file_path.joinpath(notebooks)
-
-notebooks = notebooks if notebooks else path_runtime.joinpath("notebooks")
-PATH_NOTEBOOKS: Final[Path] = Path(str(notebooks), assume_git=True)
-
-is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
-if is_job_config_from_yaml is None:
-    assert file_path is not None
-    is_job_config_from_yaml = file_config.get("job_config_from_yaml")
-
-IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
-
-is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
-if is_debugmode is None:
-    is_debugmode = file_config.get("debugmode")
-
-IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
-
-is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
-if is_devmode is None:
-    is_devmode = file_config.get("devmode")
-
-IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
-
-loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
-if loglevel is None:
-    loglevel = file_config.get("loglevel")
-
-loglevel = loglevel.upper() if loglevel else "INFO"
-if loglevel == "DEBUG":
-    _loglevel = logging.DEBUG
-elif loglevel == "INFO":
-    _loglevel = logging.INFO
-elif loglevel == "WARNING":
-    _loglevel = logging.WARNING
-elif loglevel == "ERROR":
-    _loglevel = logging.ERROR
-elif loglevel == "CRITICAL":
-    _loglevel = logging.CRITICAL
-else:
-    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
-
-LOGLEVEL = _loglevel
-
-path_config = os.environ.get("FABRICKS_CONFIG")
-if path_config is None:
-    if path_config := file_config.get("config"):
-        assert file_path is not None
-        path_config = file_path.joinpath(path_config)
-else:
-    path_config = PATH_RUNTIME.joinpath(path_config).string if path_config else None
-
-if not path_config:
-    path_config = PATH_RUNTIME.joinpath(
-        "fabricks",
-        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
-    ).string
-
-PATH_CONFIG: Final[Path] = Path(path_config, assume_git=True)

fabricks/context/config/utils.py

@@ -1,53 +0,0 @@
-def get_config_from_toml():
-    import os
-    import pathlib
-    import sys
-
-    if sys.version_info >= (3, 11):
-        import tomllib
-    else:
-        import tomli as tomllib  # type: ignore
-
-    path = pathlib.Path(os.getcwd())
-    while path is not None and not (path / "pyproject.toml").exists():
-        if path == path.parent:
-            break
-        path = path.parent
-
-    if (path / "pyproject.toml").exists():
-        with open((path / "pyproject.toml"), "rb") as f:
-            config = tomllib.load(f)
-            return path, config.get("tool", {}).get("fabricks", {})
-
-    return None, {}
-
-
-def get_config_from_json():
-    import json
-    import os
-    import pathlib
-
-    path = pathlib.Path(os.getcwd())
-    while path is not None and not (path / "fabricksconfig.json").exists():
-        if path == path.parent:
-            break
-        path = path.parent
-
-    if (path / "fabricksconfig.json").exists():
-        with open((path / "fabricksconfig.json"), "r") as f:
-            config = json.load(f)
-            return path, config
-
-    return None, {}
-
-
-def get_config_from_file():
-    json_path, json_config = get_config_from_json()
-    if json_config:
-        return json_path, json_config
-
-    pyproject_path, pyproject_config = get_config_from_toml()
-    if pyproject_config:
-        return pyproject_path, pyproject_config
-
-    return None, {}