fabricks 3.0.10__py3-none-any.whl → 3.0.12__py3-none-any.whl

fabricks/cdc/base/merger.py

@@ -89,6 +89,7 @@ class Merger(Processor):
                 sql = sql.replace("{src}", "src")
                 sql = fix_sql(sql)
                 sql = sql.replace("`src`", "{src}")
+
                 DEFAULT_LOGGER.debug("merge", extra={"label": self, "sql": sql})
 
             except Exception as e:

fabricks/cdc/base/processor.py

@@ -390,11 +390,12 @@ class Processor(Generator):
         try:
             sql = template.render(**context)
             if fix:
-                DEFAULT_LOGGER.debug("fix context", extra={"label": self, "sql": sql})
                 sql = self.fix_sql(sql)
+            else:
+                DEFAULT_LOGGER.debug("print query", extra={"label": self, "sql": sql})
 
         except (Exception, TypeError) as e:
-            DEFAULT_LOGGER.exception("fail to execute sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         row = self.spark.sql(sql).collect()[0]
@@ -425,7 +426,7 @@ class Processor(Generator):
 
         except (Exception, TypeError) as e:
             DEFAULT_LOGGER.debug("context", extra={"label": self, "context": context})
-            DEFAULT_LOGGER.exception("fail to generate sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         return sql

fabricks/context/config.py

@@ -0,0 +1,127 @@
+import logging
+import os
+from typing import Final
+
+from fabricks.context.helpers import get_config_from_file
+from fabricks.utils.log import get_logger
+from fabricks.utils.path import Path
+from fabricks.utils.spark import spark
+
+logger, _ = get_logger("logs", level=logging.DEBUG)
+file_path, file_config, origin = get_config_from_file()
+
+if file_path:
+    logger.debug(f"found {origin} config ({file_path})", extra={"label": "config"})
+
+# path to runtime
+runtime = os.environ.get("FABRICKS_RUNTIME", "none")
+runtime = None if runtime.lower() == "none" else runtime
+if runtime is None:
+    if runtime := file_config.get("runtime"):
+        assert file_path is not None
+        runtime = file_path.joinpath(runtime)
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve runtime from env", extra={"label": "config"})
+
+if runtime is None:
+    if file_path is not None:
+        runtime = file_path
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+    else:
+        raise ValueError(
+            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
+        )
+
+path_runtime = Path(runtime, assume_git=True)
+
+# path to config
+config = os.environ.get("FABRICKS_CONFIG")
+if config is None:
+    if config := file_config.get("config"):
+        assert file_path is not None
+        config = file_path.joinpath(config)
+        logger.debug(f"resolve config from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve config from env", extra={"label": "config"})
+
+if config is None:
+    logger.debug("resolve config from default path", extra={"label": "config"})
+    config = path_runtime.joinpath(
+        "fabricks",
+        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
+    ).string
+
+path_config = Path(config, assume_git=True)
+
+# path to notebooks
+notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
+notebooks = None if notebooks.lower() == "none" else notebooks
+if notebooks is None:
+    if notebooks := file_config.get("notebooks"):
+        assert file_path is not None
+        notebooks = file_path.joinpath(notebooks)
+        logger.debug(f"resolve notebooks from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve notebooks from env", extra={"label": "config"})
+
+if notebooks is None:
+    logger.debug("resolve notebooks from default path", extra={"label": "config"})
+    notebooks = path_runtime.joinpath("notebooks")
+
+path_notebooks = Path(str(notebooks), assume_git=True)
+
+# job config from yaml
+is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
+if is_job_config_from_yaml is None:
+    if is_job_config_from_yaml := file_config.get("job_config_from_yaml"):
+        logger.debug(f"resolve job_config_from_yaml from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve job_config_from_yaml from env", extra={"label": "config"})
+
+# debug mode
+is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
+if is_debugmode is None:
+    if is_debugmode := file_config.get("debugmode"):
+        logger.debug(f"resolve debugmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve debugmode from env", extra={"label": "config"})
+
+# dev mode
+is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
+if is_devmode is None:
+    if is_devmode := file_config.get("devmode"):
+        logger.debug(f"resolve devmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve devmode from env", extra={"label": "config"})
+
+# log level
+loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
+if loglevel is None:
+    if loglevel := file_config.get("loglevel"):
+        logger.debug(f"resolve loglevel from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve loglevel from env", extra={"label": "config"})
+
+loglevel = loglevel.upper() if loglevel else "INFO"
+if loglevel == "DEBUG":
+    _loglevel = logging.DEBUG
+elif loglevel == "INFO":
+    _loglevel = logging.INFO
+elif loglevel == "WARNING":
+    _loglevel = logging.WARNING
+elif loglevel == "ERROR":
+    _loglevel = logging.ERROR
+elif loglevel == "CRITICAL":
+    _loglevel = logging.CRITICAL
+else:
+    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
+
+# Constants
+PATH_CONFIG: Final[Path] = path_config
+PATH_RUNTIME: Final[Path] = path_runtime
+PATH_NOTEBOOKS: Final[Path] = path_notebooks
+IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
+IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
+IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
+LOGLEVEL: Final[int] = _loglevel

fabricks/context/{config/utils.py → helpers.py}

@@ -1,3 +1,8 @@
+from typing import List
+
+from fabricks.utils.path import Path
+
+
 def get_config_from_toml():
     import os
     import pathlib
@@ -44,10 +49,33 @@ def get_config_from_json():
 def get_config_from_file():
     json_path, json_config = get_config_from_json()
     if json_config:
-        return json_path, json_config
+        return json_path, json_config, "json"
 
     pyproject_path, pyproject_config = get_config_from_toml()
     if pyproject_config:
-        return pyproject_path, pyproject_config
+        return pyproject_path, pyproject_config, "pyproject"
 
-    return None, {}
+    return None, {}, None
+
+
+def get_storage_paths(objects: List[dict], variables: dict) -> dict:
+    d = {}
+    for o in objects:
+        if o:
+            name = o.get("name")
+            assert name
+            uri = o.get("path_options", {}).get("storage")
+            assert uri
+            d[name] = Path.from_uri(uri, regex=variables)
+    return d
+
+
+def get_runtime_path(objects: List[dict], root: Path) -> dict:
+    d = {}
+    for o in objects:
+        name = o.get("name")
+        assert name
+        uri = o.get("path_options", {}).get("runtime")
+        assert uri
+        d[name] = root.joinpath(uri)
+    return d

fabricks/context/runtime.py

@@ -1,11 +1,12 @@
-from typing import Final, List, Optional
+from typing import Final, Optional
 
 import yaml
 
-from fabricks.context.config import path_config, path_runtime
+from fabricks.context.config import PATH_CONFIG, PATH_RUNTIME
+from fabricks.context.helpers import get_runtime_path, get_storage_paths
 from fabricks.utils.path import Path
 
-with open(str(path_config)) as f:
+with open(str(PATH_CONFIG)) as f:
     data = yaml.safe_load(f)
 
 conf: dict = [d["conf"] for d in data][0]
@@ -51,67 +52,42 @@ FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_cr
 
 path_udfs = path_options.get("udfs", "fabricks/udfs")
 assert path_udfs, "path to udfs mandatory"
-PATH_UDFS: Final[Path] = path_runtime.joinpath(path_udfs)
+PATH_UDFS: Final[Path] = PATH_RUNTIME.joinpath(path_udfs)
 
 path_parsers = path_options.get("parsers", "fabricks/parsers")
 assert path_parsers, "path to parsers mandatory"
-PATH_PARSERS: Final[Path] = path_runtime.joinpath(path_parsers)
+PATH_PARSERS: Final[Path] = PATH_RUNTIME.joinpath(path_parsers)
 
 path_extenders = path_options.get("extenders", "fabricks/extenders")
 assert path_extenders, "path to extenders mandatory"
-PATH_EXTENDERS: Final[Path] = path_runtime.joinpath(path_extenders)
+PATH_EXTENDERS: Final[Path] = PATH_RUNTIME.joinpath(path_extenders)
 
 path_views = path_options.get("views", "fabricks/views")
 assert path_views, "path to views mandatory"
-PATH_VIEWS: Final[Path] = path_runtime.joinpath(path_views)
+PATH_VIEWS: Final[Path] = PATH_RUNTIME.joinpath(path_views)
 
 path_schedules = path_options.get("schedules", "fabricks/schedules")
 assert path_schedules, "path to schedules mandatory"
-PATH_SCHEDULES: Final[Path] = path_runtime.joinpath(path_schedules)
+PATH_SCHEDULES: Final[Path] = PATH_RUNTIME.joinpath(path_schedules)
 
 path_requirements = path_options.get("requirements", "fabricks/requirements")
 assert path_requirements, "path to requirements mandatory"
-PATH_REQUIREMENTS: Final[Path] = path_runtime.joinpath(path_requirements)
+PATH_REQUIREMENTS: Final[Path] = PATH_RUNTIME.joinpath(path_requirements)
 
 path_masks = path_options.get("masks", "fabricks/masks")
 assert path_masks, "path to masks mandatory"
-PATH_MASKS: Final[Path] = path_runtime.joinpath(path_masks)
-
-
-def _get_storage_paths(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        if o:
-            name = o.get("name")
-            assert name
-            uri = o.get("path_options", {}).get("storage")
-            assert uri
-            d[name] = Path.from_uri(uri, regex=variables)
-    return d
-
+PATH_MASKS: Final[Path] = PATH_RUNTIME.joinpath(path_masks)
 
 PATHS_STORAGE: Final[dict[str, Path]] = {
     "fabricks": FABRICKS_STORAGE,
-    **_get_storage_paths(BRONZE),
-    **_get_storage_paths(SILVER),
-    **_get_storage_paths(GOLD),
-    **_get_storage_paths(databases),
+    **get_storage_paths(BRONZE, variables),
+    **get_storage_paths(SILVER, variables),
+    **get_storage_paths(GOLD, variables),
+    **get_storage_paths(databases, variables),
 }
 
-
-def _get_runtime_path(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        name = o.get("name")
-        assert name
-        uri = o.get("path_options", {}).get("runtime")
-        assert uri
-        d[name] = path_runtime.joinpath(uri)
-    return d
-
-
 PATHS_RUNTIME: Final[dict[str, Path]] = {
-    **_get_runtime_path(BRONZE),
-    **_get_runtime_path(SILVER),
-    **_get_runtime_path(GOLD),
+    **get_runtime_path(BRONZE, PATH_RUNTIME),
+    **get_runtime_path(SILVER, PATH_RUNTIME),
+    **get_runtime_path(GOLD, PATH_RUNTIME),
 }

fabricks/context/secret.py

@@ -40,7 +40,7 @@ def _get_secret_from_secret_scope(secret_scope: str, name: str) -> str:
     if not _scopes or secret_scope not in _scopes:  # we get the scopes only once, unless you search for something new
         _scopes = [s.name for s in dbutils.secrets.listScopes()]
 
-    assert secret_scope in _scopes, "scope {secret_scope} not found"
+    assert secret_scope in _scopes, f"scope {secret_scope} not found"
 
     return dbutils.secrets.get(scope=secret_scope, key=name)
 

fabricks/core/dags/processor.py

@@ -10,7 +10,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_ex
 
 from fabricks.context import PATH_NOTEBOOKS
 from fabricks.core.dags.base import BaseDags
-from fabricks.core.dags.log import LOGGER
+from fabricks.core.dags.log import LOGGER, TABLE_LOG_HANDLER
 from fabricks.core.dags.run import run
 from fabricks.core.jobs.base._types import TStep
 from fabricks.core.steps.get_step import get_step
@@ -148,7 +148,9 @@ class DagProcessor(BaseDags):
                 finally:
                     j["Status"] = "ok"
                     self.table.upsert(j)
+
                     LOGGER.info("end", extra=self.extra(j))
+                    TABLE_LOG_HANDLER.flush()
 
                 dependencies = self.table.query(f"PartitionKey eq 'dependencies' and ParentId eq '{j.get('JobId')}'")
                 self.table.delete(dependencies)

fabricks/core/jobs/base/generator.py

@@ -336,7 +336,7 @@ class Generator(Configurator):
 
                 comment = self.options.table.get("comment")
                 if comment:
-                    self.table.add_comment(comment=comment)
+                    self.table.add_table_comment(comment=comment)
 
         else:
             DEFAULT_LOGGER.debug("table exists, skip creation", extra={"label": self})
@@ -389,6 +389,24 @@ class Generator(Configurator):
     def overwrite_schema(self, df: Optional[DataFrame] = None):
         self._update_schema(df=df, overwrite=True)
 
+    def update_comments(self, table: Optional[bool] = True, columns: Optional[bool] = True):
+        if self.virtual:
+            return
+
+        if self.persist:
+            self.table.drop_comments()
+
+            if table:
+                comment = self.options.table.get("comment")
+                if comment:
+                    self.table.add_table_comment(comment=comment)
+
+            if columns:
+                comments = self.options.table.get_dict("comments")
+                if comments:
+                    for col, comment in comments.items():
+                        self.table.add_column_comment(column=col, comment=comment)
+
     def get_differences_with_deltatable(self, df: Optional[DataFrame] = None):
         if df is None:
             df = self.get_data(stream=self.stream)

fabricks/deploy/views.py

@@ -276,6 +276,7 @@ def create_or_replace_logs_pivot_view():
         l.step,
         l.job,
         l.job_id,
+        -- flags
         collect_set(l.status) as statuses,
         array_contains(statuses, 'skipped') as skipped,
         array_contains(statuses, 'warned') as warned,
@@ -283,15 +284,19 @@ def create_or_replace_logs_pivot_view():
         array_contains(statuses, 'failed') or (not done and not skipped) as failed,
         not done and not failed and not skipped and array_contains(statuses, 'running') as timed_out,
         not array_contains(statuses, 'running') as cancelled,
+        --
         max(l.notebook_id) as notebook_id,
+        --
+        max(l.timestamp) filter (where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status in ('done', 'ok')) as end_time,
+        --
         max(l.timestamp) filter (where l.status = 'scheduled' ) as scheduled_time,
         max(l.timestamp) filter (where l.status = 'waiting' ) as waiting_time,
-        max(l.timestamp) filter (where l.status = 'running') as start_time,
         max(l.timestamp) filter (where l.status = 'running' ) as running_time,
         max(l.timestamp) filter (where l.status = 'done' ) as done_time,
         max(l.timestamp) filter (where l.status = 'failed' ) as failed_time,
-        max(l.timestamp) filter(where l.status = 'ok') as end_time,
-        max(l.timestamp) filter(where l.status = 'ok') as ok_time,
+        max(l.timestamp) filter (where l.status = 'ok') as ok_time,
+        --
         max(l.exception) as exception
       from
         fabricks.logs l

fabricks/metastore/table.py

@@ -666,7 +666,45 @@ class Table(DbObject):
             """
         )
 
-    def add_comment(self, comment: str):
+    def drop_comments(self):
+        self.drop_table_comment()
+        for col in self.columns:
+            self.drop_column_comment(col)
+
+    def drop_table_comment(self):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug("drop table comment", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on table {self.qualified_name}
+            is null;
+            """
+        )
+
+    def drop_column_comment(self, column: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"drop comment from column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is null;
+            """
+        )
+
+    def add_column_comment(self, column: str, comment: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"add comment '{comment}' to column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is '{comment}';
+            """
+        )
+
+    def add_table_comment(self, comment: str):
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.debug(f"add comment '{comment}'", extra={"label": self})

{fabricks-3.0.10.dist-info → fabricks-3.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.10
+Version: 3.0.12
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.10.dist-info → fabricks-3.0.12.dist-info}/RECORD

@@ -40,8 +40,8 @@ fabricks/cdc/base/_types.py,sha256=WloCDC3ATrn0aZJ6E8BRYKZx19N3EE56r6qlBYhcuvQ,2
 fabricks/cdc/base/cdc.py,sha256=9w5BqQxSVbFVEozJWmZQThqdppkE_SYi4fHSzJ7WMvA,78
 fabricks/cdc/base/configurator.py,sha256=w6Ywif87iv1WG-5OM3XkzIRrsns-_QQ6XlADpk0YLlw,6434
 fabricks/cdc/base/generator.py,sha256=pa_GJn7Pdi5vMnXN8zExmOPMpCqdZ3QoxHEB0wv0lsk,5933
-fabricks/cdc/base/merger.py,sha256=1_j-hKnKKEcbogyXX0Cm2IdyB-tpDJAIMIz1MwWnXX0,4158
-fabricks/cdc/base/processor.py,sha256=b8ATjVX-dW8JCrt4n3v8HlteTi1498jrItEla0BqynU,17689
+fabricks/cdc/base/merger.py,sha256=3qUUs0uqmwOMdXc50kV3Zo9omuQuUUFgtMLBrg4E-wk,4159
+fabricks/cdc/base/processor.py,sha256=gL3pWMaBRsc0oB93ISnH2x07WbmtM_QEIx8qrUcUoZ0,17704
 fabricks/cdc/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fabricks/cdc/templates/filter.sql.jinja,sha256=AQcOj4KRwKscVG3zepnEAN9Yxb03AM7068hqW7dtVI8,236
 fabricks/cdc/templates/merge.sql.jinja,sha256=YS9wWckCVsUI1pUYiRSFBIuQ16WU3EPWSkhZVy2niBA,221
@@ -69,13 +69,13 @@ fabricks/cdc/templates/queries/nocdc/complete.sql.jinja,sha256=cVKKCSbiuuw1K7BOz
 fabricks/cdc/templates/queries/nocdc/update.sql.jinja,sha256=mjNUwGVhZ08yUkdv9sCTkqyW60p0YavtWTqvSUVrwjA,1283
 fabricks/context/__init__.py,sha256=qfntJ9O6omzY_t6AhDP6Ndu9C5LMiVdWbo6ikhtoe7o,1446
 fabricks/context/_types.py,sha256=FzQJ35vp0uc6pAq18bc-VHwMVEWtd0VDdm8xQmNr2Sg,2681
+fabricks/context/config.py,sha256=EmLUnswuWfrncaNJMDjvdMg-1lD8aneKAY8IDna7VPE,4814
+fabricks/context/helpers.py,sha256=GV9MscE8p6CsHbjVC-Qnqtv9VBf893DoxfLa057hFT8,2061
 fabricks/context/log.py,sha256=CadrRf8iL6iXlGIGIhEIswa7wGqC-E-oLwWcGTyJ10s,2074
-fabricks/context/runtime.py,sha256=7pXU_5gPyvChwiyxP4ch59eAgyoNOH9jMUlGWxwXWn8,3915
-fabricks/context/secret.py,sha256=LEx7MXSFm29wvsBffNSIQ6p73wqqmFj2jmU2y64h-Fc,3191
+fabricks/context/runtime.py,sha256=87PtX6SqLoFd0PGxgisF6dLlxtCHaHxkMMIt34UyB2w,3479
+fabricks/context/secret.py,sha256=iRM-KU-JcJAEOLoGJ8S4Oh65-yt674W6CDTSkOE7SXw,3192
 fabricks/context/spark_session.py,sha256=BPaxKJXHZDI5oQiOPhmua_xjXnrVgluh--AVpvUgbck,2553
 fabricks/context/utils.py,sha256=EQRscdUhdjwk2htZu8gCgNZ9PfRzzrR6e1kRrIbVlBM,2786
-fabricks/context/config/__init__.py,sha256=pFEsGXBQkX5_FP0cwQMX427j6dQuTx81NR9snMxc8cU,3127
-fabricks/context/config/utils.py,sha256=7KCTUiSbqQnDD5mbCO9_o1KbUgD-Xbei_UGgpMQi9nE,1371
 fabricks/core/__init__.py,sha256=LaqDi4xuyHAoLOvS44PQdZdRfq9SmVr7mB6BDHyxYpc,209
 fabricks/core/extenders.py,sha256=39bSm9QiW4vBAyT659joE-5p_EZiNM4gi8KA3-OgX3E,917
 fabricks/core/job_schema.py,sha256=6-70oy0ZJd3V9AiXfc0Q8b8NVEynxQza_h7mB13uB-s,853
@@ -86,7 +86,7 @@ fabricks/core/dags/__init__.py,sha256=0DUKzVcXcROvxkN19P_kaOJ7da5BAM7Vt8EGQbp2KS
 fabricks/core/dags/base.py,sha256=tFj27SqeZUZ7pB_LOWkpdowZz5gj30JUANI4gWK3Pl8,3139
 fabricks/core/dags/generator.py,sha256=4fp_CRsWnl_UauM9Jx-E4UCaxnm2_Q5103J58fRws2U,4832
 fabricks/core/dags/log.py,sha256=v1xfpQGfddHDz9lflvXOWTXMde3CdERo9jzeSmNDRhY,402
-fabricks/core/dags/processor.py,sha256=dcEWk0y2fuNt7RfxkeCny5axdg5aE4UpD7QwEijcQWM,7864
+fabricks/core/dags/processor.py,sha256=IzjqrpNu6lTYp-Rl2T_8Sb5N0pSk9BhdP9vuL4sBRMg,7930
 fabricks/core/dags/run.py,sha256=RIDULb9WakObSyYzmkglh8RwFRwC8-NFC-1yPDMkBC0,1074
 fabricks/core/dags/terminator.py,sha256=Y6pV2UnSyrCIx2AQWJXoHk5Roq12gZqpennHx_Lbnzs,793
 fabricks/core/dags/utils.py,sha256=4kyobLGl4tO0Flo6LxNzYjCU_G42vns1LrkxTO5_KLY,1585
@@ -105,7 +105,7 @@ fabricks/core/jobs/base/_types.py,sha256=y66BtJlJskq7wGzn7te5XYjO-NEqeQGUC11kkbe
 fabricks/core/jobs/base/checker.py,sha256=Cdfh8rQYy4MvMFl0HyC3alGUWm8zrFXk08m2t2JMu6Y,5477
 fabricks/core/jobs/base/configurator.py,sha256=9G5F7Qg5FWHPbHgdh8Qxc85OoSX0rnjD4c9itwU5KKc,10415
 fabricks/core/jobs/base/exception.py,sha256=HrdxEuOfK5rY-ItZvEL3iywLgdpYUpmWFkjjjks7oYc,2318
-fabricks/core/jobs/base/generator.py,sha256=NlJgR1461rx4bOBfvpmBrS6PvNeHD6QH6FOqAZiXDvM,16987
+fabricks/core/jobs/base/generator.py,sha256=TI4Wy8tlVRIKEczx6qmrIf2ppGwLvhdaatb9pnCvzlI,17635
 fabricks/core/jobs/base/invoker.py,sha256=FvjfpNqi542slxC2yLu1BIu5EklNUWySxDF8cD_SqKQ,7602
 fabricks/core/jobs/base/job.py,sha256=dWmk2PpQH2NETaaDS6KoiefRnDHfDMdCyhmogkdcSFI,93
 fabricks/core/jobs/base/processor.py,sha256=qkNiJSSLaEnivKGBcd9UZyIVFexnv-n1p_5mCZIy1rA,9076
@@ -136,14 +136,14 @@ fabricks/deploy/schedules.py,sha256=0a5dU1rW6fg8aAp7TTt-l0DgR-4kmzsX2xxV2C30yaw,
 fabricks/deploy/tables.py,sha256=IF822oxOCy12r08Dz54YUK5luud6dtTPxJ4TUIHE-No,2621
 fabricks/deploy/udfs.py,sha256=7fw3O5LgOOxDEhuS3s1yFdqybgFh65r_1IdfZUYeejs,597
 fabricks/deploy/utils.py,sha256=V41r1zVT9KcsICqTLAzpb4ixRk2q2ybJMrGhkPOtG6k,5099
-fabricks/deploy/views.py,sha256=Ket511q0v8QHgkNArb4qVPFDuuQnQn3v-fvr7KO6oxM,14360
+fabricks/deploy/views.py,sha256=8cSt6IzZy-JHHkyqd91NT2hi3LTNTOolimlfSBXMCvU,14434
 fabricks/metastore/README.md,sha256=utPUGAxmjyNMGe43GfL0Gup4MjeTKKwyiUoNVSfMquI,51
 fabricks/metastore/__init__.py,sha256=RhjY2CuqtZBg8fEizzzvW8qszqCM-vSCL1tQGuzoato,174
 fabricks/metastore/_types.py,sha256=NXYxwQHP0sCllM0N6QBbaK4CdtM_m_rHFDxRNRfBcLU,1919
 fabricks/metastore/database.py,sha256=23VAKKzjrwlEaj28DNNmiOhcfdKRzYk8eEfq-PzINbg,1924
 fabricks/metastore/dbobject.py,sha256=ve8p48OqEpJYsqWNhgesGSE0emM--uY8QrvBRoR3j3g,1881
 fabricks/metastore/pyproject.toml,sha256=6RZM9RMKMDF_EAequhORZ7TD0BQNk7aBCTWAv-sRcp0,519
-fabricks/metastore/table.py,sha256=mos701lEU_EwjdwwGb2ey-RSebkg8Ubms2VcOJvIARA,27937
+fabricks/metastore/table.py,sha256=AaoNL-1mz4A0CCb3tH_0BUurYPjA1oL5pioCYlEMtu4,29113
 fabricks/metastore/utils.py,sha256=8SxhjDkz_aSH4IGUusel7hqOQxP9U8PNBCY0M7GH00Y,1355
 fabricks/metastore/view.py,sha256=f7hKJWtnH1KmZym8dkoucKOTndntzai_f2YqferxHLs,1431
 fabricks/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -171,6 +171,6 @@ fabricks/utils/schema/get_schema_for_type.py,sha256=5k-R6zCgUAtapQgxT4turcx1IQ-b
 fabricks/utils/write/__init__.py,sha256=i0UnZenXj9Aq0b0_aU3s6882vg-Vu_AyKfQhl_dTp-g,200
 fabricks/utils/write/delta.py,sha256=lTQ0CfUhcvn3xTCcT_Ns6PMDBsO5UEfa2S9XpJiLJ9c,1250
 fabricks/utils/write/stream.py,sha256=wQBpAnQtYA6nl79sPKhVM6u5m-66suX7B6VQ6tW4TOs,622
-fabricks-3.0.10.dist-info/METADATA,sha256=lkZrsNWUnqfPuY2mhdi5v5CmWsL1xEsMy22Rxwzek64,798
-fabricks-3.0.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fabricks-3.0.10.dist-info/RECORD,,
+fabricks-3.0.12.dist-info/METADATA,sha256=LYFVTMlw8ueiyViUZC5ZZHk2LZkpNnVGO2B1ZhYrdTg,798
+fabricks-3.0.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fabricks-3.0.12.dist-info/RECORD,,

fabricks/context/config/__init__.py

@@ -1,92 +0,0 @@
-import logging
-import os
-from typing import Final
-
-from fabricks.context.config.utils import get_config_from_file
-from fabricks.utils.path import Path
-from fabricks.utils.spark import spark
-
-file_path, file_config = get_config_from_file()
-
-runtime = os.environ.get("FABRICKS_RUNTIME", "none")
-runtime = None if runtime.lower() == "none" else runtime
-if runtime is None:
-    if runtime := file_config.get("runtime"):
-        assert file_path is not None
-        runtime = file_path.joinpath(runtime)
-
-if runtime is None:
-    if file_path is not None:
-        runtime = file_path
-    else:
-        raise ValueError(
-            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
-        )
-
-path_runtime = Path(runtime, assume_git=True)
-PATH_RUNTIME: Final[Path] = path_runtime
-
-notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
-notebooks = None if notebooks.lower() == "none" else notebooks
-if notebooks is None:
-    if notebooks := file_config.get("notebooks"):
-        assert file_path is not None
-        notebooks = file_path.joinpath(notebooks)
-
-notebooks = notebooks if notebooks else path_runtime.joinpath("notebooks")
-PATH_NOTEBOOKS: Final[Path] = Path(str(notebooks), assume_git=True)
-
-is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
-if is_job_config_from_yaml is None:
-    assert file_path is not None
-    is_job_config_from_yaml = file_config.get("job_config_from_yaml")
-
-IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
-
-is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
-if is_debugmode is None:
-    is_debugmode = file_config.get("debugmode")
-
-IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
-
-is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
-if is_devmode is None:
-    is_devmode = file_config.get("devmode")
-
-IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
-
-loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
-if loglevel is None:
-    loglevel = file_config.get("loglevel")
-
-loglevel = loglevel.upper() if loglevel else "INFO"
-if loglevel == "DEBUG":
-    _loglevel = logging.DEBUG
-elif loglevel == "INFO":
-    _loglevel = logging.INFO
-elif loglevel == "WARNING":
-    _loglevel = logging.WARNING
-elif loglevel == "ERROR":
-    _loglevel = logging.ERROR
-elif loglevel == "CRITICAL":
-    _loglevel = logging.CRITICAL
-else:
-    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
-
-LOGLEVEL = _loglevel
-
-path_config = os.environ.get("FABRICKS_CONFIG")
-if path_config is None:
-    if path_config := file_config.get("config"):
-        assert file_path is not None
-        path_config = file_path.joinpath(path_config)
-else:
-    path_config = PATH_RUNTIME.joinpath(path_config).string if path_config else None
-
-if not path_config:
-    path_config = PATH_RUNTIME.joinpath(
-        "fabricks",
-        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
-    ).string
-
-PATH_CONFIG: Final[Path] = Path(path_config, assume_git=True)