fabricks 3.0.11__tar.gz → 3.0.12__tar.gz

{fabricks-3.0.11 → fabricks-3.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fabricks
-Version: 3.0.11
+Version: 3.0.12
 Author-email: BMS DWH Team <bi_support@bmsuisse.ch>
 Requires-Python: <4,>=3.9
 Requires-Dist: azure-data-tables<13,>=12.5.0

{fabricks-3.0.11 → fabricks-3.0.12}/README.md

@@ -271,12 +271,8 @@ with
     from dates d
     left join newkey r on d.__timestamp between r.__valid_from and r.__valid_to and d.only_offer_id = r.only_offer_id
     group by d.only_offer_id, __timestamp, __operation
-    
 ```
 
-
-
-
 ## 🧾 Conclusion
 
 Fabricks helps you define pipelines declaratively in YAML and implement logic in SQL, while handling CDC, dependencies, logging, and table lifecycle. Start from the examples near each step, then evolve to advanced features like checks, UDFs, extenders, and semantic properties.

{fabricks-3.0.11 → fabricks-3.0.12}/fabricks/cdc/base/merger.py

@@ -89,6 +89,7 @@ class Merger(Processor):
                 sql = sql.replace("{src}", "src")
                 sql = fix_sql(sql)
                 sql = sql.replace("`src`", "{src}")
+
                 DEFAULT_LOGGER.debug("merge", extra={"label": self, "sql": sql})
 
             except Exception as e:

{fabricks-3.0.11 → fabricks-3.0.12}/fabricks/cdc/base/processor.py

@@ -390,11 +390,12 @@ class Processor(Generator):
         try:
             sql = template.render(**context)
             if fix:
-                DEFAULT_LOGGER.debug("fix context", extra={"label": self, "sql": sql})
                 sql = self.fix_sql(sql)
+            else:
+                DEFAULT_LOGGER.debug("print query", extra={"label": self, "sql": sql})
 
         except (Exception, TypeError) as e:
-            DEFAULT_LOGGER.exception("fail to execute sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         row = self.spark.sql(sql).collect()[0]
@@ -425,7 +426,7 @@ class Processor(Generator):
 
         except (Exception, TypeError) as e:
             DEFAULT_LOGGER.debug("context", extra={"label": self, "context": context})
-            DEFAULT_LOGGER.exception("fail to generate sql query", extra={"label": self, "context": context})
+            DEFAULT_LOGGER.exception("fail to render sql query", extra={"label": self, "context": context})
             raise e
 
         return sql

fabricks-3.0.12/fabricks/context/config.py

@@ -0,0 +1,127 @@
+import logging
+import os
+from typing import Final
+
+from fabricks.context.helpers import get_config_from_file
+from fabricks.utils.log import get_logger
+from fabricks.utils.path import Path
+from fabricks.utils.spark import spark
+
+logger, _ = get_logger("logs", level=logging.DEBUG)
+file_path, file_config, origin = get_config_from_file()
+
+if file_path:
+    logger.debug(f"found {origin} config ({file_path})", extra={"label": "config"})
+
+# path to runtime
+runtime = os.environ.get("FABRICKS_RUNTIME", "none")
+runtime = None if runtime.lower() == "none" else runtime
+if runtime is None:
+    if runtime := file_config.get("runtime"):
+        assert file_path is not None
+        runtime = file_path.joinpath(runtime)
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve runtime from env", extra={"label": "config"})
+
+if runtime is None:
+    if file_path is not None:
+        runtime = file_path
+        logger.debug(f"resolve runtime from {origin} file", extra={"label": "config"})
+    else:
+        raise ValueError(
+            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
+        )
+
+path_runtime = Path(runtime, assume_git=True)
+
+# path to config
+config = os.environ.get("FABRICKS_CONFIG")
+if config is None:
+    if config := file_config.get("config"):
+        assert file_path is not None
+        config = file_path.joinpath(config)
+        logger.debug(f"resolve config from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve config from env", extra={"label": "config"})
+
+if config is None:
+    logger.debug("resolve config from default path", extra={"label": "config"})
+    config = path_runtime.joinpath(
+        "fabricks",
+        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
+    ).string
+
+path_config = Path(config, assume_git=True)
+
+# path to notebooks
+notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
+notebooks = None if notebooks.lower() == "none" else notebooks
+if notebooks is None:
+    if notebooks := file_config.get("notebooks"):
+        assert file_path is not None
+        notebooks = file_path.joinpath(notebooks)
+        logger.debug(f"resolve notebooks from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve notebooks from env", extra={"label": "config"})
+
+if notebooks is None:
+    logger.debug("resolve notebooks from default path", extra={"label": "config"})
+    notebooks = path_runtime.joinpath("notebooks")
+
+path_notebooks = Path(str(notebooks), assume_git=True)
+
+# job config from yaml
+is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
+if is_job_config_from_yaml is None:
+    if is_job_config_from_yaml := file_config.get("job_config_from_yaml"):
+        logger.debug(f"resolve job_config_from_yaml from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve job_config_from_yaml from env", extra={"label": "config"})
+
+# debug mode
+is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
+if is_debugmode is None:
+    if is_debugmode := file_config.get("debugmode"):
+        logger.debug(f"resolve debugmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve debugmode from env", extra={"label": "config"})
+
+# dev mode
+is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
+if is_devmode is None:
+    if is_devmode := file_config.get("devmode"):
+        logger.debug(f"resolve devmode from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve devmode from env", extra={"label": "config"})
+
+# log level
+loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
+if loglevel is None:
+    if loglevel := file_config.get("loglevel"):
+        logger.debug(f"resolve loglevel from {origin} file", extra={"label": "config"})
+else:
+    logger.debug("resolve loglevel from env", extra={"label": "config"})
+
+loglevel = loglevel.upper() if loglevel else "INFO"
+if loglevel == "DEBUG":
+    _loglevel = logging.DEBUG
+elif loglevel == "INFO":
+    _loglevel = logging.INFO
+elif loglevel == "WARNING":
+    _loglevel = logging.WARNING
+elif loglevel == "ERROR":
+    _loglevel = logging.ERROR
+elif loglevel == "CRITICAL":
+    _loglevel = logging.CRITICAL
+else:
+    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
+
+# Constants
+PATH_CONFIG: Final[Path] = path_config
+PATH_RUNTIME: Final[Path] = path_runtime
+PATH_NOTEBOOKS: Final[Path] = path_notebooks
+IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
+IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
+IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
+LOGLEVEL: Final[int] = _loglevel

fabricks-3.0.11/fabricks/context/config/utils.py → fabricks-3.0.12/fabricks/context/helpers.py

@@ -1,3 +1,8 @@
+from typing import List
+
+from fabricks.utils.path import Path
+
+
 def get_config_from_toml():
     import os
     import pathlib
@@ -44,10 +49,33 @@ def get_config_from_json():
 def get_config_from_file():
     json_path, json_config = get_config_from_json()
     if json_config:
-        return json_path, json_config
+        return json_path, json_config, "json"
 
     pyproject_path, pyproject_config = get_config_from_toml()
     if pyproject_config:
-        return pyproject_path, pyproject_config
+        return pyproject_path, pyproject_config, "pyproject"
 
-    return None, {}
+    return None, {}, None
+
+
+def get_storage_paths(objects: List[dict], variables: dict) -> dict:
+    d = {}
+    for o in objects:
+        if o:
+            name = o.get("name")
+            assert name
+            uri = o.get("path_options", {}).get("storage")
+            assert uri
+            d[name] = Path.from_uri(uri, regex=variables)
+    return d
+
+
+def get_runtime_path(objects: List[dict], root: Path) -> dict:
+    d = {}
+    for o in objects:
+        name = o.get("name")
+        assert name
+        uri = o.get("path_options", {}).get("runtime")
+        assert uri
+        d[name] = root.joinpath(uri)
+    return d

{fabricks-3.0.11 → fabricks-3.0.12}/fabricks/context/runtime.py

@@ -1,11 +1,12 @@
-from typing import Final, List, Optional
+from typing import Final, Optional
 
 import yaml
 
-from fabricks.context.config import path_config, path_runtime
+from fabricks.context.config import PATH_CONFIG, PATH_RUNTIME
+from fabricks.context.helpers import get_runtime_path, get_storage_paths
 from fabricks.utils.path import Path
 
-with open(str(path_config)) as f:
+with open(str(PATH_CONFIG)) as f:
     data = yaml.safe_load(f)
 
 conf: dict = [d["conf"] for d in data][0]
@@ -51,67 +52,42 @@ FABRICKS_STORAGE_CREDENTIAL: Final[Optional[str]] = path_options.get("storage_cr
 
 path_udfs = path_options.get("udfs", "fabricks/udfs")
 assert path_udfs, "path to udfs mandatory"
-PATH_UDFS: Final[Path] = path_runtime.joinpath(path_udfs)
+PATH_UDFS: Final[Path] = PATH_RUNTIME.joinpath(path_udfs)
 
 path_parsers = path_options.get("parsers", "fabricks/parsers")
 assert path_parsers, "path to parsers mandatory"
-PATH_PARSERS: Final[Path] = path_runtime.joinpath(path_parsers)
+PATH_PARSERS: Final[Path] = PATH_RUNTIME.joinpath(path_parsers)
 
 path_extenders = path_options.get("extenders", "fabricks/extenders")
 assert path_extenders, "path to extenders mandatory"
-PATH_EXTENDERS: Final[Path] = path_runtime.joinpath(path_extenders)
+PATH_EXTENDERS: Final[Path] = PATH_RUNTIME.joinpath(path_extenders)
 
 path_views = path_options.get("views", "fabricks/views")
 assert path_views, "path to views mandatory"
-PATH_VIEWS: Final[Path] = path_runtime.joinpath(path_views)
+PATH_VIEWS: Final[Path] = PATH_RUNTIME.joinpath(path_views)
 
 path_schedules = path_options.get("schedules", "fabricks/schedules")
 assert path_schedules, "path to schedules mandatory"
-PATH_SCHEDULES: Final[Path] = path_runtime.joinpath(path_schedules)
+PATH_SCHEDULES: Final[Path] = PATH_RUNTIME.joinpath(path_schedules)
 
 path_requirements = path_options.get("requirements", "fabricks/requirements")
 assert path_requirements, "path to requirements mandatory"
-PATH_REQUIREMENTS: Final[Path] = path_runtime.joinpath(path_requirements)
+PATH_REQUIREMENTS: Final[Path] = PATH_RUNTIME.joinpath(path_requirements)
 
 path_masks = path_options.get("masks", "fabricks/masks")
 assert path_masks, "path to masks mandatory"
-PATH_MASKS: Final[Path] = path_runtime.joinpath(path_masks)
-
-
-def _get_storage_paths(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        if o:
-            name = o.get("name")
-            assert name
-            uri = o.get("path_options", {}).get("storage")
-            assert uri
-            d[name] = Path.from_uri(uri, regex=variables)
-    return d
-
+PATH_MASKS: Final[Path] = PATH_RUNTIME.joinpath(path_masks)
 
 PATHS_STORAGE: Final[dict[str, Path]] = {
     "fabricks": FABRICKS_STORAGE,
-    **_get_storage_paths(BRONZE),
-    **_get_storage_paths(SILVER),
-    **_get_storage_paths(GOLD),
-    **_get_storage_paths(databases),
+    **get_storage_paths(BRONZE, variables),
+    **get_storage_paths(SILVER, variables),
+    **get_storage_paths(GOLD, variables),
+    **get_storage_paths(databases, variables),
 }
 
-
-def _get_runtime_path(objects: List[dict]) -> dict:
-    d = {}
-    for o in objects:
-        name = o.get("name")
-        assert name
-        uri = o.get("path_options", {}).get("runtime")
-        assert uri
-        d[name] = path_runtime.joinpath(uri)
-    return d
-
-
 PATHS_RUNTIME: Final[dict[str, Path]] = {
-    **_get_runtime_path(BRONZE),
-    **_get_runtime_path(SILVER),
-    **_get_runtime_path(GOLD),
+    **get_runtime_path(BRONZE, PATH_RUNTIME),
+    **get_runtime_path(SILVER, PATH_RUNTIME),
+    **get_runtime_path(GOLD, PATH_RUNTIME),
 }

{fabricks-3.0.11 → fabricks-3.0.12}/fabricks/core/dags/processor.py

@@ -10,7 +10,7 @@ from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_ex
 
 from fabricks.context import PATH_NOTEBOOKS
 from fabricks.core.dags.base import BaseDags
-from fabricks.core.dags.log import LOGGER
+from fabricks.core.dags.log import LOGGER, TABLE_LOG_HANDLER
 from fabricks.core.dags.run import run
 from fabricks.core.jobs.base._types import TStep
 from fabricks.core.steps.get_step import get_step
@@ -148,7 +148,9 @@ class DagProcessor(BaseDags):
                 finally:
                     j["Status"] = "ok"
                     self.table.upsert(j)
+
                     LOGGER.info("end", extra=self.extra(j))
+                    TABLE_LOG_HANDLER.flush()
 
                 dependencies = self.table.query(f"PartitionKey eq 'dependencies' and ParentId eq '{j.get('JobId')}'")
                 self.table.delete(dependencies)

{fabricks-3.0.11 → fabricks-3.0.12}/fabricks/core/jobs/base/generator.py

@@ -336,7 +336,7 @@ class Generator(Configurator):
 
                 comment = self.options.table.get("comment")
                 if comment:
-                    self.table.add_comment(comment=comment)
+                    self.table.add_table_comment(comment=comment)
 
         else:
             DEFAULT_LOGGER.debug("table exists, skip creation", extra={"label": self})
@@ -389,6 +389,24 @@ class Generator(Configurator):
     def overwrite_schema(self, df: Optional[DataFrame] = None):
         self._update_schema(df=df, overwrite=True)
 
+    def update_comments(self, table: Optional[bool] = True, columns: Optional[bool] = True):
+        if self.virtual:
+            return
+
+        if self.persist:
+            self.table.drop_comments()
+
+            if table:
+                comment = self.options.table.get("comment")
+                if comment:
+                    self.table.add_table_comment(comment=comment)
+
+            if columns:
+                comments = self.options.table.get_dict("comments")
+                if comments:
+                    for col, comment in comments.items():
+                        self.table.add_column_comment(column=col, comment=comment)
+
     def get_differences_with_deltatable(self, df: Optional[DataFrame] = None):
         if df is None:
             df = self.get_data(stream=self.stream)

{fabricks-3.0.11 → fabricks-3.0.12}/fabricks/deploy/views.py

@@ -276,6 +276,7 @@ def create_or_replace_logs_pivot_view():
         l.step,
         l.job,
         l.job_id,
+        -- flags
         collect_set(l.status) as statuses,
         array_contains(statuses, 'skipped') as skipped,
         array_contains(statuses, 'warned') as warned,
@@ -283,15 +284,19 @@ def create_or_replace_logs_pivot_view():
         array_contains(statuses, 'failed') or (not done and not skipped) as failed,
         not done and not failed and not skipped and array_contains(statuses, 'running') as timed_out,
         not array_contains(statuses, 'running') as cancelled,
+        --
         max(l.notebook_id) as notebook_id,
+        --
+        max(l.timestamp) filter (where l.status = 'running') as start_time,
+        max(l.timestamp) filter (where l.status in ('done', 'ok')) as end_time,
+        --
         max(l.timestamp) filter (where l.status = 'scheduled' ) as scheduled_time,
         max(l.timestamp) filter (where l.status = 'waiting' ) as waiting_time,
-        max(l.timestamp) filter (where l.status = 'running') as start_time,
         max(l.timestamp) filter (where l.status = 'running' ) as running_time,
         max(l.timestamp) filter (where l.status = 'done' ) as done_time,
         max(l.timestamp) filter (where l.status = 'failed' ) as failed_time,
-        max(l.timestamp) filter(where l.status = 'ok') as end_time,
-        max(l.timestamp) filter(where l.status = 'ok') as ok_time,
+        max(l.timestamp) filter (where l.status = 'ok') as ok_time,
+        --
         max(l.exception) as exception
       from
         fabricks.logs l

{fabricks-3.0.11 → fabricks-3.0.12}/fabricks/metastore/table.py

@@ -666,7 +666,45 @@ class Table(DbObject):
             """
         )
 
-    def add_comment(self, comment: str):
+    def drop_comments(self):
+        self.drop_table_comment()
+        for col in self.columns:
+            self.drop_column_comment(col)
+
+    def drop_table_comment(self):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug("drop table comment", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on table {self.qualified_name}
+            is null;
+            """
+        )
+
+    def drop_column_comment(self, column: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"drop comment from column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is null;
+            """
+        )
+
+    def add_column_comment(self, column: str, comment: str):
+        assert self.registered, f"{self} not registered"
+
+        DEFAULT_LOGGER.debug(f"add comment '{comment}' to column {column}", extra={"label": self})
+        self.spark.sql(
+            f"""
+            comment on column {self.qualified_name}.`{column}`
+            is '{comment}';
+            """
+        )
+
+    def add_table_comment(self, comment: str):
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.debug(f"add comment '{comment}'", extra={"label": self})

{fabricks-3.0.11 → fabricks-3.0.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fabricks"
-version = "3.0.11"
+version = "3.0.12"
 description = ""
 authors = [
     { name = "BMS DWH Team", email = "bi_support@bmsuisse.ch" },

{fabricks-3.0.11 → fabricks-3.0.12}/uv.lock

@@ -645,7 +645,7 @@ wheels = [
 
 [[package]]
 name = "fabricks"
-version = "3.0.11"
+version = "3.0.12"
 source = { editable = "." }
 dependencies = [
     { name = "azure-data-tables" },

fabricks-3.0.11/fabricks/context/config/__init__.py

@@ -1,92 +0,0 @@
-import logging
-import os
-from typing import Final
-
-from fabricks.context.config.utils import get_config_from_file
-from fabricks.utils.path import Path
-from fabricks.utils.spark import spark
-
-file_path, file_config = get_config_from_file()
-
-runtime = os.environ.get("FABRICKS_RUNTIME", "none")
-runtime = None if runtime.lower() == "none" else runtime
-if runtime is None:
-    if runtime := file_config.get("runtime"):
-        assert file_path is not None
-        runtime = file_path.joinpath(runtime)
-
-if runtime is None:
-    if file_path is not None:
-        runtime = file_path
-    else:
-        raise ValueError(
-            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
-        )
-
-path_runtime = Path(runtime, assume_git=True)
-PATH_RUNTIME: Final[Path] = path_runtime
-
-notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
-notebooks = None if notebooks.lower() == "none" else notebooks
-if notebooks is None:
-    if notebooks := file_config.get("notebooks"):
-        assert file_path is not None
-        notebooks = file_path.joinpath(notebooks)
-
-notebooks = notebooks if notebooks else path_runtime.joinpath("notebooks")
-PATH_NOTEBOOKS: Final[Path] = Path(str(notebooks), assume_git=True)
-
-is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
-if is_job_config_from_yaml is None:
-    assert file_path is not None
-    is_job_config_from_yaml = file_config.get("job_config_from_yaml")
-
-IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
-
-is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
-if is_debugmode is None:
-    is_debugmode = file_config.get("debugmode")
-
-IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
-
-is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
-if is_devmode is None:
-    is_devmode = file_config.get("devmode")
-
-IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
-
-loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
-if loglevel is None:
-    loglevel = file_config.get("loglevel")
-
-loglevel = loglevel.upper() if loglevel else "INFO"
-if loglevel == "DEBUG":
-    _loglevel = logging.DEBUG
-elif loglevel == "INFO":
-    _loglevel = logging.INFO
-elif loglevel == "WARNING":
-    _loglevel = logging.WARNING
-elif loglevel == "ERROR":
-    _loglevel = logging.ERROR
-elif loglevel == "CRITICAL":
-    _loglevel = logging.CRITICAL
-else:
-    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
-
-LOGLEVEL = _loglevel
-
-path_config = os.environ.get("FABRICKS_CONFIG")
-if path_config is None:
-    if path_config := file_config.get("config"):
-        assert file_path is not None
-        path_config = file_path.joinpath(path_config)
-else:
-    path_config = PATH_RUNTIME.joinpath(path_config).string if path_config else None
-
-if not path_config:
-    path_config = PATH_RUNTIME.joinpath(
-        "fabricks",
-        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
-    ).string
-
-PATH_CONFIG: Final[Path] = Path(path_config, assume_git=True)