fabricks 3.0.5.2__py3-none-any.whl → 3.0.7__py3-none-any.whl

fabricks/core/jobs/gold.py

@@ -99,14 +99,20 @@ class Gold(BaseJob):
     def register_udfs(self):
         for u in self.get_udfs():
             if not is_registered(u):
-                DEFAULT_LOGGER.debug(f"register udf ({u})", extra={"job": self})
+                DEFAULT_LOGGER.debug(f"register udf ({u})", extra={"label": self})
                 register_udf(udf=u, spark=self.spark)
 
     def base_transform(self, df: DataFrame) -> DataFrame:
         df = df.transform(self.extend)
         return df
 
-    def get_data(self, stream=False, transform: Optional[bool] = False) -> DataFrame:
+    def get_data(
+        self,
+        stream: bool = False,
+        transform: Optional[bool] = False,
+        schema_only: Optional[bool] = False,
+        **kwargs,
+    ) -> DataFrame:
         if self.options.job.get_boolean("requirements"):
             import sys
 
@@ -116,12 +122,12 @@ class Gold(BaseJob):
             df = self.spark.createDataFrame([{}])  # type: ignore
 
         elif self.options.job.get("notebook"):
-            from databricks.sdk.runtime import dbutils
+            invokers = self.options.invokers.get_list("run")
+            assert len(invokers) <= 1, "at most one invoker allowed when notebook is true"
 
-            DEFAULT_LOGGER.debug("run notebook", extra={"job": self})
-            path = self.paths.runtime.get_notebook_path()
+            global_temp_view = self.invoke(path=self.paths.runtime, schema_only=schema_only, **kwargs)
+            assert global_temp_view is not None, "global_temp_view not found"
 
-            global_temp_view = dbutils.notebook.run(path, self.timeout, arguments={})  # type: ignore
             df = self.spark.sql(f"select * from global_temp.{global_temp_view}")
 
         elif self.options.job.get("table"):
@@ -135,6 +141,10 @@ class Gold(BaseJob):
 
         if transform:
             df = self.base_transform(df)
+
+        if schema_only:
+            df = df.where("1 == 2")
+
         return df
 
     def create_or_replace_view(self):
@@ -178,7 +188,7 @@ class Gold(BaseJob):
         from fabricks.context import CATALOG
 
         dependencies = []
-        df = self.get_data(self.stream)
+        df = self.get_data(stream=self.stream)
 
         if df is not None:
             explain_plan = self.spark.sql("explain extended select * from {df}", df=df).collect()[0][0]
@@ -194,23 +204,14 @@ class Gold(BaseJob):
         return dependencies
 
     def get_cdc_context(self, df: DataFrame, reload: Optional[bool] = None) -> dict:
-        if "__order_duplicate_by_asc" in df.columns:
-            order_duplicate_by = {"__order_duplicate_by_asc": "asc"}
-        elif "__order_duplicate_by_desc" in df.columns:
-            order_duplicate_by = {"__order_duplicate_by_desc": "desc"}
-        else:
-            order_duplicate_by = None
-
-        deduplicate = self.options.job.get_boolean(
-            "deduplicate",
-            None,
-        )  # assume no duplicate in gold (to improve performance)
-        rectify = self.options.job.get_boolean(
-            "rectify_as_upserts",
-            None,
-        )  # assume no reload in gold (to improve performance)
-        correct_valid_from = self.options.job.get_boolean("correct_valid_from", True)
-        add_metadata = self.step_conf.get("options", {}).get("metadata", False)
+        # assume no duplicate in gold (to improve performance)
+        deduplicate = self.options.job.get_boolean("deduplicate", None)
+        # assume no reload in gold (to improve performance)
+        rectify = self.options.job.get_boolean("rectify_as_upserts", None)
+
+        add_metadata = self.options.job.get_boolean("metadata", None)
+        if add_metadata is None:
+            add_metadata = self.step_conf.get("options", {}).get("metadata", False)
 
         context = {
             "add_metadata": add_metadata,
@@ -219,27 +220,37 @@ class Gold(BaseJob):
             "deduplicate_hash": True if self.slowly_changing_dimension else None,
             "deduplicate": False,
             "rectify": False,
-            "order_duplicate_by": order_duplicate_by,
-            "correct_valid_from": correct_valid_from,
         }
 
+        # force deduplicate
         if deduplicate is not None:
             context["deduplicate"] = deduplicate
             context["deduplicate_key"] = deduplicate
             context["deduplicate_hash"] = deduplicate
 
+        # force rectify
         if rectify is not None:
             context["rectify"] = rectify
 
+        # add key and hash when needed
+        if self.mode == "update" and self.change_data_capture == "nocdc":
+            if "__key" not in df.columns:
+                context["add_key"] = True
+            if "__hash" not in df.columns:
+                context["add_hash"] = True
+
+        # add key and hash when needed
         if self.slowly_changing_dimension:
             if "__key" not in df.columns:
                 context["add_key"] = True
             if "__hash" not in df.columns:
                 context["add_hash"] = True
 
+        if self.slowly_changing_dimension:
             if "__operation" not in df.columns:
+                # assume no duplicate hash
                 if deduplicate is None:
-                    context["deduplicate_hash"] = None  # assume no duplicate hash
+                    context["deduplicate_hash"] = None
 
                 if self.mode == "update":
                     context["add_operation"] = "reload"
@@ -249,16 +260,25 @@ class Gold(BaseJob):
                 else:
                     context["add_operation"] = "upsert"
 
+        # filter to get latest data
         if not reload:
             if self.mode == "update" and self.change_data_capture == "scd2":
                 context["slice"] = "update"
 
+            if self.mode == "update" and self.change_data_capture == "nocdc" and "__timestamp" in df.columns:
+                context["slice"] = "update"
+
             if self.mode == "append" and "__timestamp" in df.columns:
                 context["slice"] = "update"
 
         if self.mode == "memory":
             context["mode"] = "complete"
 
+        # correct __valid_from
+        if self.change_data_capture == "scd2":
+            context["correct_valid_from"] = self.options.job.get_boolean("correct_valid_from", True)
+
+        # add __timestamp
         if self.options.job.get_boolean("persist_last_timestamp"):
             if self.change_data_capture == "scd1":
                 if "__timestamp" not in df.columns:
@@ -267,6 +287,11 @@ class Gold(BaseJob):
                 if "__valid_from" not in df.columns:
                     context["add_timestamp"] = True
 
+        if "__order_duplicate_by_asc" in df.columns:
+            context["order_duplicate_by"] = {"__order_duplicate_by_asc": "asc"}
+        elif "__order_duplicate_by_desc" in df.columns:
+            context["order_duplicate_by"] = {"__order_duplicate_by_desc": "desc"}
+
         return context
 
     def for_each_batch(self, df: DataFrame, batch: Optional[int] = None, **kwargs):
@@ -277,20 +302,19 @@ class Gold(BaseJob):
 
         # if dataframe, reference is passed (BUG)
         name = f"{self.step}_{self.topic}_{self.item}"
-        global_temp_view = create_or_replace_global_temp_view(name=name, df=df)
+        global_temp_view = create_or_replace_global_temp_view(name=name, df=df, job=self)
         sql = f"select * from {global_temp_view}"
 
         check_df = self.spark.sql(sql)
         if check_df.isEmpty():
-            DEFAULT_LOGGER.warning("no data", extra={"job": self})
+            DEFAULT_LOGGER.warning("no data", extra={"label": self})
             return
 
         if reload:
-            DEFAULT_LOGGER.warning("force reload", extra={"job": self})
+            DEFAULT_LOGGER.warning("force reload", extra={"label": self})
             self.cdc.complete(sql, **context)
 
         elif self.mode == "update":
-            assert not isinstance(self.cdc, NoCDC), "nocdc update not allowed"
             self.cdc.update(sql, **context)
 
         elif self.mode == "append":
@@ -323,7 +347,7 @@ class Gold(BaseJob):
 
     def create(self):
         if self.mode == "invoke":
-            DEFAULT_LOGGER.info("invoke (no table nor view)", extra={"job": self})
+            DEFAULT_LOGGER.info("invoke (no table nor view)", extra={"label": self})
         else:
             self.register_udfs()
             super().create()
@@ -335,7 +359,7 @@ class Gold(BaseJob):
             self.cdc_last_timestamp.table.register()
 
         if self.mode == "invoke":
-            DEFAULT_LOGGER.info("invoke (no table nor view)", extra={"job": self})
+            DEFAULT_LOGGER.info("invoke (no table nor view)", extra={"label": self})
         else:
             super().register()
 
@@ -345,17 +369,6 @@ class Gold(BaseJob):
 
         super().drop()
 
-    def optimize(
-        self,
-        vacuum: Optional[bool] = True,
-        optimize: Optional[bool] = True,
-        analyze: Optional[bool] = True,
-    ):
-        if self.mode == "memory":
-            DEFAULT_LOGGER.debug("memory (no optimize)", extra={"job": self})
-        else:
-            super().optimize(vacuum=vacuum, optimize=optimize, analyze=analyze)
-
     @property
     def cdc_last_timestamp(self) -> NoCDC:
         assert self.mode == "update", "persist_last_timestamp only allowed in update"
@@ -387,15 +400,15 @@ class Gold(BaseJob):
         else:
             self.cdc_last_timestamp.overwrite(df)
 
-    def overwrite(self):
+    def overwrite(self, schedule: Optional[str] = None):
         if self.mode == "invoke":
-            DEFAULT_LOGGER.debug("invoke (no overwrite)", extra={"job": self})
+            DEFAULT_LOGGER.debug("invoke (no overwrite)", extra={"label": self})
             return
 
         elif self.mode == "memory":
-            DEFAULT_LOGGER.debug("memory (no overwrite)", extra={"job": self})
+            DEFAULT_LOGGER.debug("memory (no overwrite)", extra={"label": self})
             self.create_or_replace_view()
             return
 
         self.overwrite_schema()
-        self.run(reload=True)
+        self.run(reload=True, schedule=schedule)

fabricks/core/jobs/silver.py

@@ -95,7 +95,13 @@ class Silver(BaseJob):
             )
         return df
 
-    def get_data(self, stream: bool = False, transform: Optional[bool] = False) -> DataFrame:
+    def get_data(
+        self,
+        stream: bool = False,
+        transform: Optional[bool] = False,
+        schema_only: Optional[bool] = False,
+        **kwargs,
+    ) -> DataFrame:
         deps = self.get_dependencies()
         assert deps, "not dependency found"
 
@@ -139,7 +145,7 @@ class Silver(BaseJob):
                         dfs.append(df)
 
                 except Exception as e:
-                    DEFAULT_LOGGER.exception("could not get dependencies", extra={"job": self})
+                    DEFAULT_LOGGER.exception("fail to get dependencies", extra={"label": self})
                     raise e
 
             df = concat_dfs(dfs)
@@ -151,6 +157,9 @@ class Silver(BaseJob):
         if transform:
             df = self.base_transform(df)
 
+        if schema_only:
+            df = df.where("1 == 2")
+
         return df
 
     def get_dependencies(self) -> Sequence[JobDependency]:
@@ -186,7 +195,7 @@ class Silver(BaseJob):
 
             sql = f"create or replace view {self.qualified_name} as {' union all '.join(queries)}"
             sql = fix_sql(sql)
-            DEFAULT_LOGGER.debug("view", extra={"job": self, "sql": sql})
+            DEFAULT_LOGGER.debug("view", extra={"label": self, "sql": sql})
             self.spark.sql(sql)
 
         else:
@@ -195,7 +204,7 @@ class Silver(BaseJob):
             parent = deps[0].parent
             sql = f"select * from {parent}"
             sql = fix_sql(sql)
-            DEFAULT_LOGGER.debug("view", extra={"job": self, "sql": sql})
+            DEFAULT_LOGGER.debug("view", extra={"label": self, "sql": sql})
 
             df = self.spark.sql(sql)
             cdc_options = self.get_cdc_context(df)
@@ -205,7 +214,7 @@ class Silver(BaseJob):
         from py4j.protocol import Py4JJavaError
 
         try:
-            DEFAULT_LOGGER.debug("create or replace current view", extra={"job": self})
+            DEFAULT_LOGGER.debug("create or replace current view", extra={"label": self})
 
             df = self.spark.sql(f"select * from {self.qualified_name}")
 
@@ -222,23 +231,23 @@ class Silver(BaseJob):
               {where_clause}
             """
             # sql = fix_sql(sql)
-            # DEFAULT_LOGGER.debug("current view", extra={"job": self, "sql": sql})
+            # DEFAULT_LOGGER.debug("current view", extra={"label": self, "sql": sql})
             self.spark.sql(sql)
 
-        except Py4JJavaError:
-            DEFAULT_LOGGER.exception("could not create or replace view", extra={"job": self})
+        except Py4JJavaError as e:
+            DEFAULT_LOGGER.exception("fail to create nor replace view", extra={"label": self}, exc_info=e)
 
-    def overwrite(self):
+    def overwrite(self, schedule: Optional[str] = None):
         self.truncate()
-        self.run()
+        self.run(schedule=schedule)
 
     def overwrite_schema(self, df: Optional[DataFrame] = None):
-        DEFAULT_LOGGER.warning("overwrite schema not allowed", extra={"job": self})
+        DEFAULT_LOGGER.warning("overwrite schema not allowed", extra={"label": self})
 
     def get_cdc_context(self, df: DataFrame, reload: Optional[bool] = None) -> dict:
         # if dataframe, reference is passed (BUG)
         name = f"{self.step}_{self.topic}_{self.item}__check"
-        global_temp_view = create_or_replace_global_temp_view(name=name, df=df)
+        global_temp_view = create_or_replace_global_temp_view(name=name, df=df, job=self)
 
         not_append = not self.mode == "append"
         nocdc = self.change_data_capture == "nocdc"
@@ -265,12 +274,12 @@ class Silver(BaseJob):
                   1
                 """
             sql = fix_sql(sql)
-            DEFAULT_LOGGER.debug("check", extra={"job": self, "sql": sql})
+            DEFAULT_LOGGER.debug("check", extra={"label": self, "sql": sql})
 
             check_df = self.spark.sql(sql)
             if not check_df.isEmpty():
                 rectify = True
-                DEFAULT_LOGGER.debug("rectify enabled", extra={"job": self})
+                DEFAULT_LOGGER.debug("rectify enabled", extra={"label": self})
 
         context = {
             "soft_delete": self.slowly_changing_dimension,
@@ -279,29 +288,30 @@ class Silver(BaseJob):
             "order_duplicate_by": order_duplicate_by,
         }
 
-        if self.slowly_changing_dimension:
-            if "__key" not in df.columns:
-                context["add_key"] = True
-
         if self.mode == "memory":
             context["mode"] = "complete"
-        if self.mode == "latest":
-            context["slice"] = "latest"
 
-        if self.change_data_capture == "scd2":
-            context["correct_valid_from"] = True
+        if self.slowly_changing_dimension:
+            if "__key" not in df.columns:
+                context["add_key"] = True
 
-        if nocdc:
-            if "__operation" in df.columns:
-                context["except"] = ["__operation"]
         if nocdc and self.mode == "memory":
             if "__operation" not in df.columns:
                 context["add_operation"] = "upsert"
-                context["except"] = ["__operation"]
 
+        if self.mode == "latest":
+            context["slice"] = "latest"
         if not self.stream and self.mode == "update":
             context["slice"] = "update"
 
+        if self.change_data_capture == "scd2":
+            context["correct_valid_from"] = True
+
+        if "__operation" in df.columns:
+            context["exclude"] = ["__operation"]
+        if nocdc:  # operation is passed from the bronze layer
+            context["exclude"] = ["__operation"]
+
         return context
 
     def for_each_batch(self, df: DataFrame, batch: Optional[int] = None, **kwargs):
@@ -313,12 +323,12 @@ class Silver(BaseJob):
         name = f"{self.step}_{self.topic}_{self.item}"
         if batch is not None:
             name = f"{name}__{batch}"
-        global_temp_view = create_or_replace_global_temp_view(name=name, df=df)
+        global_temp_view = create_or_replace_global_temp_view(name=name, df=df, job=self)
         sql = f"select * from {global_temp_view}"
 
         check_df = self.spark.sql(sql)
         if check_df.isEmpty():
-            DEFAULT_LOGGER.warning("no data", extra={"job": self})
+            DEFAULT_LOGGER.warning("no data", extra={"label": self})
             return
 
         if self.mode == "update":
@@ -359,16 +369,5 @@ class Silver(BaseJob):
 
     def drop(self):
         super().drop()
-        DEFAULT_LOGGER.debug("drop current view", extra={"job": self})
+        DEFAULT_LOGGER.debug("drop current view", extra={"label": self})
         self.spark.sql(f"drop view if exists {self.qualified_name}__current")
-
-    def optimize(
-        self,
-        vacuum: Optional[bool] = True,
-        optimize: Optional[bool] = True,
-        analyze: Optional[bool] = True,
-    ):
-        if self.mode == "memory":
-            DEFAULT_LOGGER.debug("memory (no optimize)", extra={"job": self})
-        else:
-            super().optimize(vacuum=vacuum, optimize=optimize, analyze=analyze)

fabricks/core/masks.py

@@ -0,0 +1,52 @@
+import os
+from typing import List, Optional
+
+from pyspark.sql import SparkSession
+
+from fabricks.context import CATALOG, PATH_MASKS, SPARK
+from fabricks.context.log import DEFAULT_LOGGER
+
+
+def register_all_masks():
+    """
+    Register all masks.
+    """
+
+    DEFAULT_LOGGER.info("register masks")
+    for mask in get_masks():
+        split = mask.split(".")
+        try:
+            register_mask(mask=split[0])
+        except Exception as e:
+            DEFAULT_LOGGER.exception(f"could not register mask {mask}", exc_info=e)
+
+
+def get_masks() -> List[str]:
+    return [os.path.basename(f) for f in PATH_MASKS.walk()]
+
+
+def is_registered(mask: str, spark: Optional[SparkSession] = None) -> bool:
+    if spark is None:
+        spark = SPARK
+    assert spark is not None
+
+    df = spark.sql("show user functions in default")
+
+    if CATALOG:
+        df = df.where(f"function == '{CATALOG}.default.mask_{mask}'")
+    else:
+        df = df.where(f"function == 'spark_catalog.default.mask_{mask}'")
+
+    return not df.isEmpty()
+
+
+def register_mask(mask: str, spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = SPARK
+    assert spark is not None
+
+    if not is_registered(mask, spark):
+        DEFAULT_LOGGER.debug(f"register mask {mask}")
+
+        path = PATH_MASKS.joinpath(f"{mask}.sql")
+        spark.sql(path.get_sql())

fabricks/core/parsers/base.py

@@ -6,7 +6,7 @@ from pyspark.sql.functions import col, expr, from_json, lit
 from pyspark.sql.types import MapType, StringType
 
 from fabricks.core.parsers._types import ParserOptions
-from fabricks.core.utils import clean
+from fabricks.core.parsers.utils import clean
 from fabricks.utils.path import Path
 from fabricks.utils.read.read import read
 
@@ -26,7 +26,7 @@ class BaseParser(ABC):
             "__timestamp",
             expr("left(concat_ws('', slice(__split, __split_size - 4, 4), '00'), 14)"),
         )
-        df = df.withColumn("__timestamp", expr("to_timestamp(__timestamp, 'yyyyMMddHHmmss')"))
+        df = df.withColumn("__timestamp", expr("try_to_timestamp(__timestamp, 'yyyyMMddHHmmss')"))
         df = df.drop("__split", "__split_size")
 
         return df

fabricks/core/schedules/__init__.py

@@ -0,0 +1,14 @@
+from fabricks.core.schedules.generate import generate
+from fabricks.core.schedules.process import process
+from fabricks.core.schedules.run import run
+from fabricks.core.schedules.terminate import terminate
+from fabricks.core.schedules.views import create_or_replace_view, create_or_replace_views
+
+__all__ = [
+    "process",
+    "generate",
+    "terminate",
+    "run",
+    "create_or_replace_view",
+    "create_or_replace_views",
+]

fabricks/core/schedules/diagrams.py

@@ -0,0 +1,46 @@
+from pyspark.sql import DataFrame
+
+
+def get_dependencies(name: str) -> DataFrame:
+    from fabricks.core.dags import DagGenerator
+
+    g = DagGenerator(schedule=name)
+    return g.get_dependencies()
+
+
+def get_mermaid_diagram(name: str) -> str:
+    df = get_dependencies(name)
+
+    df = df.withColumnRenamed("ParentId", "parent_id")
+    df = df.withColumnRenamed("Parent", "parent")
+    df = df.withColumnRenamed("JobId", "job_id")
+    df = df.withColumnRenamed("Job", "job")
+
+    dependencies = df.select("parent_id", "parent", "job_id", "job").collect()
+
+    out = "flowchart TD\n"
+
+    unique_nodes = set()
+
+    for row in dependencies:
+        parent_id = str(row["parent_id"])
+        parent_name = str(row["parent"])
+        child_id = str(row["job_id"])
+        child_name = str(row["job"])
+
+        if parent_id != "0" and parent_id is not None:
+            if parent_id not in unique_nodes:
+                out += f"    {parent_id}[{parent_name}]\n"
+                unique_nodes.add(parent_id)
+
+            if child_id not in unique_nodes:
+                out += f"    {child_id}[{child_name}]\n"
+                unique_nodes.add(child_id)
+
+            out += f"    {parent_id} --> {child_id}\n"
+        else:
+            if child_id not in unique_nodes:
+                out += f"    {child_id}[{child_name}]\n"
+                unique_nodes.add(child_id)
+
+    return out

fabricks/core/schedules/get_schedule.py

@@ -0,0 +1,5 @@
+from fabricks.core.jobs.get_schedule import get_schedule  # void circular import
+
+__all__ = [
+    "get_schedule",
+]

fabricks/core/schedules/get_schedules.py

@@ -0,0 +1,9 @@
+from fabricks.core.jobs.get_schedules import (  # void circular import
+    get_schedules,
+    get_schedules_df,
+)
+
+__all__ = [
+    "get_schedules",
+    "get_schedules_df",
+]

fabricks/core/schedules/run.py

@@ -0,0 +1,3 @@
+from fabricks.core.dags.run import run
+
+__all__ = ["run"]

fabricks/core/schedules/views.py

@@ -0,0 +1,61 @@
+from fabricks.context import SPARK
+from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.core.schedules.get_schedule import get_schedule
+from fabricks.core.schedules.get_schedules import get_schedules_df
+from fabricks.utils.sqlglot import fix as fix_sql
+
+
+def create_or_replace_view_internal(name: str, options: dict):
+    step = "-- no step provided"
+    tag = "-- no tag provided"
+    view = "-- no view provided"
+
+    assert isinstance(options, dict), "options must be a dict"
+
+    if options.get("steps") is not None:
+        steps = [f"'{s}'" for s in options.get("steps")]  # type: ignore
+        step = f"and j.step in ({', '.join(steps)})"
+
+    if options.get("tag") is not None:
+        tag = f"""and array_contains(j.tags, '{options.get("tag")}')"""
+
+    if options.get("view") is not None:
+        view = f"""inner join fabricks.{options.get("view")} v on j.job_id = v.job_id"""
+
+    sql = f"""
+    create or replace view fabricks.{name}_schedule
+    as
+    select
+        j.*
+    from
+        fabricks.jobs j
+        {view}
+    where
+        true 
+        {step}
+        {tag}
+        and j.type not in ('manual')
+    """
+    sql = fix_sql(sql)
+    DEFAULT_LOGGER.debug("create or replace (schedule) view", extra={"label": f"fabricks.{name}_schedule", "sql": sql})
+
+    SPARK.sql(sql)
+
+
+def create_or_replace_view(name: str):
+    sc = get_schedule(name=name)
+    try:
+        create_or_replace_view_internal(sc["name"], sc["options"])
+    except Exception as e:
+        DEFAULT_LOGGER.exception(f"could not create nor replace view {sc['name']}", exc_info=e)
+
+
+def create_or_replace_views():
+    DEFAULT_LOGGER.info("create or replace (schedule) views")
+
+    df = get_schedules_df()
+    for row in df.collect():
+        try:
+            create_or_replace_view_internal(row.name, row.options.asDict())
+        except Exception as e:
+            DEFAULT_LOGGER.exception(f"could not create nor replace view {row.name}", exc_info=e)