fabricks 3.0.11__py3-none-any.whl

fabricks/core/jobs/silver.py

@@ -0,0 +1,373 @@
+from typing import Optional, Sequence, Union, cast
+
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import expr
+from pyspark.sql.types import Row
+
+from fabricks.cdc.nocdc import NoCDC
+from fabricks.context.log import DEFAULT_LOGGER
+from fabricks.core.jobs.base._types import JobDependency, TBronze, TSilver
+from fabricks.core.jobs.base.job import BaseJob
+from fabricks.core.jobs.bronze import Bronze
+from fabricks.metastore.view import create_or_replace_global_temp_view
+from fabricks.utils.helpers import concat_dfs
+from fabricks.utils.read.read import read
+from fabricks.utils.sqlglot import fix as fix_sql
+
+
+class Silver(BaseJob):
+    def __init__(
+        self,
+        step: TSilver,
+        topic: Optional[str] = None,
+        item: Optional[str] = None,
+        job_id: Optional[str] = None,
+        conf: Optional[Union[dict, Row]] = None,
+    ):  # type: ignore
+        super().__init__(
+            "silver",
+            step=step,
+            topic=topic,
+            item=item,
+            job_id=job_id,
+            conf=conf,
+        )
+
+    _parent_step: Optional[TBronze] = None
+    _stream: Optional[bool] = None
+
+    @classmethod
+    def from_job_id(cls, step: str, job_id: str, *, conf: Optional[Union[dict, Row]] = None):
+        return cls(step=cast(TSilver, step), job_id=job_id, conf=conf)
+
+    @classmethod
+    def from_step_topic_item(cls, step: str, topic: str, item: str, *, conf: Optional[Union[dict, Row]] = None):
+        return cls(step=cast(TSilver, step), topic=topic, item=item, conf=conf)
+
+    @property
+    def stream(self) -> bool:
+        if not self._stream:
+            _stream = self.options.job.get("stream")
+            if _stream is None:
+                _stream = self.step_conf.get("options", {}).get("stream")
+            self._stream = _stream if _stream is not None else True
+        return self._stream  # type: ignore
+
+    @property
+    def schema_drift(self) -> bool:
+        return True
+
+    @property
+    def persist(self) -> bool:
+        return self.mode in ["update", "append", "latest"]
+
+    @property
+    def virtual(self) -> bool:
+        return self.mode in ["combine", "memory"]
+
+    @property
+    def parent_step(self) -> TBronze:
+        if not self._parent_step:
+            _parent_step = self.step_conf.get("options", {}).get("parent")
+            _parent_step = cast(TBronze, _parent_step)
+            assert _parent_step is not None
+            self._parent_step = _parent_step
+        return self._parent_step
+
+    def base_transform(self, df: DataFrame) -> DataFrame:
+        df = df.transform(self.extend)
+
+        if "__metadata" in df.columns:
+            df = df.withColumn(
+                "__metadata",
+                expr(
+                    """
+                    struct(
+                        __metadata.file_path as file_path,
+                        __metadata.file_name as file_name,
+                        __metadata.file_size as file_size,            
+                        __metadata.file_modification_time as file_modification_time,
+                        __metadata.inserted as inserted,
+                    cast(current_timestamp() as timestamp) as updated
+                    )
+                    """
+                ),
+            )
+        return df
+
+    def get_data(
+        self,
+        stream: bool = False,
+        transform: Optional[bool] = False,
+        schema_only: Optional[bool] = False,
+        **kwargs,
+    ) -> DataFrame:
+        deps = self.get_dependencies()
+        assert deps, "not dependency found"
+
+        if self.mode == "memory":
+            assert len(deps) == 1, f"more than 1 dependency not allowed ({deps})"
+
+            parent = deps[0].parent
+            df = self.spark.sql(f"select * from {parent}")
+
+        elif self.mode == "combine":
+            dfs = []
+
+            for row in sorted(deps, key=lambda x: x.parent_id):
+                df = self.spark.sql(f"select * from {row.parent}")
+                dfs.append(df)
+
+            df = concat_dfs(dfs)
+            assert df is not None
+
+        else:
+            dfs = []
+
+            for row in sorted(deps, key=lambda x: x.parent_id):
+                try:
+                    bronze = Bronze.from_job_id(step=self.parent_step, job_id=row.parent_id)
+                    if bronze.mode in ["memory", "register"]:
+                        # data already transformed if bronze is persisted
+                        df = bronze.get_data(stream=stream, transform=True)
+                    else:
+                        df = read(
+                            stream=stream,
+                            path=bronze.table.deltapath,
+                            file_format="delta",
+                            metadata=False,
+                            spark=self.spark,
+                        )
+
+                    if df:
+                        if len(deps) > 1:
+                            assert "__source" in df.columns, "__source not found"
+                        dfs.append(df)
+
+                except Exception as e:
+                    DEFAULT_LOGGER.exception("fail to get dependencies", extra={"label": self})
+                    raise e
+
+            df = concat_dfs(dfs)
+            assert df is not None
+
+        # transforms
+        df = self.filter_where(df)
+        df = self.encrypt(df)
+        if transform:
+            df = self.base_transform(df)
+
+        if schema_only:
+            df = df.where("1 == 2")
+
+        return df
+
+    def get_dependencies(self) -> Sequence[JobDependency]:
+        dependencies = []
+
+        parents = self.options.job.get_list("parents") or []
+        if parents:
+            for p in parents:
+                dependencies.append(JobDependency.from_parts(self.job_id, p, "job"))
+
+        else:
+            p = f"{self.parent_step}.{self.topic}_{self.item}"
+            dependencies.append(JobDependency.from_parts(self.job_id, p, "parser"))
+
+        return dependencies
+
+    def create_or_replace_view(self):
+        assert self.mode in ["memory", "combine"], f"{self.mode} not allowed"
+
+        deps = self.get_dependencies()
+        assert deps, "dependency not found"
+
+        if self.mode == "combine":
+            queries = []
+
+            for row in deps:
+                columns = self.get_data().columns
+                df = self.spark.sql(f"select * from {row.parent}")
+                cols = [f"`{c}`" if c in df.columns else f"null as `{c}`" for c in columns if c not in ["__source"]]
+                source = "__source" if "__source" in df.columns else f"'{row.parent}' as __source"
+                query = f"select {', '.join(cols)}, {source} from {row.parent}"
+                queries.append(query)
+
+            sql = f"create or replace view {self.qualified_name} as {' union all '.join(queries)}"
+            sql = fix_sql(sql)
+            DEFAULT_LOGGER.debug("view", extra={"label": self, "sql": sql})
+            self.spark.sql(sql)
+
+        else:
+            assert len(deps) == 1, "only one dependency allowed"
+
+            parent = deps[0].parent
+            sql = f"select * from {parent}"
+            sql = fix_sql(sql)
+            DEFAULT_LOGGER.debug("view", extra={"label": self, "sql": sql})
+
+            df = self.spark.sql(sql)
+            cdc_options = self.get_cdc_context(df)
+            self.cdc.create_or_replace_view(sql, **cdc_options)
+
+    def create_or_replace_current_view(self):
+        from py4j.protocol import Py4JJavaError
+
+        try:
+            DEFAULT_LOGGER.debug("create or replace current view", extra={"label": self})
+
+            df = self.spark.sql(f"select * from {self.qualified_name}")
+
+            where_clause = "-- no where clause"
+            if "__is_current" in df.columns:
+                where_clause = "where __is_current"
+
+            sql = f"""
+            create or replace view {self.qualified_name}__current with schema evolution as
+            select
+              *
+            from
+              {self.qualified_name}
+              {where_clause}
+            """
+            # sql = fix_sql(sql)
+            # DEFAULT_LOGGER.debug("current view", extra={"label": self, "sql": sql})
+            self.spark.sql(sql)
+
+        except Py4JJavaError as e:
+            DEFAULT_LOGGER.exception("fail to create nor replace view", extra={"label": self}, exc_info=e)
+
+    def overwrite(self, schedule: Optional[str] = None):
+        self.truncate()
+        self.run(schedule=schedule)
+
+    def overwrite_schema(self, df: Optional[DataFrame] = None):
+        DEFAULT_LOGGER.warning("overwrite schema not allowed", extra={"label": self})
+
+    def get_cdc_context(self, df: DataFrame, reload: Optional[bool] = None) -> dict:
+        # if dataframe, reference is passed (BUG)
+        name = f"{self.step}_{self.topic}_{self.item}__check"
+        global_temp_view = create_or_replace_global_temp_view(name=name, df=df, job=self)
+
+        not_append = not self.mode == "append"
+        nocdc = self.change_data_capture == "nocdc"
+        order_duplicate_by = self.options.job.get_dict("order_duplicate_by") or {}
+
+        rectify = False
+        if not_append and not nocdc:
+            if not self.stream and self.mode == "update" and self.table.exists():
+                timestamp = "__valid_from" if self.change_data_capture == "scd2" else "__timestamp"
+                extra_check = f" and __timestamp > coalesce((select max({timestamp}) from {self}), cast('0001-01-01' as timestamp))"
+            else:
+                extra_check = "-- no extra check"
+
+            sql = f"""
+                select 
+                  __operation
+                from
+                  {global_temp_view}
+                where
+                  true
+                  and __operation == 'reload'
+                  {extra_check}
+                limit 
+                  1
+                """
+            sql = fix_sql(sql)
+            DEFAULT_LOGGER.debug("check", extra={"label": self, "sql": sql})
+
+            check_df = self.spark.sql(sql)
+            if not check_df.isEmpty():
+                rectify = True
+                DEFAULT_LOGGER.debug("rectify enabled", extra={"label": self})
+
+        context = {
+            "soft_delete": self.slowly_changing_dimension,
+            "deduplicate": self.options.job.get_boolean("deduplicate", not_append),
+            "rectify": rectify,
+            "order_duplicate_by": order_duplicate_by,
+        }
+
+        if self.mode == "memory":
+            context["mode"] = "complete"
+
+        if self.slowly_changing_dimension:
+            if "__key" not in df.columns:
+                context["add_key"] = True
+
+        if nocdc and self.mode == "memory":
+            if "__operation" not in df.columns:
+                context["add_operation"] = "upsert"
+
+        if self.mode == "latest":
+            context["slice"] = "latest"
+        if not self.stream and self.mode == "update":
+            context["slice"] = "update"
+
+        if self.change_data_capture == "scd2":
+            context["correct_valid_from"] = True
+
+        if "__operation" in df.columns:
+            context["exclude"] = ["__operation"]
+        if nocdc:  # operation is passed from the bronze layer
+            context["exclude"] = ["__operation"]
+
+        return context
+
+    def for_each_batch(self, df: DataFrame, batch: Optional[int] = None, **kwargs):
+        assert self.persist, f"{self.mode} not allowed"
+
+        context = self.get_cdc_context(df)
+
+        # if dataframe, reference is passed (BUG)
+        name = f"{self.step}_{self.topic}_{self.item}"
+        if batch is not None:
+            name = f"{name}__{batch}"
+        global_temp_view = create_or_replace_global_temp_view(name=name, df=df, job=self)
+        sql = f"select * from {global_temp_view}"
+
+        check_df = self.spark.sql(sql)
+        if check_df.isEmpty():
+            DEFAULT_LOGGER.warning("no data", extra={"label": self})
+            return
+
+        if self.mode == "update":
+            assert not isinstance(self.cdc, NoCDC)
+            self.cdc.update(sql, **context)
+
+        elif self.mode == "append":
+            assert isinstance(self.cdc, NoCDC)
+            self.cdc.append(sql, **context)
+
+        elif self.mode == "latest":
+            assert isinstance(self.cdc, NoCDC)
+            check_df = self.spark.sql(
+                f"""
+                select 
+                  __operation
+                from
+                  {global_temp_view}
+                where
+                  __operation <> 'reload'
+                limit 
+                  1
+                """
+            )
+            assert check_df.isEmpty(), f"{check_df.collect()[0][0]} not allowed"
+            self.cdc.complete(sql, **context)
+
+        else:
+            raise ValueError(f"{self.mode} - not allowed")
+
+    def create(self):
+        super().create()
+        self.create_or_replace_current_view()
+
+    def register(self):
+        super().register()
+        self.create_or_replace_current_view()
+
+    def drop(self):
+        super().drop()
+        DEFAULT_LOGGER.debug("drop current view", extra={"label": self})
+        self.spark.sql(f"drop view if exists {self.qualified_name}__current")

fabricks/core/masks.py

@@ -0,0 +1,52 @@
+import os
+from typing import List, Optional
+
+from pyspark.sql import SparkSession
+
+from fabricks.context import CATALOG, PATH_MASKS, SPARK
+from fabricks.context.log import DEFAULT_LOGGER
+
+
+def register_all_masks():
+    """
+    Register all masks.
+    """
+
+    DEFAULT_LOGGER.info("register masks")
+    for mask in get_masks():
+        split = mask.split(".")
+        try:
+            register_mask(mask=split[0])
+        except Exception as e:
+            DEFAULT_LOGGER.exception(f"could not register mask {mask}", exc_info=e)
+
+
+def get_masks() -> List[str]:
+    return [os.path.basename(f) for f in PATH_MASKS.walk()]
+
+
+def is_registered(mask: str, spark: Optional[SparkSession] = None) -> bool:
+    if spark is None:
+        spark = SPARK
+    assert spark is not None
+
+    df = spark.sql("show user functions in default")
+
+    if CATALOG:
+        df = df.where(f"function == '{CATALOG}.default.mask_{mask}'")
+    else:
+        df = df.where(f"function == 'spark_catalog.default.mask_{mask}'")
+
+    return not df.isEmpty()
+
+
+def register_mask(mask: str, spark: Optional[SparkSession] = None):
+    if spark is None:
+        spark = SPARK
+    assert spark is not None
+
+    if not is_registered(mask, spark):
+        DEFAULT_LOGGER.debug(f"register mask {mask}")
+
+        path = PATH_MASKS.joinpath(f"{mask}.sql")
+        spark.sql(path.get_sql())

fabricks/core/parsers/__init__.py

@@ -0,0 +1,12 @@
+from fabricks.core.parsers._types import ParserOptions
+from fabricks.core.parsers.base import PARSERS, BaseParser
+from fabricks.core.parsers.decorator import parser
+from fabricks.core.parsers.get_parser import get_parser
+
+__all__ = [
+    "BaseParser",
+    "get_parser",
+    "parser",
+    "ParserOptions",
+    "PARSERS",
+]

fabricks/core/parsers/_types.py

@@ -0,0 +1,6 @@
+from typing import Optional, TypedDict
+
+
+class ParserOptions(TypedDict):
+    file_format: Optional[str]
+    read_options: Optional[dict[str, str]]

fabricks/core/parsers/base.py

@@ -0,0 +1,95 @@
+from abc import ABC
+from typing import Callable, Optional, final
+
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.functions import col, expr, from_json, lit
+from pyspark.sql.types import MapType, StringType
+
+from fabricks.core.parsers._types import ParserOptions
+from fabricks.core.parsers.utils import clean
+from fabricks.utils.path import Path
+from fabricks.utils.read.read import read
+
+
+class BaseParser(ABC):
+    def __init__(self, options: Optional[ParserOptions], file_format: str):
+        self.options = options or {}
+        self.file_format = file_format
+
+    def add_timestamp_from_file_path(self, df: DataFrame) -> DataFrame:
+        df = df.withColumn(
+            "__split",
+            expr("split(replace(__metadata.file_path, __metadata.file_name), '/')"),
+        )
+        df = df.withColumn("__split_size", expr("size(__split)"))
+        df = df.withColumn(
+            "__timestamp",
+            expr("left(concat_ws('', slice(__split, __split_size - 4, 4), '00'), 14)"),
+        )
+        df = df.withColumn("__timestamp", expr("try_to_timestamp(__timestamp, 'yyyyMMddHHmmss')"))
+        df = df.drop("__split", "__split_size")
+
+        return df
+
+    def parse(
+        self,
+        data_path: Path,
+        schema_path: Path,
+        spark: SparkSession,
+        stream: bool,
+    ) -> DataFrame:
+        df = read(
+            stream=stream,
+            path=data_path,
+            file_format=self.file_format,
+            schema_path=schema_path,
+            options=self.options.get("read_options"),
+            spark=spark,
+        )
+
+        if "__timestamp" not in df.columns:
+            df = self.add_timestamp_from_file_path(df)
+
+        return df
+
+    @final
+    def get_data(
+        self,
+        data_path: Path,
+        schema_path: Path,
+        spark: SparkSession,
+        stream: bool,
+    ) -> DataFrame:
+        """
+        Retrieves and processes data from the specified data path using the provided schema.
+
+        Args:
+            data_path (Path): The path to the data file.
+            schema_path (Path): The path to the schema file.
+            spark (SparkSession): The SparkSession object.
+            stream (bool): Indicates whether the data should be processed as a stream.
+
+        Returns:
+            DataFrame: The processed data as a DataFrame.
+
+        Raises:
+            AssertionError: If the "__timestamp" column is missing in the DataFrame.
+            AssertionError: If the "__metadata.file_path" column is missing in the DataFrame.
+        """
+        df = self.parse(data_path=data_path, schema_path=schema_path, spark=spark, stream=stream)
+        df = df.transform(clean)
+
+        if "__rescued_data" not in df.columns:
+            df = df.withColumn("__rescued_data", lit(None).cast(StringType()))
+
+        df = df.withColumn("__rescued_data", from_json(col("__rescued_data"), MapType(StringType(), StringType())))  # type: ignore
+
+        assert "__timestamp" in df.columns, "__timestamp mandatory in dataframe"
+        assert df.select("__metadata.file_path"), "file_path mandatory in struct __metadata in dataframe"
+        return df
+
+    def __str__(self):
+        return f"{type(self).__name__} ({self.file_format})"
+
+
+PARSERS: dict[str, Callable[[Optional[ParserOptions]], BaseParser]] = {}

fabricks/core/parsers/decorator.py

@@ -0,0 +1,11 @@
+from typing import Callable, Optional
+
+from fabricks.core.parsers._types import ParserOptions
+from fabricks.core.parsers.base import PARSERS, BaseParser
+
+
+def parser(name: str):
+    def decorator(parser: Callable[[Optional[ParserOptions]], BaseParser]):
+        PARSERS[name] = parser
+
+    return decorator

fabricks/core/parsers/get_parser.py

@@ -0,0 +1,26 @@
+from importlib.util import module_from_spec, spec_from_file_location
+from typing import Optional
+
+from fabricks.context import PATH_PARSERS
+from fabricks.core.parsers._types import ParserOptions
+from fabricks.core.parsers.base import PARSERS, BaseParser
+
+
+def get_parser(name: str, parser_options: Optional[ParserOptions] = None) -> BaseParser:
+    if name not in ["json", "parquet", "avro", "csv", "tsv", "delta", "table"]:
+        path = PATH_PARSERS.joinpath(name).append(".py")
+        assert path.exists(), f"parser not found ({path})"
+
+        spec = spec_from_file_location(name, path.string)
+        assert spec, f"parser not found ({path})"
+        assert spec.loader is not None
+
+        mod = module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        parser = PARSERS[name](parser_options)
+
+    else:
+        parser = BaseParser(parser_options, name)
+
+    assert parser
+    return parser

fabricks/core/parsers/utils.py

@@ -0,0 +1,69 @@
+from pyspark.sql import DataFrame
+from pyspark.sql.functions import length, lower
+from pyspark.sql.functions import trim as _trim
+from pyspark.sql.functions import when
+from pyspark.sql.types import DoubleType, FloatType, IntegerType
+
+
+def value_to_none(df: DataFrame) -> DataFrame:
+    cols = [name for name, dtype in df.dtypes if not name.startswith("__")]
+    for c in cols:
+        df = df.withColumn(
+            c,
+            when(length(df[f"`{c}`"].cast("string")) == 0, None)
+            .when(lower(df[f"`{c}`"].cast("string")) == "none", None)
+            .when(lower(df[f"`{c}`"].cast("string")) == "null", None)
+            .when(lower(df[f"`{c}`"].cast("string")) == "blank", None)
+            .when(lower(df[f"`{c}`"].cast("string")) == "(none)", None)
+            .when(lower(df[f"`{c}`"].cast("string")) == "(null)", None)
+            .when(lower(df[f"`{c}`"].cast("string")) == "(blank)", None)
+            .otherwise(df[f"`{c}`"]),
+        )
+    return df
+
+
+def decimal_to_float(df: DataFrame) -> DataFrame:
+    cols = [name for name, dtype in df.dtypes if dtype.startswith("decimal") and not name.startswith("__")]
+    for c in cols:
+        df = df.withColumn(c, df[f"`{c}`"].cast(FloatType()))
+    return df
+
+
+def decimal_to_double(df: DataFrame) -> DataFrame:
+    cols = [name for name, dtype in df.dtypes if dtype.startswith("decimal") and not name.startswith("__")]
+    for c in cols:
+        df = df.withColumn(c, df[f"`{c}`"].cast(DoubleType()))
+    return df
+
+
+def tinyint_to_int(df: DataFrame) -> DataFrame:
+    cols = [name for name, dtype in df.dtypes if dtype.startswith("tinyint") and not name.startswith("__")]
+    for c in cols:
+        df = df.withColumn(c, df[f"`{c}`"].cast(IntegerType()))
+    return df
+
+
+def trim(df: DataFrame) -> DataFrame:
+    cols = [name for name, dtype in df.dtypes if dtype.startswith("string") and not name.startswith("__")]
+    for c in cols:
+        df = df.withColumn(c, _trim(df[f"`{c}`"]))
+    return df
+
+
+def clean(df: DataFrame) -> DataFrame:
+    """
+    Cleans the given DataFrame by performing the following operations:
+    1. Trims whitespace from all string columns.
+    2. Converts empty strings to None.
+    3. Converts decimal values to double.
+
+    Args:
+        df (pandas.DataFrame): The DataFrame to be cleaned.
+
+    Returns:
+        pandas.DataFrame: The cleaned DataFrame.
+    """
+    df = trim(df)
+    df = value_to_none(df)
+    df = decimal_to_double(df)
+    return df

fabricks/core/schedules/__init__.py

@@ -0,0 +1,14 @@
+from fabricks.core.schedules.generate import generate
+from fabricks.core.schedules.process import process
+from fabricks.core.schedules.run import run
+from fabricks.core.schedules.terminate import terminate
+from fabricks.core.schedules.views import create_or_replace_view, create_or_replace_views
+
+__all__ = [
+    "process",
+    "generate",
+    "terminate",
+    "run",
+    "create_or_replace_view",
+    "create_or_replace_views",
+]

fabricks/core/schedules/diagrams.py

@@ -0,0 +1,21 @@
+from pyspark.sql import DataFrame
+
+
+def get_dependencies(name: str) -> DataFrame:
+    from fabricks.core.dags import DagGenerator
+
+    g = DagGenerator(schedule=name)
+    return g.get_dependencies()
+
+
+def get_mermaid_diagram(name: str) -> str:
+    from fabricks.utils.mermaid import get_mermaid_diagram as get_diagram
+
+    df = get_dependencies(name)
+
+    df = df.withColumnRenamed("ParentId", "parent_id")
+    df = df.withColumnRenamed("Parent", "parent")
+    df = df.withColumnRenamed("JobId", "job_id")
+    df = df.withColumnRenamed("Job", "job")
+
+    return get_diagram(df)