fabricks 3.0.19__py3-none-any.whl → 4.0.1__py3-none-any.whl

fabricks/core/jobs/get_jobs.py

@@ -1,27 +1,26 @@
-from dataclasses import dataclass
-from typing import List, Literal, Optional, TypedDict, Union, overload
+from typing import List, Literal, Optional, Union, overload
 
+from pydantic import BaseModel
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import expr
 from pyspark.sql.types import Row
+from sparkdantic import create_spark_schema
 
 from fabricks.context import IS_JOB_CONFIG_FROM_YAML, PATHS_RUNTIME, SPARK
-from fabricks.core.jobs.base._types import AllowedModes, TStep
 from fabricks.core.jobs.base.job import BaseJob
 from fabricks.core.jobs.get_job import get_job, get_job_internal
+from fabricks.models import AllowedModes
 from fabricks.utils.helpers import concat_dfs, run_in_parallel
-from fabricks.utils.path import Path
+from fabricks.utils.path import GitPath
 from fabricks.utils.read import read_yaml
-from fabricks.utils.schema import get_schema_for_type
 
 
-class GenericOptions(TypedDict):
+class GenericOptions(BaseModel):
     mode: AllowedModes
 
 
-@dataclass
-class JobConfGeneric:
-    step: TStep
+class JobConfGeneric(BaseModel):
+    step: str
     job_id: str
     topic: str
     item: str
@@ -39,9 +38,9 @@ def get_jobs_internal():
 
 def get_jobs_internal_df() -> DataFrame:
     if IS_JOB_CONFIG_FROM_YAML:
-        schema = get_schema_for_type(JobConfGeneric)
+        schema = create_spark_schema(JobConfGeneric)
 
-        def _read_yaml(path: Path):
+        def _read_yaml(path: GitPath):
             df = SPARK.createDataFrame(read_yaml(path, root="job"), schema=schema)  # type: ignore
             if df:
                 df = df.withColumn("job_id", expr("md5(concat(step,'.',topic,'_',item))"))

fabricks/core/jobs/get_schedules.py

@@ -1,23 +1,9 @@
-from typing import List, Optional, TypedDict
-
 from pyspark.sql import DataFrame
+from sparkdantic import create_spark_schema
 
 from fabricks.context import PATH_SCHEDULES, SPARK
-from fabricks.core.jobs.base._types import TStep
+from fabricks.models.schedule import Schedule
 from fabricks.utils.read.read_yaml import read_yaml
-from fabricks.utils.schema import get_schema_for_type
-
-
-class Options(TypedDict):
-    steps: Optional[List[TStep]]
-    tag: Optional[str]
-    view: Optional[str]
-    variables: Optional[dict[str, str]]
-
-
-class Schedule(TypedDict):
-    name: str
-    options: Options
 
 
 def get_schedules():
@@ -25,7 +11,7 @@ def get_schedules():
 
 
 def get_schedules_df() -> DataFrame:
-    schema = get_schema_for_type(Schedule)
+    schema = create_spark_schema(Schedule)
     df = SPARK.createDataFrame(list(get_schedules()), schema=schema)  # type: ignore
 
     assert df, "no schedules found"

fabricks/core/jobs/gold.py

@@ -1,6 +1,6 @@
 import re
 from collections.abc import Sequence
-from typing import List, Optional, Union, cast
+from typing import List, Literal, Optional, Union, cast
 
 from pyspark.sql import DataFrame
 from pyspark.sql.types import Row
@@ -8,18 +8,18 @@ from typing_extensions import deprecated
 
 from fabricks.cdc.nocdc import NoCDC
 from fabricks.context.log import DEFAULT_LOGGER
-from fabricks.core.jobs.base._types import JobDependency, TGold
 from fabricks.core.jobs.base.job import BaseJob
-from fabricks.core.udfs import is_registered, register_udf, udf_prefix
+from fabricks.core.udfs import UDF_PREFIX, is_registered, register_udf
 from fabricks.metastore.view import create_or_replace_global_temp_view
-from fabricks.utils.path import Path
+from fabricks.models import JobDependency, JobGoldOptions, StepGoldConf, StepGoldOptions
+from fabricks.utils.path import GitPath
 from fabricks.utils.sqlglot import fix, get_tables
 
 
 class Gold(BaseJob):
     def __init__(
         self,
-        step: TGold,
+        step: str,
         topic: Optional[str] = None,
         item: Optional[str] = None,
         job_id: Optional[str] = None,
@@ -35,16 +35,31 @@ class Gold(BaseJob):
         )
 
     _sql: Optional[str] = None
-    _sql_path: Optional[Path] = None
+    _sql_path: Optional[GitPath] = None
     _schema_drift: Optional[bool] = None
 
     @classmethod
     def from_job_id(cls, step: str, job_id: str, *, conf: Optional[Union[dict, Row]] = None):
-        return cls(step=cast(TGold, step), job_id=job_id)
+        return cls(step=step, job_id=job_id)
 
     @classmethod
     def from_step_topic_item(cls, step: str, topic: str, item: str, *, conf: Optional[Union[dict, Row]] = None):
-        return cls(step=cast(TGold, step), topic=topic, item=item)
+        return cls(step=step, topic=topic, item=item)
+
+    @property
+    def options(self) -> JobGoldOptions:
+        """Direct access to typed gold job options."""
+        return self.conf.options  # type: ignore
+
+    @property
+    def step_conf(self) -> StepGoldConf:
+        """Direct access to typed gold step conf."""
+        return self.base_step_conf  # type: ignore
+
+    @property
+    def step_options(self) -> StepGoldOptions:
+        """Direct access to typed gold step options."""
+        return self.base_step_conf.options  # type: ignore
 
     @property
     def stream(self) -> bool:
@@ -53,7 +68,7 @@ class Gold(BaseJob):
     @property
     def schema_drift(self) -> bool:
         if not self._schema_drift:
-            _schema_drift = self.step_conf.get("options", {}).get("schema_drift", False)
+            _schema_drift = self.step_conf.options.schema_drift or False
             assert _schema_drift is not None
             self._schema_drift = cast(bool, _schema_drift)
         return self._schema_drift
@@ -68,7 +83,7 @@ class Gold(BaseJob):
 
     @property
     def sql(self) -> str:
-        sql = self.paths.runtime.get_sql()
+        sql = self.paths.to_runtime.get_sql()
         return fix(sql, keep_comments=False)
 
     @deprecated("use sql instead")
@@ -81,17 +96,17 @@ class Gold(BaseJob):
             return []
 
         # udf not allowed in notebook
-        elif self.options.job.get("notebook"):
+        elif self.options.notebook:
             return []
 
         # udf not allowed in table
-        elif self.options.job.get("table"):
+        elif self.options.table:
             return []
 
         else:
             matches = []
-            if f"{udf_prefix}" in self.sql:
-                r = re.compile(rf"(?<={udf_prefix})\w*(?=\()")
+            if f"{UDF_PREFIX}" in self.sql:
+                r = re.compile(rf"(?<={UDF_PREFIX})\w*(?=\()")
                 matches = re.findall(r, self.sql)
                 matches = set(matches)
                 matches = list(matches)
@@ -114,7 +129,7 @@ class Gold(BaseJob):
         schema_only: Optional[bool] = False,
         **kwargs,
     ) -> DataFrame:
-        if self.options.job.get_boolean("requirements"):
+        if self.options.requirements:
             import sys
 
             sys.path.append("/dbfs/mnt/fabricks/site-packages")
@@ -122,28 +137,28 @@ class Gold(BaseJob):
         if self.mode == "invoke":
             df = self.spark.createDataFrame([{}])  # type: ignore
 
-        elif self.options.job.get("notebook"):
-            invokers = self.options.invokers.get_list("run")
+        elif self.options.notebook:
+            invokers = self.invoker_options.run or [] if self.invoker_options else []
             assert len(invokers) <= 1, "at most one invoker allowed when notebook is true"
 
             path = None
             if invokers:
-                notebook = invokers[0].get("notebook")
-                if notebook:
-                    from fabricks.context import PATH_RUNTIME
+                from fabricks.context import PATH_RUNTIME
 
-                    path = PATH_RUNTIME.joinpath(notebook)
+                path = PATH_RUNTIME.joinpath(invokers[0].notebook) if invokers[0].notebook else None
 
             if path is None:
-                path = self.paths.runtime
+                path = self.paths.to_runtime
+
+            assert path is not None, "path could not be resolved"
 
             global_temp_view = self.invoke(path=path, schema_only=schema_only, **kwargs)
             assert global_temp_view is not None, "global_temp_view not found"
 
             df = self.spark.sql(f"select * from global_temp.{global_temp_view}")
 
-        elif self.options.job.get("table"):
-            table = self.options.job.get("table")
+        elif self.options.table:
+            table = self.options.table
             df = self.spark.read.table(table)  # type: ignore
 
         else:
@@ -168,11 +183,11 @@ class Gold(BaseJob):
 
     def get_dependencies(self) -> Sequence[JobDependency]:
         data = []
-        parents = self.options.job.get_list("parents") or []
+        parents = self.options.parents or []
 
         if self.mode == "invoke":
             dependencies = []
-        elif self.options.job.get("notebook"):
+        elif self.options.notebook:
             dependencies = self._get_notebook_dependencies()
         else:
             dependencies = self._get_sql_dependencies()
@@ -189,7 +204,7 @@ class Gold(BaseJob):
         return data
 
     def _get_sql_dependencies(self) -> List[str]:
-        from fabricks.core.jobs.base._types import Steps
+        from fabricks.context import Steps
 
         steps = [str(s) for s in Steps]
         return get_tables(self.sql, allowed_databases=steps)
@@ -217,13 +232,13 @@ class Gold(BaseJob):
 
     def get_cdc_context(self, df: DataFrame, reload: Optional[bool] = None) -> dict:
         # assume no duplicate in gold (to improve performance)
-        deduplicate = self.options.job.get_boolean("deduplicate", None)
+        deduplicate = self.options.deduplicate
         # assume no reload in gold (to improve performance)
-        rectify = self.options.job.get_boolean("rectify_as_upserts", None)
+        rectify = self.options.rectify_as_upserts
 
-        add_metadata = self.options.job.get_boolean("metadata", None)
+        add_metadata = self.options.metadata
         if add_metadata is None:
-            add_metadata = self.step_conf.get("options", {}).get("metadata", False)
+            add_metadata = self.step_conf.options.metadata or False
 
         context = {
             "add_metadata": add_metadata,
@@ -288,10 +303,12 @@ class Gold(BaseJob):
 
         # correct __valid_from
         if self.change_data_capture == "scd2":
-            context["correct_valid_from"] = self.options.job.get_boolean("correct_valid_from", True)
+            context["correct_valid_from"] = (
+                self.options.correct_valid_from if self.options.correct_valid_from is not None else True
+            )
 
         # add __timestamp
-        if self.options.job.get_boolean("persist_last_timestamp"):
+        if self.options.persist_last_timestamp:
             if self.change_data_capture == "scd1":
                 if "__timestamp" not in df.columns:
                     context["add_timestamp"] = True
@@ -299,6 +316,14 @@ class Gold(BaseJob):
                 if "__valid_from" not in df.columns:
                     context["add_timestamp"] = True
 
+        # add __updated
+        if self.options.persist_last_updated_timestamp:
+            if "__last_updated" not in df.columns:
+                context["add_last_updated"] = True
+        if self.options.last_updated:
+            if "__last_updated" not in df.columns:
+                context["add_last_updated"] = True
+
         if "__order_duplicate_by_asc" in df.columns:
             context["order_duplicate_by"] = {"__order_duplicate_by_asc": "asc"}
         elif "__order_duplicate_by_desc" in df.columns:
@@ -345,7 +370,10 @@ class Gold(BaseJob):
 
     def for_each_run(self, **kwargs):
         last_version = None
-        if self.options.job.get_boolean("persist_last_timestamp"):
+
+        if self.options.persist_last_timestamp:
+            last_version = self.table.get_last_version()
+        if self.options.persist_last_updated_timestamp:
             last_version = self.table.get_last_version()
 
         if self.mode == "invoke":
@@ -354,8 +382,11 @@ class Gold(BaseJob):
         else:
             super().for_each_run(**kwargs)
 
-        if self.options.job.get_boolean("persist_last_timestamp"):
-            self._update_last_timestamp(last_version=last_version)
+        if self.options.persist_last_timestamp:
+            self._persist_timestamp(field="__timestamp", last_version=last_version)
+
+        if self.options.persist_last_updated_timestamp:
+            self._persist_timestamp(field="__last_updated", last_version=last_version)
 
     def create(self):
         if self.mode == "invoke":
@@ -363,11 +394,11 @@ class Gold(BaseJob):
         else:
             self.register_udfs()
             super().create()
-            if self.options.job.get_boolean("persist_last_timestamp"):
-                self._update_last_timestamp(create=True)
+            if self.options.persist_last_timestamp:
+                self._persist_timestamp(create=True)
 
     def register(self):
-        if self.options.job.get_boolean("persist_last_timestamp"):
+        if self.options.persist_last_timestamp:
             self.cdc_last_timestamp.table.register()
 
         if self.mode == "invoke":
@@ -376,7 +407,7 @@ class Gold(BaseJob):
             super().register()
 
     def drop(self):
-        if self.options.job.get_boolean("persist_last_timestamp"):
+        if self.options.persist_last_timestamp:
             self.cdc_last_timestamp.drop()
 
         super().drop()
@@ -389,14 +420,25 @@ class Gold(BaseJob):
         cdc = NoCDC(self.step, self.topic, f"{self.item}__last_timestamp")
         return cdc
 
-    def _update_last_timestamp(self, last_version: Optional[int] = None, create: bool = False):
+    def _persist_timestamp(
+        self,
+        field: Literal["__timestamp", "__last_updated"] = "__timestamp",
+        last_version: Optional[int] = None,
+        create: bool = False,
+    ):
         df = self.spark.sql(f"select * from {self} limit 1")
 
         fields = []
-        if self.change_data_capture == "scd1":
-            fields.append("max(__timestamp) :: timestamp as __timestamp")
-        elif self.change_data_capture == "scd2":
-            fields.append("max(__valid_from) :: timestamp as __timestamp")
+
+        if field == "__last_updated":
+            fields.append("max(__last_updated) :: timestamp as __last_updated")
+
+        elif field == "__timestamp":
+            if self.change_data_capture == "scd1":
+                fields.append("max(__timestamp) :: timestamp as __timestamp")
+            elif self.change_data_capture == "scd2":
+                fields.append("max(__valid_from) :: timestamp as __timestamp")
+
         if "__source" in df.columns:
             fields.append("__source")
 
@@ -412,7 +454,7 @@ class Gold(BaseJob):
         else:
             self.cdc_last_timestamp.overwrite(df)
 
-    def overwrite(self, schedule: Optional[str] = None):
+    def overwrite(self, schedule: Optional[str] = None, invoke: Optional[bool] = False):
         if self.mode == "invoke":
             DEFAULT_LOGGER.debug("invoke (no overwrite)", extra={"label": self})
             return
@@ -423,4 +465,4 @@ class Gold(BaseJob):
             return
 
         self.overwrite_schema()
-        self.run(reload=True, schedule=schedule)
+        self.run(reload=True, schedule=schedule, invoke=invoke)

fabricks/core/jobs/silver.py

@@ -1,4 +1,4 @@
-from typing import Optional, Sequence, Union, cast
+from typing import Optional, Sequence, Union
 
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import expr
@@ -6,10 +6,10 @@ from pyspark.sql.types import Row
 
 from fabricks.cdc.nocdc import NoCDC
 from fabricks.context.log import DEFAULT_LOGGER
-from fabricks.core.jobs.base._types import JobDependency, TBronze, TSilver
 from fabricks.core.jobs.base.job import BaseJob
 from fabricks.core.jobs.bronze import Bronze
 from fabricks.metastore.view import create_or_replace_global_temp_view
+from fabricks.models import JobDependency, JobSilverOptions, StepSilverConf, StepSilverOptions
 from fabricks.utils.helpers import concat_dfs
 from fabricks.utils.read.read import read
 from fabricks.utils.sqlglot import fix as fix_sql
@@ -18,7 +18,7 @@ from fabricks.utils.sqlglot import fix as fix_sql
 class Silver(BaseJob):
     def __init__(
         self,
-        step: TSilver,
+        step: str,
         topic: Optional[str] = None,
         item: Optional[str] = None,
         job_id: Optional[str] = None,
@@ -33,23 +33,38 @@ class Silver(BaseJob):
             conf=conf,
         )
 
-    _parent_step: Optional[TBronze] = None
+    _parent_step: Optional[str] = None
     _stream: Optional[bool] = None
 
     @classmethod
     def from_job_id(cls, step: str, job_id: str, *, conf: Optional[Union[dict, Row]] = None):
-        return cls(step=cast(TSilver, step), job_id=job_id, conf=conf)
+        return cls(step=step, job_id=job_id, conf=conf)
 
     @classmethod
     def from_step_topic_item(cls, step: str, topic: str, item: str, *, conf: Optional[Union[dict, Row]] = None):
-        return cls(step=cast(TSilver, step), topic=topic, item=item, conf=conf)
+        return cls(step=step, topic=topic, item=item, conf=conf)
+
+    @property
+    def options(self) -> JobSilverOptions:
+        """Direct access to typed silver job options."""
+        return self.conf.options  # type: ignore
+
+    @property
+    def step_conf(self) -> StepSilverConf:
+        """Direct access to typed silver step conf."""
+        return self.base_step_conf  # type: ignore
+
+    @property
+    def step_options(self) -> StepSilverOptions:
+        """Direct access to typed silver step options."""
+        return self.base_step_conf.options  # type: ignore
 
     @property
     def stream(self) -> bool:
         if not self._stream:
-            _stream = self.options.job.get("stream")
+            _stream = self.options.stream
             if _stream is None:
-                _stream = self.step_conf.get("options", {}).get("stream")
+                _stream = self.step_conf.options.stream
             self._stream = _stream if _stream is not None else True
         return self._stream  # type: ignore
 
@@ -66,18 +81,17 @@ class Silver(BaseJob):
         return self.mode in ["combine", "memory"]
 
     @property
-    def parent_step(self) -> TBronze:
+    def parent_step(self) -> str:
         if not self._parent_step:
-            _parent_step = self.step_conf.get("options", {}).get("parent")
-            _parent_step = cast(TBronze, _parent_step)
+            _parent_step = self.step_conf.options.parent
             assert _parent_step is not None
-            self._parent_step = _parent_step
+            self._parent_step = str(_parent_step)
         return self._parent_step
 
-    def base_transform(self, df: DataFrame) -> DataFrame:
-        df = df.transform(self.extend)
-
+    def update_metadata(self, df: DataFrame) -> DataFrame:
         if "__metadata" in df.columns:
+            DEFAULT_LOGGER.debug("update metadata", extra={"label": self})
+
             df = df.withColumn(
                 "__metadata",
                 expr(
@@ -88,11 +102,18 @@ class Silver(BaseJob):
                         __metadata.file_size as file_size,            
                         __metadata.file_modification_time as file_modification_time,
                         __metadata.inserted as inserted,
-                    cast(current_timestamp() as timestamp) as updated
+                        cast(current_timestamp() as timestamp) as updated
                     )
                     """
                 ),
             )
+
+        return df
+
+    def base_transform(self, df: DataFrame) -> DataFrame:
+        df = df.transform(self.extend)
+        df = self.update_metadata(df)
+
         return df
 
     def get_data(
@@ -153,7 +174,6 @@ class Silver(BaseJob):
 
         # transforms
         df = self.filter_where(df)
-        df = self.encrypt(df)
         if transform:
             df = self.base_transform(df)
 
@@ -165,7 +185,7 @@ class Silver(BaseJob):
     def get_dependencies(self) -> Sequence[JobDependency]:
         dependencies = []
 
-        parents = self.options.job.get_list("parents") or []
+        parents = self.options.parents or []
         if parents:
             for p in parents:
                 dependencies.append(JobDependency.from_parts(self.job_id, p, "job"))
@@ -237,9 +257,9 @@ class Silver(BaseJob):
         except Py4JJavaError as e:
             DEFAULT_LOGGER.exception("fail to create nor replace view", extra={"label": self}, exc_info=e)
 
-    def overwrite(self, schedule: Optional[str] = None):
+    def overwrite(self, schedule: Optional[str] = None, invoke: Optional[bool] = False):
         self.truncate()
-        self.run(schedule=schedule)
+        self.run(schedule=schedule, invoke=invoke)
 
     def overwrite_schema(self, df: Optional[DataFrame] = None):
         DEFAULT_LOGGER.warning("overwrite schema not allowed", extra={"label": self})
@@ -251,7 +271,7 @@ class Silver(BaseJob):
 
         not_append = not self.mode == "append"
         nocdc = self.change_data_capture == "nocdc"
-        order_duplicate_by = self.options.job.get_dict("order_duplicate_by") or {}
+        order_duplicate_by = self.options.order_duplicate_by or {}
 
         rectify = False
         if not_append and not nocdc:
@@ -283,7 +303,7 @@ class Silver(BaseJob):
 
         context = {
             "soft_delete": self.slowly_changing_dimension,
-            "deduplicate": self.options.job.get_boolean("deduplicate", not_append),
+            "deduplicate": self.options.deduplicate if self.options.deduplicate is not None else not_append,
             "rectify": rectify,
             "order_duplicate_by": order_duplicate_by,
         }

fabricks/core/masks.py

@@ -3,22 +3,25 @@ from typing import List, Optional
 
 from pyspark.sql import SparkSession
 
-from fabricks.context import CATALOG, PATH_MASKS, SPARK
+from fabricks.context import CATALOG, CONF_RUNTIME, PATH_MASKS, SPARK
 from fabricks.context.log import DEFAULT_LOGGER
 
+MASK_SCHEMA = CONF_RUNTIME.mask_options.schema_name or "default" if CONF_RUNTIME.mask_options else "default"
+MASK_PREFIX = CONF_RUNTIME.mask_options.prefix or "mask_" if CONF_RUNTIME.mask_options else "mask_"
+
 
 def register_all_masks(override: bool = False):
     """
     Register all masks.
     """
 
-    DEFAULT_LOGGER.info("register masks")
+    DEFAULT_LOGGER.info("register masks", extra={"label": "fabricks"})
     for mask in get_masks():
         split = mask.split(".")
         try:
             register_mask(mask=split[0], override=override)
         except Exception as e:
-            DEFAULT_LOGGER.exception(f"could not register mask {mask}", exc_info=e)
+            DEFAULT_LOGGER.exception(f"could not register mask {mask}", exc_info=e, extra={"label": "fabricks"})
 
 
 def get_masks() -> List[str]:
@@ -30,12 +33,12 @@ def is_registered(mask: str, spark: Optional[SparkSession] = None) -> bool:
         spark = SPARK
     assert spark is not None
 
-    df = spark.sql("show user functions in default")
+    df = spark.sql(f"show user functions in {MASK_SCHEMA}")
 
     if CATALOG:
-        df = df.where(f"function == '{CATALOG}.default.mask_{mask}'")
+        df = df.where(f"function == '{CATALOG}.{MASK_SCHEMA}.{MASK_PREFIX}{mask}'")
     else:
-        df = df.where(f"function == 'spark_catalog.default.mask_{mask}'")
+        df = df.where(f"function == 'spark_catalog.{MASK_SCHEMA}.{MASK_PREFIX}{mask}'")
 
     return not df.isEmpty()
 
@@ -47,9 +50,9 @@ def register_mask(mask: str, override: Optional[bool] = False, spark: Optional[S
 
     if not is_registered(mask, spark) or override:
         if override:
-            DEFAULT_LOGGER.debug(f"override mask {mask}")
+            DEFAULT_LOGGER.debug(f"drop mask {mask}", extra={"label": "fabricks"})
         else:
-            DEFAULT_LOGGER.debug(f"register mask {mask}")
+            DEFAULT_LOGGER.debug(f"register mask {mask}", extra={"label": "fabricks"})
 
         path = PATH_MASKS.joinpath(f"{mask}.sql")
         spark.sql(path.get_sql())

fabricks/core/parsers/__init__.py

@@ -1,4 +1,3 @@
-from fabricks.core.parsers._types import ParserOptions
 from fabricks.core.parsers.base import PARSERS, BaseParser
 from fabricks.core.parsers.decorator import parser
 from fabricks.core.parsers.get_parser import get_parser
@@ -7,6 +6,5 @@ __all__ = [
     "BaseParser",
     "get_parser",
     "parser",
-    "ParserOptions",
     "PARSERS",
 ]