fabricks 3.0.5.2__py3-none-any.whl → 3.0.7__py3-none-any.whl

fabricks/core/jobs/base/configurator.py

@@ -4,12 +4,13 @@ from typing import Optional, Union, cast
 
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import Row
+from typing_extensions import deprecated
 
-from fabricks.cdc import SCD1, SCD2, ChangeDataCaptures, NoCDC
+from fabricks.cdc import SCD1, SCD2, AllowedChangeDataCaptures, NoCDC
 from fabricks.context import CONF_RUNTIME, PATHS_RUNTIME, PATHS_STORAGE, STEPS
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.context.spark_session import build_spark_session
-from fabricks.core.jobs.base._types import Modes, Options, Paths, TStep
+from fabricks.core.jobs.base._types import AllowedModes, Options, Paths, TStep
 from fabricks.core.jobs.get_job_conf import get_job_conf
 from fabricks.core.jobs.get_job_id import get_job_id
 from fabricks.metastore.table import Table
@@ -52,36 +53,30 @@ class Configurator(ABC):
     _root: Optional[Path] = None
 
     _cdc: Optional[Union[NoCDC, SCD1, SCD2]] = None
-    _change_data_capture: Optional[ChangeDataCaptures] = None
-    _mode: Optional[Modes] = None
+    _change_data_capture: Optional[AllowedChangeDataCaptures] = None
+    _mode: Optional[AllowedModes] = None
 
     @property
     @abstractmethod
-    def stream(self) -> bool:
-        raise NotImplementedError()
+    def stream(self) -> bool: ...
 
     @property
     @abstractmethod
-    def schema_drift(self) -> bool:
-        raise NotImplementedError()
+    def schema_drift(self) -> bool: ...
 
     @property
     @abstractmethod
-    def persist(self) -> bool:
-        raise NotImplementedError()
+    def persist(self) -> bool: ...
 
     @property
     @abstractmethod
-    def virtual(self) -> bool:
-        raise NotImplementedError()
+    def virtual(self) -> bool: ...
 
     @classmethod
-    def from_step_topic_item(cls, step: str, topic: str, item: str):
-        raise NotImplementedError()
+    def from_step_topic_item(cls, step: str, topic: str, item: str): ...
 
     @classmethod
-    def from_job_id(cls, step: str, job_id: str):
-        raise NotImplementedError()
+    def from_job_id(cls, step: str, job_id: str): ...
 
     @property
     def spark(self) -> SparkSession:
@@ -93,22 +88,22 @@ class Configurator(ABC):
             step_conf_options = step_options.get("conf", {})
             if step_sql_options:
                 for key, value in step_sql_options.items():
-                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"step": self.step})
+                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"label": self.step})
                     spark.sql(f"set {key} = {value}")
             if step_conf_options:
                 for key, value in step_conf_options.items():
-                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"step": self.step})
+                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"label": self.step})
                     spark.conf.set(f"{key}", f"{value}")
 
             job_sql_options = self.options.spark.get_dict("sql")
             job_conf_options = self.options.spark.get_dict("conf")
             if job_sql_options:
                 for key, value in job_sql_options.items():
-                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"job": self})
+                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"label": self})
                     spark.sql(f"set {key} = {value}")
             if job_conf_options:
                 for key, value in job_conf_options.items():
-                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"job": self})
+                    DEFAULT_LOGGER.debug(f"add {key} = {value}", extra={"label": self})
                     spark.conf.set(f"{key}", f"{value}")
 
             self._spark = spark
@@ -195,9 +190,9 @@ class Configurator(ABC):
         return self._options
 
     @property
-    def change_data_capture(self) -> ChangeDataCaptures:
+    def change_data_capture(self) -> AllowedChangeDataCaptures:
         if not self._change_data_capture:
-            cdc: ChangeDataCaptures = self.options.job.get("change_data_capture") or "nocdc"
+            cdc: AllowedChangeDataCaptures = self.options.job.get("change_data_capture") or "nocdc"
             self._change_data_capture = cdc
         return self._change_data_capture
 
@@ -220,49 +215,34 @@ class Configurator(ABC):
         return self.change_data_capture in ["scd1", "scd2"]
 
     @abstractmethod
-    def get_cdc_context(self, df: DataFrame, reload: Optional[bool] = False) -> dict:
-        raise NotImplementedError()
+    def get_cdc_context(self, df: DataFrame, reload: Optional[bool] = False) -> dict: ...
 
     def get_cdc_data(self, stream: bool = False) -> Optional[DataFrame]:
-        df = self.get_data(stream)
+        df = self.get_data(stream=stream)
         if df:
             cdc_context = self.get_cdc_context(df)
             cdc_df = self.cdc.get_data(src=df, **cdc_context)
             return cdc_df
 
     @property
-    def mode(self) -> Modes:
+    def mode(self) -> AllowedModes:
         if not self._mode:
             _mode = self.options.job.get("mode")
             assert _mode is not None
-            self._mode = cast(Modes, _mode)
+            self._mode = cast(AllowedModes, _mode)
         return self._mode
 
     @abstractmethod
-    def get_data(self, stream: bool = False, transform: Optional[bool] = False) -> Optional[DataFrame]:
-        """
-        Retrieves the data for the job.
-
-        Args:
-            stream (bool, optional): If True, the data will be streamed. Defaults to False.
-            transform (bool, optional): If True, the data will be transformed. Defaults to False.
-
-        Returns:
-            DataFrame or None: The retrieved data as a DataFrame, or None if the data is not available.
-        """
-        raise NotImplementedError()
+    def get_data(self, stream: bool = False, transform: Optional[bool] = None, **kwargs) -> Optional[DataFrame]: ...
 
     @abstractmethod
-    def for_each_batch(self, df: DataFrame, batch: Optional[int] = None, **kwargs):
-        raise NotImplementedError()
+    def for_each_batch(self, df: DataFrame, batch: Optional[int] = None, **kwargs): ...
 
     @abstractmethod
-    def for_each_run(self, **kwargs):
-        raise NotImplementedError()
+    def for_each_run(self, **kwargs): ...
 
     @abstractmethod
-    def base_transform(self, df: DataFrame) -> DataFrame:
-        raise NotImplementedError()
+    def base_transform(self, df: DataFrame) -> DataFrame: ...
 
     @abstractmethod
     def run(
@@ -271,47 +251,41 @@ class Configurator(ABC):
         schedule: Optional[str] = None,
         schedule_id: Optional[str] = None,
         invoke: Optional[bool] = True,
-    ):
-        raise NotImplementedError()
+    ): ...
 
+    @deprecated("use maintain instead")
     def optimize(
         self,
         vacuum: Optional[bool] = True,
         optimize: Optional[bool] = True,
         analyze: Optional[bool] = True,
     ):
-        """
-        Optimize the table by performing vacuum, optimizing CDC, and analyzing the table.
-
-        If the mode is set to 'memory', no optimization is performed.
-
-        The retention days for optimization are determined in the following order:
-        1. If 'retention_days' is specified in the job options table, it is used.
-        2. If 'retention_days' is specified in the step configuration table options, it is used.
-        3. If 'retention_days' is specified in the CONF_RUNTIME options, it is used.
-
-        After determining the retention days, the table is vacuumed with the specified retention days,
-        CDC is optimized for the table, and the table is analyzed.
+        return self.maintain(
+            vacuum=vacuum,
+            optimize=optimize,
+            compute_statistics=analyze,
+        )
 
-        Note: This method assumes that either 'runtime' or 'step' or 'job' is specified.
-
-        Returns:
-            None
-        """
+    def maintain(
+        self,
+        vacuum: Optional[bool] = True,
+        optimize: Optional[bool] = True,
+        compute_statistics: Optional[bool] = True,
+    ):
         if self.mode == "memory":
-            DEFAULT_LOGGER.debug("memory (no optimize)", extra={"job": self})
+            DEFAULT_LOGGER.debug("could not maintain (memory)", extra={"label": self})
 
         else:
             if vacuum:
                 self.vacuum()
             if optimize:
                 self.cdc.optimize_table()
-            if analyze:
+            if compute_statistics:
                 self.table.compute_statistics()
 
     def vacuum(self):
         if self.mode == "memory":
-            DEFAULT_LOGGER.debug("memory (no vacuum)", extra={"job": self})
+            DEFAULT_LOGGER.debug("could not vacuum (memory)", extra={"label": self})
 
         else:
             job = self.options.table.get("retention_days")

fabricks/core/jobs/base/generator.py

@@ -4,7 +4,7 @@ from typing import Optional, Sequence, Union, cast
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import lit
 
-from fabricks.cdc import SCD1
+from fabricks.cdc import NoCDC
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.core.jobs.base._types import JobDependency
 from fabricks.core.jobs.base.configurator import Configurator
@@ -14,17 +14,16 @@ from fabricks.metastore.view import create_or_replace_global_temp_view
 
 class Generator(Configurator):
     def update_dependencies(self):
-        DEFAULT_LOGGER.info("update dependencies", extra={"job": self})
+        DEFAULT_LOGGER.info("update dependencies", extra={"label": self})
 
         deps = self.get_dependencies()
         if deps:
             df = self.spark.createDataFrame([d.model_dump() for d in deps])  # type: ignore
-            scd1 = SCD1("fabricks", self.step, "dependencies")
-            scd1.delete_missing(df, keys=["dependency_id"], update_where=f"job_id = '{self.job_id}'", uuid=True)
+            cdc = NoCDC("fabricks", self.step, "dependencies")
+            cdc.delete_missing(df, keys=["dependency_id"], update_where=f"job_id = '{self.job_id}'", uuid=True)
 
     @abstractmethod
-    def get_dependencies(self) -> Sequence[JobDependency]:
-        raise NotImplementedError()
+    def get_dependencies(self) -> Sequence[JobDependency]: ...
 
     def rm(self):
         """
@@ -33,7 +32,7 @@ class Generator(Configurator):
         If the schema folder exists, it will be deleted. The method also calls the `rm_checkpoints` method to remove any checkpoints associated with the generator.
         """
         if self.paths.schema.exists():
-            DEFAULT_LOGGER.info("delete schema folder", extra={"job": self})
+            DEFAULT_LOGGER.info("delete schema folder", extra={"label": self})
             self.paths.schema.rm()
         self.rm_checkpoints()
 
@@ -44,7 +43,7 @@ class Generator(Configurator):
         This method checks if the checkpoints folder exists and deletes it if it does.
         """
         if self.paths.checkpoints.exists():
-            DEFAULT_LOGGER.info("delete checkpoints folder", extra={"job": self})
+            DEFAULT_LOGGER.info("delete checkpoints folder", extra={"label": self})
             self.paths.checkpoints.rm()
 
     def rm_commit(self, id: Union[str, int]):
@@ -59,7 +58,7 @@ class Generator(Configurator):
         """
         path = self.paths.commits.joinpath(str(id))
         if path.exists():
-            DEFAULT_LOGGER.warning(f"delete commit {id}", extra={"job": self})
+            DEFAULT_LOGGER.warning(f"delete commit {id}", extra={"label": self})
             path.rm()
 
     def truncate(self):
@@ -72,7 +71,7 @@ class Generator(Configurator):
         Returns:
             None
         """
-        DEFAULT_LOGGER.warning("truncate", extra={"job": self})
+        DEFAULT_LOGGER.warning("truncate", extra={"label": self})
         self.rm()
         if self.persist:
             self.table.truncate()
@@ -92,6 +91,9 @@ class Generator(Configurator):
         Returns:
                 None
         """
+        if self.options.job.get("no_drop"):
+            raise ValueError("no_drop is set, cannot drop the job")
+
         try:
             row = self.spark.sql(
                 f"""
@@ -106,7 +108,7 @@ class Generator(Configurator):
                 """
             ).collect()[0]
             if cast(int, row.count) > 0:
-                DEFAULT_LOGGER.warning(f"{row.count} children found", extra={"job": self, "content": row.children})
+                DEFAULT_LOGGER.warning(f"{row.count} children found", extra={"label": self, "content": row.children})
 
         except Exception:
             pass
@@ -162,7 +164,7 @@ class Generator(Configurator):
         Raises:
             NotImplementedError: This method is meant to be overridden by subclasses.
         """
-        raise NotImplementedError()
+        ...
 
     def create_table(self):
         def _create_table(df: DataFrame, batch: Optional[int] = 0):
@@ -185,12 +187,29 @@ class Generator(Configurator):
             elif step_powerbi is not None:
                 powerbi = step_powerbi
 
-            if powerbi:
+            # first take from job options, then from step options
+            job_masks = self.options.table.get("masks", None)
+            step_masks = self.step_conf.get("table_options", {}).get("masks", None)
+            if job_masks is not None:
+                masks = job_masks
+            elif step_masks is not None:
+                masks = step_masks
+            else:
+                masks = None
+
+            maximum_compatibility = self.options.table.get_boolean("maximum_compatibility", False)
+
+            if maximum_compatibility:
+                default_properties = {
+                    "delta.minReaderVersion": "1",
+                    "delta.minWriterVersion": "7",
+                    "delta.columnMapping.mode": "none",
+                }
+            elif powerbi:
                 default_properties = {
                     "delta.columnMapping.mode": "name",
                     "delta.minReaderVersion": "2",
                     "delta.minWriterVersion": "5",
-                    "fabricks.last_version": "0",
                 }
             else:
                 default_properties = {
@@ -200,9 +219,10 @@ class Generator(Configurator):
                     "delta.minReaderVersion": "2",
                     "delta.minWriterVersion": "5",
                     "delta.feature.timestampNtz": "supported",
-                    "fabricks.last_version": "0",
                 }
 
+            default_properties["fabricks.last_version"] = "0"
+
             if "__identity" in df.columns:
                 identity = False
             else:
@@ -234,9 +254,7 @@ class Generator(Configurator):
                             cluster_by.append("__hash")
 
                     if not cluster_by:
-                        DEFAULT_LOGGER.warning(
-                            "liquid clustering disabled (no clustering columns found)", extra={"job": self}
-                        )
+                        DEFAULT_LOGGER.debug("could not determine clustering column", extra={"label": self})
                         liquid_clustering = False
                         cluster_by = None
 
@@ -257,9 +275,13 @@ class Generator(Configurator):
             if properties is None:
                 properties = default_properties
 
+            primary_key = self.options.table.get_dict("primary_key")
+            foreign_keys = self.options.table.get_dict("foreign_keys")
+            comments = self.options.table.get_dict("comments")
+
             # if dataframe, reference is passed (BUG)
             name = f"{self.step}_{self.topic}_{self.item}__init"
-            global_temp_view = create_or_replace_global_temp_view(name=name, df=df.where("1 == 2"))
+            global_temp_view = create_or_replace_global_temp_view(name=name, df=df.where("1 == 2"), job=self)
             sql = f"select * from {global_temp_view}"
 
             self.cdc.create_table(
@@ -270,11 +292,17 @@ class Generator(Configurator):
                 partitioning=partitioning,
                 partition_by=partition_by,
                 properties=properties,
+                masks=masks,
+                primary_key=primary_key,
+                foreign_keys=foreign_keys,
+                comments=comments,
                 **cdc_options,
             )
 
         if not self.table.exists():
-            df = self.get_data(self.stream)
+            DEFAULT_LOGGER.debug("create table", extra={"label": self})
+
+            df = self.get_data(stream=self.stream, schema_only=True)
             if df:
                 if self.stream:
                     # add dummy stream to be sure that the writeStream will start
@@ -310,6 +338,9 @@ class Generator(Configurator):
                 if comment:
                     self.table.add_comment(comment=comment)
 
+        else:
+            DEFAULT_LOGGER.debug("table exists, skip creation", extra={"label": self})
+
     def _update_schema(
         self,
         df: Optional[DataFrame] = None,
@@ -328,7 +359,7 @@ class Generator(Configurator):
                 _update_schema(df)
 
             else:
-                df = self.get_data(self.stream)
+                df = self.get_data(stream=self.stream, schema_only=True)
                 assert df is not None
                 df = self.base_transform(df)
 
@@ -360,7 +391,7 @@ class Generator(Configurator):
 
     def get_differences_with_deltatable(self, df: Optional[DataFrame] = None):
         if df is None:
-            df = self.get_data(self.stream)
+            df = self.get_data(stream=self.stream)
             assert df is not None
             df = self.base_transform(df)
 
@@ -370,7 +401,7 @@ class Generator(Configurator):
 
     def get_schema_differences(self, df: Optional[DataFrame] = None) -> Optional[Sequence[SchemaDiff]]:
         if df is None:
-            df = self.get_data(self.stream)
+            df = self.get_data(stream=self.stream)
             assert df is not None
             df = self.base_transform(df)
 
@@ -413,4 +444,4 @@ class Generator(Configurator):
                 else:
                     self.table.enable_liquid_clustering(auto=True)
         else:
-            DEFAULT_LOGGER.debug("liquid clustering not enabled", extra={"job": self})
+            DEFAULT_LOGGER.debug("could not enable liquid clustering", extra={"label": self})

fabricks/core/jobs/base/invoker.py

@@ -7,13 +7,17 @@ from fabricks.context import PATH_RUNTIME
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.core.jobs.base.checker import Checker
 from fabricks.core.jobs.base.exception import PostRunInvokeException, PreRunInvokeException
-from fabricks.core.schedules import get_schedules
+from fabricks.core.jobs.get_schedule import get_schedule
 from fabricks.utils.path import Path
 
 
 class Invoker(Checker):
-    def invoke(self, schedule: Optional[str] = None):
-        self._invoke_job(position="run", schedule=schedule)
+    def invoke(self, schedule: Optional[str] = None, **kwargs):
+        return self._invoke_job(
+            position="run",
+            schedule=schedule,
+            **kwargs,
+        )  # kwargs and return needed for get_data in gold
 
     def invoke_pre_run(self, schedule: Optional[str] = None):
         self._invoke_job(position="pre_run", schedule=schedule)
@@ -23,30 +27,50 @@ class Invoker(Checker):
         self._invoke_job(position="post_run", schedule=schedule)
         self._invoke_step(position="post_run", schedule=schedule)
 
-    def _invoke_job(self, position: str, schedule: Optional[str] = None):
+    def _invoke_job(self, position: str, schedule: Optional[str] = None, **kwargs):
         invokers = self.options.invokers.get_list(position)
+        if position == "run":
+            invokers = invokers if len(invokers) > 0 else [{}]  # run must work even without run invoker options
 
         errors = []
 
         if invokers:
-            for i in invokers:
-                DEFAULT_LOGGER.info(f"{position}-invoke", extra={"job": self})
+            for i, invoker in enumerate(invokers):
+                DEFAULT_LOGGER.debug(f"invoke ({i}, {position})", extra={"label": self})
                 try:
-                    notebook = i.get("notebook")
-                    assert notebook, "notebook mandatory"
-                    path = PATH_RUNTIME.joinpath(notebook)
-
-                    arguments = i.get("arguments") or {}
-                    timeout = i.get("timeout")
-
-                    self._run_notebook(
-                        path=path,
-                        arguments=arguments,
-                        timeout=timeout,
-                        schedule=schedule,
-                    )
+                    path = kwargs.get("path")
+                    if path is None:
+                        notebook = invoker.get("notebook")
+                        assert notebook, "notebook mandatory"
+                        path = PATH_RUNTIME.joinpath(notebook)
+
+                    assert path is not None, "path mandatory"
+
+                    arguments = invoker.get("arguments") or {}
+                    timeout = invoker.get("timeout")
+
+                    schema_only = kwargs.get("schema_only")
+                    if schema_only is not None:
+                        arguments["schema_only"] = schema_only
+
+                    if len(invokers) == 1 and position == "run":
+                        return self._run_notebook(
+                            path=path,
+                            arguments=arguments,
+                            timeout=timeout,
+                            schedule=schedule,
+                        )
+                    else:
+                        self._run_notebook(
+                            path=path,
+                            arguments=arguments,
+                            timeout=timeout,
+                            schedule=schedule,
+                        )
 
                 except Exception as e:
+                    DEFAULT_LOGGER.warning(f"fail to run invoker ({i}, {position})", extra={"label": self})
+
                     if position == "pre_run":
                         errors.append(PreRunInvokeException(e))
                     elif position == "post_run":
@@ -63,15 +87,15 @@ class Invoker(Checker):
         errors = []
 
         if invokers:
-            for i in invokers:
-                DEFAULT_LOGGER.info(f"{position}-invoke", extra={"step": self.step})
+            for i, invoker in enumerate(invokers):
+                DEFAULT_LOGGER.debug(f"invoke by step ({i}, {position})", extra={"label": self})
                 try:
-                    notebook = i.get("notebook")
+                    notebook = invoker.get("notebook")
                     assert notebook, "notebook mandatory"
                     path = PATH_RUNTIME.joinpath(notebook)
 
-                    arguments = i.get("arguments", {})
-                    timeout = i.get("timeout")
+                    arguments = invoker.get("arguments", {})
+                    timeout = invoker.get("timeout")
 
                     self._run_notebook(
                         path=path,
@@ -81,6 +105,8 @@ class Invoker(Checker):
                     )
 
                 except Exception as e:
+                    DEFAULT_LOGGER.warning(f"fail to run invoker by step ({i}, {position})", extra={"label": self})
+
                     if position == "pre_run":
                         errors.append(PreRunInvokeException(e))
                     elif position == "post_run":
@@ -125,9 +151,7 @@ class Invoker(Checker):
 
         variables = None
         if schedule is not None:
-            variables = (
-                next(s for s in get_schedules() if s.get("name") == schedule).get("options", {}).get("variables", {})
-            )
+            variables = get_schedule(name=schedule).get("options", {}).get("variables", {})
 
         if variables is None:
             variables = {}
@@ -135,7 +159,7 @@ class Invoker(Checker):
         if arguments is None:
             arguments = {}
 
-        dbutils.notebook.run(
+        return dbutils.notebook.run(
             path=path.get_notebook_path(),  # type: ignore
             timeout_seconds=timeout,  # type: ignore
             arguments={  # type: ignore
@@ -154,7 +178,7 @@ class Invoker(Checker):
         extenders = self.options.extenders
         for e in extenders:
             name = e.get("extender")
-            DEFAULT_LOGGER.info(f"calling {name}", extra={"job": self})
+            DEFAULT_LOGGER.debug(f"extend ({name})", extra={"label": self})
             arguments = e.get("arguments") or {}
 
             extender = get_extender(name)
@@ -168,7 +192,7 @@ class Invoker(Checker):
         extenders = self.step_conf.get("extender_options", {})
         for e in extenders:
             name = e.get("extender")
-            DEFAULT_LOGGER.info(f"calling {name}", extra={"step": self.step})
+            DEFAULT_LOGGER.debug(f"extend by step ({name})", extra={"label": self})
             arguments = e.get("arguments", {})
 
             extender = get_extender(name)