fabricks 3.0.18__py3-none-any.whl → 4.0.0__py3-none-any.whl

fabricks/api/context.py

@@ -1,5 +1,17 @@
-from fabricks.context import BRONZE, DBUTILS, GOLD, SECRET_SCOPE, SILVER, SPARK, init_spark_session, pprint_runtime
-from fabricks.core.jobs.base._types import Bronzes, Golds, Silvers, Steps
+from fabricks.context import (
+    BRONZE,
+    CONF_RUNTIME,
+    DBUTILS,
+    GOLD,
+    SILVER,
+    SPARK,
+    Bronzes,
+    Golds,
+    Silvers,
+    Steps,
+    init_spark_session,
+    pprint_runtime,
+)
 
 # step
 BRONZES = Bronzes
@@ -18,7 +30,7 @@ __all__ = [
     "GOLDS",
     "init_spark_session",
     "pprint_runtime",
-    "SECRET_SCOPE",
+    "CONF_RUNTIME",
     "SILVER",
     "Silvers",
     "SILVERS",

fabricks/api/notebooks/schedule.py

@@ -4,14 +4,13 @@
 # COMMAND ----------
 
 from logging import DEBUG
-from typing import Any, cast
+from typing import Any
 
 from databricks.sdk.runtime import dbutils, display, spark
 
 from fabricks.context import PATH_NOTEBOOKS
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.core import get_step
-from fabricks.core.jobs.base._types import TStep
 from fabricks.core.schedules import generate, terminate
 from fabricks.utils.helpers import run_in_parallel, run_notebook
 
@@ -51,7 +50,7 @@ steps = [row.step for row in spark.sql("select step from {df} group by step", df
 
 
 def _schedule(task: Any):
-    step = get_step(step=cast(TStep, task))
+    step = get_step(step=task)
     run_notebook(
         PATH_NOTEBOOKS.joinpath("process"),
         timeout=step.timeouts.step,

fabricks/api/parsers.py

@@ -1,3 +1,4 @@
-from fabricks.core.parsers import BaseParser, ParserOptions, parser
+from fabricks.core.parsers import BaseParser, parser
+from fabricks.models import ParserOptions
 
 __all__ = ["BaseParser", "ParserOptions", "parser"]

fabricks/api/utils.py

@@ -1,9 +1,11 @@
 from fabricks.utils.helpers import concat_dfs, concat_ws, run_in_parallel
-from fabricks.utils.path import Path
+from fabricks.utils.path import FileSharePath, GitPath, Path
 
 __all__ = [
     "concat_dfs",
     "concat_ws",
+    "FileSharePath",
+    "GitPath",
     "Path",
     "run_in_parallel",
 ]

fabricks/cdc/__init__.py

@@ -1,4 +1,4 @@
-from fabricks.cdc.base import AllowedChangeDataCaptures, BaseCDC
+from fabricks.cdc.base import BaseCDC
 from fabricks.cdc.cdc import CDC
 from fabricks.cdc.nocdc import NoCDC
 from fabricks.cdc.scd1 import SCD1
@@ -7,7 +7,6 @@ from fabricks.cdc.scd2 import SCD2
 __all__ = [
     "BaseCDC",
     "CDC",
-    "AllowedChangeDataCaptures",
     "NoCDC",
     "SCD1",
     "SCD2",

fabricks/cdc/base/__init__.py

@@ -1,4 +1,3 @@
-from fabricks.cdc.base._types import AllowedChangeDataCaptures
 from fabricks.cdc.base.cdc import BaseCDC
 
-__all__ = ["BaseCDC", "AllowedChangeDataCaptures"]
+__all__ = ["BaseCDC"]

fabricks/cdc/base/_types.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
-from typing import Literal, Union
+from typing import Union
 
 from pyspark.sql import DataFrame
+from pyspark.sql.types import StructType
 
 from fabricks.metastore.table import Table
 
-AllowedChangeDataCaptures = Literal["nocdc", "scd1", "scd2"]
-AllowedSources = Union[DataFrame, Table, str]
+# Import from models for consistency
+
+AllowedSources = Union[DataFrame, Table, str, StructType]

fabricks/cdc/base/configurator.py

@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
 from typing import List, Optional, Union
 
 from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.types import StructType
 
 from fabricks.cdc.base._types import AllowedSources
 from fabricks.context import SPARK
@@ -111,6 +112,7 @@ class Configurator(ABC):
         cols = [
             "__operation",
             "__metadata",
+            "__last_updated",
             "__rescued_data",
         ]
 
@@ -135,6 +137,7 @@ class Configurator(ABC):
             # Trailing
             "__operation",
             "__metadata",
+            "__last_updated",
             "__rescued_data",
         ]
 
@@ -149,6 +152,8 @@ class Configurator(ABC):
             df = self.table.dataframe
         elif isinstance(src, str):
             df = self.spark.sql(src)
+        elif isinstance(src, StructType):
+            df = self.spark.createDataFrame([], schema=src)
         else:
             raise ValueError(f"{src} not allowed")
 

fabricks/cdc/base/generator.py

@@ -4,6 +4,7 @@ from typing import Any, List, Optional, Sequence, Union, cast
 
 from py4j.protocol import Py4JJavaError
 from pyspark.sql import DataFrame
+from pyspark.sql.types import StructType
 
 from fabricks.cdc.base._types import AllowedSources
 from fabricks.cdc.base.configurator import Configurator
@@ -25,11 +26,11 @@ class Generator(Configurator):
         identity: Optional[bool] = False,
         liquid_clustering: Optional[bool] = False,
         cluster_by: Optional[Union[List[str], str]] = None,
-        properties: Optional[dict[str, str]] = None,
+        properties: Optional[dict[str, str | bool | int]] = None,
         masks: Optional[dict[str, str]] = None,
         primary_key: Optional[dict[str, Any]] = None,
         foreign_keys: Optional[dict[str, Any]] = None,
-        comments: Optional[dict[str, str]] = None,
+        comments: Optional[dict[str, Any]] = None,
         **kwargs,
     ):
         kwargs["mode"] = "complete"
@@ -145,6 +146,7 @@ class Generator(Configurator):
         d = self.get_schema_differences(src, **kwargs)
         if d is None:
             return None
+
         return len(d) > 0
 
     def _update_schema(
@@ -155,7 +157,9 @@ class Generator(Configurator):
         **kwargs,
     ):
         if self.is_view:
-            assert not isinstance(src, DataFrameLike), "dataframe not allowed"
+            assert not isinstance(src, DataFrameLike) and not isinstance(src, StructType), (
+                "dataframe and structtype not allowed"
+            )
             self.create_or_replace_view(src=src)
 
         else:

fabricks/cdc/base/merger.py

@@ -7,6 +7,7 @@ from pyspark.sql import DataFrame
 
 from fabricks.cdc.base._types import AllowedSources
 from fabricks.cdc.base.processor import Processor
+from fabricks.context.config import IS_DEBUGMODE
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.metastore.view import create_or_replace_global_temp_view
 from fabricks.utils._types import DataFrameLike
@@ -56,6 +57,7 @@ class Merger(Processor):
         assert "__key" or keys, f"{self} - __key or keys not found"
 
         return {
+            "debugmode": IS_DEBUGMODE,
             "src": src,
             "format": format,
             "tgt": self.table,

fabricks/cdc/base/processor.py

@@ -7,6 +7,7 @@ from pyspark.sql import DataFrame
 
 from fabricks.cdc.base._types import AllowedSources
 from fabricks.cdc.base.generator import Generator
+from fabricks.context.config import IS_DEBUGMODE
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.metastore.table import Table
 from fabricks.metastore.view import create_or_replace_global_temp_view
@@ -65,6 +66,7 @@ class Processor(Generator):
         add_key = kwargs.get("add_key", None)
         add_hash = kwargs.get("add_hash", None)
         add_timestamp = kwargs.get("add_timestamp", None)
+        add_last_updated = kwargs.get("add_last_updated", None)
         add_metadata = kwargs.get("add_metadata", None)
 
         has_order_by = None if not order_duplicate_by else True
@@ -78,6 +80,7 @@ class Processor(Generator):
         has_hash = add_hash or "__hash" in inputs
         has_identity = "__identity" in inputs
         has_rescued_data = "__rescued_data" in inputs
+        has_last_updated = add_last_updated or "__last_updated" in inputs
 
         soft_delete = kwargs.get("soft_delete", None)
         delete_missing = kwargs.get("delete_missing", None)
@@ -152,6 +155,10 @@ class Processor(Generator):
         if add_hash and "__hash" in inputs:
             overwrite.append("__hash")
 
+        # override __last_updated if added and found in df
+        if add_last_updated and "__last_updated" in inputs:
+            overwrite.append("__last_updated")
+
         # override metadata if added and found in df
         if add_metadata and "__metadata" in inputs:
             overwrite.append("__metadata")
@@ -219,6 +226,11 @@ class Processor(Generator):
                 outputs.append("__metadata")
             if "__metadata" not in intermediates:
                 intermediates.append("__metadata")
+        if has_last_updated:
+            if "__last_updated" not in outputs:
+                outputs.append("__last_updated")
+            if "__last_updated" not in intermediates:
+                intermediates.append("__last_updated")
         if has_source:
             if "__source" not in outputs:
                 outputs.append("__source")
@@ -311,6 +323,7 @@ class Processor(Generator):
         parent_final = "__final"
 
         return {
+            "debugmode": IS_DEBUGMODE,
             "src": src,
             "format": format,
             "tgt": tgt,
@@ -337,6 +350,7 @@ class Processor(Generator):
             "has_rows": has_rows,
             "has_source": has_source,
             "has_metadata": has_metadata,
+            "has_last_updated": has_last_updated,
             "has_timestamp": has_timestamp,
             "has_operation": has_operation,
             "has_identity": has_identity,
@@ -347,6 +361,7 @@ class Processor(Generator):
             # default add
             "add_metadata": add_metadata,
             "add_timestamp": add_timestamp,
+            "add_last_updated": add_last_updated,
             "add_key": add_key,
             "add_hash": add_hash,
             # value add