fabricks 3.0.19__py3-none-any.whl → 4.0.0__py3-none-any.whl

fabricks/deploy/views.py

@@ -1,11 +1,10 @@
-from fabricks.context import SPARK
+from fabricks.context import SPARK, Steps
 from fabricks.context.log import DEFAULT_LOGGER
-from fabricks.core.jobs.base._types import Steps
 from fabricks.utils.sqlglot import fix as fix_sql
 
 
 def deploy_views():
-    DEFAULT_LOGGER.info("create or replace fabricks (default) views")
+    DEFAULT_LOGGER.info("create or replace fabricks (default) views", extra={"label": "fabricks"})
 
     create_or_replace_jobs_view()
     create_or_replace_tables_view()
@@ -69,7 +68,7 @@ def create_or_replace_jobs_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.debug(f"could not find fabricks.{table}")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{table}", extra={"label": "fabricks"})
 
     sql = f"""create or replace view fabricks.jobs with schema evolution as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)
@@ -96,7 +95,7 @@ def create_or_replace_tables_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_tables")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_tables", extra={"label": "fabricks"})
 
     sql = f"""create or replace view fabricks.tables with schema evolution as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)
@@ -123,7 +122,7 @@ def create_or_replace_views_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_views")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_views", extra={"label": "fabricks"})
 
     sql = f"""create or replace view fabricks.views with schema evolution as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)
@@ -153,7 +152,7 @@ def create_or_replace_dependencies_view():
             dmls.append(dml)
 
         except Exception:
-            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_dependencies")
+            DEFAULT_LOGGER.debug(f"could not find fabricks.{step}_dependencies", extra={"label": "fabricks"})
 
     sql = f"""create or replace view fabricks.dependencies with schema evolution  as {" union all ".join(dmls)}"""
     sql = fix_sql(sql)
@@ -180,7 +179,7 @@ def create_or_replace_dependencies_flat_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.dependencies_flat", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.dependencies_flat", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -221,7 +220,7 @@ def create_or_replace_dependencies_unpivot_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.dependencies_unpivot", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.dependencies_unpivot", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -262,7 +261,7 @@ def create_or_replace_dependencies_circular_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.dependencies_circular", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.dependencies_circular", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -334,7 +333,7 @@ def create_or_replace_logs_pivot_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.logs_pivot", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.logs_pivot", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -361,7 +360,7 @@ def create_or_replace_last_schedule_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.last_schedule", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.last_schedule", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -388,7 +387,7 @@ def create_or_replace_last_status_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.last_status", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.last_status", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -427,7 +426,7 @@ def create_or_replace_previous_schedule_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.previous_schedule", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.previous_schedule", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -453,7 +452,7 @@ def create_or_replace_schedules_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.schedules", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.schedules", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)
 
 
@@ -510,5 +509,5 @@ def create_or_replace_jobs_to_be_updated_view():
     """
     sql = fix_sql(sql)
 
-    DEFAULT_LOGGER.debug("create or replace fabricks.jobs_to_be_updated", extra={"sql": sql})
+    DEFAULT_LOGGER.debug("create or replace fabricks.jobs_to_be_updated", extra={"sql": sql, "label": "fabricks"})
     SPARK.sql(sql)

fabricks/metastore/database.py

@@ -7,7 +7,7 @@ from typing_extensions import deprecated
 from fabricks.context import PATHS_STORAGE, SPARK
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.metastore.utils import get_tables, get_views
-from fabricks.utils.path import Path
+from fabricks.utils.path import FileSharePath
 
 
 class Database:
@@ -25,11 +25,11 @@ class Database:
 
     @property
     @deprecated("use delta_path instead")
-    def deltapath(self) -> Path:
+    def deltapath(self) -> FileSharePath:
         return self.storage.joinpath("delta")
 
     @property
-    def delta_path(self) -> Path:
+    def delta_path(self) -> FileSharePath:
         return self.storage.joinpath("delta")
 
     def create(self):

fabricks/metastore/table.py

@@ -1,5 +1,5 @@
 import re
-from typing import Any, List, Optional, Sequence, Union, overload
+from typing import Sequence, overload
 
 from delta import DeltaTable
 from pyspark.errors.exceptions.base import AnalysisException
@@ -11,21 +11,22 @@ from fabricks.context import SPARK
 from fabricks.context.log import DEFAULT_LOGGER
 from fabricks.metastore._types import AddedColumn, ChangedColumn, DroppedColumn, SchemaDiff
 from fabricks.metastore.dbobject import DbObject
-from fabricks.utils.path import Path
+from fabricks.models import ForeignKey, PrimaryKey
+from fabricks.utils.path import FileSharePath
 from fabricks.utils.sqlglot import fix
 
 
 class Table(DbObject):
     @classmethod
-    def from_step_topic_item(cls, step: str, topic: str, item: str, spark: Optional[SparkSession] = SPARK):
+    def from_step_topic_item(cls, step: str, topic: str, item: str, spark: SparkSession | None = SPARK):
         return cls(step, topic, item, spark=spark)
 
     @property
-    def deltapath(self) -> Path:
+    def deltapath(self) -> FileSharePath:
         return self.database.delta_path.joinpath("/".join(self.levels))
 
     @property
-    def delta_path(self) -> Path:
+    def delta_path(self) -> FileSharePath:
         return self.database.delta_path.joinpath("/".join(self.levels))
 
     @property
@@ -43,7 +44,7 @@ class Table(DbObject):
         return self.spark.sql(f"select * from {self}")
 
     @property
-    def columns(self) -> List[str]:
+    def columns(self) -> list[str]:
         assert self.registered, f"{self} not registered"
 
         return self.dataframe.columns
@@ -98,16 +99,16 @@ class Table(DbObject):
         self,
         df: DataFrame,
         *,
-        partitioning: Optional[bool] = False,
-        partition_by: Optional[Union[List[str], str]] = None,
-        identity: Optional[bool] = False,
-        liquid_clustering: Optional[bool] = False,
-        cluster_by: Optional[Union[List[str], str]] = None,
-        properties: Optional[dict[str, str]] = None,
-        masks: Optional[dict[str, str]] = None,
-        primary_key: Optional[dict[str, Any]] = None,
-        foreign_keys: Optional[dict[str, Any]] = None,
-        comments: Optional[dict[str, str]] = None,
+        partitioning: bool | None = False,
+        partition_by: list[str] | str | None = None,
+        identity: bool | None = False,
+        liquid_clustering: bool | None = False,
+        cluster_by: list[str] | str | None = None,
+        properties: dict[str, str | bool | int] | None = None,
+        masks: dict[str, str] | None = None,
+        primary_key: dict[str, PrimaryKey] | None = None,
+        foreign_keys: dict[str, ForeignKey] | None = None,
+        comments: dict[str, str] | None = None,
     ): ...
 
     @overload
@@ -115,32 +116,32 @@ class Table(DbObject):
         self,
         *,
         schema: StructType,
-        partitioning: Optional[bool] = False,
-        partition_by: Optional[Union[List[str], str]] = None,
-        identity: Optional[bool] = False,
-        liquid_clustering: Optional[bool] = False,
-        cluster_by: Optional[Union[List[str], str]] = None,
-        properties: Optional[dict[str, str]] = None,
-        masks: Optional[dict[str, str]] = None,
-        primary_key: Optional[dict[str, Any]] = None,
-        foreign_keys: Optional[dict[str, Any]] = None,
-        comments: Optional[dict[str, str]] = None,
+        partitioning: bool | None = False,
+        partition_by: list[str] | str | None = None,
+        identity: bool | None = False,
+        liquid_clustering: bool | None = False,
+        cluster_by: list[str] | str | None = None,
+        properties: dict[str, str | bool | int] | None = None,
+        masks: dict[str, str] | None = None,
+        primary_key: dict[str, PrimaryKey] | None = None,
+        foreign_keys: dict[str, ForeignKey] | None = None,
+        comments: dict[str, str] | None = None,
     ): ...
 
     def create(
         self,
-        df: Optional[DataFrame] = None,
-        schema: Optional[StructType] = None,
-        partitioning: Optional[bool] = False,
-        partition_by: Optional[Union[List[str], str]] = None,
-        identity: Optional[bool] = False,
-        liquid_clustering: Optional[bool] = False,
-        cluster_by: Optional[Union[List[str], str]] = None,
-        properties: Optional[dict[str, str]] = None,
-        masks: Optional[dict[str, str]] = None,
-        primary_key: Optional[dict[str, Any]] = None,
-        foreign_keys: Optional[dict[str, Any]] = None,
-        comments: Optional[dict[str, str]] = None,
+        df: DataFrame | None = None,
+        schema: StructType | None = None,
+        partitioning: bool | None = False,
+        partition_by: list[str] | str | None = None,
+        identity: bool | None = False,
+        liquid_clustering: bool | None = False,
+        cluster_by: list[str] | str | None = None,
+        properties: dict[str, str | bool | int] | None = None,
+        masks: dict[str, str] | None = None,
+        primary_key: dict[str, PrimaryKey] | None = None,
+        foreign_keys: dict[str, ForeignKey] | None = None,
+        comments: dict[str, str] | None = None,
     ):
         self._create(
             df=df,
@@ -158,8 +159,11 @@ class Table(DbObject):
         )
 
     def _get_ddl_columns(
-        self, df: DataFrame, masks: Optional[dict[str, str]], comments: Optional[dict[str, str]]
-    ) -> List[str]:
+        self,
+        df: DataFrame,
+        masks: dict[str, str] | None,
+        comments: dict[str, str] | None,
+    ) -> list[str]:
         def _backtick(name: str, dtype: str) -> str:
             j = df.schema[name].jsonValue()
             r = re.compile(r"(?<='name': ')[^']+(?=',)")
@@ -188,18 +192,18 @@ class Table(DbObject):
 
     def _create(
         self,
-        df: Optional[DataFrame] = None,
-        schema: Optional[StructType] = None,
-        partitioning: Optional[bool] = False,
-        partition_by: Optional[Union[List[str], str]] = None,
-        identity: Optional[bool] = False,
-        liquid_clustering: Optional[bool] = False,
-        cluster_by: Optional[Union[List[str], str]] = None,
-        properties: Optional[dict[str, str]] = None,
-        masks: Optional[dict[str, str]] = None,
-        primary_key: Optional[dict[str, Any]] = None,
-        foreign_keys: Optional[dict[str, Any]] = None,
-        comments: Optional[dict[str, str]] = None,
+        df: DataFrame | None = None,
+        schema: StructType | None = None,
+        partitioning: bool | None = False,
+        partition_by: list[str] | str | None = None,
+        identity: bool | None = False,
+        liquid_clustering: bool | None = False,
+        cluster_by: list[str] | str | None = None,
+        properties: dict[str, str | bool | int] | None = None,
+        masks: dict[str, str] | None = None,
+        primary_key: dict[str, PrimaryKey] | None = None,
+        foreign_keys: dict[str, ForeignKey] | None = None,
+        comments: dict[str, str] | None = None,
     ):
         DEFAULT_LOGGER.info("create table", extra={"label": self})
         if not df:
@@ -238,19 +242,21 @@ class Table(DbObject):
             assert len(primary_key) == 1, "only one primary key allowed"
 
             for key, value in primary_key.items():
-                keys = value["keys"]
+                keys = value.keys
                 if isinstance(keys, str):
                     keys = [keys]
+
                 ddl_primary_key = f", constraint {key} primary key (" + ", ".join(keys) + ")"
 
         if foreign_keys:
             fks = []
 
             for key, value in foreign_keys.items():
-                reference = value["reference"]
-                keys = value["keys"]
+                reference = value.reference
+                keys = value.keys
                 if isinstance(keys, str):
                     keys = [keys]
+
                 keys = ", ".join([f"`{k}`" for k in keys])
                 fk = f"constraint {key} foreign key ({keys}) references {reference}"
                 fks.append(fk)
@@ -301,7 +307,13 @@ class Table(DbObject):
 
     @property
     def is_deltatable(self) -> bool:
-        return DeltaTable.isDeltaTable(self.spark, str(self.delta_path))
+        try:
+            return DeltaTable.isDeltaTable(self.spark, str(self.delta_path))
+        except Exception as e:
+            if "PERMISSION_DENIED" in str(e) or "row filter or column mask" in str(e):
+                return True
+            else:
+                raise e
 
     @property
     def column_mapping_enabled(self) -> bool:
@@ -329,13 +341,17 @@ class Table(DbObject):
         self.create_restore_point()
         self.spark.sql(f"truncate table {self.qualified_name}")
 
-    def schema_drifted(self, df: DataFrame, exclude_columns_with_prefix: Optional[str] = None) -> bool:
+    def schema_drifted(self, df: DataFrame, exclude_columns_with_prefix: list[str] | None = None) -> bool:
         assert self.registered, f"{self} not registered"
 
-        diffs = self.get_schema_differences(df)
+        diffs = self.get_schema_differences(df, exclude_columns_with_prefix=exclude_columns_with_prefix)
         return len(diffs) > 0
 
-    def get_schema_differences(self, df: DataFrame) -> Sequence[SchemaDiff]:
+    def get_schema_differences(
+        self,
+        df: DataFrame,
+        exclude_columns_with_prefix: list[str] | None = None,
+    ) -> Sequence[SchemaDiff]:
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.debug("get schema differences", extra={"label": self, "df": df})
@@ -346,6 +362,9 @@ class Table(DbObject):
                 df1 = df1.drop("__identity")
 
         all_columns = set(df1.columns).union(set(df.columns))
+        if exclude_columns_with_prefix:
+            for excluded in exclude_columns_with_prefix:
+                all_columns = {c for c in all_columns if not c.startswith(excluded)}
 
         df1_dict = {name: dtype for name, dtype in df1.dtypes}
         df2_dict = {name: dtype for name, dtype in df.dtypes}
@@ -378,8 +397,16 @@ class Table(DbObject):
 
         return diffs
 
-    def update_schema(self, df: DataFrame, widen_types: bool = False):
+    def update_schema(self, df: DataFrame | None = None, schema: StructType | None = None, widen_types: bool = False):
+        if df is None and schema is None:
+            raise ValueError("Either df or schema must be provided")
+
+        if df is None and schema is not None:
+            df = self.spark.createDataFrame([], schema)
+
+        assert df is not None
         assert self.registered, f"{self} not registered"
+
         if not self.column_mapping_enabled:
             self.enable_column_mapping()
 
@@ -428,8 +455,16 @@ class Table(DbObject):
                 except Exception:
                     pass
 
-    def overwrite_schema(self, df: DataFrame):
+    def overwrite_schema(self, df: DataFrame | None = None, schema: StructType | None = None):
+        if df is None and schema is None:
+            raise ValueError("Either df or schema must be provided")
+
+        if df is None and schema is not None:
+            df = self.spark.createDataFrame([], schema)
+
+        assert df is not None
         assert self.registered, f"{self} not registered"
+
         if not self.column_mapping_enabled:
             self.enable_column_mapping()
 
@@ -473,7 +508,7 @@ class Table(DbObject):
             pass
         self.spark.sql("SET self.spark.databricks.delta.retentionDurationCheck.enabled = True")
 
-    def optimize(self, columns: Optional[Union[str, List[str]]] = None):
+    def optimize(self, columns: str | list[str] | None = None):
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.info("optimize", extra={"label": self})
@@ -579,7 +614,7 @@ class Table(DbObject):
 
         return self.spark.sql(f"describe detail {self.qualified_name}")
 
-    def get_partitions(self) -> List[str]:
+    def get_partitions(self) -> list[str]:
         assert self.registered, f"{self} not registered"
 
         try:
@@ -610,7 +645,7 @@ class Table(DbObject):
         version = df.select(max("version")).collect()[0][0]
         return version
 
-    def get_property(self, key: str) -> Optional[str]:
+    def get_property(self, key: str) -> str | None:
         assert self.registered, f"{self} not registered"
 
         try:
@@ -652,7 +687,7 @@ class Table(DbObject):
                 """
             )
 
-    def set_property(self, key: Union[str, int], value: Union[str, int]):
+    def set_property(self, key: str | int, value: str | int):
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.debug(f"set property {key} = {value}", extra={"label": self})
@@ -735,7 +770,7 @@ class Table(DbObject):
             """
         )
 
-    def add_column(self, name: str, type: str, after: Optional[str] = None):
+    def add_column(self, name: str, type: str, after: str | None = None):
         assert self.registered, f"{self} not registered"
 
         DEFAULT_LOGGER.info(f"add column {name} ({type})", extra={"label": self})
@@ -747,7 +782,7 @@ class Table(DbObject):
             """
         )
 
-    def create_bloomfilter_index(self, columns: Union[str, List[str]]):
+    def create_bloomfilter_index(self, columns: str | list[str]):
         assert self.registered, f"{self} not registered"
 
         if isinstance(columns, str):
@@ -790,7 +825,7 @@ class Table(DbObject):
         df = self.spark.sql(f"describe history {self.qualified_name}")
         return df
 
-    def enable_liquid_clustering(self, columns: Optional[Union[str, List[str]]] = None, auto: Optional[bool] = False):
+    def enable_liquid_clustering(self, columns: str | list[str] | None = None, auto: bool | None = False):
         assert self.registered, f"{self} not registered"
 
         if auto:

fabricks/models/__init__.py

@@ -0,0 +1,125 @@
+"""Fabricks models module - All Pydantic models for jobs, steps, and runtime configuration."""
+
+# Common types and aliases
+from fabricks.models.common import (
+    AllowedChangeDataCaptures,
+    AllowedConstraintOptions,
+    AllowedFileFormats,
+    AllowedForeignKeyOptions,
+    AllowedModes,
+    AllowedModesBronze,
+    AllowedModesGold,
+    AllowedModesSilver,
+    AllowedOperations,
+    AllowedOrigins,
+    AllowedTypes,
+    Database,
+    DatabasePathOptions,
+    ExtenderOptions,
+    InvokerOptions,
+    SparkOptions,
+)
+from fabricks.models.dependency import JobDependency, SchemaDependencies
+
+# Job models
+from fabricks.models.job import BronzeOptions as JobBronzeOptions
+from fabricks.models.job import CheckOptions
+from fabricks.models.job import GoldOptions as JobGoldOptions
+from fabricks.models.job import JobConf, JobConfBase, JobConfBronze, JobConfGold, JobConfSilver, ParserOptions
+from fabricks.models.job import SilverOptions as JobSilverOptions
+from fabricks.models.job import TOptions
+from fabricks.models.path import Paths
+
+# Runtime models
+from fabricks.models.runtime import RuntimeConf, RuntimeOptions, RuntimePathOptions, RuntimeTimeoutOptions
+
+# Schedule models
+from fabricks.models.schedule import Schedule, ScheduleOptions
+
+# Step models
+from fabricks.models.step import BronzeConf as StepBronzeConf
+from fabricks.models.step import BronzeOptions as StepBronzeOptions
+from fabricks.models.step import GoldConf as StepGoldConf
+from fabricks.models.step import GoldOptions as StepGoldOptions
+from fabricks.models.step import PowerBI
+from fabricks.models.step import SilverConf as StepSilverConf
+from fabricks.models.step import SilverOptions as StepSilverOptions
+from fabricks.models.step import Step, StepOptions, StepPathOptions, StepTimeoutOptions
+
+# Table models
+from fabricks.models.table import (
+    ForeignKey,
+    ForeignKeyOptions,
+    PrimaryKey,
+    PrimaryKeyOptions,
+    StepTableOptions,
+    TableOptions,
+)
+
+# Utility functions
+from fabricks.models.utils import get_dependency_id, get_job_id
+
+__all__ = [
+    # Common types
+    "AllowedChangeDataCaptures",
+    "AllowedConstraintOptions",
+    "AllowedFileFormats",
+    "AllowedForeignKeyOptions",
+    "AllowedModes",
+    "AllowedModesBronze",
+    "AllowedModesGold",
+    "AllowedModesSilver",
+    "AllowedOperations",
+    "AllowedOrigins",
+    "AllowedTypes",
+    "Database",
+    "DatabasePathOptions",
+    "ExtenderOptions",
+    "SparkOptions",
+    # Job models
+    "CheckOptions",
+    "InvokerOptions",
+    "JobBronzeOptions",
+    "JobConf",
+    "JobConfBase",
+    "JobConfBronze",
+    "JobConfGold",
+    "JobConfSilver",
+    "JobDependency",
+    "JobGoldOptions",
+    "JobSilverOptions",
+    "Paths",
+    "SchemaDependencies",
+    "TOptions",
+    # Runtime models
+    "RuntimeConf",
+    "RuntimeOptions",
+    "RuntimePathOptions",
+    "RuntimeTimeoutOptions",
+    # Step models
+    "PowerBI",
+    "Step",
+    "StepBronzeConf",
+    "StepBronzeOptions",
+    "StepGoldConf",
+    "StepGoldOptions",
+    "StepOptions",
+    "StepPathOptions",
+    "StepSilverConf",
+    "StepSilverOptions",
+    "StepTimeoutOptions",
+    # Table models
+    "ForeignKey",
+    "ForeignKeyOptions",
+    "PrimaryKey",
+    "PrimaryKeyOptions",
+    "StepTableOptions",
+    "TableOptions",
+    "ParserOptions",
+    # Schedule models
+    "ScheduleOptions",
+    "Schedule",
+    # Utility functions
+    "get_dependency_id",
+    "get_job_id",
+]