fabricks 3.0.19__py3-none-any.whl → 4.0.1__py3-none-any.whl

fabricks/models/runtime.py

@@ -0,0 +1,182 @@
+"""Runtime configuration models."""
+
+from typing import ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field, computed_field
+
+from fabricks.models.common import Database, ExtenderOptions, SparkOptions
+from fabricks.models.config import ConfigOptions
+from fabricks.models.step import BronzeConf, GoldConf, PowerBI, SilverConf
+from fabricks.utils.path import FileSharePath, GitPath, resolve_fileshare_path, resolve_git_path
+
+
+class RuntimePathOptions(BaseModel):
+    """Path configuration for runtime components."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    storage: str
+    udfs: str
+    parsers: str
+    schedules: str
+    views: str
+    requirements: str
+    storage_credential: str | None = None
+    extenders: str | None = None
+    masks: str | None = None
+
+
+class UDFOptions(BaseModel):
+    prefix: str | None = None
+    schema_name: str | None = Field(None, alias="schema")
+
+
+class MaskOptions(BaseModel):
+    prefix: str | None = None
+    schema_name: str | None = Field(None, alias="schema")
+
+
+class RuntimeResolvedPathOptions(BaseModel):
+    """Resolved path objects for runtime components."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True, arbitrary_types_allowed=True)
+
+    storage: FileSharePath
+    udfs: GitPath
+    parsers: GitPath
+    schedules: GitPath
+    views: GitPath
+    requirements: GitPath
+    extenders: GitPath
+    masks: GitPath
+
+    storages: dict[str, FileSharePath]
+    runtimes: dict[str, GitPath]
+
+
+class RuntimeTimeoutOptions(BaseModel):
+    """Timeout settings for runtime operations."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    step: int
+    job: int
+    pre_run: int
+    post_run: int
+
+
+class RuntimeOptions(BaseModel):
+    """Main runtime configuration options."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    secret_scope: str
+    encryption_key: str | None = None
+    unity_catalog: bool | None = None
+    type_widening: bool | None = None
+    catalog: str | None = None
+    workers: int
+    timeouts: RuntimeTimeoutOptions
+    retention_days: int
+    timezone: str | None = None
+
+
+class RuntimeConf(BaseModel):
+    """Complete runtime configuration."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True, arbitrary_types_allowed=True)
+
+    name: str
+    options: RuntimeOptions
+    path_options: RuntimePathOptions
+    extender_options: ExtenderOptions | None = None
+    spark_options: SparkOptions | None = None
+    udf_options: UDFOptions | None = None
+    mask_options: MaskOptions | None = None
+    bronze: list[BronzeConf] | None = None
+    silver: list[SilverConf] | None = None
+    gold: list[GoldConf] | None = None
+    powerbi: list[PowerBI] | None = None
+    databases: list[Database] | None = None
+    variables: dict[str, str] | None = None
+    credentials: dict[str, str] | None = None
+
+    config: ClassVar[ConfigOptions] = ConfigOptions()
+
+    @computed_field
+    @property
+    def resolved_path_options(self) -> RuntimeResolvedPathOptions:
+        """Get all runtime paths resolved as Path objects."""
+        return self._resolve_paths()
+
+    def _resolve_paths(self) -> RuntimeResolvedPathOptions:
+        """
+        Get all runtime paths resolved as Path objects.
+
+        Args:
+            runtime: The base runtime path (e.g., PATH_RUNTIME)
+
+        Returns:
+            RuntimeResolvedPathOptions with all paths resolved
+        """
+        # Collect all storage paths with variable substitution
+        storage_paths: dict[str, FileSharePath] = {
+            "fabricks": resolve_fileshare_path(self.path_options.storage, variables=self.variables),
+        }
+
+        # Add storage paths for bronze/silver/gold/databases
+        for objects in [self.bronze, self.silver, self.gold, self.databases]:
+            if objects:
+                for obj in objects:
+                    storage_paths[obj.name] = resolve_fileshare_path(
+                        obj.path_options.storage,
+                        variables=self.variables,
+                    )
+
+        root = self.config.resolved_paths.runtime
+
+        # Collect all runtime paths with base path joining
+        runtime_paths: dict[str, GitPath] = {}
+        for objects in [self.bronze, self.silver, self.gold]:
+            if objects:
+                for obj in objects:
+                    runtime_paths[obj.name] = resolve_git_path(
+                        obj.path_options.runtime,
+                        base=root,
+                    )
+
+        return RuntimeResolvedPathOptions(
+            storage=storage_paths["fabricks"],
+            udfs=resolve_git_path(
+                path=self.path_options.udfs,
+                base=root,
+            ),
+            parsers=resolve_git_path(
+                path=self.path_options.parsers,
+                base=root,
+            ),
+            schedules=resolve_git_path(
+                path=self.path_options.schedules,
+                base=root,
+            ),
+            views=resolve_git_path(
+                path=self.path_options.views,
+                base=root,
+            ),
+            requirements=resolve_git_path(
+                path=self.path_options.requirements,
+                base=root,
+            ),
+            extenders=resolve_git_path(
+                path=self.path_options.extenders,
+                base=root,
+                default="fabricks/extenders",
+            ),
+            masks=resolve_git_path(
+                path=self.path_options.masks,
+                base=root,
+                default="fabricks/masks",
+            ),
+            storages=storage_paths,
+            runtimes=runtime_paths,
+        )

fabricks/models/schedule.py

@@ -0,0 +1,21 @@
+from pydantic import BaseModel, ConfigDict
+
+
+class ScheduleOptions(BaseModel):
+    """Options for scheduling a notebook run."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    steps: list[str] | None = None
+    tag: str | None = None
+    view: str | None = None
+    variables: dict[str, str | bool | int] | None = None
+
+
+class Schedule(BaseModel):
+    """Schedule model representing a notebook schedule."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    name: str
+    options: ScheduleOptions

fabricks/models/step.py

@@ -0,0 +1,103 @@
+"""Step configuration models."""
+
+from pydantic import BaseModel, ConfigDict
+
+from fabricks.models.common import BaseInvokerOptions, ExtenderOptions, SparkOptions
+from fabricks.models.table import StepTableOptions
+
+
+class StepInvokerOptions(BaseModel):
+    """Grouped invoker operations for pre/run/post execution."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    pre_run: list[BaseInvokerOptions] | None = None
+    post_run: list[BaseInvokerOptions] | None = None
+
+
+class StepTimeoutOptions(BaseModel):
+    """Optional timeout overrides for individual steps."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    step: int | None = None
+    job: int | None = None
+    pre_run: int | None = None
+    post_run: int | None = None
+
+
+class StepPathOptions(BaseModel):
+    """Path configuration for steps."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    runtime: str
+    storage: str
+
+
+class StepOptions(BaseModel):
+    """Base step configuration options."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    order: int
+    workers: int | None = None
+    timeouts: StepTimeoutOptions | None = None
+
+
+class BronzeOptions(StepOptions):
+    """Bronze layer step options."""
+
+    clean: bool | None = None
+
+
+class SilverOptions(StepOptions):
+    """Silver layer step options."""
+
+    parent: str
+    stream: bool | None = None
+    local_checkpoint: bool | None = None
+
+
+class GoldOptions(StepOptions):
+    """Gold layer step options."""
+
+    schema_drift: bool | None = None
+    metadata: bool | None = None
+
+
+class Step(BaseModel):
+    """Base step configuration."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    name: str
+    path_options: StepPathOptions
+    table_options: StepTableOptions | None = None
+    extender_options: list[ExtenderOptions] | None = None
+    invoker_options: StepInvokerOptions | None = None
+    spark_options: SparkOptions | None = None
+
+
+class BronzeConf(Step):
+    """Bronze layer step configuration."""
+
+    options: BronzeOptions
+
+
+class SilverConf(Step):
+    """Silver layer step configuration."""
+
+    options: SilverOptions
+
+
+class GoldConf(Step):
+    """Gold layer step configuration."""
+
+    options: GoldOptions
+
+
+class PowerBI(Step):
+    """PowerBI configuration."""
+
+    path_options: StepPathOptions | None = None

fabricks/models/table.py

@@ -0,0 +1,77 @@
+"""Table-related options and constraint models."""
+
+from pydantic import BaseModel, ConfigDict
+
+from fabricks.models.common import AllowedConstraintOptions, AllowedForeignKeyOptions
+
+
+class ForeignKeyOptions(BaseModel):
+    """Options for foreign key constraints."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    foreign_key: AllowedForeignKeyOptions | None = None
+    constraint: AllowedConstraintOptions | None = None
+
+
+class PrimaryKeyOptions(BaseModel):
+    """Options for primary key constraints."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    constraint: AllowedConstraintOptions | None = None
+
+
+class ForeignKey(BaseModel):
+    """Foreign key constraint definition."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    keys: list[str]
+    reference: str
+    options: ForeignKeyOptions | None = None
+
+
+class PrimaryKey(BaseModel):
+    """Primary key constraint definition."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    keys: list[str]
+    options: PrimaryKeyOptions | None = None
+
+
+class TableOptions(BaseModel):
+    """Comprehensive table configuration options for jobs."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    identity: bool | None = None
+    liquid_clustering: bool | None = None
+    partition_by: list[str] | None = None
+    zorder_by: list[str] | None = None
+    cluster_by: list[str] | None = None
+    powerbi: bool | None = None
+    maximum_compatibility: bool | None = None
+    bloomfilter_by: list[str] | None = None
+    constraints: dict[str, str | bool | int] | None = None
+    properties: dict[str, str | bool | int] | None = None
+    comment: str | None = None
+    calculated_columns: dict[str, str | bool | int] | None = None
+    masks: dict[str, str] | None = None
+    comments: dict[str, str | bool | int] | None = None
+    retention_days: int | None = None
+    primary_key: dict[str, PrimaryKey] | None = None
+    foreign_keys: dict[str, ForeignKey] | None = None
+
+
+class StepTableOptions(BaseModel):
+    """Simplified table options for step-level configuration."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    powerbi: bool | None = None
+    liquid_clustering: bool | None = None
+    properties: dict[str, str | bool | int] | None = None
+    retention_days: int | None = None
+    masks: dict[str, str] | None = None

fabricks/{core/jobs/get_job_id.py → models/utils.py}

@@ -1,3 +1,5 @@
+"""Utility functions for job and dependency ID generation."""
+
 from typing import Optional, overload
 
 from fabricks.utils.helpers import md5

fabricks/utils/helpers.py

@@ -8,7 +8,7 @@ from pyspark.sql import DataFrame
 from typing_extensions import deprecated
 
 from fabricks.utils._types import DataFrameLike
-from fabricks.utils.path import Path
+from fabricks.utils.path import GitPath
 from fabricks.utils.spark import spark
 
 
@@ -197,12 +197,12 @@ def run_in_parallel(
     return results
 
 
-def run_notebook(path: Path, timeout: Optional[int] = None, **kwargs):
+def run_notebook(path: GitPath, timeout: Optional[int] = None, **kwargs):
     """
     Runs a notebook located at the given path.
 
     Args:
-        path (Path): The path to the notebook file.
+        path (GitPath): The path to the notebook file.
         timeout (Optional[int]): The maximum execution time for the notebook in seconds. Defaults to None.
         **kwargs: Additional keyword arguments to be passed to the notebook.
 
@@ -229,10 +229,11 @@ def md5(s: Any) -> str:
     return md5.hexdigest()
 
 
-def load_module_from_path(name: str, path: Path):
+def load_module_from_path(name: str, path: GitPath):
     from importlib.util import module_from_spec, spec_from_file_location
 
-    sys.path.append(str(path.parent))
+    if path.parent not in sys.path:
+        sys.path.insert(0, str(path.parent))
 
     spec = spec_from_file_location(name, path.string)
     assert spec, f"no valid module found in {path.string}"

fabricks/utils/log.py

@@ -60,7 +60,28 @@ class LogFormatter(logging.Formatter):
         extra = ""
         if hasattr(record, "exc_info") and record.exc_info:
             exc_info = record.__dict__.get("exc_info", None)
-            extra += f" [{self.COLORS[logging.ERROR]}{exc_info[0].__name__}{self.RESET}]"
+            extra += f" [{self.COLORS[logging.ERROR]}{exc_info[0].__name__}{self.RESET}]"  # type: ignore
+
+        if hasattr(record, "df"):
+            df = record.__dict__.get("df")
+            if isinstance(df, DataFrame):
+                try:
+                    pandas_df = df.toPandas()
+                except Exception:
+                    # Handle timestamp precision/timezone issues by casting to string
+                    from pyspark.sql.functions import col
+                    from pyspark.sql.types import TimestampType
+
+                    for field in df.schema.fields:
+                        if isinstance(field.dataType, TimestampType):
+                            df = df.withColumn(field.name, col(field.name).cast("string"))
+                    pandas_df = df.toPandas()
+
+                extra += f"\n---\n%df\n{pandas_df.to_string(index=True)}\n---"
+
+        if hasattr(record, "json"):
+            json_data = record.__dict__.get("json")
+            extra += f"\n---\n{json.dumps(json_data, indent=2, default=str)}\n---"
 
         if self.debugmode:
             if hasattr(record, "sql"):
@@ -72,11 +93,6 @@ class LogFormatter(logging.Formatter):
             if hasattr(record, "context"):
                 extra += f"\n---\n{json.dumps(record.__dict__.get('context'), indent=2, default=str)}\n---"
 
-            if hasattr(record, "df"):
-                df = record.__dict__.get("df")
-                if isinstance(df, DataFrame):
-                    extra += f"\n---\n%df\n{df.toPandas().to_string(index=True)}\n---"
-
         record.levelname = levelname_formatted
         record.prefix = prefix
         record.timestamp = self.formatTime(record)
@@ -156,6 +172,9 @@ class AzureTableLogHandler(logging.Handler):
                     }
                     r["Exception"] = json.dumps(d)
 
+            if hasattr(record, "json"):
+                r["Data"] = json.dumps(record.__dict__.get("json", ""))
+
             if self.debugmode:
                 if hasattr(record, "content"):
                     r["Content"] = json.dumps(record.__dict__.get("content", ""))[:1000]