fabricks 3.0.18__py3-none-any.whl → 4.0.0__py3-none-any.whl

fabricks/models/common.py

@@ -0,0 +1,79 @@
+"""Common types and type aliases used across all models."""
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+# Mode type definitions
+AllowedModesBronze = Literal["memory", "append", "register"]
+AllowedModesSilver = Literal["memory", "append", "latest", "update", "combine"]
+AllowedModesGold = Literal["memory", "append", "complete", "update", "invoke"]
+AllowedModes = Literal[AllowedModesBronze, AllowedModesSilver, AllowedModesGold]
+
+# File and operation types
+AllowedFileFormats = Literal["json_array", "json", "jsonl", "csv", "parquet", "delta"]
+AllowedOperations = Literal["upsert", "reload", "delete"]
+AllowedTypes = Literal["manual", "default"]
+AllowedOrigins = Literal["parser", "job"]
+
+# Constraint types
+AllowedConstraintOptions = Literal["not enforced", "deferrable", "initially deferred", "norely", "rely"]
+AllowedForeignKeyOptions = Literal["match full", "on update no action", "on delete no action"]
+
+# Change Data Capture types
+AllowedChangeDataCaptures = Literal["nocdc", "scd1", "scd2", "none"]
+
+
+class SparkOptions(BaseModel):
+    """Spark SQL and configuration options."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    sql: dict[str, str | bool | int] | None = None
+    conf: dict[str, str | bool | int] | None = None
+
+
+class BaseInvokerOptions(BaseModel):
+    """Options for invoking notebooks during pre/post run operations."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    notebook: str | None = None
+    timeout: int | None = None
+    arguments: dict[str, str | bool | int] | None = None
+
+
+class InvokerOptions(BaseModel):
+    """Grouped invoker operations for pre/run/post execution."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    pre_run: list[BaseInvokerOptions] | None = None
+    run: list[BaseInvokerOptions] | None = None
+    post_run: list[BaseInvokerOptions] | None = None
+
+
+class ExtenderOptions(BaseModel):
+    """Configuration for runtime extenders."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    extender: str
+    arguments: dict[str, str] | None = None
+
+
+class DatabasePathOptions(BaseModel):
+    """Path configuration for databases."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    storage: str
+
+
+class Database(BaseModel):
+    """Database configuration."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    name: str
+    path_options: DatabasePathOptions

fabricks/models/config.py

@@ -0,0 +1,225 @@
+import logging
+import os
+import pathlib
+from pathlib import Path as PathLibPath
+
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
+
+from fabricks.utils.path import GitPath, resolve_git_path
+
+
+class HierarchicalFileSettingsSource(PydanticBaseSettingsSource):
+    """Custom settings source for hierarchical file configuration."""
+
+    def get_field_value(self, field):
+        # Not used in this implementation
+        return None, None, False
+
+    def __call__(self):
+        """Load settings from hierarchical file search."""
+        data = self._load_hierarchical_file()
+        return data
+
+    def _load_hierarchical_file(self):
+        """Search up directory hierarchy for configuration files."""
+
+        def pyproject_settings(base: PathLibPath):
+            pyproject_path = base / "pyproject.toml"
+            if pyproject_path.exists():
+                import sys
+
+                if sys.version_info >= (3, 11):
+                    import tomllib
+                else:
+                    import tomli as tomllib  # type: ignore
+
+                with open(pyproject_path, "rb") as f:
+                    data = tomllib.load(f)
+
+                data = data.get("tool", {}).get("fabricks", {})
+                data["base"] = str(base)
+                return data
+
+            return None
+
+        def json_settings(base: PathLibPath):
+            json_path = base / "fabricksconfig.json"
+            if json_path.exists():
+                import json
+
+                with open(json_path, "r") as f:
+                    data = json.load(f)
+
+                data["base"] = str(base)
+                return data
+
+            return None
+
+        path = pathlib.Path(os.getcwd())
+        data = None
+
+        while not data:
+            data = json_settings(path)
+            if data:
+                break
+
+            data = pyproject_settings(path)
+            if data:
+                break
+
+            if path == path.parent:
+                break
+
+            path = path.parent
+
+        return data or {}
+
+
+class ResolvedPathOptions(BaseModel):
+    """Resolved path objects for main configuration."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True, arbitrary_types_allowed=True)
+
+    base: GitPath
+    config: GitPath
+    runtime: GitPath
+    notebooks: GitPath
+
+
+class ConfigOptions(BaseSettings):
+    """Main configuration options for Fabricks framework."""
+
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    base: str = Field(
+        validation_alias=AliasChoices("FABRICKS_BASE", "base"),
+        default="none",
+    )
+    config: str = Field(
+        validation_alias=AliasChoices("FABRICKS_CONFIG", "config"),
+        default="none",
+    )
+    runtime: str = Field(
+        validation_alias=AliasChoices("FABRICKS_RUNTIME", "runtime"),
+        default="none",
+    )
+    notebooks: str = Field(
+        validation_alias=AliasChoices("FABRICKS_NOTEBOOKS", "notebooks"),
+        default="none",
+    )
+    job_config_from_yaml: bool = Field(
+        validation_alias=AliasChoices("FABRICKS_IS_JOB_CONFIG_FROM_YAML", "job_config_from_yaml"),
+        default=False,
+    )
+    debugmode: bool = Field(
+        validation_alias=AliasChoices("FABRICKS_IS_DEBUGMODE", "debugmode"),
+        default=False,
+    )
+    funmode: bool = Field(
+        validation_alias=AliasChoices("FABRICKS_IS_FUNMODE", "funmode"),
+        default=False,
+    )
+    devmode: bool = Field(
+        validation_alias=AliasChoices("FABRICKS_IS_DEVMODE", "devmode"),
+        default=False,
+    )
+    loglevel: int = Field(
+        validation_alias=AliasChoices("FABRICKS_LOGLEVEL", "loglevel"),
+        default=20,
+    )
+
+    @field_validator("job_config_from_yaml", "debugmode", "funmode", "devmode", mode="before")
+    @classmethod
+    def validate_bool(cls, v):
+        """
+        Convert common string representations of boolean values to bool.
+
+        Accepted case-insensitive string values are:
+        - "true", "1", "yes" -> True
+        - "false", "0", "no" -> False
+
+        Non-string inputs or strings not matching the above values are returned unchanged.
+        """
+        if isinstance(v, bool):
+            return v
+
+        if isinstance(v, str):
+            if v.lower() in ("true", "1", "yes"):
+                return True
+            elif v.lower() in ("false", "0", "no"):
+                return False
+
+        return v
+
+    @field_validator("loglevel", mode="before")
+    @classmethod
+    def validate_loglevel(cls, v):
+        """Validate log level."""
+        if isinstance(v, str):
+            levels = {
+                "DEBUG": logging.DEBUG,
+                "INFO": logging.INFO,
+                "WARNING": logging.WARNING,
+                "ERROR": logging.ERROR,
+                "CRITICAL": logging.CRITICAL,
+            }
+            v_upper = v.upper()
+            if v_upper in levels:
+                return levels[v_upper]
+
+            return logging.INFO  # Default log level
+
+        return v
+
+    @field_validator("notebooks", mode="before")
+    @classmethod
+    def validate_notebooks(cls, v):
+        """Set default notebooks path if not provided."""
+        if not v or v == "none":
+            return "runtime/notebooks"
+
+        return v
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ):
+        # Order: env vars > hierarchical file > defaults
+        return (
+            init_settings,
+            env_settings,
+            HierarchicalFileSettingsSource(settings_cls),
+            file_secret_settings,
+        )
+
+    def _resolve_paths(self) -> ResolvedPathOptions:
+        """
+        Get all paths resolved as Path objects.
+
+        Args:
+            runtime: The base runtime path (e.g., PATH_RUNTIME)
+
+        Returns:
+            ResolvedPathOptions with all paths resolved
+        """
+        # Collect all storage paths with variable substitution
+        root = GitPath(self.base)
+
+        return ResolvedPathOptions(
+            base=resolve_git_path(path=self.base),
+            config=resolve_git_path(path=self.config, base=root),
+            runtime=resolve_git_path(path=self.runtime, base=root),
+            notebooks=resolve_git_path(path=self.notebooks, base=root),
+        )
+
+    @computed_field
+    @property
+    def resolved_paths(self) -> ResolvedPathOptions:
+        """Get all paths resolved as Path objects."""
+        return self._resolve_paths()

fabricks/models/dependency.py

@@ -0,0 +1,50 @@
+"""Job dependency tracking models."""
+
+from pydantic import BaseModel, ConfigDict, model_validator
+from pyspark.sql.types import StringType, StructField, StructType
+
+from fabricks.models.common import AllowedOrigins
+from fabricks.models.utils import get_dependency_id, get_job_id
+
+
+class JobDependency(BaseModel):
+    """Job dependency tracking."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    origin: AllowedOrigins
+    job_id: str
+    parent: str
+    parent_id: str
+    dependency_id: str
+
+    def __str__(self) -> str:
+        return f"{self.job_id} -> {self.parent}"
+
+    @model_validator(mode="after")
+    def check_no_circular_dependency(self):
+        if self.job_id == self.parent_id:
+            raise ValueError("Circular dependency detected")
+        return self
+
+    @staticmethod
+    def from_parts(job_id: str, parent: str, origin: AllowedOrigins):
+        parent = parent.removesuffix("__current")
+        return JobDependency(
+            job_id=job_id,
+            origin=origin,
+            parent=parent,
+            parent_id=get_job_id(job=parent),
+            dependency_id=get_dependency_id(parent=parent, job_id=job_id),
+        )
+
+
+SchemaDependencies = StructType(
+    [
+        StructField("dependency_id", StringType(), True),
+        StructField("origin", StringType(), True),
+        StructField("job_id", StringType(), True),
+        StructField("parent_id", StringType(), True),
+        StructField("parent", StringType(), True),
+    ]
+)

fabricks/models/job.py

@@ -0,0 +1,157 @@
+"""Job configuration models."""
+
+from pydantic import BaseModel, ConfigDict, Field, computed_field
+
+from fabricks.models.common import (
+    AllowedChangeDataCaptures,
+    AllowedModes,
+    AllowedModesBronze,
+    AllowedModesGold,
+    AllowedModesSilver,
+    AllowedOperations,
+    AllowedTypes,
+    ExtenderOptions,
+    InvokerOptions,
+    SparkOptions,
+)
+from fabricks.models.table import TableOptions
+from fabricks.models.utils import get_job_id
+
+
+class CheckOptions(BaseModel):
+    """Data quality check options for jobs."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    skip: bool | None = None
+    pre_run: bool | None = None
+    post_run: bool | None = None
+    min_rows: int | None = None
+    max_rows: int | None = None
+    count_must_equal: str | None = None
+
+
+class ParserOptions(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    file_format: str | None = None
+    read_options: dict[str, str] | None = None
+
+
+class BaseOptions(BaseModel):
+    """Base job options."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    mode: AllowedModes
+    change_data_capture: AllowedChangeDataCaptures | None = Field(default="none")
+
+    parents: list[str] | None = None
+    optimize: bool | None = None
+    compute_statistics: bool | None = None
+    vacuum: bool | None = None
+    no_drop: bool | None = None
+    timeout: int | None = None
+
+
+class BronzeOptions(BaseOptions):
+    """Bronze layer job options."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    mode: AllowedModesBronze
+    type: AllowedTypes | None = None
+
+    uri: str
+    keys: list[str] | None = None
+
+    parser: str | None = None
+    source: str | None = None
+    filter_where: str | None = None
+    encrypted_columns: list[str] | None = None
+    calculated_columns: dict[str, str] | None = None
+    operation: AllowedOperations | None = None
+
+
+class SilverOptions(BaseOptions):
+    """Silver layer job options."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    mode: AllowedModesSilver
+    type: AllowedTypes | None = None
+
+    filter_where: str | None = None
+    deduplicate: bool | None = None
+    stream: bool | None = None
+    order_duplicate_by: dict[str, str] | None = None
+
+
+class GoldOptions(BaseOptions):
+    """Gold layer job options."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    mode: AllowedModesGold
+    type: AllowedTypes | None = None
+
+    update_where: str | None = None
+    deduplicate: bool | None = None
+    rectify_as_upserts: bool | None = None
+    correct_valid_from: bool | None = None
+    persist_last_timestamp: bool | None = None
+    persist_last_updated_timestamp: bool | None = None
+    table: str | None = None
+    notebook: bool | None = None
+    requirements: bool | None = None
+    metadata: bool | None = None
+    last_updated: bool | None = None
+
+
+TOptions = BronzeOptions | SilverOptions | GoldOptions
+
+
+class JobConfBase(BaseModel):
+    """Base job configuration with computed fields."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    step: str
+    topic: str
+    item: str
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def job_id(self) -> str:
+        """Computed job_id from step, topic, and item."""
+        return get_job_id(step=self.step, topic=self.topic, item=self.item)
+
+    options: TOptions
+    table_options: TableOptions | None = None
+    check_options: CheckOptions | None = None
+    spark_options: SparkOptions | None = None
+    invoker_options: InvokerOptions | None = None
+    extender_options: list[ExtenderOptions] | None = None
+    tags: list[str] | None = None
+    comment: str | None = None
+
+
+class JobConfBronze(JobConfBase):
+    """Bronze-specific job configuration."""
+
+    options: BronzeOptions
+    parser_options: ParserOptions | None = None
+
+
+class JobConfSilver(JobConfBase):
+    """Silver-specific job configuration."""
+
+    options: SilverOptions
+
+
+class JobConfGold(JobConfBase):
+    """Gold-specific job configuration."""
+
+    options: GoldOptions
+
+
+JobConf = JobConfBronze | JobConfSilver | JobConfGold

fabricks/models/path.py

@@ -0,0 +1,17 @@
+"""Path configuration models."""
+
+from dataclasses import dataclass
+
+from fabricks.utils.path import FileSharePath, GitPath
+
+
+@dataclass(frozen=True)
+class Paths:
+    """Runtime path references."""
+
+    to_storage: "FileSharePath"
+    to_tmp: "FileSharePath"
+    to_checkpoints: "FileSharePath"
+    to_commits: "FileSharePath"
+    to_schema: "FileSharePath"
+    to_runtime: "GitPath"