fabricks 3.0.5.2__py3-none-any.whl → 3.0.7__py3-none-any.whl

fabricks/api/notebooks/optimize.py

@@ -1,29 +0,0 @@
-# Databricks notebook source
-# MAGIC %run ./add_fabricks
-
-# COMMAND ----------
-
-from databricks.sdk.runtime import dbutils
-from pyspark.errors.exceptions.base import IllegalArgumentException
-
-from fabricks.core.scripts import optimize
-
-# COMMAND ----------
-
-dbutils.widgets.text("schedule_id", "---")
-
-# COMMAND ----------
-
-try:
-    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
-except (TypeError, IllegalArgumentException, ValueError):
-    schedule_id = dbutils.widgets.get("schedule_id")
-    schedule_id = None if schedule_id == "---" else schedule_id
-
-# COMMAND ----------
-
-optimize(schedule_id=schedule_id)
-
-# COMMAND ----------
-
-dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/api/notebooks/vacuum.py

@@ -1,29 +0,0 @@
-# Databricks notebook source
-# MAGIC %run ./add_fabricks
-
-# COMMAND ----------
-
-from databricks.sdk.runtime import dbutils
-from pyspark.errors.exceptions.base import IllegalArgumentException
-
-from fabricks.core.scripts import vacuum
-
-# COMMAND ----------
-
-dbutils.widgets.text("schedule_id", "---")
-
-# COMMAND ----------
-
-try:
-    schedule_id = dbutils.jobs.taskValues.get(taskKey="initialize", key="schedule_id")
-except (TypeError, IllegalArgumentException, ValueError):
-    schedule_id = dbutils.widgets.get("schedule_id")
-    schedule_id = None if schedule_id == "---" else schedule_id
-
-# COMMAND ----------
-
-vacuum(schedule_id=schedule_id)
-
-# COMMAND ----------
-
-dbutils.notebook.exit(value="exit (0)")  # type: ignore

fabricks/cdc/templates/query/context.sql.jinja

@@ -1,101 +0,0 @@
-/*
-⛷️🧀🍫🏔️
-
-👀🏁
-{%- if format %}
-☐ format: {{format}}{% endif %}
-{%- if tgt %}
-☐ tgt: {{tgt}}{% endif %}
-{%- if cdc %}
-☐ cdc: {{cdc}}{% endif %}
-{%- if mode %}
-☐ mode: {{mode}}{% endif %}
-{%- if slice %}
-🗹 slice: {{slice}}{% endif %}
-{%- if slices %}
-☐ slices: {{slices}}{% endif %}
-{%- if rectify %}
-🗹 rectify: {{rectify}}{% endif %}
-{%- if deduplicate %}
-🗹 deduplicate: {{deduplicate}}{% endif %}
-{%- if deduplicate_key %}
-🗹 deduplicate_key: {{deduplicate_key}}{% endif %}
-{%- if deduplicate_hash %}
-🗹 deduplicate_hash: {{deduplicate_hash}}{% endif %}
-{%- if soft_delete %}
-🗹 soft_delete: {{soft_delete}}{% endif %}
-{%- if correct_valid_from %}
-🗹 correct_valid_from: {{correct_valid_from}}{% endif %}
-{%- if has_data %}
-🗹 has_data: {{has_data}}{% else %}☒ has_data: {{has_data}}{% endif %}
-{%- if has_rows %}
-🗹 has_rows: {{has_rows}}{% endif %}
-{%- if has_source %}
-🗹 has_source: {{has_source}}{% endif %}
-{%- if sources %}
-🗹 sources: {{sources}}{% endif %}
-{%- if has_metadata %}
-🗹 has_metadata: {{has_metadata}}{% endif %}
-{%- if has_timestamp %}
-🗹 has_timestamp: {{has_timestamp}}{% endif %}
-{%- if has_identity %}
-🗹 has_identity: {{has_identity}}{% endif %}
-{%- if has_key %}
-🗹 has_key: {{has_key}}{% endif %}
-{%- if has_hash %}
-🗹 has_hash: {{has_hash}}{% endif %}
-{%- if has_order_by %}
-🗹 has_order_by: {{has_order_by}}{% endif %}
-{%- if has_rescued_data %}
-🗹 has_rescued_data: {{has_rescued_data}}{% endif %} 
-{%- if add_metadata %}
-🗹 add_metadata: {{add_metadata}}{% endif %}
-{%- if add_timestamp %}
-🗹 add_timestamp: {{add_timestamp}}{% endif %}
-{%- if add_key %}
-🗹 add_key: {{add_key}}{% endif %}
-{%- if add_hash %}
-🗹 add_hash: {{add_hash}}{% endif %}
-{%- if add_operation %}
-☐ add_operation: {{add_operation}}{% endif %}
-{%- if add_source %}
-☐ add_source: {{add_source}}{% endif %}
-{%- if add_calculated_columns %}
-☐ add_calculated_columns: {{add_calculated_columns}}{% endif %}
-{%- if order_duplicate_by %}
-🗹 order_duplicate_by: {{order_duplicate_by}}{% endif %}
-{%- if all_except %}
-☐ all_except: {{all_except}}{% endif %}
-{%- if all_overwrite %}
-☐ all_overwrite: {{all_overwrite}}{% endif %}
-{%- if filter_where %}
-☐ filter_where: {{filter_where}}{% endif %}
-{%- if update_where %}
-☐ update_where: {{update_where}}{% endif %}
-{%- if parent_slice %}
-☐ parent_slice: {{parent_slice}}{% endif %}
-{%- if parent_rectify %}
-☐ parent_rectify: {{parent_rectify}}{% endif %}
-{%- if parent_deduplicate_key %}
-☐ parent_deduplicate_key: {{parent_deduplicate_key}}{% endif %}
-{%- if parent_deduplicate_hash %}
-☐ parent_deduplicate_hash: {{parent_deduplicate_hash}}{% endif %}
-{%- if parent_cdc %}
-☐ parent_cdc: {{parent_cdc}}{% endif %}
-{%- if parent_final %}
-☐ parent_final: {{parent_final}}{% endif %}
-👀🏳️
-
-👁️🏁
-{%- if src %}
-☐ src: {{src}}{% endif %}
-{%- if fields %}
-☐ fields: {{fields}}{% endif %}
-{%- if keys %}
-☐ keys: {{keys}}{% endif %}
-{%- if hashes %}
-☐ hashes: {{hashes}}{% endif %}
-👁️🏳️
-
-*/
-

fabricks/cdc/templates/query/current.sql.jinja

@@ -1,32 +0,0 @@
-{% import 'query/hash.sql.jinja' as h -%}
-
-__current as (
-    select
-        {% for field in fields %} {{ field }}, {% endfor %}
-        {% if has_data %} 'current'
-        {% else %} 'delete'
-        {% endif %} as __operation,
-        {% if has_timestamp %}
-            {% if cdc == "nocdc" %} __timestamp as __timestamp, {% endif %}
-            {% if cdc == "scd1" %} __timestamp as __timestamp, {% endif %}
-            {% if cdc == "scd2" %} __valid_from as __timestamp, {% endif %}
-        {% else %} cast('0001-01-01' as timestamp) as __timestamp,
-        {% endif %}
-        {% if has_hash %} __hash,
-        {% else %} {{ h.hash(fields=hashes) }} as __hash,
-        {% endif %}
-        {% if has_identity %} __identity, {% endif %}
-        {% if has_key %} __key,
-        {% else %} {{ h.hash(fields=keys) }} as __key,
-        {% endif %}
-        {% if has_source %} __source, {% endif %}
-        {% if has_metadata %} __metadata, {% endif %}
-        {% if has_rescued_data %} __rescued_data, {% endif %}
-    from {{ tgt }} t
-    where
-        true
-        {% if cdc == "scd2" %} and __is_current {% endif %}
-        {% if cdc == "scd1" %} {% if soft_delete %} and __is_current {% endif %} {% endif %}
-        {% if sources %} and ({{ sources }}) {% endif %}
-        {% if update_where %} and {{ update_where }} {% endif %}
-),

fabricks/cdc/templates/query/deduplicate_hash.sql.jinja

@@ -1,21 +0,0 @@
-__deduplicate_hash as (
-    select
-        *,
-        lag(__hash) over (
-            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
-        ) as __deduplicate_hash_previous__hash,
-        lag(__operation) over (
-            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
-        ) as __deduplicate_hash_previous_operation
-    from {{ parent_deduplicate_hash }}
-    where true
-),
-__deduplicated_hash as (
-    select *
-    from __deduplicate_hash
-    where
-        true
-        and not (
-            __hash <=> __deduplicate_hash_previous__hash and __operation <=> __deduplicate_hash_previous_operation
-        )
-),

fabricks/cdc/templates/query/deduplicate_key.sql.jinja

@@ -1,14 +0,0 @@
-__deduplicate_key as (
-    select
-        *,
-        row_number() over (
-            partition by {% if has_source %} __source, {% endif %} __key, __timestamp
-            order by
-                /* prioritize delete over upsert */
-                __operation asc,
-                {% if has_order_by %} {% for o in order_duplicate_by %} {{ o }}, {% endfor %} {% endif %}
-        ) as __deduplicate_key_rn
-    from {{ parent_deduplicate_key }}
-    where true
-),
-__deduplicated_key as (select *, from __deduplicate_key where __deduplicate_key_rn == 1),

fabricks/cdc/templates/query/hash.sql.jinja

@@ -1 +0,0 @@
-{% macro hash(fields) -%} md5(array_join(array({% for f in fields %}{{ f }}, {% endfor %}), '*', '-1')) {%- endmacro %}

fabricks/cdc/templates/query/slice.sql.jinja

@@ -1,14 +0,0 @@
-__sliced as (
-    select
-        {% for field in fields %} {{ field }}, {% endfor %}
-        s.__operation,
-        s.__timestamp,
-        s.__hash,
-        s.__key,
-        {% if has_identity %} s.__identity, {% endif %}
-        {% if has_source %} s.__source, {% endif %}
-        {% if has_metadata %} s.__metadata, {% endif %}
-        {% if has_rescued_data %} __rescued_data, {% endif %}
-    from {{ parent_slice }} s
-    where true and ({{ slices }})
-),

fabricks/config/base.py

@@ -1,8 +0,0 @@
-from __future__ import annotations
-
-from pydantic import BaseModel, ConfigDict
-
-
-class ModelBase(BaseModel):
-    # Ignore extra/unknown fields (TypedDict-like strictness)
-    model_config = ConfigDict(extra="ignore")

fabricks/config/fabricks/__init__.py

@@ -1,26 +0,0 @@
-from __future__ import annotations
-
-from pydantic_settings import BaseSettings
-
-from fabricks.config.fabricks.base import BaseConfig
-from fabricks.config.fabricks.environment import EnvironmentConfig
-from fabricks.config.fabricks.pyproject import PyprojectConfig
-
-
-class FabricksConfig(BaseConfig, BaseSettings):
-    @classmethod
-    def load(cls) -> FabricksConfig:
-        pyproject = PyprojectConfig.load()
-        environ = EnvironmentConfig()  # type: ignore
-
-        data = {}
-
-        if pyproject:
-            dump = pyproject.model_dump(exclude_none=True)
-            data.update(dump)
-
-        # Override with environment settings
-        dump = environ.model_dump(exclude_none=True)
-        data.update(dump)
-
-        return cls(**data)

fabricks/config/fabricks/base.py

@@ -1,90 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Optional
-
-from pydantic import Field, field_validator
-
-from fabricks.config.base import ModelBase
-
-
-class BaseConfig(ModelBase):
-    root: Optional[Path] = Field(None, description="Root directory")
-
-    runtime: Optional[str] = Field(None, description="Runtime path")
-    notebooks: Optional[str] = Field(None, description="Notebooks path")
-    is_job_config_from_yaml: Optional[bool] = Field(None, description="Load job config from YAML")
-    is_debugmode: Optional[bool] = Field(None, description="Enable debug mode")
-    loglevel: Optional[str] = Field(None, description="Logging level")
-    config: Optional[str] = Field(None, description="Config file path")
-
-    @field_validator("runtime", "notebooks", "config", mode="before")
-    @classmethod
-    def handle_none(cls, v):
-        if isinstance(v, str) and v.lower() == "none":
-            return None
-
-        return v
-
-    @field_validator("is_job_config_from_yaml", "is_debugmode", mode="before")
-    @classmethod
-    def handle_bool(cls, v):
-        if v is None:
-            return None
-
-        if isinstance(v, str):
-            return v.lower() in ("true", "1", "yes")
-
-        return bool(v)
-
-    @field_validator("loglevel", mode="before")
-    @classmethod
-    def handle_loglevel(cls, v):
-        if v is None:
-            return None
-
-        if v.upper() not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}:
-            raise ValueError(f"{v.upper()} not allowed. Use DEBUG, INFO, WARNING, ERROR or CRITICAL")
-
-        return v.upper()
-
-    def resolve_runtime_path(self) -> Path:
-        runtime = self.runtime
-
-        # Use environment/explicit setting if available
-        if runtime is not None:
-            return Path(runtime)
-
-        # Fall back to pyproject.toml location
-        if self.root is not None:
-            return self.root
-
-        # Final fallback
-        raise ValueError("No pyproject.toml nor FABRICKS_RUNTIME")
-
-    def resolve_notebooks_path(self) -> Path:
-        notebooks = self.notebooks
-        runtime = self.resolve_runtime_path()
-
-        if notebooks is not None:
-            if self.root is not None and not Path(notebooks).is_absolute():
-                return self.root.joinpath(notebooks)
-
-            return Path(notebooks)
-
-        # Default to runtime/notebooks
-        return runtime.joinpath("notebooks")
-
-    def resolve_config_path(self, cluster_id: Optional[str] = None) -> Path:
-        config = self.config
-        runtime = self.resolve_runtime_path()
-
-        if config is not None:
-            if self.root is not None and not Path(config).is_absolute():
-                return self.root.joinpath(config)
-
-            return Path(config)
-
-        # default to fabricks/conf.yml
-        assert cluster_id is not None
-        return runtime.joinpath(f"fabricks/conf.{cluster_id}.yml")

fabricks/config/fabricks/environment.py

@@ -1,9 +0,0 @@
-from __future__ import annotations
-
-from pydantic_settings import BaseSettings, SettingsConfigDict
-
-from fabricks.config.fabricks.base import BaseConfig
-
-
-class EnvironmentConfig(BaseConfig, BaseSettings):
-    model_config = SettingsConfigDict(env_prefix="FABRICKS_", case_sensitive=False, extra="ignore")

fabricks/config/fabricks/pyproject.py

@@ -1,47 +0,0 @@
-from __future__ import annotations
-
-import os
-import tomllib  # type: ignore
-from pathlib import Path
-from typing import Optional
-
-from pydantic import Field
-from pydantic_settings import BaseSettings, SettingsConfigDict
-
-from fabricks.config.fabricks.base import BaseConfig
-
-
-class PyprojectConfig(BaseConfig, BaseSettings):
-    model_config = SettingsConfigDict(
-        env_prefix="",
-        case_sensitive=False,
-        extra="ignore",
-    )
-
-    is_job_config_from_yaml: Optional[bool] = Field(None, alias="job_config_from_yaml")
-    is_debugmode: Optional[bool] = Field(None, alias="debugmode")
-
-    @classmethod
-    def _from_pyproject(cls, root: Path) -> PyprojectConfig:
-        path = root / "pyproject.toml"
-        if not path.exists():
-            return cls()  # type: ignore
-
-        with open(path, "rb") as f:
-            pyproject = tomllib.load(f)
-            config = pyproject.get("tool", {}).get("fabricks", {})
-
-        return cls(**config, root=root)
-
-    @classmethod
-    def load(cls) -> PyprojectConfig:
-        path = Path(os.getcwd())
-
-        while path is not None:
-            if (path / "pyproject.toml").exists():
-                break
-            if path == path.parent:
-                break
-            path = path.parent
-
-        return cls._from_pyproject(path)

fabricks/config/jobs/__init__.py

@@ -1,6 +0,0 @@
-from fabricks.config.jobs.base import BaseJobConfig
-from fabricks.config.jobs.bronze import BronzeJobConfig
-from fabricks.config.jobs.gold import GoldJobConfig
-from fabricks.config.jobs.silver import SilverJobConfig
-
-__all__ = ["BaseJobConfig", "BronzeJobConfig", "SilverJobConfig", "GoldJobConfig"]

fabricks/config/jobs/base.py

@@ -1,101 +0,0 @@
-from typing import List, Literal, Optional
-
-from fabricks.config.base import ModelBase
-
-FileFormats = Literal["json_array", "json", "jsonl", "csv", "parquet", "delta"]
-Operations = Literal["upsert", "reload", "delete"]
-Types = Literal["manual", "default"]
-Origins = Literal["parser", "job"]
-ChangeDataCaptures = Literal["none", "nocdc", "scd1", "scd2"]
-Modes = Literal[
-    "memory",
-    "append",
-    "complete",
-    "update",
-    "invoke",
-    "memory",
-    "append",
-    "register",
-    "memory",
-    "append",
-    "latest",
-    "update",
-    "combine",
-]
-Steps = Literal["bronze", "silver", "gold"]
-
-
-class SparkOptions(ModelBase):
-    sql: Optional[dict[str, str]] = None
-    conf: Optional[dict[str, str]] = None
-
-
-class TableOptions(ModelBase):
-    identity: Optional[bool] = None
-    liquid_clustering: Optional[bool] = None
-    partition_by: Optional[List[str]] = None
-    zorder_by: Optional[List[str]] = None
-    cluster_by: Optional[List[str]] = None
-    powerbi: Optional[bool] = None
-    bloomfilter_by: Optional[List[str]] = None
-    constraints: Optional[dict[str, str]] = None
-    properties: Optional[dict[str, str]] = None
-    comment: Optional[str] = None
-    calculated_columns: Optional[dict[str, str]] = None
-    retention_days: Optional[int] = None
-
-
-class InvokeOptions(ModelBase):
-    notebook: str
-    timeout: int
-    arguments: Optional[dict[str, str]] = None
-
-
-class InvokerOptions(ModelBase):
-    pre_run: Optional[List[InvokeOptions]] = None
-    run: Optional[List[InvokeOptions]] = None
-    post_run: Optional[List[InvokeOptions]] = None
-
-
-class ExtenderOptions(ModelBase):
-    extender: str
-    arguments: Optional[dict[str, str]] = None
-
-
-class CheckOptions(ModelBase):
-    skip: Optional[bool] = None
-    pre_run: Optional[bool] = None
-    post_run: Optional[bool] = None
-    min_rows: Optional[int] = None
-    max_rows: Optional[int] = None
-    count_must_equal: Optional[str] = None
-
-
-class DefaultOptions(ModelBase):
-    type: Optional[Types] = None
-    mode: Modes
-    change_data_capture: Optional[ChangeDataCaptures]
-    # extra
-    parents: Optional[List[str]] = None
-    filter_where: Optional[str] = None
-    timeout: Optional[int] = None
-
-
-class BaseJobConfig(ModelBase):
-    job_id: str
-
-    extend: Steps
-    step: Steps
-
-    topic: str
-    item: str
-
-    options: Optional[DefaultOptions] = None
-    table_options: Optional[TableOptions] = None
-    check_options: Optional[CheckOptions] = None
-    spark_options: Optional[SparkOptions] = None
-    invoker_options: Optional[InvokerOptions] = None
-    extender_options: Optional[List[ExtenderOptions]] = None
-
-    tags: Optional[List[str]] = None
-    comment: Optional[str] = None

fabricks/config/jobs/bronze.py

@@ -1,38 +0,0 @@
-from typing import List, Literal, Optional
-
-from fabricks.config.base import ModelBase
-from fabricks.config.jobs.base import BaseJobConfig, ChangeDataCaptures, DefaultOptions, Operations
-
-BronzeModes = Literal["memory", "append", "register"]
-
-
-class ParserOptions(ModelBase):
-    file_format: Optional[str]
-    read_options: Optional[dict[str, str]]
-
-
-class BronzeOptions(DefaultOptions):
-    # default
-    mode: BronzeModes
-    change_data_capture: ChangeDataCaptures = "none"
-
-    # mandatory
-    uri: str
-    parser: str
-    source: str
-
-    # preferred
-    keys: Optional[List[str]] = None
-
-    # optional
-    encrypted_columns: Optional[List[str]] = None
-    calculated_columns: Optional[dict[str, str]] = None
-    operation: Optional[Operations] = None
-
-
-class BronzeJobConfig(BaseJobConfig):
-    extend: Literal["bronze"] = "bronze"
-    step: Literal["bronze"]
-
-    options: Optional[BronzeOptions] = None
-    parser_options: Optional[ParserOptions] = None

fabricks/config/jobs/gold.py

@@ -1,27 +0,0 @@
-from typing import Literal, Optional
-
-from fabricks.config.jobs.base import BaseJobConfig, DefaultOptions
-
-GoldModes = Literal["memory", "append", "complete", "update", "invoke"]
-
-
-class GoldOptions(DefaultOptions):
-    # default
-    mode: GoldModes
-
-    # optional
-    update_where: Optional[str] = None
-    deduplicate: Optional[bool] = None  # remove duplicates on the keys and on the hash
-    rectify_as_upserts: Optional[bool] = None  # convert reloads into upserts and deletes
-    correct_valid_from: Optional[bool] = None
-    persist_last_timestamp: Optional[bool] = None
-    table: Optional[str] = None
-    notebook: Optional[bool] = None
-    requirements: Optional[bool] = None
-
-
-class GoldJobConfig(BaseJobConfig):
-    extend: Literal["gold"] = "gold"
-    step: Literal["gold"]
-
-    options: Optional[GoldOptions] = None

fabricks/config/jobs/silver.py

@@ -1,22 +0,0 @@
-from typing import Literal, Optional
-
-from fabricks.config.jobs.base import BaseJobConfig, DefaultOptions
-
-SilverModes = Literal["memory", "append", "latest", "update", "combine"]
-
-
-class SilverOptions(DefaultOptions):
-    # default
-    mode: SilverModes
-
-    # optional
-    deduplicate: Optional[bool] = None
-    stream: Optional[bool] = None
-    order_duplicate_by: Optional[dict[str, str]] = None
-
-
-class SilverJobConfig(BaseJobConfig):
-    extend: Literal["silver"] = "silver"
-    step: Literal["silver"]
-
-    options: Optional[SilverOptions] = None