fabricks 3.0.11__py3-none-any.whl

fabricks/cdc/templates/queries/scd1.sql.jinja

@@ -0,0 +1,85 @@
+__scd1_base as (
+    select
+        *,
+        {% if not rectify %} __operation as __original_operation, {% endif %}
+        lead(__operation) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+        ) as __scd1_next_operation
+    from {{ parent_cdc }}
+),
+__scd1_last_key as (
+    -- take last update as it is the latest picture
+    select
+        *,
+        row_number() over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp desc
+        ) as __scd1_rn
+    from __scd1_base
+    where true and __operation == 'upsert'
+    {% if mode == "update" %}
+        {% if has_rows %}
+            -- take first delete ONLY if no upsert is present
+            union all
+            select
+                *,
+                row_number() over (
+                    partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+                ) as __scd1_rn
+            from __scd1_base b
+            where
+                true
+                and __operation == 'delete'
+                and not exists (
+                    select 1
+                    from __scd1_base b2
+                    where
+                        true and b.__key == b2.__key
+                        {% if has_source %} and b.__source == b2.__source {% endif %} and b2.__operation == 'upsert'
+                )
+        {% endif %}
+    {% endif %}
+),
+__scd1 as (
+    select
+        *,
+        __scd1_next_operation <=> 'delete' or __operation == 'delete' as __is_deleted,
+        not (__scd1_next_operation <=> 'delete' or __operation == 'delete') as __is_current
+    from __scd1_last_key
+    where true and __scd1_rn == 1
+),
+{% if mode == "complete" %}
+    __final as (
+        select {% for output in outputs %} `{{ output }}`, {% endfor %}
+        from __scd1 s
+        where true {% if not soft_delete %} and s.__is_current {% endif %}
+    )
+{% else %}
+    __merge_condition as (
+        select s.*, s.__key as __merge_key, o.__merge_condition
+        from __scd1 s
+        left join
+            (
+                select 'upsert' as __operation, 'upsert' as __merge_condition
+                {% if has_rows %}
+                    union all
+                    select 'delete' as __operation, 'delete' as __merge_condition
+                {% endif %}
+            ) o
+            on s.__operation = o.__operation
+    ),
+    {% if has_rows %}
+        __scd1_no_fake_update as (
+            select *
+            from __merge_condition m
+            left anti join
+                __current c on m.__key == c.__key and m.__hash = c.__hash
+                {% if has_source %} and m.__source = c.__source {% endif %} and m.__operation == 'upsert'
+        ),
+    {% endif %}
+    __final as (
+        select __merge_key, __merge_condition, {% for output in outputs %} `{{ output }}`, {% endfor %}
+        {% if has_rows %} from __scd1_no_fake_update m
+        {% else %} from __merge_condition m
+        {% endif %}
+    )
+{% endif %}

fabricks/cdc/templates/queries/scd2.sql.jinja

@@ -0,0 +1,98 @@
+__scd2_base as (
+    select
+        *,
+        {% if not rectify %} __operation as __original_operation, {% endif %}
+        lead(__operation) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp
+        ) as __scd2_next_operation,
+        lead(__timestamp) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp
+        ) as __scd2_next_timestamp
+    from {{ parent_cdc }}
+),
+__scd2 as (
+    select
+        *,
+        __timestamp as __valid_from,
+        coalesce(__scd2_next_timestamp - interval 1 second, cast('9999-12-31' as timestamp)) as __valid_to,
+        __operation <> 'delete' and __valid_to <=> '9999-12-31' as __is_current,
+        __operation == 'delete' or __scd2_next_operation <=> 'delete' as __is_deleted,
+        {% if mode == "update" %}
+            row_number() over (
+                partition by __key{% if has_source %}, __source{% endif %} order by __timestamp asc
+            ) as __scd2_rn
+        {% endif %}
+    from __scd2_base
+),
+{% if mode == "complete" %}
+    __complete as (select s.* from __scd2 s where true and not __operation <=> 'delete'),
+    {% if correct_valid_from %}
+        __correct_valid_from as (
+            select
+                * except (__valid_from),
+                if(
+                    __valid_from == min(__valid_from) over (partition by null),
+                    cast('1900-01-01' as timestamp),
+                    __valid_from
+                ) as __valid_from
+            from __complete
+        ),
+    {% endif %}
+    __final as (
+        select {% for output in outputs %} `{{ output }}`, {% endfor %}
+        {% if correct_valid_from %} from __correct_valid_from
+        {% else %} from __complete
+        {% endif %}
+    )
+{% else %}
+    {% if has_rows %}
+        __scd2_no_fake_update as (
+            select
+                * except (__scd2_rn),
+                row_number() over (partition by `__key` order by `__timestamp` asc) as `__scd2_rn`
+            from __scd2 s
+            left anti join
+                __current c on s.__key == c.__key and s.__hash == c.__hash
+                {% if has_source %} and s.__source == c.__source {% endif %}
+                and s.__operation == 'upsert'
+                and s.__scd2_rn == 1
+        ),
+    {% endif %}
+    __merge_condition as (
+        select s.*, if(__merge_condition == 'insert', null, __key) as __merge_key, o.__merge_condition
+        {% if has_rows %} from __scd2_no_fake_update s
+        {% else %} from __scd2 s
+        {% endif %}
+        inner join
+            (
+                select 'upsert' as __operation, 'insert' as __merge_condition
+                {% if has_rows %}
+                    union all
+                    select 'upsert' as __operation, 'update' as __merge_condition
+                    union all
+                    select 'delete' as __operation, 'delete' as __merge_condition
+                {% endif %}
+            ) o
+            on s.__operation = o.__operation
+        -- only the first record can be an update or a delete
+        where (s.__scd2_rn == 1 and o.__merge_condition in ('update', 'delete')) or o.__merge_condition == 'insert'
+    ),
+    {% if correct_valid_from %}
+        __correct_valid_from as (
+            select
+                * except (__valid_from),
+                if(
+                    __valid_from == min(__valid_from) over (partition by null),
+                    cast('1900-01-01' as timestamp),
+                    __valid_from
+                ) as __valid_from
+            from __merge_condition
+        ),
+    {% endif %}
+    __final as (
+        select __merge_key, __merge_condition, {% for output in outputs %} `{{ output }}`, {% endfor %}
+        {% if correct_valid_from %} from __correct_valid_from
+        {% else %} from __merge_condition
+        {% endif %}
+    )
+{% endif %}

fabricks/cdc/templates/query.sql.jinja

@@ -0,0 +1,15 @@
+{% include 'queries/context.sql.jinja' %}
+{% include 'ctes/base.sql.jinja' %}
+{% if slice %} {% include 'ctes/slice.sql.jinja' %} {% endif %}
+{% if deduplicate_key %} {% include 'ctes/deduplicate_key.sql.jinja' %} {% endif %}
+{% if mode == "update" %} {% if has_rows %} {% include 'ctes/current.sql.jinja' %} {% endif %} {% endif %}
+{% if rectify %} {% include 'ctes/rectify.sql.jinja' %} {% endif %}
+{% if deduplicate_hash %} {% include 'ctes/deduplicate_hash.sql.jinja' %} {% endif %}
+{% if cdc == "nocdc" %}
+    {% if mode == "update" %} {% include 'queries/nocdc/update.sql.jinja' %}
+    {% else %} {% include 'queries/nocdc/complete.sql.jinja' %}
+    {% endif %}
+{% endif %}
+{% if cdc == "scd1" %} {% include 'queries/scd1.sql.jinja' %} {% endif %}
+{% if cdc == "scd2" %} {% include 'queries/scd2.sql.jinja' %} {% endif %}
+{% include 'queries/final.sql.jinja' %}

fabricks/context/__init__.py

@@ -0,0 +1,72 @@
+from fabricks.context.config import (
+    IS_DEBUGMODE,
+    IS_DEVMODE,
+    IS_JOB_CONFIG_FROM_YAML,
+    LOGLEVEL,
+    PATH_CONFIG,
+    PATH_NOTEBOOKS,
+    PATH_RUNTIME,
+)
+from fabricks.context.runtime import (
+    BRONZE,
+    CATALOG,
+    CONF_RUNTIME,
+    FABRICKS_STORAGE,
+    FABRICKS_STORAGE_CREDENTIAL,
+    GOLD,
+    IS_TYPE_WIDENING,
+    IS_UNITY_CATALOG,
+    PATH_EXTENDERS,
+    PATH_MASKS,
+    PATH_PARSERS,
+    PATH_REQUIREMENTS,
+    PATH_SCHEDULES,
+    PATH_UDFS,
+    PATH_VIEWS,
+    PATHS_RUNTIME,
+    PATHS_STORAGE,
+    SECRET_SCOPE,
+    SILVER,
+    STEPS,
+    TIMEZONE,
+    VARIABLES,
+)
+from fabricks.context.spark_session import DBUTILS, SPARK, build_spark_session, init_spark_session
+from fabricks.context.utils import pprint_runtime
+
+__all__ = [
+    "BRONZE",
+    "build_spark_session",
+    "CATALOG",
+    "CONF_RUNTIME",
+    "DBUTILS",
+    "FABRICKS_STORAGE_CREDENTIAL",
+    "FABRICKS_STORAGE",
+    "GOLD",
+    "init_spark_session",
+    "IS_DEBUGMODE",
+    "IS_DEVMODE",
+    "IS_JOB_CONFIG_FROM_YAML",
+    "IS_TYPE_WIDENING",
+    "IS_UNITY_CATALOG",
+    "LOGLEVEL",
+    "PATH_CONFIG",
+    "PATH_EXTENDERS",
+    "PATH_MASKS",
+    "PATH_NOTEBOOKS",
+    "PATH_PARSERS",
+    "PATH_REQUIREMENTS",
+    "PATH_RUNTIME",
+    "PATH_SCHEDULES",
+    "PATH_UDFS",
+    "PATH_VIEWS",
+    "PATHS_RUNTIME",
+    "pprint_runtime",
+    "PATHS_STORAGE",
+    "SECRET_SCOPE",
+    "SILVER",
+    "SPARK",
+    "STEPS",
+    "TIMEZONE",
+    "VARIABLES",
+]

fabricks/context/_types.py

@@ -0,0 +1,133 @@
+from typing import List, Optional, TypedDict
+
+
+class RuntimePathOptions(TypedDict):
+    storage: str
+    udfs: str
+    parsers: str
+    schedules: str
+    views: str
+    requirements: str
+
+
+class RuntimeTimeoutOptions(TypedDict):
+    step: int
+    job: int
+    pre_run: int
+    post_run: int
+
+
+class StepTimeoutOptions(TypedDict):
+    step: Optional[int]
+    job: Optional[int]
+    pre_run: Optional[int]
+    post_run: Optional[int]
+
+
+class RuntimeOptions(TypedDict):
+    secret_scope: str
+    unity_catalog: Optional[bool]
+    type_widening: Optional[bool]
+    catalog: Optional[str]
+    workers: int
+    timeouts: RuntimeTimeoutOptions
+    retention_days: int
+    timezone: Optional[str]
+
+
+class SparkOptions(TypedDict):
+    sql: dict
+    conf: dict
+
+
+class StepPathOptions(TypedDict):
+    runtime: str
+    storage: str
+
+
+class InvokeOptions(TypedDict):
+    notebook: str
+    arguments: Optional[dict[str, str]]
+
+
+class ExtenderOptions(TypedDict):
+    extender: str
+    arguments: Optional[dict[str, str]]
+
+
+class StepOptions(TypedDict):
+    order: int
+    workers: Optional[int]
+    timeouts: StepTimeoutOptions
+    extenders: Optional[List[str]]
+    pre_run: Optional[InvokeOptions]
+    post_run: Optional[InvokeOptions]
+
+
+class SilverOptions(StepOptions):
+    parent: str
+    stream: Optional[bool]
+    local_checkpoint: Optional[bool]
+
+
+class GoldOptions(StepOptions):
+    schema_drift: Optional[bool]
+    metadata: Optional[bool]
+
+
+class Step(TypedDict):
+    name: str
+
+
+class TableOptions(TypedDict):
+    powerbi: Optional[bool]
+    liquid_clustering: Optional[bool]
+    properties: Optional[dict[str, str]]
+    retention_days: Optional[int]
+    masks: Optional[dict[str, str]]
+
+
+class Bronze(Step):
+    options: StepOptions
+    path_options: StepPathOptions
+    table_options: Optional[TableOptions]
+
+
+class Silver(Step):
+    options: SilverOptions
+    path_options: StepPathOptions
+    table_options: Optional[TableOptions]
+
+
+class Gold(Step):
+    options: GoldOptions
+    path_options: StepPathOptions
+    table_options: Optional[TableOptions]
+
+
+class PowerBI(Step):
+    pass
+
+
+class DatabasePathOptions(TypedDict):
+    storage: str
+
+
+class Database(TypedDict):
+    name: str
+    path_options: DatabasePathOptions
+
+
+class Conf(TypedDict):
+    name: str
+    options: RuntimeOptions
+    path_options: RuntimePathOptions
+    extender_options: Optional[ExtenderOptions]
+    spark_options: SparkOptions
+    bronze: Optional[List[Bronze]]
+    silver: Optional[List[Silver]]
+    gold: Optional[List[Gold]]
+    powerbi: Optional[List[PowerBI]]
+    databases: Optional[List[Database]]
+    variables: Optional[List[dict[str, str]]]
+    credentials: Optional[List[dict[str, str]]]

fabricks/context/config/__init__.py

@@ -0,0 +1,92 @@
+import logging
+import os
+from typing import Final
+
+from fabricks.context.config.utils import get_config_from_file
+from fabricks.utils.path import Path
+from fabricks.utils.spark import spark
+
+file_path, file_config = get_config_from_file()
+
+runtime = os.environ.get("FABRICKS_RUNTIME", "none")
+runtime = None if runtime.lower() == "none" else runtime
+if runtime is None:
+    if runtime := file_config.get("runtime"):
+        assert file_path is not None
+        runtime = file_path.joinpath(runtime)
+
+if runtime is None:
+    if file_path is not None:
+        runtime = file_path
+    else:
+        raise ValueError(
+            "could not resolve runtime (could not find pyproject.toml nor fabricksconfig.json nor FABRICKS_RUNTIME)"
+        )
+
+path_runtime = Path(runtime, assume_git=True)
+PATH_RUNTIME: Final[Path] = path_runtime
+
+notebooks = os.environ.get("FABRICKS_NOTEBOOKS", "none")
+notebooks = None if notebooks.lower() == "none" else notebooks
+if notebooks is None:
+    if notebooks := file_config.get("notebooks"):
+        assert file_path is not None
+        notebooks = file_path.joinpath(notebooks)
+
+notebooks = notebooks if notebooks else path_runtime.joinpath("notebooks")
+PATH_NOTEBOOKS: Final[Path] = Path(str(notebooks), assume_git=True)
+
+is_job_config_from_yaml = os.environ.get("FABRICKS_IS_JOB_CONFIG_FROM_YAML", None)
+if is_job_config_from_yaml is None:
+    assert file_path is not None
+    is_job_config_from_yaml = file_config.get("job_config_from_yaml")
+
+IS_JOB_CONFIG_FROM_YAML: Final[bool] = str(is_job_config_from_yaml).lower() in ("true", "1", "yes")
+
+is_debugmode = os.environ.get("FABRICKS_IS_DEBUGMODE", None)
+if is_debugmode is None:
+    is_debugmode = file_config.get("debugmode")
+
+IS_DEBUGMODE: Final[bool] = str(is_debugmode).lower() in ("true", "1", "yes")
+
+is_devmode = os.environ.get("FABRICKS_IS_DEVMODE", None)
+if is_devmode is None:
+    is_devmode = file_config.get("devmode")
+
+IS_DEVMODE: Final[bool] = str(is_devmode).lower() in ("true", "1", "yes")
+
+loglevel = os.environ.get("FABRICKS_LOGLEVEL", None)
+if loglevel is None:
+    loglevel = file_config.get("loglevel")
+
+loglevel = loglevel.upper() if loglevel else "INFO"
+if loglevel == "DEBUG":
+    _loglevel = logging.DEBUG
+elif loglevel == "INFO":
+    _loglevel = logging.INFO
+elif loglevel == "WARNING":
+    _loglevel = logging.WARNING
+elif loglevel == "ERROR":
+    _loglevel = logging.ERROR
+elif loglevel == "CRITICAL":
+    _loglevel = logging.CRITICAL
+else:
+    raise ValueError(f"could not resolve {loglevel} (DEBUG, INFO, WARNING, ERROR or CRITICAL)")
+
+LOGLEVEL = _loglevel
+
+path_config = os.environ.get("FABRICKS_CONFIG")
+if path_config is None:
+    if path_config := file_config.get("config"):
+        assert file_path is not None
+        path_config = file_path.joinpath(path_config)
+else:
+    path_config = PATH_RUNTIME.joinpath(path_config).string if path_config else None
+
+if not path_config:
+    path_config = PATH_RUNTIME.joinpath(
+        "fabricks",
+        f"conf.{spark.conf.get('spark.databricks.clusterUsageTags.clusterOwnerOrgId')}.yml",
+    ).string
+
+PATH_CONFIG: Final[Path] = Path(path_config, assume_git=True)

fabricks/context/config/utils.py

@@ -0,0 +1,53 @@
+def get_config_from_toml():
+    import os
+    import pathlib
+    import sys
+
+    if sys.version_info >= (3, 11):
+        import tomllib
+    else:
+        import tomli as tomllib  # type: ignore
+
+    path = pathlib.Path(os.getcwd())
+    while path is not None and not (path / "pyproject.toml").exists():
+        if path == path.parent:
+            break
+        path = path.parent
+
+    if (path / "pyproject.toml").exists():
+        with open((path / "pyproject.toml"), "rb") as f:
+            config = tomllib.load(f)
+            return path, config.get("tool", {}).get("fabricks", {})
+
+    return None, {}
+
+
+def get_config_from_json():
+    import json
+    import os
+    import pathlib
+
+    path = pathlib.Path(os.getcwd())
+    while path is not None and not (path / "fabricksconfig.json").exists():
+        if path == path.parent:
+            break
+        path = path.parent
+
+    if (path / "fabricksconfig.json").exists():
+        with open((path / "fabricksconfig.json"), "r") as f:
+            config = json.load(f)
+            return path, config
+
+    return None, {}
+
+
+def get_config_from_file():
+    json_path, json_config = get_config_from_json()
+    if json_config:
+        return json_path, json_config
+
+    pyproject_path, pyproject_config = get_config_from_toml()
+    if pyproject_config:
+        return pyproject_path, pyproject_config
+
+    return None, {}

fabricks/context/log.py

@@ -0,0 +1,77 @@
+import json
+import logging
+from typing import Final, Literal, Optional
+
+import requests
+
+from fabricks.context import IS_DEBUGMODE, LOGLEVEL, SECRET_SCOPE, TIMEZONE
+from fabricks.utils.log import get_logger
+
+logger, _ = get_logger(
+    "logs",
+    LOGLEVEL,
+    table=None,
+    debugmode=IS_DEBUGMODE,
+    timezone=TIMEZONE,
+)
+logging.getLogger("SQLQueryContextLogger").setLevel(logging.CRITICAL)
+
+DEFAULT_LOGGER: Final[logging.Logger] = logger
+
+
+def send_message_to_channel(
+    channel: str,
+    title: str,
+    message: str,
+    color: Optional[str] = None,
+    loglevel: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO",
+) -> bool:
+    """
+    Send a message to Microsoft Teams via webhook
+
+    Args:
+        webhook_url (str): The webhook URL for your Teams channel
+        message (str): The message to send
+        title (str, optional): Title for the message card
+        color (str, optional): Hex color for the message card)
+
+    Returns:
+        bool: True if message was sent successfully, False otherwise
+    """
+    from databricks.sdk.runtime import dbutils
+
+    channel = channel.lower()
+    channel = channel.replace(" ", "-")
+    webhook_url = dbutils.secrets.get(scope=SECRET_SCOPE, key=f"{channel}-webhook-url")
+
+    teams_message = {
+        "@type": "MessageCard",
+        "@context": "http://schema.org/extensions",
+        "summary": title,
+    }
+
+    if title:
+        teams_message["title"] = title
+
+    if color:
+        teams_message["themeColor"] = color
+    else:
+        COLORS = {
+            "DEBUG": "#00FFFF",
+            "INFO": "#00FF00 ",
+            "WARNING": "#FFFF00 ",
+            "ERROR": "#FF0000 ",
+            "CRITICAL": "#FF0000",
+        }
+        color = COLORS[loglevel]
+        teams_message["themeColor"] = color
+
+    teams_message["text"] = message
+
+    teams_message_json = json.dumps(teams_message)
+
+    response = requests.post(webhook_url, data=teams_message_json, headers={"Content-Type": "application/json"})
+    if response.status_code == 200:
+        return True
+    else:
+        return False