fabricks 2024.7.1.5__py3-none-any.whl

fabricks/cdc/templates/query/current.sql.jinja

@@ -0,0 +1,32 @@
+{% import 'query/hash.sql.jinja' as h -%}
+
+__current as (
+    select
+        {% for field in fields %} {{ field }}, {% endfor %}
+        'current' as __operation,
+        'current' as __original_operation,
+        {% if has_timestamp %}
+            {% if cdc == "nocdc" %} __timestamp as __timestamp, {% endif %}
+            {% if cdc == "scd1" %} __timestamp as __timestamp, {% endif %}
+            {% if cdc == "scd2" %} __valid_from as __timestamp, {% endif %}
+        {% else %} cast('0001-01-01' as timestamp) as __timestamp,
+        {% endif %}
+        __timestamp as __original_timestamp,
+        {% if has_hash %} __hash,
+        {% else %} {{ h.hash(fields=hashes) }} as __hash,
+        {% endif %}
+        {% if has_identity %} __identity, {% endif %}
+        {% if has_key %} __key,
+        {% else %} {{ h.hash(fields=keys) }} as __key,
+        {% endif %}
+        {% if has_source %} __source, {% endif %}
+        {% if has_metadata %} __metadata, {% endif %}
+        {% if has_rescued_data %} __rescued_data, {% endif %}
+    from {{ tgt }} t
+    where
+        true
+        {% if cdc == "scd2" %} and __is_current {% endif %}
+        {% if cdc == "scd1" %} {% if soft_delete %} and __is_current {% endif %} {% endif %}
+        {% if has_source %} and exists (select 1 from __base s where s.__source == t.__source) {% endif %}
+        {% if update_where %} and {{ update_where }} {% endif %}
+),

fabricks/cdc/templates/query/deduplicate_hash.sql.jinja

@@ -0,0 +1,21 @@
+__deduplicate_hash as (
+    select
+        *,
+        lag(__hash) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+        ) as __deduplicate_hash_previous__hash,
+        lag(__operation) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+        ) as __deduplicate_hash_previous_operation
+    from {{ parent_deduplicate_hash }}
+    where true
+),
+__deduplicated_hash as (
+    select *
+    from __deduplicate_hash
+    where
+        true
+        and not (
+            __hash <=> __deduplicate_hash_previous__hash and __operation <=> __deduplicate_hash_previous_operation
+        )
+),

fabricks/cdc/templates/query/deduplicate_key.sql.jinja

@@ -0,0 +1,14 @@
+__deduplicate_key as (
+    select
+        *,
+        row_number() over (
+            partition by {% if has_source %} __source, {% endif %} __key, __timestamp
+            order by
+                /* prioritize delete over upsert */
+                __operation asc,
+                {% if has_order_by %} {% for o in order_duplicate_by %} {{ o }}, {% endfor %} {% endif %}
+        ) as __deduplicate_key_rn
+    from {{ parent_deduplicate_key }}
+    where true
+),
+__deduplicated_key as (select *, from __deduplicate_key where __deduplicate_key_rn == 1),

fabricks/cdc/templates/query/filter.sql.jinja

@@ -0,0 +1,71 @@
+{% if filter == "latest" %}
+    __filter_latest_timestamp as (
+        select {% if has_source %} t.__source, {% endif %} max(t.__timestamp) as __max_timestamp
+        from {{ parent_filter }} t
+        {% if has_source %} group by t.__source {% endif %}
+    ),
+    __filtered as (
+        select
+            {% for field in fields %} {{ field }}, {% endfor %}
+            s.__operation,
+            s.__timestamp,
+            s.__hash,
+            s.__key,
+            {% if has_source %} s.__source, {% endif %}
+            {% if has_metadata %} s.__metadata, {% endif %}
+            {% if has_rescued_data %} __rescued_data, {% endif %}
+        from {{ parent_filter }} s
+        where
+            true
+            and exists (
+                select 1
+                from __filter_latest_timestamp m
+                where
+                    true and s.__timestamp = m.__max_timestamp
+                    {% if has_source %} and s.__source == m.__source {% endif %}
+            )
+    ),
+{% else %}
+    __filter_updated_timestamp as (
+        select
+            {% if has_source %} t.__source,
+            {% endif %}
+            {% if cdc == "nocdc" %} coalesce(max(t.__timestamp), cast('0001-01-01' as timestamp)) as __max_timestamp
+            {% endif %}
+            {% if cdc == "scd1" %} coalesce(max(t.__timestamp), cast('0001-01-01' as timestamp)) as __max_timestamp
+            {% endif %}
+            {% if cdc == "scd2" %} coalesce(max(t.__valid_from), cast('0001-01-01' as timestamp)) as __max_timestamp
+            {% endif %}
+        from {{ tgt }} t
+        where true
+        {% if has_source %}
+                and exists (select 1 from {{ parent_filter }} s where s.__source == t.__source) group by t.__source
+        {% endif %}
+    ),
+    __filtered as (
+        select
+            {% for field in fields %} {{ field }},
+            {% endfor %}
+            s.__operation,
+            s.__timestamp,
+            s.__hash,
+            s.__key,
+            {% if has_source %} s.__source,
+            {% endif %}
+            {% if has_metadata %} s.__metadata,
+            {% endif %}
+            {% if has_rescued_data %} __rescued_data,
+            {% endif %}
+        from {{ parent_filter }} s
+        where
+            true
+            and exists (
+                select 1
+                from __filter_updated_timestamp m
+                where
+                    true and s.__timestamp > m.__max_timestamp
+                    {% if has_source %} and s.__source == m.__source
+                    {% endif %}
+            )
+    ),
+{% endif %}

fabricks/cdc/templates/query/final.sql.jinja

@@ -0,0 +1 @@
+select * {% if all_except %} except ({% for e in all_except %}{{ e }}, {% endfor %}), {% endif %} from __final

fabricks/cdc/templates/query/hash.sql.jinja

@@ -0,0 +1 @@
+{% macro hash(fields) -%} md5(array_join(array({% for f in fields %}{{ f }}, {% endfor %}), '*', '-1')) {%- endmacro %}

fabricks/cdc/templates/query/nocdc.sql.jinja

@@ -0,0 +1,10 @@
+__final as (
+    select *
+    from {{ parent_cdc }}
+    {% if filter %}
+        where
+            true
+            -- operation current added by filter
+            and __operation <> 'current'
+    {% endif %}
+)

fabricks/cdc/templates/query/rectify.sql.jinja

@@ -0,0 +1,120 @@
+{% import 'query/hash.sql.jinja' as h -%}
+
+{% if mode == "update" %}
+    __rectified_base as (
+        select
+            {% for field in fields %} {{ field }}, {% endfor %}
+            s.__operation,
+            s.__operation as __original_operation,
+            s.__timestamp,
+            s.__timestamp as __original_timestamp,
+            s.__hash,
+            s.__key,
+            {% if has_identity %} s.__identity, {% endif %}
+            {% if has_source %} s.__source, {% endif %}
+            {% if has_metadata %} s.__metadata, {% endif %}
+            {% if has_rescued_data %} s.__rescued_data, {% endif %}
+        from {{ parent_rectify }} s
+        {% if has_rows %}
+            union all
+            select *
+            from __current
+        {% endif %}
+    ),
+{% endif %}
+__rectified_next_operation as (
+    select
+        *,
+        lead(__operation) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+        ) as __rectified_next_operation
+    {% if mode == "update" %} from __rectified_base p
+    {% else %} from {{ parent_rectify }}
+    {% endif %}
+),
+__rectified_timestamps as (
+    select
+        {% if has_source %} __source, {% endif %}
+        __timestamp,
+        lead(__timestamp) over (
+            {% if has_source %}partition by __source {% endif %} order by __timestamp asc
+        ) as __rectified_next_timestamp,
+        lead(if(max(__operation) == 'reload', __timestamp, null)) ignore nulls over (
+            {% if has_source %}partition by __source {% endif %} order by __timestamp asc
+        ) as __rectified_next_timestamp_reload,
+    from __rectified_next_operation
+    group by {% if has_source %} __source, {% endif %} __timestamp
+),
+__rectified_is_deleted_next as (
+    select
+        cur.*,
+        t.* except ({% if has_source %}t.__source, {% endif %} t.__timestamp),
+        -- there is more reload
+        __rectified_next_timestamp_reload is not null as __rectify_more_reload,
+        -- the next operation is bedore the next reload
+        if(
+            __rectify_more_reload, t.__rectified_next_timestamp < t.__rectified_next_timestamp_reload, true
+        ) as __rectify_key_next_operation_before_next_reload,
+        -- the record is deleted
+        cur.__operation == 'delete' as __rectified_key_is_deleted,
+        -- the record is not found in next reload
+        __rectify_more_reload and nxt.__timestamp is null as __rectified_key_not_found_in_next_reload,
+        -- there is no more operation
+        t.__rectified_next_timestamp is null as __rectified_no_more_operation,
+        -- the record is deleted before next reload
+        __rectify_key_next_operation_before_next_reload
+        and cur.__rectified_next_operation <=> 'delete' as __rectified_key_is_deleted_next,
+        -- the record is updated before next reload
+        __rectify_key_next_operation_before_next_reload
+        and cur.__rectified_next_operation <=> 'upsert' as __rectified_key_is_updated_next,
+        case
+            when __rectified_key_is_deleted
+            then false
+            when __rectified_key_is_updated_next
+            then false
+            when __rectified_key_is_deleted_next
+            then false
+            when __rectified_no_more_operation
+            then false
+            when __rectified_key_not_found_in_next_reload
+            then true
+            else false
+        end as __rectified_is_deleted_next
+    from __rectified_next_operation cur
+    left join
+        __rectified_timestamps t on cur.__timestamp = t.__timestamp
+        {% if has_source %} and cur.__source == t.__source {% endif %}
+    left join
+        __rectified_next_operation nxt
+        on t.__rectified_next_timestamp_reload = nxt.__timestamp
+        and cur.__key == nxt.__key
+        {% if has_source %} and cur.__source == nxt.__source {% endif %}
+),
+{% if mode == "complete" %} __rectified as ( {% else %} __rectified_operation as (
+{% endif %}
+    select
+        c.* except (c.__operation, c.__timestamp),
+        if(c.__operation == 'delete', 'delete', d2.__rectified_operation) as __operation,
+        case
+            when c.__operation == 'delete'
+            then c.__timestamp
+            when d2.__rectified_operation == 'upsert'
+            then c.__timestamp
+            else c.__rectified_next_timestamp_reload
+        end as __timestamp
+    from __rectified_is_deleted_next c
+    cross join
+        (
+            select false as __is_deleted, 'upsert' as __rectified_operation
+            union all
+            select true as __is_deleted, 'delete' as __rectified_operation
+            union all
+            select true as __is_deleted, 'upsert' as __rectified_operation
+        ) d2
+        on c.__rectified_is_deleted_next = d2.__is_deleted
+),
+{% if mode == "update" %}
+    __rectified as (
+        select * from __rectified_operation where not (__original_operation == 'current' and __operation == 'upsert')
+    ),
+{% endif %}

fabricks/cdc/templates/query/scd1.sql.jinja

@@ -0,0 +1,112 @@
+__scd1_base as (
+    select
+        *,
+        {% if not rectify %} __operation as __original_operation, {% endif %}
+        lead(__operation) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+        ) as __scd1_next_operation
+    from {{ parent_cdc }}
+),
+__scd1_last_key as (
+    -- take last update as it is the latest picture
+    select
+        *,
+        row_number() over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp desc
+        ) as __scd1_rn
+    from __scd1_base
+    where true and __operation == 'upsert'
+    {% if mode == "update" %}
+        {% if has_rows %}
+            -- take first delete ONLY if no upsert is present
+            union all
+            select
+                *,
+                row_number() over (
+                    partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+                ) as __scd1_rn
+            from __scd1_base b
+            where
+                true
+                and __operation == 'delete'
+                and not exists (
+                    select 1
+                    from __scd1_base b2
+                    where
+                        true and b.__key == b2.__key
+                        {% if has_source %} and b.__source == b2.__source {% endif %} and b2.__operation == 'upsert'
+                )
+        {% endif %}
+    {% endif %}
+),
+__scd1 as (
+    select
+        *,
+        __scd1_next_operation <=> 'delete'
+        or __operation == 'delete' as __is_deleted,
+        not (__scd1_next_operation <=> 'delete' or __operation == 'delete') as __is_current
+    from __scd1_last_key
+    where true and __scd1_rn == 1
+),
+{% if mode == "complete" %}
+    __final as (
+        select
+            {% for field in fields %} s.{{ field }}, {% endfor %},
+            {% if has_identity %} __identity, {% endif %}
+            s.__key,
+            s.__timestamp,
+            {% if soft_delete %} s.__is_current, s.__is_deleted, {% endif %}
+            s.__hash,
+            {% if has_source %} s.__source, {% endif %}
+            {% if has_metadata %} s.__metadata, {% endif %}
+            {% if has_rescued_data %} s.__rescued_data, {% endif %}
+        from __scd1 s
+        where true {% if not soft_delete %} and s.__is_current {% endif %}
+    )
+{% else %}
+    __merge_condition as (
+        select s.*, s.__key as __merge_key, o.__merge_condition
+        from __scd1 s
+        left join
+            (
+                select 'upsert' as __operation, 'upsert' as __merge_condition
+                {% if has_rows %}
+                    union all
+                    select 'delete' as __operation, 'delete' as __merge_condition
+                {% endif %}
+            ) o
+            on s.__operation = o.__operation
+    ),
+    {% if has_rows %}
+        __scd1_no_fake_update as (
+            select *
+            from __merge_condition m left
+            anti join
+                __current c on m.__key == c.__key and m.__hash = c.__hash
+                {% if has_source %} and m.__source = c.__source {% endif %} and m.__operation == 'upsert'
+        ),
+    {% endif %}
+    __final as (
+        select
+            __merge_key,
+            __merge_condition,
+            {% for field in fields %} {{ field }},
+            {% endfor %},
+            {% if has_identity %} __identity,
+            {% endif %}
+            __key,
+            __timestamp,
+            {% if soft_delete %} __is_current, __is_deleted,
+            {% endif %}
+            __hash,
+            {% if has_source %} __source,
+            {% endif %}
+            {% if has_metadata %} __metadata,
+            {% endif %}
+            {% if has_rescued_data %} __rescued_data,
+            {% endif %}
+        {% if has_rows %} from __scd1_no_fake_update m
+        {% else %} from __merge_condition m
+        {% endif %}
+    )
+{% endif %}

fabricks/cdc/templates/query/scd2.sql.jinja

@@ -0,0 +1,114 @@
+__scd2_base as (
+    select
+        *,
+        {% if not rectify %} __operation as __original_operation, {% endif %}
+        lead(__operation) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp
+        ) as __scd2_next_operation,
+        lead(__timestamp) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp
+        ) as __scd2_next_timestamp
+    from {{ parent_cdc }}
+),
+__scd2 as (
+    select
+        *,
+        __timestamp as __valid_from,
+        coalesce(__scd2_next_timestamp - interval 1 second, cast('9999-12-31' as timestamp)) as __valid_to,
+        __operation <> 'delete' and __valid_to <=> '9999-12-31' as __is_current,
+        __operation == 'delete' or __scd2_next_operation <=> 'delete' as __is_deleted,
+        {% if mode == "update" %}
+            row_number() over (
+                partition by __key{% if has_source %}, __source{% endif %} order by __timestamp asc
+            ) as __scd2_rn
+        {% endif %}
+    from __scd2_base
+),
+{% if mode == "complete" %}
+    __complete as (select s.* from __scd2 s where true and not __operation <=> 'delete'),
+    __final as (
+        select
+            {% for field in fields %} {{ field }}, {% endfor %},
+            {% if has_identity %} __identity, {% endif %}
+            __key,
+            if(
+                __valid_from == min(__valid_from) over (partition by null),
+                cast('1900-01-01' as timestamp),
+                __valid_from
+            ) as __valid_from,
+            __valid_to,
+            __is_current,
+            {% if soft_delete %} __is_deleted, {% endif %}
+            __hash,
+            {% if has_source %} __source, {% endif %}
+            {% if has_metadata %} __metadata, {% endif %}
+            {% if has_rescued_data %} __rescued_data, {% endif %}
+        from __complete
+    )
+{% else %}
+    {% if has_rows %}
+        __scd2_no_fake_update as (
+            select
+                * except (__scd2_rn),
+                row_number() over (partition by `__key` order by `__timestamp` asc) as `__scd2_rn`
+            from __scd2 s left
+            anti join
+                __current c on s.__key == c.__key and s.__hash == c.__hash
+                {% if has_source %} and s.__source == c.__source {% endif %}
+                and s.__operation == 'upsert'
+                and s.__scd2_rn == 1
+        ),
+    {% endif %}
+    __merge_condition as (
+        select s.*, if(__merge_condition == 'insert', null, __key) as __merge_key, o.__merge_condition
+        {% if has_rows %} from __scd2_no_fake_update s
+        {% else %} from __scd2 s
+        {% endif %}
+        inner join
+            (
+                select 'upsert' as __operation, 'insert' as __merge_condition
+                {% if has_rows %}
+                    union all
+                    select 'upsert' as __operation, 'update' as __merge_condition
+                    union all
+                    select 'delete' as __operation, 'delete' as __merge_condition
+                {% endif %}
+            ) o
+            on s.__operation = o.__operation
+        -- only the first record can be an update or a delete
+        where (s.__scd2_rn == 1 and o.__merge_condition in ('update', 'delete')) or o.__merge_condition == 'insert'
+    ),
+    __final as (
+        select
+            __merge_key,
+            __merge_condition,
+            {% for field in fields %} {{ field }},
+            {% endfor %},
+            {% if has_identity %} __identity,
+            {% endif %}
+            __key,
+            {% if fix_valid_from %}
+                {% if not has_rows %}
+                    if(
+                        __valid_from == min(__valid_from) over (partition by null),
+                        cast('1900-01-01' as timestamp),
+                        __valid_from
+                    ) as __valid_from,
+                {% else %} __valid_from,
+                {% endif %}
+            {% else %} __valid_from,
+            {% endif %}
+            __valid_to,
+            __is_current and __merge_condition == 'insert' as __is_current,
+            {% if soft_delete %} __is_deleted,
+            {% endif %}
+            __hash,
+            {% if has_source %} __source,
+            {% endif %}
+            {% if has_metadata %} __metadata,
+            {% endif %}
+            {% if has_rescued_data %} __rescued_data,
+            {% endif %}
+        from __merge_condition m
+    )
+{% endif %}

fabricks/cdc/templates/query.sql.jinja

@@ -0,0 +1,11 @@
+{% include 'query/context.sql.jinja' %}
+{% include 'query/base.sql.jinja' %}
+{% if filter %} {% include 'query/filter.sql.jinja' %} {% endif %}
+{% if deduplicate_key %} {% include 'query/deduplicate_key.sql.jinja' %} {% endif %}
+{% if mode == "update" %} {% if has_rows %} {% include 'query/current.sql.jinja' %} {% endif %} {% endif %}
+{% if rectify %} {% include 'query/rectify.sql.jinja' %} {% endif %}
+{% if deduplicate_hash %} {% include 'query/deduplicate_hash.sql.jinja' %} {% endif %}
+{% if cdc == "nocdc" %} {% include 'query/nocdc.sql.jinja' %} {% endif %}
+{% if cdc == "scd1" %} {% include 'query/scd1.sql.jinja' %} {% endif %}
+{% if cdc == "scd2" %} {% include 'query/scd2.sql.jinja' %} {% endif %}
+{% include 'query/final.sql.jinja' %}

fabricks/context/__init__.py

@@ -0,0 +1,51 @@
+from fabricks.context.runtime import (
+    BRONZE,
+    CONF_RUNTIME,
+    FABRICKS_STORAGE,
+    GOLD,
+    IS_DEBUG,
+    IS_LIVE,
+    IS_TEST,
+    PATH_EXTENDERS,
+    PATH_LIBRARIES,
+    PATH_PARSERS,
+    PATH_REQUIREMENTS,
+    PATH_RUNTIME,
+    PATH_SCHEDULES,
+    PATH_UDFS,
+    PATH_VIEWS,
+    PATHS_RUNTIME,
+    PATHS_STORAGE,
+    SECRET_SCOPE,
+    SILVER,
+    STEPS,
+    VARIABLES,
+    VERSION,
+)
+from fabricks.context.spark import build_spark_session
+
+__all__ = [
+    "BRONZE",
+    "build_spark_session",
+    "CONF_RUNTIME",
+    "FABRICKS_STORAGE",
+    "GOLD",
+    "IS_DEBUG",
+    "IS_LIVE",
+    "IS_TEST",
+    "PATH_EXTENDERS",
+    "PATH_LIBRARIES",
+    "PATH_PARSERS",
+    "PATH_REQUIREMENTS",
+    "PATH_RUNTIME",
+    "PATH_SCHEDULES",
+    "PATH_UDFS",
+    "PATH_VIEWS",
+    "PATHS_RUNTIME",
+    "PATHS_STORAGE",
+    "SECRET_SCOPE",
+    "SILVER",
+    "STEPS",
+    "VARIABLES",
+    "VERSION",
+]

fabricks/context/log.py

@@ -0,0 +1,26 @@
+import logging
+from functools import wraps
+from typing import Callable, cast
+
+from fabricks.context.runtime import FABRICKS_STORAGE, SECRET_SCOPE
+from fabricks.utils.azure_table import AzureTable
+from fabricks.utils.log import get_logger
+from fabricks.utils.secret import AccessKey, get_secret_from_secret_scope
+
+storage_account = FABRICKS_STORAGE.get_storage_account()
+secret = get_secret_from_secret_scope(SECRET_SCOPE, f"{storage_account}-access-key")
+access_key = cast(AccessKey, secret).key
+
+table = AzureTable("logs", storage_account=storage_account, access_key=access_key)
+Logger, TableLogger = get_logger("logs", logging.DEBUG, table=table)
+
+
+def flush(func: Callable):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        finally:
+            TableLogger.flush()
+
+    return wrapper