fabricks 3.0.11__py3-none-any.whl

fabricks/cdc/templates/ctes/rectify.sql.jinja

@@ -0,0 +1,113 @@
+{% if mode == "update" %}
+    __rectified_base as (
+        select
+            {% for i in intermediates %} `{{ i }}`, {% endfor %}
+            __timestamp as __original_timestamp,
+            __operation as __original_operation,
+        from {{ parent_rectify }}
+        {% if has_rows %}
+            union all
+            select
+                {% for i in intermediates %} `{{ i }}`, {% endfor %}
+                __timestamp as __original_timestamp,
+                __operation as __original_operation,
+            from __current
+        {% endif %}
+    ),
+{% endif %}
+__rectified_next_operation as (
+    select
+        *,
+        lead(__operation) over (
+            partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+        ) as __rectified_next_operation
+    {% if mode == "update" %} from __rectified_base p
+    {% else %} from {{ parent_rectify }}
+    {% endif %}
+),
+__rectified_timestamps as (
+    select
+        {% if has_source %} __source, {% endif %}
+        __timestamp,
+        lead(__timestamp) over (
+            {% if has_source %}partition by __source {% endif %} order by __timestamp asc
+        ) as __rectified_next_timestamp,
+        lead(if(max(__operation) == 'reload', __timestamp, null)) ignore nulls over (
+            {% if has_source %}partition by __source {% endif %} order by __timestamp asc
+        ) as __rectified_next_timestamp_reload,
+    from __rectified_next_operation
+    group by {% if has_source %} __source, {% endif %} __timestamp
+),
+__rectified_is_deleted_next as (
+    select
+        cur.*,
+        t.* except ({% if has_source %}t.__source, {% endif %} t.__timestamp),
+        -- there is more reload
+        __rectified_next_timestamp_reload is not null as __rectify_more_reload,
+        -- the next operation is bedore the next reload
+        if(
+            __rectify_more_reload, t.__rectified_next_timestamp < t.__rectified_next_timestamp_reload, true
+        ) as __rectify_key_next_operation_before_next_reload,
+        -- the record is deleted
+        cur.__operation == 'delete' as __rectified_key_is_deleted,
+        -- the record is not found in next reload
+        __rectify_more_reload and nxt.__timestamp is null as __rectified_key_not_found_in_next_reload,
+        -- there is no more operation
+        t.__rectified_next_timestamp is null as __rectified_no_more_operation,
+        -- the record is deleted before next reload
+        __rectify_key_next_operation_before_next_reload
+        and cur.__rectified_next_operation <=> 'delete' as __rectified_key_is_deleted_next,
+        -- the record is updated before next reload
+        __rectify_key_next_operation_before_next_reload
+        and cur.__rectified_next_operation <=> 'upsert' as __rectified_key_is_updated_next,
+        case
+            when __rectified_key_is_deleted
+            then false
+            when __rectified_key_is_updated_next
+            then false
+            when __rectified_key_is_deleted_next
+            then false
+            when __rectified_no_more_operation
+            then false
+            when __rectified_key_not_found_in_next_reload
+            then true
+            else false
+        end as __rectified_is_deleted_next
+    from __rectified_next_operation cur
+    left join
+        __rectified_timestamps t on cur.__timestamp = t.__timestamp
+        {% if has_source %} and cur.__source == t.__source {% endif %}
+    left join
+        __rectified_next_operation nxt
+        on t.__rectified_next_timestamp_reload = nxt.__timestamp
+        and cur.__key == nxt.__key
+        {% if has_source %} and cur.__source == nxt.__source {% endif %}
+),
+{% if mode == "complete" %} __rectified as ( {% else %} __rectified_operation as (
+{% endif %}
+    select
+        c.* except (c.__operation, c.__timestamp),
+        if(c.__operation == 'delete', 'delete', d2.__rectified_operation) as __operation,
+        case
+            when c.__operation == 'delete'
+            then c.__timestamp
+            when d2.__rectified_operation == 'upsert'
+            then c.__timestamp
+            else c.__rectified_next_timestamp_reload
+        end as __timestamp
+    from __rectified_is_deleted_next c
+    cross join
+        (
+            select false as __is_deleted, 'upsert' as __rectified_operation
+            union all
+            select true as __is_deleted, 'delete' as __rectified_operation
+            union all
+            select true as __is_deleted, 'upsert' as __rectified_operation
+        ) d2
+        on c.__rectified_is_deleted_next = d2.__is_deleted
+),
+{% if mode == "update" %}
+    __rectified as (
+        select * from __rectified_operation where not (__original_operation == 'current' and __operation == 'upsert')
+    ),
+{% endif %}

fabricks/cdc/templates/ctes/slice.sql.jinja

@@ -0,0 +1 @@
+__sliced as (select * from {{ parent_slice }} s where true and ({{ slices }})),

fabricks/cdc/templates/filter.sql.jinja

@@ -0,0 +1,4 @@
+{% include 'ctes/base.sql.jinja' %}
+{% if slice == "update" %} {% include 'filters/update.sql.jinja' %} {% endif %}
+{% if slice == "latest" %} {% include 'filters/latest.sql.jinja' %} {% endif %}
+{% include 'filters/final.sql.jinja' %}

fabricks/cdc/templates/filters/final.sql.jinja

@@ -0,0 +1,4 @@
+select
+    array_join(collect_list(`slices`), ' or ') as `slices`,
+    {% if has_source %} array_join(collect_list(`sources`), ' or ') as `sources` {% endif %}
+from __final

fabricks/cdc/templates/filters/latest.sql.jinja

@@ -0,0 +1,17 @@
+__latest as (
+    select {% if has_source %} __source, {% endif %} max(__timestamp) as __max_timestamp
+    from {{ parent_slice }}
+    {% if has_source %} group by __source {% endif %}
+),
+__final as (
+    select
+        {% if has_source %} concat('t.__source == \'', `__source`, '\'') as sources, {% endif %}
+        concat_ws(
+            ' ',
+            ' (',
+            concat('s.__timestamp == \'', `__max_timestamp`, '\''),
+            {% if has_source %} concat('and s.__source == \'', `__source`, '\''), {% endif %}
+            ' )'
+        ) as `slices`
+    from __latest
+)

fabricks/cdc/templates/filters/update.sql.jinja

@@ -0,0 +1,30 @@
+{% if has_source %} __update_source as (select __source from {{ parent_slice }} group by __source), {% endif %}
+__update as (
+    select
+        {% if has_source %} s.__source, {% endif %}
+        {% if cdc == "nocdc" %}
+            coalesce(max(t.__timestamp), cast('0001-01-01' as timestamp)) as __max_timestamp
+        {% endif %}
+        {% if cdc == "scd1" %}
+            coalesce(max(t.__timestamp), cast('0001-01-01' as timestamp)) as __max_timestamp
+        {% endif %}
+        {% if cdc == "scd2" %}
+            coalesce(max(t.__valid_from), cast('0001-01-01' as timestamp)) as __max_timestamp
+        {% endif %}
+    from {{ tgt }} t
+    {% if has_source %} right join __update_source s on s.__source == t.__source {% endif %}
+    where true
+    {% if has_source %} group by s.__source {% endif %}
+),
+__final as (
+    select
+        {% if has_source %} concat('t.__source == \'', `__source`, '\'') as sources, {% endif %}
+        concat_ws(
+            ' ',
+            ' (',
+            concat('s.__timestamp > \'', `__max_timestamp`, '\''),
+            {% if has_source %} concat('and s.__source == \'', `__source`, '\''), {% endif %}
+            ' )'
+        ) as `slices`
+    from __update
+)

fabricks/cdc/templates/macros/bactick.sql.jinja

@@ -0,0 +1 @@
+{% macro backtick(field) -%} `{{ field }}` {%- endmacro %}

fabricks/cdc/templates/macros/hash.sql.jinja

@@ -0,0 +1,18 @@
+{% macro add_hash(fields) -%}
+    md5(
+        array_join(
+            array(
+                {% for f in fields %}
+                    {% if f == "__operation" %}cast(`{{ f }}` <=> 'delete' as string)  -- reloads and upserts should have the same hash, not deletes,
+                    {% else %}`{{ f }}`::string,
+                    {% endif %}
+                {% endfor %}
+            ),
+            '*',
+            '-1'
+        )
+    )
+{%- endmacro %}
+{% macro add_key(fields) -%}
+    md5(array_join(array({% for f in fields %}`{{ f }}`::string, {% endfor %}), '*', '-1'))
+{%- endmacro %}

fabricks/cdc/templates/merge.sql.jinja

@@ -0,0 +1,3 @@
+{% if cdc == "scd1" %} {% include 'merges/scd1.sql.jinja' %} {% endif %}
+{% if cdc == "scd2" %} {% include 'merges/scd2.sql.jinja' %} {% endif %}
+{% if cdc == "nocdc" %} {% include 'merges/nocdc.sql.jinja' %} {% endif %}

fabricks/cdc/templates/merges/nocdc.sql.jinja

@@ -0,0 +1,41 @@
+{% if format == "dataframe" %}
+  merge into {{ tgt }} t using {{ "{src}" }} s
+{% endif %}
+{% if format == "view" %}
+  merge into {{ tgt }} t using {{ src }} s
+{% endif %}
+{% if has_key %}
+  on t.__key == s.__merge_key
+{% else %}
+  on
+    {% for k in keys %}
+  t.{{ k }} <=> s.{{ k }}
+    {% endfor %}
+{% endif %}
+{% if has_source %}
+  and t.__source == s.__source 
+{% endif %}
+{% if update_where %} {{ update_where }} {% endif %}
+  when matched
+  and __merge_condition == 'upsert' then
+  update
+  set
+{% for c in columns %}
+    {{ c }} = s.{{c}},
+{% endfor %}
+-- delete
+  when matched
+  and __merge_condition == 'delete' then
+  delete
+  when not matched
+  and __merge_condition == 'upsert' then
+  insert (
+{% for c in columns %}
+    {{ c }},
+{% endfor %}
+  )
+  values (
+{% for c in columns %}
+    s.{{ c }},
+{% endfor %}
+  )

fabricks/cdc/templates/merges/scd1.sql.jinja

@@ -0,0 +1,73 @@
+{% if format == "dataframe" %}
+  merge into {{ tgt }} t using {{ "{src}" }} s
+{% endif %}
+{% if format == "view" %}
+  merge into {{ tgt }} t using {{ src }} s
+{% endif %}
+{% if has_key %}
+  on t.__key == s.__merge_key
+{% else %}
+  on
+    {% for k in keys %}
+  t.{{ k }} <=> s.{{ k }}
+    {% endfor %}
+{% endif %}
+{% if has_source %}
+  and t.__source == s.__source 
+{% endif %}
+{% if update_where %} {{ update_where }} {% endif %}
+  when matched
+  and __merge_condition == 'upsert' then
+  update
+  set
+{% for f in fields %}
+    {{ f }} = s.{{f}},
+{% endfor %}
+{% if has_timestamp %}
+    __timestamp = s.__timestamp,
+{% endif %}
+{% if has_metadata %}
+    __metadata.updated = cast(current_timestamp() as timestamp),    
+{% endif %}
+{% if has_hash %}
+    __hash = s.__hash,
+{% endif %}
+{% if has_rescued_data %}
+    __rescued_data = s.__rescued_data,
+{% endif %}
+{% if soft_delete %}
+    __is_current = s.__is_current,
+    __is_deleted = s.__is_deleted,
+{% endif %}
+{% if soft_delete %}
+    -- soft delete
+  when matched
+  and __merge_condition == 'delete' then
+  update
+  set
+    __is_current = False,
+    __is_deleted = True,
+    {% if has_timestamp %}
+    __timestamp = s.__timestamp,
+    {% endif %}
+    {% if has_metadata %}
+    __metadata.updated = cast(current_timestamp() as timestamp),
+    {% endif %}
+{% else %}
+    -- delete
+  when matched
+  and __merge_condition == 'delete' then
+  delete
+{% endif %}
+  when not matched
+  and __merge_condition == 'upsert' then
+  insert (
+{% for c in columns %}
+    {{ c }},
+{% endfor %}
+  )
+  values (
+{% for c in columns %}
+    s.{{ c }},
+{% endfor %}
+  )

fabricks/cdc/templates/merges/scd2.sql.jinja

@@ -0,0 +1,54 @@
+{% if format == "dataframe" %}
+  merge into {{ tgt }} t using {{ "{src}" }} s
+{% endif %}
+{% if format == "view" %}
+  merge into {{ tgt }} t using {{ src }} s
+{% endif %}
+{% if has_key %}
+  on t.__key == s.__merge_key
+{% else %}
+  on
+    {% for k in keys %}
+  t.{{ k }} <=> s.{{ k }} and
+    {% endfor %}
+{% endif %}
+  and t.__is_current 
+{% if has_source %}
+  and t.__source == s.__source 
+{% endif %}
+  when matched
+  and __merge_condition == 'update' then
+  update
+  set
+    __valid_to = s.__valid_from - interval 1 seconds,
+    __is_current = False,
+{% if soft_delete %}
+    __is_deleted = False,
+{% endif %}
+{% if has_metadata %}
+    __metadata.updated = cast(current_timestamp() as timestamp),
+{% endif %}
+  when matched
+  and __merge_condition == 'delete' then
+  update
+  set
+    __valid_to = s.__valid_from - interval 1 seconds,
+    __is_current = False,
+{% if soft_delete %}
+    __is_deleted = True,
+{% endif %}
+{% if has_metadata %}
+    __metadata.updated = cast(current_timestamp() as timestamp),
+{% endif %}
+  when not matched
+  and __merge_condition == 'insert' then
+  insert (
+{% for c in columns %}
+    {{ c }},
+{% endfor %}
+  )
+  values (
+{% for c in columns %}
+    s.{{ c }},
+{% endfor %}
+  )

fabricks/cdc/templates/queries/context.sql.jinja

@@ -0,0 +1,186 @@
+/*
+
+⚙️ BASE
+{%- if cdc %}
+☐ cdc: {{ cdc }}
+{%- endif %}
+{%- if mode %}
+☐ mode: {{ mode }}
+{%- endif %}
+
+🎯 SOURCE & TARTGET
+{%- if format %}
+☐ format: {{ format }}
+{%- endif %}
+{%- if src %}
+☐ src: {{ src | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if tgt %}
+☐ tgt: {{ tgt }}
+{%- endif %}
+
+📊 CTE's
+{%- if slice %}
+🗹 slice?
+{%- endif %}
+{%- if deduplicate %}
+🗹 deduplicate?
+{%- endif %}
+{%- if advanced_deduplication %}
+🗹 advanced deduplication?
+{%- endif %}
+{%- if deduplicate_key %}
+🗹 deduplicate key?
+{%- endif %}
+{%- if deduplicate_hash %}
+🗹 deduplicate hash?
+{%- endif %}
+{%- if order_duplicate_by %}
+🗹 order duplicate by?
+{%- endif %}
+{%- if rectify %}
+🗹 rectify?
+{%- endif %}
+{%- if correct_valid_from %}
+🗹 correct valid from?
+{%- endif %}
+
+🔪 FILTERING
+{%- if filter_where %}
+☐ filter where: {{ filter_where }}
+{%- endif %}
+{%- if update_where %}
+☐ update where: {{ update_where }}
+{%- endif %}
+{%- if slices %}
+☐ slices: {{ slices }}
+{%- endif %}
+{%- if sources %}
+☐ sources: {{ sources }}
+{%- endif %}
+
+🗑️ DELETES
+{%- if delete_missing %}
+🗹 delete missing?
+{%- endif %}
+{%- if soft_delete %}
+🗹 soft delete?
+{%- endif %}
+
+✅ DATA VALIDATION
+{%- if has_no_data %}
+☒ has_data?
+{%- else %}
+🗹 has data?
+{%- endif %}
+{%- if has_rows %}
+🗹 has rows?
+{%- else %}
+☒ has rows?
+{%- endif %}
+{%- if has_source %}
+🗹 has source?
+{%- endif %}
+
+🏷️ HAS FIELDS
+{%- if has_metadata %}
+🗹 has metadata?
+{%- endif %}
+{%- if has_timestamp %}
+🗹 has timestamp?
+{%- endif %}
+{%- if has_identity %}
+🗹 has identity?
+{%- endif %}
+{%- if has_key %}
+🗹 has key?
+{%- endif %}
+{%- if has_hash %}
+🗹 has hash?
+{%- endif %}
+{%- if has_operation %}
+🗹 has operation?
+{%- endif %}
+{%- if has_order_by %}
+🗹 has order by?
+{%- endif %}
+{%- if has_rescued_data %}
+🗹 has rescued data?
+{%- endif %}
+
+➕ ADD COLUMNS
+{%- if add_metadata %}
+🗹 add metadata?
+{%- endif %}
+{%- if add_timestamp %}
+🗹 add timestamp?
+{%- endif %}
+{%- if add_key %}
+🗹 add key?
+{%- endif %}
+{%- if add_hash %}
+🗹 add hash?
+{%- endif %}
+{%- if add_operation %}
+☐ add_operation: {{ add_operation }}
+{%- endif %}
+{%- if add_source %}
+☐ add_source: {{ add_source }}
+{%- endif %}
+{%- if add_calculated_columns %}
+☐ add_calculated_columns: {{ add_calculated_columns }}
+{%- endif %}
+
+🔄 EXTRA COLUMN OPERATIONs
+{%- if all_except %}
+☐ all_except: {{ all_except | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if all_overwrite %}
+☐ all_overwrite: {{ all_overwrite | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if overwrite %}
+☐ overwrite: {{ overwrite | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+
+👨‍👩‍👧 PARENTS
+{%- if parent_slice %}
+☐ parent_slice: {{ parent_slice }}
+{%- endif %}
+{%- if parent_rectify %}
+☐ parent_rectify: {{ parent_rectify }}
+{%- endif %}
+{%- if parent_deduplicate_key %}
+☐ parent_deduplicate_key: {{ parent_deduplicate_key }}
+{%- endif %}
+{%- if parent_deduplicate_hash %}
+☐ parent_deduplicate_hash: {{ parent_deduplicate_hash }}
+{%- endif %}
+{%- if parent_cdc %}
+☐ parent_cdc: {{ parent_cdc }}
+{%- endif %}
+{%- if parent_final %}
+☐ parent_final: {{ parent_final }}
+{%- endif %}
+
+📦 LAYOUT
+{%- if columns %}
+☐ columns: {{ columns | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if inputs %}
+☐ inputs: {{ inputs | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if intermediates %}
+☐ intermediates: {{ intermediates | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if outputs %}
+☐ outputs: {{ outputs | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if keys %}
+☐ keys: {{ keys | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if hashes %}
+☐ hashes: {{ hashes | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+
+*/
+

fabricks/cdc/templates/queries/final.sql.jinja

@@ -0,0 +1 @@
+select * {% if all_except %} except ({% for e in all_except %}{{ e }}, {% endfor %}), {% endif %} from __final

fabricks/cdc/templates/queries/nocdc/complete.sql.jinja

@@ -0,0 +1,10 @@
+__final as (
+    select {% for output in outputs %} `{{ output }}`, {% endfor %},
+    from {{ parent_cdc }}
+    {% if filter %}
+        where
+            true
+            -- operation current added by filter
+            and __operation <> 'current'
+    {% endif %}
+)

fabricks/cdc/templates/queries/nocdc/update.sql.jinja

@@ -0,0 +1,34 @@
+{% if has_rows %}
+    __merge_condition as (
+        select
+            s.__key as __merge_key,
+            'upsert' as __merge_condition,
+            {% for output in outputs %} s.`{{ output }}`, {% endfor %}
+        from {{ parent_cdc }} s
+        left anti join
+            __current c on s.__key == c.__key and s.__hash == c.__hash
+            {% if has_source %} and s.__source = c.__source {% endif %}
+        {% if delete_missing %}
+            union all
+            select
+                c.__key as __merge_key,
+                'delete' as __merge_condition,
+                {% for output in outputs %} c.`{{ output }}`, {% endfor %}
+            from __current c
+            left anti join
+                {{ parent_cdc }} s on s.__key == c.__key {% if has_source %} and s.__source = c.__source {% endif %}
+        {% endif %}
+    ),
+{% else %}
+    __merge_condition as (select s.__key as __merge_key, 'upsert' as __merge_condition, s.* from {{ parent_cdc }} s),
+{% endif %}
+__final as (
+    select __merge_key, __merge_condition, {% for output in outputs %} `{{ output }}`, {% endfor %}
+    from __merge_condition
+    {% if filter %}
+        where
+            true
+            -- operation current added by filter
+            and __operation <> 'current'
+    {% endif %}
+)