fabricks 3.0.5.2__py3-none-any.whl → 3.0.6__py3-none-any.whl

fabricks/cdc/templates/{query → ctes}/base.sql.jinja

@@ -1,19 +1,20 @@
-{% import 'query/hash.sql.jinja' as h -%}
+{% import 'macros/hash.sql.jinja' as h -%}
 
 with
     {% if format == "query" %} __query as ({{ src }}), {% endif %}
     __base as (
         select
             *
-            {% if all_overwrite %} except ({% for o in all_overwrite %}{{ o }}, {% endfor %}),
-            {% else %},
-            {% endif %}
+            {% if overwrite %}
+                -- will be overwritten below
+                except ({% for o in overwrite %}{{ o }}, {% endfor %})
+            {% endif %},
             {% if add_calculated_columns %} {% for c in add_calculated_columns %} {{ c }}, {% endfor %} {% endif %}
             {% if add_timestamp %} cast(current_date() as timestamp) as __timestamp, {% endif %}
             {% if add_operation %} cast('{{ add_operation }}' as string) as __operation, {% endif %}
             {% if add_source %} cast('{{ add_source }}' as string) as __source, {% endif %}
-            {% if add_hash %} {{ h.hash(fields=hashes) }} as __hash, {% endif %}
-            {% if add_key %} {{ h.hash(fields=keys) }} as __key, {% endif %}
+            {% if add_hash %} {{ h.add_hash(fields=hashes) }} as __hash, {% endif %}
+            {% if add_key %} {{ h.add_hash(fields=keys) }} as __key, {% endif %}
             {% if add_metadata %}
                 struct(
                     {% if cdc == "nocdc" %}current_timestamp() as inserted,

fabricks/cdc/templates/ctes/current.sql.jinja

@@ -0,0 +1,28 @@
+{% import 'macros/hash.sql.jinja' as h -%}
+
+__current as (
+    select
+        {% for i in intermediates %}
+            {% if i == "__timestamp" %}
+                {% if add_timestamp %} cast('0001-01-01' as timestamp) as __timestamp,
+                {% elif cdc == "nocdc" %} __timestamp,
+                {% elif cdc == "scd1" %} __timestamp,
+                {% elif cdc == "scd2" %} __valid_from as __timestamp,
+                {% endif %}
+            {% elif i == "__operation" %}
+                {% if has_no_data %} 'delete' as __operation, {% else %} 'current' as __operation, {% endif %}
+            {% elif i == "__hash" %}
+                {% if add_hash %} {{ h.add_hash(fields=hashes) }} as __hash, {% else %} __hash, {% endif %}
+            {% elif i == "__key" %}
+                {% if add_key %} {{ h.add_key(fields=keys) }} as __key, {% else %} __key, {% endif %}
+            {% else %} `{{ i }}`,
+            {% endif %}
+        {% endfor %}
+    from {{ tgt }} t
+    where
+        true
+        {% if cdc == "scd2" %} and __is_current {% endif %}
+        {% if cdc == "scd1" %} {% if soft_delete %} and __is_current {% endif %} {% endif %}
+        {% if sources %} and ({{ sources }}) {% endif %}
+        {% if update_where %} and {{ update_where }} {% endif %}
+),

fabricks/cdc/templates/ctes/deduplicate_hash.sql.jinja

@@ -0,0 +1,32 @@
+{% if advanced_deduplication %}
+    __deduplicate_hash as (
+        select
+            *,
+            lag(__hash) over (
+                partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+            ) as __deduplicate_hash_previous__hash,
+            lag(__operation) over (
+                partition by {% if has_source %} __source, {% endif %} __key order by __timestamp asc
+            ) as __deduplicate_hash_previous_operation
+        from {{ parent_deduplicate_hash }}
+        where true
+    ),
+    __deduplicated_hash as (
+        select *
+        from __deduplicate_hash
+        where
+            true
+            and not (
+                __hash <=> __deduplicate_hash_previous__hash and __operation <=> __deduplicate_hash_previous_operation
+            )
+    ),
+{% else %}
+    __deduplicated_hash as (
+        select *
+        from {{ parent_deduplicate_hash }}
+        where true
+        qualify
+            not lag(__hash) over (partition by {% if has_source %} __source, {% endif %} __key order by null)
+            <=> __hash
+    ),
+{% endif %}

fabricks/cdc/templates/ctes/deduplicate_key.sql.jinja

@@ -0,0 +1,31 @@
+{% if advanced_deduplication %}
+    __deduplicate_key as (
+        select
+            *,
+            row_number() over (
+                partition by {% if has_source %} __source, {% endif %} __key, __timestamp
+                order by
+                    /* prioritize delete over upsert */
+                    __operation asc,
+                    {% if has_order_by %} {% for o in order_duplicate_by %} {{ o }}, {% endfor %} {% endif %}
+            ) as __deduplicate_key_rn
+        from {{ parent_deduplicate_key }}
+        where true
+    ),
+    __deduplicated_key as (select *, from __deduplicate_key where __deduplicate_key_rn == 1),
+{% else %}
+    __deduplicated_key as (
+        select *
+        from {{ parent_deduplicate_key }}
+        where true
+        qualify
+            row_number() over (
+                partition by {% if has_source %} __source, {% endif %} __key
+                order by
+                    {% if has_order_by %} {% for o in order_duplicate_by %} {{ o }}, {% endfor %}
+                    {% else %} null
+                    {% endif %}
+            )
+            == 1
+    ),
+{% endif %}

fabricks/cdc/templates/{query → ctes}/rectify.sql.jinja

@@ -1,34 +1,16 @@
-{% import 'query/hash.sql.jinja' as h -%}
-
 {% if mode == "update" %}
     __rectified_base as (
         select
-            {% for field in fields %} {{ field }}, {% endfor %}
-            __operation,
-            __operation as __original_operation,
-            __timestamp,
+            {% for i in intermediates %} `{{ i }}`, {% endfor %}
             __timestamp as __original_timestamp,
-            __hash,
-            __key,
-            {% if has_identity %} __identity, {% endif %}
-            {% if has_source %} __source, {% endif %}
-            {% if has_metadata %} __metadata, {% endif %}
-            {% if has_rescued_data %} __rescued_data, {% endif %}
+            __operation as __original_operation,
         from {{ parent_rectify }}
         {% if has_rows %}
             union all
             select
-                {% for field in fields %} {{ field }}, {% endfor %}
-                __operation,
-                __operation as __original_operation,
-                __timestamp,
+                {% for i in intermediates %} `{{ i }}`, {% endfor %}
                 __timestamp as __original_timestamp,
-                __hash,
-                __key,
-                {% if has_identity %} __identity, {% endif %}
-                {% if has_source %} __source, {% endif %}
-                {% if has_metadata %} __metadata, {% endif %}
-                {% if has_rescued_data %} __rescued_data, {% endif %}
+                __operation as __original_operation,
             from __current
         {% endif %}
     ),

fabricks/cdc/templates/ctes/slice.sql.jinja

@@ -0,0 +1 @@
+__sliced as (select * from {{ parent_slice }} s where true and ({{ slices }})),

fabricks/cdc/templates/filter.sql.jinja

@@ -1,4 +1,4 @@
-{% include 'query/base.sql.jinja' %}
-{% if slice == "update" %} {% include 'filter/update.sql.jinja' %} {% endif %}
-{% if slice == "latest" %} {% include 'filter/latest.sql.jinja' %} {% endif %}
-{% include 'filter/final.sql.jinja' %}
+{% include 'ctes/base.sql.jinja' %}
+{% if slice == "update" %} {% include 'filters/update.sql.jinja' %} {% endif %}
+{% if slice == "latest" %} {% include 'filters/latest.sql.jinja' %} {% endif %}
+{% include 'filters/final.sql.jinja' %}

fabricks/cdc/templates/macros/bactick.sql.jinja

@@ -0,0 +1 @@
+{% macro backtick(field) -%} `{{ field }}` {%- endmacro %}

fabricks/cdc/templates/macros/hash.sql.jinja

@@ -0,0 +1,18 @@
+{% macro add_hash(fields) -%}
+    md5(
+        array_join(
+            array(
+                {% for f in fields %}
+                    {% if f == "__operation" %}cast(`{{ f }}` <=> 'delete' as string)  -- reloads and upserts should have the same hash, not deletes,
+                    {% else %}`{{ f }}`::string,
+                    {% endif %}
+                {% endfor %}
+            ),
+            '*',
+            '-1'
+        )
+    )
+{%- endmacro %}
+{% macro add_key(fields) -%}
+    md5(array_join(array({% for f in fields %}`{{ f }}`::string, {% endfor %}), '*', '-1'))
+{%- endmacro %}

fabricks/cdc/templates/merge.sql.jinja

@@ -1,2 +1,3 @@
-{% if cdc == "scd1" %} {% include 'merge/scd1.sql.jinja' %} {% endif %}
-{% if cdc == "scd2" %} {% include 'merge/scd2.sql.jinja' %} {% endif %}
+{% if cdc == "scd1" %} {% include 'merges/scd1.sql.jinja' %} {% endif %}
+{% if cdc == "scd2" %} {% include 'merges/scd2.sql.jinja' %} {% endif %}
+{% if cdc == "nocdc" %} {% include 'merges/nocdc.sql.jinja' %} {% endif %}

fabricks/cdc/templates/merges/nocdc.sql.jinja

@@ -0,0 +1,41 @@
+{% if format == "dataframe" %}
+  merge into {{ tgt }} t using {{ "{src}" }} s
+{% endif %}
+{% if format == "view" %}
+  merge into {{ tgt }} t using {{ src }} s
+{% endif %}
+{% if has_key %}
+  on t.__key == s.__merge_key
+{% else %}
+  on
+    {% for k in keys %}
+  t.{{ k }} <=> s.{{ k }}
+    {% endfor %}
+{% endif %}
+{% if has_source %}
+  and t.__source == s.__source 
+{% endif %}
+{% if update_where %} {{ update_where }} {% endif %}
+  when matched
+  and __merge_condition == 'upsert' then
+  update
+  set
+{% for c in columns %}
+    {{ c }} = s.{{c}},
+{% endfor %}
+-- delete
+  when matched
+  and __merge_condition == 'delete' then
+  delete
+  when not matched
+  and __merge_condition == 'upsert' then
+  insert (
+{% for c in columns %}
+    {{ c }},
+{% endfor %}
+  )
+  values (
+{% for c in columns %}
+    s.{{ c }},
+{% endfor %}
+  )

fabricks/cdc/templates/queries/context.sql.jinja

@@ -0,0 +1,186 @@
+/*
+
+⚙️ BASE
+{%- if cdc %}
+☐ cdc: {{ cdc }}
+{%- endif %}
+{%- if mode %}
+☐ mode: {{ mode }}
+{%- endif %}
+
+🎯 SOURCE & TARTGET
+{%- if format %}
+☐ format: {{ format }}
+{%- endif %}
+{%- if src %}
+☐ src: {{ src | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if tgt %}
+☐ tgt: {{ tgt }}
+{%- endif %}
+
+📊 CTE's
+{%- if slice %}
+🗹 slice?
+{%- endif %}
+{%- if deduplicate %}
+🗹 deduplicate?
+{%- endif %}
+{%- if advanced_deduplication %}
+🗹 advanced deduplication?
+{%- endif %}
+{%- if deduplicate_key %}
+🗹 deduplicate key?
+{%- endif %}
+{%- if deduplicate_hash %}
+🗹 deduplicate hash?
+{%- endif %}
+{%- if order_duplicate_by %}
+🗹 order duplicate by?
+{%- endif %}
+{%- if rectify %}
+🗹 rectify?
+{%- endif %}
+{%- if correct_valid_from %}
+🗹 correct valid from?
+{%- endif %}
+
+🔪 FILTERING
+{%- if filter_where %}
+☐ filter where: {{ filter_where }}
+{%- endif %}
+{%- if update_where %}
+☐ update where: {{ update_where }}
+{%- endif %}
+{%- if slices %}
+☐ slices: {{ slices }}
+{%- endif %}
+{%- if sources %}
+☐ sources: {{ sources }}
+{%- endif %}
+
+🗑️ DELETES
+{%- if delete_missing %}
+🗹 delete missing?
+{%- endif %}
+{%- if soft_delete %}
+🗹 soft delete?
+{%- endif %}
+
+✅ DATA VALIDATION
+{%- if has_no_data %}
+☒ has_data?
+{%- else %}
+🗹 has data?
+{%- endif %}
+{%- if has_rows %}
+🗹 has rows?
+{%- else %}
+☒ has rows?
+{%- endif %}
+{%- if has_source %}
+🗹 has source?
+{%- endif %}
+
+🏷️ HAS FIELDS
+{%- if has_metadata %}
+🗹 has metadata?
+{%- endif %}
+{%- if has_timestamp %}
+🗹 has timestamp?
+{%- endif %}
+{%- if has_identity %}
+🗹 has identity?
+{%- endif %}
+{%- if has_key %}
+🗹 has key?
+{%- endif %}
+{%- if has_hash %}
+🗹 has hash?
+{%- endif %}
+{%- if has_operation %}
+🗹 has operation?
+{%- endif %}
+{%- if has_order_by %}
+🗹 has order by?
+{%- endif %}
+{%- if has_rescued_data %}
+🗹 has rescued data?
+{%- endif %}
+
+➕ ADD COLUMNS
+{%- if add_metadata %}
+🗹 add metadata?
+{%- endif %}
+{%- if add_timestamp %}
+🗹 add timestamp?
+{%- endif %}
+{%- if add_key %}
+🗹 add key?
+{%- endif %}
+{%- if add_hash %}
+🗹 add hash?
+{%- endif %}
+{%- if add_operation %}
+☐ add_operation: {{ add_operation }}
+{%- endif %}
+{%- if add_source %}
+☐ add_source: {{ add_source }}
+{%- endif %}
+{%- if add_calculated_columns %}
+☐ add_calculated_columns: {{ add_calculated_columns }}
+{%- endif %}
+
+🔄 EXTRA COLUMN OPERATIONs
+{%- if all_except %}
+☐ all_except: {{ all_except | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if all_overwrite %}
+☐ all_overwrite: {{ all_overwrite | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if overwrite %}
+☐ overwrite: {{ overwrite | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+
+👨‍👩‍👧 PARENTS
+{%- if parent_slice %}
+☐ parent_slice: {{ parent_slice }}
+{%- endif %}
+{%- if parent_rectify %}
+☐ parent_rectify: {{ parent_rectify }}
+{%- endif %}
+{%- if parent_deduplicate_key %}
+☐ parent_deduplicate_key: {{ parent_deduplicate_key }}
+{%- endif %}
+{%- if parent_deduplicate_hash %}
+☐ parent_deduplicate_hash: {{ parent_deduplicate_hash }}
+{%- endif %}
+{%- if parent_cdc %}
+☐ parent_cdc: {{ parent_cdc }}
+{%- endif %}
+{%- if parent_final %}
+☐ parent_final: {{ parent_final }}
+{%- endif %}
+
+📦 LAYOUT
+{%- if columns %}
+☐ columns: {{ columns | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if inputs %}
+☐ inputs: {{ inputs | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if intermediates %}
+☐ intermediates: {{ intermediates | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if outputs %}
+☐ outputs: {{ outputs | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if keys %}
+☐ keys: {{ keys | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+{%- if hashes %}
+☐ hashes: {{ hashes | join(", ") | truncate(100, killwords=True) }}
+{%- endif %}
+
+*/
+

fabricks/cdc/templates/{query/nocdc.sql.jinja → queries/nocdc/complete.sql.jinja}

@@ -1,5 +1,5 @@
 __final as (
-    select *
+    select {% for output in outputs %} `{{ output }}`, {% endfor %},
     from {{ parent_cdc }}
     {% if filter %}
         where

fabricks/cdc/templates/queries/nocdc/update.sql.jinja

@@ -0,0 +1,35 @@
+{% if has_rows %}
+    __merge_condition as (
+        select
+            s.__key as __merge_key,
+            'upsert' as __merge_condition,
+            {% for output in outputs %} s.`{{ output }}`, {% endfor %}
+        from {{ parent_cdc }} s
+        left anti join
+            __current c on s.__key == c.__key and s.__hash == c.__hash
+            {% if has_source %} and s.__source = c.__source {% endif %}
+        {% if delete_missing %}
+            union all
+            select
+                c.__key as __merge_key,
+                'delete' as __merge_condition,
+                {% for output in outputs %} c.`{{ output }}`, {% endfor %}
+            from __current c
+            left anti join
+                {{ parent_cdc }} s on s.__key == c.__key and s.__hash == c.__hash
+                {% if has_source %} and s.__source = c.__source {% endif %}
+        {% endif %}
+    ),
+{% else %}
+    __merge_condition as (select s.__key as __merge_key, 'upsert' as __merge_condition, s.* from {{ parent_cdc }} s),
+{% endif %}
+__final as (
+    select __merge_key, __merge_condition, {% for output in outputs %} `{{ output }}`, {% endfor %}
+    from __merge_condition
+    {% if filter %}
+        where
+            true
+            -- operation current added by filter
+            and __operation <> 'current'
+    {% endif %}
+)

fabricks/cdc/templates/{query → queries}/scd1.sql.jinja

@@ -49,16 +49,7 @@ __scd1 as (
 ),
 {% if mode == "complete" %}
     __final as (
-        select
-            {% for field in fields %} s.{{ field }}, {% endfor %},
-            {% if has_identity %} __identity, {% endif %}
-            s.__key,
-            s.__timestamp,
-            {% if soft_delete %} s.__is_current, s.__is_deleted, {% endif %}
-            s.__hash,
-            {% if has_source %} s.__source, {% endif %}
-            {% if has_metadata %} s.__metadata, {% endif %}
-            {% if has_rescued_data %} s.__rescued_data, {% endif %}
+        select {% for output in outputs %} `{{ output }}`, {% endfor %}
         from __scd1 s
         where true {% if not soft_delete %} and s.__is_current {% endif %}
     )
@@ -86,24 +77,7 @@ __scd1 as (
         ),
     {% endif %}
     __final as (
-        select
-            __merge_key,
-            __merge_condition,
-            {% for field in fields %} {{ field }},
-            {% endfor %},
-            {% if has_identity %} __identity,
-            {% endif %}
-            __key,
-            __timestamp,
-            {% if soft_delete %} __is_current, __is_deleted,
-            {% endif %}
-            __hash,
-            {% if has_source %} __source,
-            {% endif %}
-            {% if has_metadata %} __metadata,
-            {% endif %}
-            {% if has_rescued_data %} __rescued_data,
-            {% endif %}
+        select __merge_key, __merge_condition, {% for output in outputs %} `{{ output }}`, {% endfor %}
         {% if has_rows %} from __scd1_no_fake_update m
         {% else %} from __merge_condition m
         {% endif %}

fabricks/cdc/templates/{query → queries}/scd2.sql.jinja

@@ -26,27 +26,23 @@ __scd2 as (
 ),
 {% if mode == "complete" %}
     __complete as (select s.* from __scd2 s where true and not __operation <=> 'delete'),
-    __final as (
-        select
-            {% for field in fields %} {{ field }}, {% endfor %},
-            {% if has_identity %} __identity, {% endif %}
-            __key,
-            {% if correct_valid_from %}
+    {% if correct_valid_from %}
+        __correct_valid_from as (
+            select
+                * except (__valid_from),
                 if(
                     __valid_from == min(__valid_from) over (partition by null),
                     cast('1900-01-01' as timestamp),
                     __valid_from
-                ) as __valid_from,
-            {% else %} __valid_from,
-            {% endif %}
-            __valid_to,
-            __is_current,
-            {% if soft_delete %} __is_deleted, {% endif %}
-            __hash,
-            {% if has_source %} __source, {% endif %}
-            {% if has_metadata %} __metadata, {% endif %}
-            {% if has_rescued_data %} __rescued_data, {% endif %}
-        from __complete
+                ) as __valid_from
+            from __complete
+        ),
+    {% endif %}
+    __final as (
+        select {% for output in outputs %} `{{ output }}`, {% endfor %}
+        {% if correct_valid_from %} from __correct_valid_from
+        {% else %} from __complete
+        {% endif %}
     )
 {% else %}
     {% if has_rows %}
@@ -81,37 +77,22 @@ __scd2 as (
         -- only the first record can be an update or a delete
         where (s.__scd2_rn == 1 and o.__merge_condition in ('update', 'delete')) or o.__merge_condition == 'insert'
     ),
+    {% if correct_valid_from %}
+        __correct_valid_from as (
+            select
+                * except (__valid_from),
+                if(
+                    __valid_from == min(__valid_from) over (partition by null),
+                    cast('1900-01-01' as timestamp),
+                    __valid_from
+                ) as __valid_from
+            from __merge_condition
+        ),
+    {% endif %}
     __final as (
-        select
-            __merge_key,
-            __merge_condition,
-            {% for field in fields %} {{ field }},
-            {% endfor %},
-            {% if has_identity %} __identity,
-            {% endif %}
-            __key,
-            {% if correct_valid_from %}
-                {% if not has_rows %}
-                    if(
-                        __valid_from == min(__valid_from) over (partition by null),
-                        cast('1900-01-01' as timestamp),
-                        __valid_from
-                    ) as __valid_from,
-                {% else %} __valid_from,
-                {% endif %}
-            {% else %} __valid_from,
-            {% endif %}
-            __valid_to,
-            __is_current and __merge_condition == 'insert' as __is_current,
-            {% if soft_delete %} __is_deleted,
-            {% endif %}
-            __hash,
-            {% if has_source %} __source,
-            {% endif %}
-            {% if has_metadata %} __metadata,
-            {% endif %}
-            {% if has_rescued_data %} __rescued_data,
-            {% endif %}
-        from __merge_condition m
+        select __merge_key, __merge_condition, {% for output in outputs %} `{{ output }}`, {% endfor %}
+        {% if correct_valid_from %} from __correct_valid_from
+        {% else %} from __merge_condition
+        {% endif %}
     )
 {% endif %}

fabricks/cdc/templates/query.sql.jinja

@@ -1,11 +1,15 @@
-{% include 'query/context.sql.jinja' %}
-{% include 'query/base.sql.jinja' %}
-{% if slice %} {% include 'query/slice.sql.jinja' %} {% endif %}
-{% if deduplicate_key %} {% include 'query/deduplicate_key.sql.jinja' %} {% endif %}
-{% if mode == "update" %} {% if has_rows %} {% include 'query/current.sql.jinja' %} {% endif %} {% endif %}
-{% if rectify %} {% include 'query/rectify.sql.jinja' %} {% endif %}
-{% if deduplicate_hash %} {% include 'query/deduplicate_hash.sql.jinja' %} {% endif %}
-{% if cdc == "nocdc" %} {% include 'query/nocdc.sql.jinja' %} {% endif %}
-{% if cdc == "scd1" %} {% include 'query/scd1.sql.jinja' %} {% endif %}
-{% if cdc == "scd2" %} {% include 'query/scd2.sql.jinja' %} {% endif %}
-{% include 'query/final.sql.jinja' %}
+{% include 'queries/context.sql.jinja' %}
+{% include 'ctes/base.sql.jinja' %}
+{% if slice %} {% include 'ctes/slice.sql.jinja' %} {% endif %}
+{% if deduplicate_key %} {% include 'ctes/deduplicate_key.sql.jinja' %} {% endif %}
+{% if mode == "update" %} {% if has_rows %} {% include 'ctes/current.sql.jinja' %} {% endif %} {% endif %}
+{% if rectify %} {% include 'ctes/rectify.sql.jinja' %} {% endif %}
+{% if deduplicate_hash %} {% include 'ctes/deduplicate_hash.sql.jinja' %} {% endif %}
+{% if cdc == "nocdc" %}
+    {% if mode == "update" %} {% include 'queries/nocdc/update.sql.jinja' %}
+    {% else %} {% include 'queries/nocdc/complete.sql.jinja' %}
+    {% endif %}
+{% endif %}
+{% if cdc == "scd1" %} {% include 'queries/scd1.sql.jinja' %} {% endif %}
+{% if cdc == "scd2" %} {% include 'queries/scd2.sql.jinja' %} {% endif %}
+{% include 'queries/final.sql.jinja' %}