quollio-core 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.6.3"
+__version__ = "0.6.5"
 __author__ = "Quollio Technologies, Inc"

quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -2,9 +2,10 @@
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
 {%- set grant_config = config.get('grants') %}
+{%- set max_columns_per_view = config.get('max_columns_per_view', 100) %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
--- `BEGIN` happens here:
+-- BEGIN happens here:
 {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
 -- fetch target_tables
@@ -41,45 +42,60 @@
 
 -- create view for each table
 {%- for stats_target_table in stats_target_tables -%}
-  -- build sql for column value aggregation.
-  {%- set sql_for_column_stats %}
-  {% set columns_json = fromjson(stats_target_table[3]) %}
-  {%- for col_name, attr in columns_json.items() -%}
-    {%- if not loop.first %}UNION{% endif %}
-    SELECT
-      DISTINCT
-      '{{stats_target_table[0]}}' as db_name
-      , '{{stats_target_table[1]}}' as schema_name
-      , '{{stats_target_table[2]}}' as table_name
-      , '{{col_name}}' as column_name
-      , {% if attr["is_calclable"] == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
-      , {% if attr["is_calclable"] == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
-      , COUNT_IF("{{col_name}}" IS NULL) AS null_count
-      , {% if attr["can_approx_count"] == True %}APPROX_COUNT_DISTINCT("{{col_name}}"){% else %}NULL{% endif %} AS cardinality
-      , {% if attr["is_calclable"] == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
-      , {% if attr["is_calclable"] == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
-      , {% if attr["is_calclable"] == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
-      , {% if attr["is_calclable"] == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
-    FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
-  {% endfor -%}
-  {%- endset %}
+  {%- set columns_json = fromjson(stats_target_table[3]) %}
+  {%- set column_list = columns_json.keys() | list %}
+  {%- set chunk_count = ((column_list | length) / max_columns_per_view) | round(0, 'ceil') | int %}
+  
+  {%- for chunk_index in range(chunk_count) %}
+    {%- set start_idx = chunk_index * max_columns_per_view %}
+    {%- set end_idx = start_idx + max_columns_per_view %}
+    {%- set chunk_columns = column_list[start_idx:end_idx] %}
+    
+    -- build sql for column value aggregation.
+    {%- set sql_for_column_stats %}
+    {%- for col_name in chunk_columns -%}
+      {%- set attr = columns_json[col_name] %}
+      {%- if not loop.first %}UNION{% endif %}
+      SELECT
+        DISTINCT
+        '{{stats_target_table[0]}}' as db_name
+        , '{{stats_target_table[1]}}' as schema_name
+        , '{{stats_target_table[2]}}' as table_name
+        , '{{col_name}}' as column_name
+        , {% if attr["IS_CALCULABLE"] == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
+        , {% if attr["IS_CALCULABLE"] == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
+        , COUNT_IF("{{col_name}}" IS NULL) AS null_count
+        , {% if attr["CAN_APPROX_COUNT"] == True %}APPROX_COUNT_DISTINCT("{{col_name}}"){% else %}NULL{% endif %} AS cardinality
+        , {% if attr["IS_CALCULABLE"] == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
+        , {% if attr["IS_CALCULABLE"] == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
+        , {% if attr["IS_CALCULABLE"] == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
+        , {% if attr["IS_CALCULABLE"] == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
+      FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
+    {% endfor -%}
+    {%- endset %}
 
-  -- create a view with a index as suffix
-  {%- set stats_view_identifier = "\"%s_%s_%s_%s\"" | format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) | upper %}
-  {%- set schema_name = "\"%s\""|format(schema) %}
-  {%- set db_name = "\"%s\""|format(database) %}
-  {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema_name, database=db_name, type='view') %}
-  {% call statement("main") %}
-    {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
-  {% endcall %}
-  {%- set full_refresh_mode = (should_full_refresh()) -%}
-  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
-  {%- do apply_grants(target_relation, grant_config, should_revoke) %}
-  {%- set target_relations = target_relations.append(target_relation) %}
+    -- create a view with a index as suffix and chunk indicator
+    {%- set chunk_suffix = "" if chunk_count == 1 else "_PART" ~ (chunk_index + 1) %}
+    {%- set stats_view_identifier = "\"%s_%s_%s_%s%s\"" | format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2], chunk_suffix) | upper %}
+    {%- set schema_name = "\"%s\""|format(schema) %}
+    {%- set db_name = "\"%s\""|format(database) %}
+    {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema_name, database=db_name, type='view') %}
+    
+    {{ log("Creating view " ~ stats_view_identifier ~ " with " ~ chunk_columns | length ~ " columns (chunk " ~ (chunk_index + 1) ~ " of " ~ chunk_count ~ ")", info=True) }}
+    
+    {% call statement("main") %}
+      {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
+    {% endcall %}
+    
+    {%- set full_refresh_mode = (should_full_refresh()) -%}
+    {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+    {%- do apply_grants(target_relation, grant_config, should_revoke) %}
+    {%- set target_relations = target_relations.append(target_relation) %}
+  {%- endfor %}
 {%- endfor -%}
 
 {{ run_hooks(post_hooks, inside_transaction=True) }}
--- `COMMIT` happens here:
+-- COMMIT happens here:
 {{ adapter.commit() }}
 {{ run_hooks(post_hooks, inside_transaction=False) }}
 

{quollio_core-0.6.3.dist-info → quollio_core-0.6.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quollio-core
-Version: 0.6.3
+Version: 0.6.5
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>

{quollio_core-0.6.3.dist-info → quollio_core-0.6.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-quollio_core/__init__.py,sha256=Stw2KgV7GaDmXB1xmDqQL-Jaa1F15gFBj2a8sOdBMSw,83
+quollio_core/__init__.py,sha256=koimld9v7KxDLohX-mmvAh-Rd5GBygo2-_CqKN_L5lE,83
 quollio_core/bigquery.py,sha256=6Oq4DVGpa3X21Es_nbrsb8pK3vaxwb9Egnvq3huo95k,5894
 quollio_core/bricks.py,sha256=8h3kbI2b6lGH2s-56jE_Q5-R5-nIsQYMfvtRrkFOzoU,10784
 quollio_core/redshift.py,sha256=KcdljY95xYf9JYrsaMOBoP_XxQQ8wFVE5ue_XEMVSFc,11504
@@ -47,7 +47,7 @@ quollio_core/dbt_projects/snowflake/packages_hub.yml,sha256=p9Bl2C44gdC6iYTUkz_1
 quollio_core/dbt_projects/snowflake/packages_local.yml,sha256=ryyJSXv83gYFu48xmzG5Z1l746jGCUBE6hs7pUNwuXE,43
 quollio_core/dbt_projects/snowflake/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=LO0FHBBtGrbuBuuJ_wbpNYxWuY6q_pifGtYIDQvf3Mg,3986
+quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=WjN_baGb_FuDX831rR5G7Payw5t91VaoQobEkTJR3jQ,4880
 quollio_core/dbt_projects/snowflake/macros/materialization/get_imported_databases.sql,sha256=WAFl9CyM-G07O7vZD2MkA1hncpjV_gSMGc2V8nRJRPk,1435
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.sql,sha256=Zhj0EXF1K8S-OkFxz3IBHe2olXktYrvly0LwZBOAUXw,5333
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.yml,sha256=a2uNIAh-xw51eu-GmHVuAnGnTbwK7h8-DjDeQtK3KaQ,711
@@ -90,7 +90,7 @@ quollio_core/repository/redshift.py,sha256=p2ouEuYcDCjx1oBhc6H1ekQsvEqHGd3bFu3PW
 quollio_core/repository/snowflake.py,sha256=yCYXrYf4I5GL_ITNTXoggj0xNbQsdwxPSmsVvZYwUVU,3869
 quollio_core/repository/ssm.py,sha256=xpm1FzbBnIsBptuYPUNnPgkKU2AH3XxI-ZL0bEetvW0,2182
 quollio_core/repository/teradata.py,sha256=1AExxRBTswpSyF4OVyAUkoiZ0yVRfqt4T99FdllkTEI,3763
-quollio_core-0.6.3.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.6.3.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-quollio_core-0.6.3.dist-info/METADATA,sha256=APAC9-Vbz9W-upz1WX4A0icMTblyK4HWRBiJZWTIxVg,7023
-quollio_core-0.6.3.dist-info/RECORD,,
+quollio_core-0.6.5.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.6.5.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+quollio_core-0.6.5.dist-info/METADATA,sha256=hYXyuls1a0DulGYU33B98YwO3Y9yBg2KOhXIIxOGrVk,7023
+quollio_core-0.6.5.dist-info/RECORD,,