quollio-core 0.4.6__tar.gz → 0.4.8__tar.gz

{quollio_core-0.4.6 → quollio_core-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.6
+Version: 0.4.8
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>
@@ -31,6 +31,11 @@ Requires-Dist: snowflake-connector-python==3.5.0
 Requires-Dist: databricks-sdk==0.17.0
 Requires-Dist: databricks-sql-connector==2.9.5
 Requires-Dist: sqlglot==20.8.0
+Requires-Dist: google-cloud==0.34.0
+Requires-Dist: google-cloud-bigquery==3.22.0
+Requires-Dist: google-cloud-datacatalog==3.19.0
+Requires-Dist: google-cloud-datacatalog-lineage==0.3.6
+Requires-Dist: google-api-python-client==2.131.0
 Requires-Dist: black>=22.3.0 ; extra == "test"
 Requires-Dist: coverage>=7.3.2 ; extra == "test"
 Requires-Dist: isort>=5.10.1 ; extra == "test"
@@ -74,7 +79,7 @@ To see available commands and options, please run the following command. (ex: Sn
 コマンドやオプションの詳細については、下記のコマンドを実行してください。(例: Snowflake)
 
 ```
-$ python3 -m quollio_core.snowflake -h
+$ python -m quollio_core.snowflake -h
 ```
 
 Then run commands with the options provided.

{quollio_core-0.4.6 → quollio_core-0.4.8}/README.md

@@ -31,7 +31,7 @@ To see available commands and options, please run the following command. (ex: Sn
 コマンドやオプションの詳細については、下記のコマンドを実行してください。(例: Snowflake)
 
 ```
-$ python3 -m quollio_core.snowflake -h
+$ python -m quollio_core.snowflake -h
 ```
 
 Then run commands with the options provided.

{quollio_core-0.4.6 → quollio_core-0.4.8}/pyproject.toml

@@ -43,6 +43,11 @@ dependencies = [
     ,"databricks-sdk==0.17.0"
     ,"databricks-sql-connector==2.9.5"
     ,"sqlglot==20.8.0"
+    ,"google-cloud==0.34.0"
+    ,"google-cloud-bigquery==3.22.0"
+    ,"google-cloud-datacatalog==3.19.0"
+    ,"google-cloud-datacatalog-lineage==0.3.6"
+    ,"google-api-python-client==2.131.0"
 ]
 dynamic = ["version", "description"]
 

{quollio_core-0.4.6 → quollio_core-0.4.8}/quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.6"
+__version__ = "0.4.8"
 __author__ = "Quollio Technologies, Inc"

quollio_core-0.4.8/quollio_core/bigquery.py

@@ -0,0 +1,114 @@
+import argparse
+import json
+import logging
+
+from quollio_core.helper.env_default import env_default
+from quollio_core.profilers.bigquery import bigquery_table_lineage
+from quollio_core.repository import qdc
+from quollio_core.repository.bigquery import get_credentials, get_org_id
+
+logger = logging.getLogger(__name__)
+
+
+def load_lineage(
+    qdc_client: qdc.QDCExternalAPIClient, project_id: str, regions: list, tenant_id: str, credentials: dict, org_id: str
+):
+    bigquery_table_lineage(
+        qdc_client=qdc_client,
+        tenant_id=tenant_id,
+        project_id=project_id,
+        regions=regions,
+        credentials=credentials,
+        org_id=org_id,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Quollio Intelligence Agent for Google BigQuery",
+        description="Collect lineage and stats from Google BigQuery and load to Quollio Data Catalog",
+        epilog="Copyright (c) 2024 Quollio Technologies, Inc.",
+    )
+    parser.add_argument(
+        "commands",
+        choices=["load_lineage"],
+        type=str,
+        nargs="+",
+        help="""
+        The command to execute.
+        'load_lineage': Load lineage data from Google Data Catalog to Quollio,
+        """,
+    )
+    parser.add_argument(
+        "--credentials",
+        type=str,
+        action=env_default("GOOGLE_APPLICATION_CREDENTIALS"),
+        help="Crendentials for Google Cloud Platform",
+    )
+    parser.add_argument(
+        "--tenant_id",
+        type=str,
+        action=env_default("TENANT_ID"),
+        required=False,
+        help="The tenant id (company id) where the lineage and stats are loaded",
+    )
+    parser.add_argument(
+        "--api_url",
+        type=str,
+        action=env_default("QDC_API_URL"),
+        required=False,
+        help="The base URL of Quollio External API",
+    )
+    parser.add_argument(
+        "--client_id",
+        type=str,
+        action=env_default("QDC_CLIENT_ID"),
+        required=False,
+        help="The client id that is created on Quollio console to let clients access Quollio External API",
+    )
+    parser.add_argument(
+        "--client_secret",
+        type=str,
+        action=env_default("QDC_CLIENT_SECRET"),
+        required=False,
+        help="The client secret that is created on Quollio console to let clients access Quollio External API",
+    )
+    parser.add_argument(
+        "--project_id",
+        type=str,
+        action=env_default("GCP_PROJECT_ID"),
+        required=False,
+        help="GCP Project ID",
+    )
+    parser.add_argument(
+        "--regions",
+        type=str,
+        action=env_default("GCP_REGIONS"),
+        required=False,
+        help="GCP regions where the data is located. Multiple regions can be provided separated by space.",
+        nargs="+",
+    )
+
+    args = parser.parse_args()
+
+    if len(args.commands) == 0:
+        raise ValueError("No command is provided")
+
+    if "load_lineage" in args.commands:
+
+        qdc_client = qdc.QDCExternalAPIClient(
+            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+        )
+
+        credentials_json = json.loads(args.credentials)
+        credentials = get_credentials(credentials_json=credentials_json)
+        org_id = get_org_id(credentials_json=credentials_json)
+
+        load_lineage(
+            qdc_client=qdc_client,
+            project_id=args.project_id,
+            regions=args.regions,
+            tenant_id=args.tenant_id,
+            credentials=credentials,
+            org_id=org_id,
+        )

{quollio_core-0.4.6 → quollio_core-0.4.8}/quollio_core/bricks.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import os
 
-from quollio_core.helper.core import setup_dbt_profile
+from quollio_core.helper.core import setup_dbt_profile, trim_prefix
 from quollio_core.helper.env_default import env_default
 from quollio_core.profilers.databricks import (
     databricks_column_level_lineage,
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
 
 def build_view(
     conn: db.DatabricksConnectionConfig,
-    target_tables: str,
+    target_tables: str = "",
     log_level: str = "info",
 ) -> None:
     logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
@@ -41,7 +41,13 @@ def build_view(
         options=["--no-use-colors", "--log-level", log_level],
     )
 
-    run_options = ["--no-use-colors", "--log-level", log_level, "--select", target_tables]
+    run_options = ["--no-use-colors", "--log-level", log_level]
+
+    if target_tables is not None:
+        target_tables_str = " ".join(target_tables)
+        run_options.append("--select")
+        run_options.append(target_tables_str)
+
     dbt_client.invoke(
         cmd="run",
         project_dir=project_path,
@@ -53,20 +59,35 @@ def build_view(
 
 def load_lineage(
     conn: db.DatabricksConnectionConfig,
+    endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    enable_column_lineage: bool = False,
 ) -> None:
     logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
 
     logger.info("Generate Databricks table to table lineage.")
     databricks_table_level_lineage(
-        conn=conn, qdc_client=qdc_client, tenant_id=tenant_id, dbt_table_name="quollio_lineage_table_level"
+        conn=conn,
+        endpoint=endpoint,
+        qdc_client=qdc_client,
+        tenant_id=tenant_id,
+        dbt_table_name="quollio_lineage_table_level",
     )
 
-    logger.info("Generate Databricks column to column lineage.")
-    databricks_column_level_lineage(
-        conn=conn, qdc_client=qdc_client, tenant_id=tenant_id, dbt_table_name="quollio_lineage_column_level"
-    )
+    if enable_column_lineage:
+        logger.info(
+            f"enable_column_lineage is set to {enable_column_lineage}.Generate Databricks column to column lineage."
+        )
+        databricks_column_level_lineage(
+            conn=conn,
+            endpoint=endpoint,
+            qdc_client=qdc_client,
+            tenant_id=tenant_id,
+            dbt_table_name="quollio_lineage_column_level",
+        )
+    else:
+        logger.info("Skip column lineage ingestion. Set enable_column_lineage to True if you ingest column lineage.")
 
     logger.info("Lineage data is successfully loaded.")
     return
@@ -74,6 +95,7 @@ def load_lineage(
 
 def load_column_stats(
     conn: db.DatabricksConnectionConfig,
+    endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
 ) -> None:
@@ -82,6 +104,7 @@ def load_column_stats(
     logger.info("Generate Databricks column stats.")
     databricks_column_stats(
         conn=conn,
+        endpoint=endpoint,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
     )
@@ -106,7 +129,6 @@ if __name__ == "__main__":
         'build_view': Build views using dbt,
         'load_lineage': Load lineage data from created views to Quollio,
         'load_stats': Load stats from created views to Quollio,
-        'load_sqllineage': Load lineage data from sql parse result(alpha),
         """,
     )
     parser.add_argument(
@@ -193,8 +215,8 @@ if __name__ == "__main__":
     parser.add_argument(
         "--target_tables",
         type=str,
-        nargs="*",
-        choices=["quollio_lineage_table_level", "quollio_lineage_view_level"],
+        nargs="+",
+        choices=["quollio_lineage_table_level", "quollio_lineage_column_level"],
         action=env_default("DATABRICKS_TARGET_TABLES"),
         required=False,
         help="Target tables you want to create with dbt module. \
@@ -202,11 +224,29 @@ if __name__ == "__main__":
               Please specify table name with blank delimiter like tableA tableB \
               if you want to create two or more tables",
     )
+    parser.add_argument(
+        "--monitoring_table_suffix",
+        type=str,
+        action=env_default("DATABRICKS_MONITORING_TABLE_SUFFIX"),
+        required=False,
+        help="Sets the monitoring tables suffix for databricks. \
+              This is used to identify the monitoring tables created by the databricks monitoring tool. \
+              Default value is _profile_metrics",
+    )
+    parser.add_argument(
+        "--enable_column_lineage",
+        type=bool,
+        action=env_default("ENABLE_COLUMN_LINEAGE", store_true=True),
+        default=False,
+        required=False,
+        help="Whether to ingest column lineage into QDIC or not. Default value is False",
+    )
 
     args = parser.parse_args()
 
     conn = db.DatabricksConnectionConfig(
-        host=args.host,
+        # MEMO: Metadata agent allows the string 'https://' as a host name but is not allowed by intelligence agent.
+        host=trim_prefix(args.host, "https://"),
         http_path=args.http_path,
         client_id=args.databricks_client_id,
         client_secret=args.databricks_client_secret,
@@ -228,10 +268,22 @@ if __name__ == "__main__":
         qdc_client = qdc.QDCExternalAPIClient(
             base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
         )
-        load_lineage(conn=conn, qdc_client=qdc_client, tenant_id=args.tenant_id)
+        load_lineage(
+            conn=conn,
+            endpoint=args.host,
+            qdc_client=qdc_client,
+            tenant_id=args.tenant_id,
+            enable_column_lineage=args.enable_column_lineage,
+        )
 
     if "load_stats" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
             base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
         )
-        databricks_column_stats(conn=conn, qdc_client=qdc_client, tenant_id=args.tenant_id)
+        databricks_column_stats(
+            conn=conn,
+            endpoint=args.host,
+            qdc_client=qdc_client,
+            tenant_id=args.tenant_id,
+            monitoring_table_suffix=args.monitoring_table_suffix,
+        )

{quollio_core-0.4.6 → quollio_core-0.4.8}/quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.yml

@@ -1,7 +1,7 @@
 version: 2
 
 model:
-  - name: quollio_lineage_column_level
+  - name: quollio_lineage_table_level
     columns: 
       - name: UPSTREAM_TABLES
         description: 'String column with all upstream tables in JSON format'

quollio_core-0.4.8/quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql

@@ -0,0 +1,136 @@
+{%- materialization divided_view, default %}
+{%- set identifier = model['alias'] %}
+{%- set target_relations = [] %}
+{%- set grant_config = config.get('grants') %}
+
+{{ run_hooks(pre_hooks, inside_transaction=False) }}
+-- `BEGIN` happens here:
+{{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+-- fetch target_tables
+{%- set query_stats_target_tables -%}
+    SELECT
+      distinct
+      database_name
+      , schema_name
+      , table_name
+    FROM
+      {{ ref('quollio_stats_profiling_columns') }}
+    WHERE
+      table_name not like 'quollio_%%'
+{%- endset -%}
+{%- set results = run_query(query_stats_target_tables) -%}
+{%- if execute -%}
+{%- set stats_target_tables = results.rows -%}
+{%- else -%}
+{%- set stats_target_tables = [] -%}
+{%- endif -%}
+
+-- skip creating views if the target profiling columns don't exist.
+{%- if stats_target_tables | length == 0 -%}
+  {% call statement("main") %}
+    {{ log("No records found. Just execute select stmt for skipping call statement.", info=True) }}
+    select null
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+{%- endif -%}
+
+-- build sql
+{%- for stats_target_table in stats_target_tables -%}
+  -- get columns for statistics. 
+  -- LISTAGG function can't be used for sys table, then it's necessary to get column for each table. 
+  -- See https://docs.aws.amazon.com/redshift/latest/dg/c_join_PG.html.
+  {%- set stats_target_columns %}
+      SELECT
+        database_name
+        , schema_name
+        , table_name
+        , column_name
+        , is_bool
+        , is_calculable
+      FROM
+        {{ ref('quollio_stats_profiling_columns') }}
+      WHERE
+        database_name = '{{stats_target_table[0]}}'
+        AND schema_name = '{{stats_target_table[1]}}'
+        AND table_name = '{{stats_target_table[2]}}'
+  {%- endset -%}
+
+  {%- set results = run_query(stats_target_columns) -%}
+  {%- set stats_target_columns = results.rows -%}
+
+  {%- set sql_for_column_stats %}
+  {%- for stats_target_column in stats_target_columns -%}
+    {%- if not loop.first -%}UNION{% endif %}
+    SELECT
+      main.db_name
+      , main.schema_name
+      , main.table_name
+      , main.column_name
+      , main.max_value
+      , main.min_value
+      , main.null_count
+      , main.cardinality
+      , main.avg_value
+      , main.median_value
+      , mode.mode_value
+      , main.stddev_value
+    FROM
+      (
+      SELECT
+        DISTINCT
+        '{{stats_target_column[0]}}'::varchar as db_name
+        , '{{stats_target_column[1]}}'::varchar as schema_name
+        , '{{stats_target_column[2]}}'::varchar as table_name
+        , '{{stats_target_column[3]}}'::varchar as column_name
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(max("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS max_value
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(min("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS min_value
+        -- requires full table scan
+        , {% if var("aggregate_all") == True %}cast(SUM(NVL2("{{stats_target_column[3]}}", 0, 1)) as integer){% else %}null::integer{% endif %} AS null_count
+        , APPROXIMATE COUNT(DISTINCT "{{stats_target_column[3]}}") AS cardinality
+        -- requires full table scan
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(avg("{{stats_target_column[3]}}")as varchar){% else %}null::varchar{% endif %} AS avg_value
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(median("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS median_value
+        -- requires full table scan
+        , {% if stats_target_column[5] == True %}cast(STDDEV_SAMP("{{stats_target_column[3]}}") as integer){% else %}null::integer{% endif %} AS stddev_value
+      FROM {{ stats_target_column[0] }}.{{ stats_target_column[1] }}.{{ stats_target_column[2] }}
+    ) main, (
+      {%- if var("aggregate_all") == True and stats_target_column[4] == false %}
+        SELECT
+          cast("{{stats_target_column[3]}}" as varchar) mode_value
+        FROM (
+           SELECT
+            DISTINCT
+            "{{stats_target_column[3]}}"
+            , ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS row_num
+          FROM {{ stats_target_column[0] }}.{{ stats_target_column[1] }}.{{ stats_target_column[2] }}
+          GROUP BY
+            "{{stats_target_column[3]}}"
+        )
+        WHERE
+          row_num = 1
+      {% else %}
+        SELECT null as mode_value {%- endif -%}
+    ) mode
+  {% endfor -%}
+  {%- endset %}
+  -- create a view with a index as suffix
+  {%- set target_identifier = "%s_%s_%s_%s"|format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) %}
+  {%- set target_relation = api.Relation.create(identifier=target_identifier, schema=schema, database=database, type='view') %}
+  -- {{ drop_relation_if_exists(target_relation) }}
+  {% call statement("main") %}
+    {{ get_replace_view_sql(target_relation, sql_for_column_stats) }}
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+  {%- do apply_grants(target_relation, grant_config, should_revoke) %}
+  {%- set target_relations = target_relations.append(target_relation) %}
+{%- endfor -%}
+
+{{ run_hooks(post_hooks, inside_transaction=True) }}
+{{ adapter.commit() }}
+{{ run_hooks(post_hooks, inside_transaction=False) }}
+
+{{ return({'relations': target_relations}) }}
+{%- endmaterialization -%}

{quollio_core-0.4.6 → quollio_core-0.4.8}/quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql

@@ -1,7 +1,6 @@
 {{
     config(
-        materialized='divided_view',
-        chunk=20
+        materialized='divided_view'
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core-0.4.8/quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -0,0 +1,85 @@
+{%- materialization divided_view, default %}
+{%- set identifier = model['alias'] %}
+{%- set target_relations = [] %}
+{%- set grant_config = config.get('grants') %}
+
+{{ run_hooks(pre_hooks, inside_transaction=False) }}
+-- `BEGIN` happens here:
+{{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+-- fetch target_tables
+{%- set query_stats_target_tables -%}
+    SELECT
+      TABLE_CATALOG
+      , TABLE_SCHEMA
+      , TABLE_NAME
+      , OBJECT_AGG(COLUMN_NAME, IS_CALCULABLE) AS COLUMNS_OBJ
+    FROM
+      {{ ref('quollio_stats_profiling_columns') }}
+    WHERE NOT startswith(table_name, 'QUOLLIO_')
+    GROUP BY
+      TABLE_CATALOG
+      , TABLE_SCHEMA
+      , TABLE_NAME
+{%- endset -%}
+{%- set results = run_query(query_stats_target_tables) -%}
+{%- if execute -%}
+{%- set stats_target_tables = results.rows -%}
+{%- else -%}
+{%- set stats_target_tables = [] -%}
+{%- endif -%}
+
+-- skip creating views if the target profiling columns don't exist.
+{%- if stats_target_tables | length == 0 -%}
+  {% call statement("main") %}
+    {{ log("No records found. Just execute select stmt for skipping call statement.", info=True) }}
+    select null
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+{%- endif -%}
+
+-- create view for each table
+{%- for stats_target_table in stats_target_tables -%}
+  -- build sql for column value aggregation.
+  {%- set sql_for_column_stats %}
+  {% set columns_json = fromjson(stats_target_table[3]) %}
+  {%- for col_name, is_calclable in columns_json.items() -%}
+    {%- if not loop.first %}UNION{% endif %}
+    SELECT
+      DISTINCT
+      '{{stats_target_table[0]}}' as db_name
+      , '{{stats_target_table[1]}}' as schema_name
+      , '{{stats_target_table[2]}}' as table_name
+      , '{{col_name}}' as column_name
+      , {% if is_calclable == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
+      , {% if is_calclable == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
+      , COUNT_IF("{{col_name}}" IS NULL) AS null_count
+      , APPROX_COUNT_DISTINCT("{{col_name}}") AS cardinality
+      , {% if is_calclable == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
+      , {% if is_calclable == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
+      , {% if is_calclable == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
+      , {% if is_calclable == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
+    FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
+  {% endfor -%}
+  {%- endset %}
+
+  -- create a view with a index as suffix
+  {%- set stats_view_identifier = "%s_%s_%s_%s"|format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) %}
+  {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema, database=database, type='view') %}
+  {% call statement("main") %}
+    {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+  {%- do apply_grants(target_relation, grant_config, should_revoke) %}
+  {%- set target_relations = target_relations.append(target_relation) %}
+{%- endfor -%}
+
+{{ run_hooks(post_hooks, inside_transaction=True) }}
+-- `COMMIT` happens here:
+{{ adapter.commit() }}
+{{ run_hooks(post_hooks, inside_transaction=False) }}
+
+{{ return({'relations': target_relations}) }}
+{%- endmaterialization -%}

{quollio_core-0.4.6 → quollio_core-0.4.8}/quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql

@@ -1,7 +1,6 @@
 {{
     config(
-        materialized='divided_view',
-        chunk=20
+        materialized='divided_view'
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core-0.4.8/quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql

@@ -0,0 +1,96 @@
+WITH columns AS (
+    SELECT
+      table_catalog
+      , table_schema
+      , table_name
+      , column_name
+      , data_type
+    FROM
+      {{ source('account_usage', 'COLUMNS') }}
+    WHERE
+      deleted is null
+      AND table_name NOT LIKE 'QUOLLIO_%%'
+    GROUP BY
+      table_catalog
+      , table_schema
+      , table_name
+      , column_name
+      , data_type
+    ORDER BY
+      table_catalog
+      , table_schema
+      , table_name
+), accessible_tables AS (
+    SELECT
+      table_catalog
+      , table_schema
+      , name
+    FROM
+      {{ source('account_usage', 'GRANTS_TO_ROLES') }}
+    WHERE
+      granted_on in ('TABLE', 'MATERIALIZED VIEW')
+      AND grantee_name = '{{ var("query_role") }}'
+      AND privilege in ('SELECT', 'OWNERSHIP', 'REFERENCES')
+      AND deleted_on IS NULL
+    GROUP BY
+      table_catalog
+      , table_schema
+      , name  
+), m_view_sys_columns AS (
+  SELECT
+    cols.table_catalog
+    , cols.table_schema
+    , cols.table_name
+    , cols.column_name
+    , cols.data_type
+  FROM
+    {{ source('account_usage', 'COLUMNS') }} cols
+  LEFT OUTER JOIN
+    {{ source('account_usage', 'TABLES') }} tbls
+  ON
+    cols.table_catalog = tbls.table_catalog
+    AND cols.table_schema = tbls.table_schema
+    AND cols.table_name = tbls.table_name
+  WHERE
+    tbls.table_type = 'MATERIALIZED VIEW'
+    AND cols.column_name = 'SYS_MV_SOURCE_PARTITION'
+), implicit_columns_removed AS (
+  SELECT
+    c.table_catalog
+    , c.table_schema
+    , c.table_name
+    , c.column_name
+    , c.data_type
+  FROM
+    columns c
+  INNER JOIN
+    accessible_tables a
+  ON
+    c.table_catalog = a.table_catalog
+    AND c.table_schema = a.table_schema
+    AND c.table_name = a.name
+  MINUS
+  SELECT
+    table_catalog
+    , table_schema
+    , table_name
+    , column_name
+    , data_type
+  FROM
+    m_view_sys_columns
+), final AS (
+  SELECT
+    table_catalog
+    , table_schema
+    , table_name
+    , column_name
+    , data_type
+    , case when data_type in('NUMBER','DECIMAL', 'DEC', 'NUMERIC',
+                             'INT', 'INTEGER', 'BIGINT', 'SMALLINT',
+                             'TINYINT', 'BYTEINT')
+                             THEN true
+           else false END AS is_calculable
+  FROM
+    implicit_columns_removed
+)
+select * from final