quollio-core 0.4.6__py3-none-any.whl → 0.4.8__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.6"
+__version__ = "0.4.8"
 __author__ = "Quollio Technologies, Inc"

quollio_core/bigquery.py

@@ -0,0 +1,114 @@
+import argparse
+import json
+import logging
+
+from quollio_core.helper.env_default import env_default
+from quollio_core.profilers.bigquery import bigquery_table_lineage
+from quollio_core.repository import qdc
+from quollio_core.repository.bigquery import get_credentials, get_org_id
+
+logger = logging.getLogger(__name__)
+
+
+def load_lineage(
+    qdc_client: qdc.QDCExternalAPIClient, project_id: str, regions: list, tenant_id: str, credentials: dict, org_id: str
+):
+    bigquery_table_lineage(
+        qdc_client=qdc_client,
+        tenant_id=tenant_id,
+        project_id=project_id,
+        regions=regions,
+        credentials=credentials,
+        org_id=org_id,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Quollio Intelligence Agent for Google BigQuery",
+        description="Collect lineage and stats from Google BigQuery and load to Quollio Data Catalog",
+        epilog="Copyright (c) 2024 Quollio Technologies, Inc.",
+    )
+    parser.add_argument(
+        "commands",
+        choices=["load_lineage"],
+        type=str,
+        nargs="+",
+        help="""
+        The command to execute.
+        'load_lineage': Load lineage data from Google Data Catalog to Quollio,
+        """,
+    )
+    parser.add_argument(
+        "--credentials",
+        type=str,
+        action=env_default("GOOGLE_APPLICATION_CREDENTIALS"),
+        help="Crendentials for Google Cloud Platform",
+    )
+    parser.add_argument(
+        "--tenant_id",
+        type=str,
+        action=env_default("TENANT_ID"),
+        required=False,
+        help="The tenant id (company id) where the lineage and stats are loaded",
+    )
+    parser.add_argument(
+        "--api_url",
+        type=str,
+        action=env_default("QDC_API_URL"),
+        required=False,
+        help="The base URL of Quollio External API",
+    )
+    parser.add_argument(
+        "--client_id",
+        type=str,
+        action=env_default("QDC_CLIENT_ID"),
+        required=False,
+        help="The client id that is created on Quollio console to let clients access Quollio External API",
+    )
+    parser.add_argument(
+        "--client_secret",
+        type=str,
+        action=env_default("QDC_CLIENT_SECRET"),
+        required=False,
+        help="The client secret that is created on Quollio console to let clients access Quollio External API",
+    )
+    parser.add_argument(
+        "--project_id",
+        type=str,
+        action=env_default("GCP_PROJECT_ID"),
+        required=False,
+        help="GCP Project ID",
+    )
+    parser.add_argument(
+        "--regions",
+        type=str,
+        action=env_default("GCP_REGIONS"),
+        required=False,
+        help="GCP regions where the data is located. Multiple regions can be provided separated by space.",
+        nargs="+",
+    )
+
+    args = parser.parse_args()
+
+    if len(args.commands) == 0:
+        raise ValueError("No command is provided")
+
+    if "load_lineage" in args.commands:
+
+        qdc_client = qdc.QDCExternalAPIClient(
+            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+        )
+
+        credentials_json = json.loads(args.credentials)
+        credentials = get_credentials(credentials_json=credentials_json)
+        org_id = get_org_id(credentials_json=credentials_json)
+
+        load_lineage(
+            qdc_client=qdc_client,
+            project_id=args.project_id,
+            regions=args.regions,
+            tenant_id=args.tenant_id,
+            credentials=credentials,
+            org_id=org_id,
+        )

quollio_core/bricks.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import os
 
-from quollio_core.helper.core import setup_dbt_profile
+from quollio_core.helper.core import setup_dbt_profile, trim_prefix
 from quollio_core.helper.env_default import env_default
 from quollio_core.profilers.databricks import (
     databricks_column_level_lineage,
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
 
 def build_view(
     conn: db.DatabricksConnectionConfig,
-    target_tables: str,
+    target_tables: str = "",
     log_level: str = "info",
 ) -> None:
     logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
@@ -41,7 +41,13 @@ def build_view(
         options=["--no-use-colors", "--log-level", log_level],
     )
 
-    run_options = ["--no-use-colors", "--log-level", log_level, "--select", target_tables]
+    run_options = ["--no-use-colors", "--log-level", log_level]
+
+    if target_tables is not None:
+        target_tables_str = " ".join(target_tables)
+        run_options.append("--select")
+        run_options.append(target_tables_str)
+
     dbt_client.invoke(
         cmd="run",
         project_dir=project_path,
@@ -53,20 +59,35 @@ def build_view(
 
 def load_lineage(
     conn: db.DatabricksConnectionConfig,
+    endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    enable_column_lineage: bool = False,
 ) -> None:
     logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
 
     logger.info("Generate Databricks table to table lineage.")
     databricks_table_level_lineage(
-        conn=conn, qdc_client=qdc_client, tenant_id=tenant_id, dbt_table_name="quollio_lineage_table_level"
+        conn=conn,
+        endpoint=endpoint,
+        qdc_client=qdc_client,
+        tenant_id=tenant_id,
+        dbt_table_name="quollio_lineage_table_level",
     )
 
-    logger.info("Generate Databricks column to column lineage.")
-    databricks_column_level_lineage(
-        conn=conn, qdc_client=qdc_client, tenant_id=tenant_id, dbt_table_name="quollio_lineage_column_level"
-    )
+    if enable_column_lineage:
+        logger.info(
+            f"enable_column_lineage is set to {enable_column_lineage}.Generate Databricks column to column lineage."
+        )
+        databricks_column_level_lineage(
+            conn=conn,
+            endpoint=endpoint,
+            qdc_client=qdc_client,
+            tenant_id=tenant_id,
+            dbt_table_name="quollio_lineage_column_level",
+        )
+    else:
+        logger.info("Skip column lineage ingestion. Set enable_column_lineage to True if you ingest column lineage.")
 
     logger.info("Lineage data is successfully loaded.")
     return
@@ -74,6 +95,7 @@ def load_lineage(
 
 def load_column_stats(
     conn: db.DatabricksConnectionConfig,
+    endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
 ) -> None:
@@ -82,6 +104,7 @@ def load_column_stats(
     logger.info("Generate Databricks column stats.")
     databricks_column_stats(
         conn=conn,
+        endpoint=endpoint,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
     )
@@ -106,7 +129,6 @@ if __name__ == "__main__":
         'build_view': Build views using dbt,
         'load_lineage': Load lineage data from created views to Quollio,
         'load_stats': Load stats from created views to Quollio,
-        'load_sqllineage': Load lineage data from sql parse result(alpha),
         """,
     )
     parser.add_argument(
@@ -193,8 +215,8 @@ if __name__ == "__main__":
     parser.add_argument(
         "--target_tables",
         type=str,
-        nargs="*",
-        choices=["quollio_lineage_table_level", "quollio_lineage_view_level"],
+        nargs="+",
+        choices=["quollio_lineage_table_level", "quollio_lineage_column_level"],
         action=env_default("DATABRICKS_TARGET_TABLES"),
         required=False,
         help="Target tables you want to create with dbt module. \
@@ -202,11 +224,29 @@ if __name__ == "__main__":
               Please specify table name with blank delimiter like tableA tableB \
               if you want to create two or more tables",
     )
+    parser.add_argument(
+        "--monitoring_table_suffix",
+        type=str,
+        action=env_default("DATABRICKS_MONITORING_TABLE_SUFFIX"),
+        required=False,
+        help="Sets the monitoring tables suffix for databricks. \
+              This is used to identify the monitoring tables created by the databricks monitoring tool. \
+              Default value is _profile_metrics",
+    )
+    parser.add_argument(
+        "--enable_column_lineage",
+        type=bool,
+        action=env_default("ENABLE_COLUMN_LINEAGE", store_true=True),
+        default=False,
+        required=False,
+        help="Whether to ingest column lineage into QDIC or not. Default value is False",
+    )
 
     args = parser.parse_args()
 
     conn = db.DatabricksConnectionConfig(
-        host=args.host,
+        # MEMO: Metadata agent allows the string 'https://' as a host name but is not allowed by intelligence agent.
+        host=trim_prefix(args.host, "https://"),
         http_path=args.http_path,
         client_id=args.databricks_client_id,
         client_secret=args.databricks_client_secret,
@@ -228,10 +268,22 @@ if __name__ == "__main__":
         qdc_client = qdc.QDCExternalAPIClient(
             base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
         )
-        load_lineage(conn=conn, qdc_client=qdc_client, tenant_id=args.tenant_id)
+        load_lineage(
+            conn=conn,
+            endpoint=args.host,
+            qdc_client=qdc_client,
+            tenant_id=args.tenant_id,
+            enable_column_lineage=args.enable_column_lineage,
+        )
 
     if "load_stats" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
             base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
         )
-        databricks_column_stats(conn=conn, qdc_client=qdc_client, tenant_id=args.tenant_id)
+        databricks_column_stats(
+            conn=conn,
+            endpoint=args.host,
+            qdc_client=qdc_client,
+            tenant_id=args.tenant_id,
+            monitoring_table_suffix=args.monitoring_table_suffix,
+        )

quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.yml

@@ -1,7 +1,7 @@
 version: 2
 
 model:
-  - name: quollio_lineage_column_level
+  - name: quollio_lineage_table_level
     columns: 
       - name: UPSTREAM_TABLES
         description: 'String column with all upstream tables in JSON format'

quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql

@@ -1,28 +1,67 @@
 {%- materialization divided_view, default %}
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
-{%- set chunk = config.get('chunk') %}
 {%- set grant_config = config.get('grants') %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
 -- `BEGIN` happens here:
 {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
--- fetch records
-{%- set query_quollio_stats_profiling_columns -%}
-SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE table_name not like 'quollio_%'
+-- fetch target_tables
+{%- set query_stats_target_tables -%}
+    SELECT
+      distinct
+      database_name
+      , schema_name
+      , table_name
+    FROM
+      {{ ref('quollio_stats_profiling_columns') }}
+    WHERE
+      table_name not like 'quollio_%%'
 {%- endset -%}
-{%- set results = run_query(query_quollio_stats_profiling_columns) -%}
+{%- set results = run_query(query_stats_target_tables) -%}
 {%- if execute -%}
-{%- set records = results.rows -%}
+{%- set stats_target_tables = results.rows -%}
 {%- else -%}
-{%- set records = [] -%}
+{%- set stats_target_tables = [] -%}
+{%- endif -%}
+
+-- skip creating views if the target profiling columns don't exist.
+{%- if stats_target_tables | length == 0 -%}
+  {% call statement("main") %}
+    {{ log("No records found. Just execute select stmt for skipping call statement.", info=True) }}
+    select null
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
 {%- endif -%}
 
 -- build sql
-{%- for i in range(0, records|length, chunk) -%}
-  {%- set build_sql %}
-  {%- for record in records[i: i+chunk] -%}
+{%- for stats_target_table in stats_target_tables -%}
+  -- get columns for statistics. 
+  -- LISTAGG function can't be used for sys table, then it's necessary to get column for each table. 
+  -- See https://docs.aws.amazon.com/redshift/latest/dg/c_join_PG.html.
+  {%- set stats_target_columns %}
+      SELECT
+        database_name
+        , schema_name
+        , table_name
+        , column_name
+        , is_bool
+        , is_calculable
+      FROM
+        {{ ref('quollio_stats_profiling_columns') }}
+      WHERE
+        database_name = '{{stats_target_table[0]}}'
+        AND schema_name = '{{stats_target_table[1]}}'
+        AND table_name = '{{stats_target_table[2]}}'
+  {%- endset -%}
+
+  {%- set results = run_query(stats_target_columns) -%}
+  {%- set stats_target_columns = results.rows -%}
+
+  {%- set sql_for_column_stats %}
+  {%- for stats_target_column in stats_target_columns -%}
     {%- if not loop.first -%}UNION{% endif %}
     SELECT
       main.db_name
@@ -41,33 +80,33 @@ SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE table_name no
       (
       SELECT
         DISTINCT
-        '{{record[0]}}'::varchar as db_name
-        , '{{record[1]}}'::varchar as schema_name
-        , '{{record[2]}}'::varchar as table_name
-        , '{{record[3]}}'::varchar as column_name
-        , {% if var("skip_heavy") == false and record[5] == true %}cast(max("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS max_value
-        , {% if var("skip_heavy") == false and record[5] == true %}cast(min("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS min_value
+        '{{stats_target_column[0]}}'::varchar as db_name
+        , '{{stats_target_column[1]}}'::varchar as schema_name
+        , '{{stats_target_column[2]}}'::varchar as table_name
+        , '{{stats_target_column[3]}}'::varchar as column_name
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(max("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS max_value
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(min("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS min_value
         -- requires full table scan
-        , {% if var("skip_heavy") == false %}cast(SUM(NVL2("{{record[3]}}", 0, 1)) as integer){% else %}null::integer{% endif %} AS null_count
-        , APPROXIMATE COUNT(DISTINCT "{{record[3]}}") AS cardinality
+        , {% if var("aggregate_all") == True %}cast(SUM(NVL2("{{stats_target_column[3]}}", 0, 1)) as integer){% else %}null::integer{% endif %} AS null_count
+        , APPROXIMATE COUNT(DISTINCT "{{stats_target_column[3]}}") AS cardinality
         -- requires full table scan
-        , {% if var("skip_heavy") == false and record[5] == true %}cast(avg("{{record[3]}}")as varchar){% else %}null::varchar{% endif %} AS avg_value
-        , {% if var("skip_heavy") == false and record[5] == true %}cast(median("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS median_value
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(avg("{{stats_target_column[3]}}")as varchar){% else %}null::varchar{% endif %} AS avg_value
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(median("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS median_value
         -- requires full table scan
-        , {% if record[5] == true %}cast(STDDEV_SAMP("{{record[3]}}") as integer){% else %}null::integer{% endif %} AS stddev_value
-      FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }}
+        , {% if stats_target_column[5] == True %}cast(STDDEV_SAMP("{{stats_target_column[3]}}") as integer){% else %}null::integer{% endif %} AS stddev_value
+      FROM {{ stats_target_column[0] }}.{{ stats_target_column[1] }}.{{ stats_target_column[2] }}
     ) main, (
-      {%- if var("skip_heavy") == false and record[4] == false %}
+      {%- if var("aggregate_all") == True and stats_target_column[4] == false %}
         SELECT
-          cast("{{record[3]}}" as varchar) mode_value
+          cast("{{stats_target_column[3]}}" as varchar) mode_value
         FROM (
            SELECT
             DISTINCT
-            "{{record[3]}}"
+            "{{stats_target_column[3]}}"
             , ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS row_num
-          FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }}
+          FROM {{ stats_target_column[0] }}.{{ stats_target_column[1] }}.{{ stats_target_column[2] }}
           GROUP BY
-            "{{record[3]}}"
+            "{{stats_target_column[3]}}"
         )
         WHERE
           row_num = 1
@@ -77,11 +116,11 @@ SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE table_name no
   {% endfor -%}
   {%- endset %}
   -- create a view with a index as suffix
-  {%- set target_identifier = "%s_%d"|format(model['name'], loop.index) %}
+  {%- set target_identifier = "%s_%s_%s_%s"|format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) %}
   {%- set target_relation = api.Relation.create(identifier=target_identifier, schema=schema, database=database, type='view') %}
   -- {{ drop_relation_if_exists(target_relation) }}
   {% call statement("main") %}
-    {{ get_replace_view_sql(target_relation, build_sql) }}
+    {{ get_replace_view_sql(target_relation, sql_for_column_stats) }}
   {% endcall %}
   {%- set full_refresh_mode = (should_full_refresh()) -%}
   {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}

quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql

@@ -1,7 +1,6 @@
 {{
     config(
-        materialized='divided_view',
-        chunk=20
+        materialized='divided_view'
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -1,51 +1,74 @@
 {%- materialization divided_view, default %}
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
-{%- set chunk = config.get('chunk') %}
 {%- set grant_config = config.get('grants') %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
 -- `BEGIN` happens here:
 {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
--- fetch records
-{%- set query_quollio_stats_profiling_columns -%}
-SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE NOT startswith(table_name, 'QUOLLIO_')
+-- fetch target_tables
+{%- set query_stats_target_tables -%}
+    SELECT
+      TABLE_CATALOG
+      , TABLE_SCHEMA
+      , TABLE_NAME
+      , OBJECT_AGG(COLUMN_NAME, IS_CALCULABLE) AS COLUMNS_OBJ
+    FROM
+      {{ ref('quollio_stats_profiling_columns') }}
+    WHERE NOT startswith(table_name, 'QUOLLIO_')
+    GROUP BY
+      TABLE_CATALOG
+      , TABLE_SCHEMA
+      , TABLE_NAME
 {%- endset -%}
-{%- set results = run_query(query_quollio_stats_profiling_columns) -%}
+{%- set results = run_query(query_stats_target_tables) -%}
 {%- if execute -%}
-{%- set records = results.rows -%}
+{%- set stats_target_tables = results.rows -%}
 {%- else -%}
-{%- set records = [] -%}
+{%- set stats_target_tables = [] -%}
+{%- endif -%}
+
+-- skip creating views if the target profiling columns don't exist.
+{%- if stats_target_tables | length == 0 -%}
+  {% call statement("main") %}
+    {{ log("No records found. Just execute select stmt for skipping call statement.", info=True) }}
+    select null
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
 {%- endif -%}
 
--- build sql
-{%- for i in range(0, records|length, chunk) -%}
-  {%- set build_sql %}
-  {%- for record in records[i: i+chunk] -%}
+-- create view for each table
+{%- for stats_target_table in stats_target_tables -%}
+  -- build sql for column value aggregation.
+  {%- set sql_for_column_stats %}
+  {% set columns_json = fromjson(stats_target_table[3]) %}
+  {%- for col_name, is_calclable in columns_json.items() -%}
     {%- if not loop.first %}UNION{% endif %}
     SELECT
       DISTINCT
-      '{{record[0]}}' as db_name
-      , '{{record[1]}}' as schema_name
-      , '{{record[2]}}' as table_name
-      , '{{record[3]}}' as column_name
-      , {% if record[5] == true %}CAST(max("{{record[3]}}") AS STRING){% else %}null{% endif %} AS max_value
-      , {% if record[5] == true %}CAST(min("{{record[3]}}") AS STRING){% else %}null{% endif %} AS min_value
-      , COUNT_IF("{{record[3]}}" IS NULL) AS null_count
-      , APPROX_COUNT_DISTINCT("{{record[3]}}") AS cardinality
-      , {% if record[5] == true %}avg("{{record[3]}}"){% else %}null{% endif %} AS avg_value
-      , {% if record[5] == true %}median("{{record[3]}}"){% else %}null{% endif %} AS median_value
-      , {% if record[5] == true %}approx_top_k("{{record[3]}}")[0][0]{% else %}null{% endif %} AS mode_value
-      , {% if record[5] == true %}stddev("{{record[3]}}"){% else %}null{% endif %} AS stddev_value
-    FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }} {{ var("sample_method") }}
+      '{{stats_target_table[0]}}' as db_name
+      , '{{stats_target_table[1]}}' as schema_name
+      , '{{stats_target_table[2]}}' as table_name
+      , '{{col_name}}' as column_name
+      , {% if is_calclable == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
+      , {% if is_calclable == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
+      , COUNT_IF("{{col_name}}" IS NULL) AS null_count
+      , APPROX_COUNT_DISTINCT("{{col_name}}") AS cardinality
+      , {% if is_calclable == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
+      , {% if is_calclable == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
+      , {% if is_calclable == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
+      , {% if is_calclable == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
+    FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
   {% endfor -%}
   {%- endset %}
+
   -- create a view with a index as suffix
-  {%- set target_identifier = "%s_%d"|format(model['name'], loop.index) %}
-  {%- set target_relation = api.Relation.create(identifier=target_identifier, schema=schema, database=database, type='view') %}
+  {%- set stats_view_identifier = "%s_%s_%s_%s"|format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) %}
+  {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema, database=database, type='view') %}
   {% call statement("main") %}
-    {{ get_create_view_as_sql(target_relation, build_sql) }}
+    {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
   {% endcall %}
   {%- set full_refresh_mode = (should_full_refresh()) -%}
   {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}

quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql

@@ -1,7 +1,6 @@
 {{
     config(
-        materialized='divided_view',
-        chunk=20
+        materialized='divided_view'
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql

@@ -36,24 +36,61 @@ WITH columns AS (
       table_catalog
       , table_schema
       , name  
+), m_view_sys_columns AS (
+  SELECT
+    cols.table_catalog
+    , cols.table_schema
+    , cols.table_name
+    , cols.column_name
+    , cols.data_type
+  FROM
+    {{ source('account_usage', 'COLUMNS') }} cols
+  LEFT OUTER JOIN
+    {{ source('account_usage', 'TABLES') }} tbls
+  ON
+    cols.table_catalog = tbls.table_catalog
+    AND cols.table_schema = tbls.table_schema
+    AND cols.table_name = tbls.table_name
+  WHERE
+    tbls.table_type = 'MATERIALIZED VIEW'
+    AND cols.column_name = 'SYS_MV_SOURCE_PARTITION'
+), implicit_columns_removed AS (
+  SELECT
+    c.table_catalog
+    , c.table_schema
+    , c.table_name
+    , c.column_name
+    , c.data_type
+  FROM
+    columns c
+  INNER JOIN
+    accessible_tables a
+  ON
+    c.table_catalog = a.table_catalog
+    AND c.table_schema = a.table_schema
+    AND c.table_name = a.name
+  MINUS
+  SELECT
+    table_catalog
+    , table_schema
+    , table_name
+    , column_name
+    , data_type
+  FROM
+    m_view_sys_columns
+), final AS (
+  SELECT
+    table_catalog
+    , table_schema
+    , table_name
+    , column_name
+    , data_type
+    , case when data_type in('NUMBER','DECIMAL', 'DEC', 'NUMERIC',
+                             'INT', 'INTEGER', 'BIGINT', 'SMALLINT',
+                             'TINYINT', 'BYTEINT')
+                             THEN true
+           else false END AS is_calculable
+  FROM
+    implicit_columns_removed
 )
-
-SELECT
-  c.table_catalog
-  , c.table_schema
-  , c.table_name
-  , c.column_name
-  , c.data_type
-  , case when c.data_type in('NUMBER','DECIMAL', 'DEC', 'NUMERIC',
-                           'INT', 'INTEGER', 'BIGINT', 'SMALLINT',
-                           'TINYINT', 'BYTEINT')
-                           THEN true
-         else false END AS is_calculable
-FROM
-  columns c
-INNER JOIN
-  accessible_tables a
-ON
-  c.table_catalog = a.table_catalog
-  AND c.table_schema = a.table_schema
-  AND c.table_name = a.name
+select * from final

quollio_core/helper/core.py

@@ -31,3 +31,7 @@ def setup_dbt_profile(connections_json: Dict[str, str], template_path: str, temp
     with open(profile_path, "w") as profiles:
         yaml.dump(yaml.safe_load(profiles_body), profiles, default_flow_style=False, allow_unicode=True)
     return
+
+
+def trim_prefix(s: str, prefix: str) -> str:
+    return s.lstrip(prefix)

quollio_core/helper/env_default.py

@@ -6,6 +6,8 @@ Currently requires explicit naming of env vars to check for
 
 import argparse
 import os
+from distutils.util import strtobool
+from typing import Union
 
 
 # Courtesy of http://stackoverflow.com/a/10551190 with env-var retrieval fixed
@@ -28,9 +30,30 @@ class EnvDefault(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
+class EnvStoreTrue(argparse._StoreTrueAction):
+    """An argparse action class that auto-sets missing default values from env vars for store_true."""
+
+    def __init__(self, envvar, required=True, default=None, **kwargs):
+        # Only pass the arguments that argparse._StoreTrueAction expects
+        action_kwargs = {key: value for key, value in kwargs.items() if key in ("option_strings", "dest")}
+        if envvar in os.environ:
+            default = _convert_value_to_bool(os.environ[envvar])
+        if required and default:
+            required = False
+        super(EnvStoreTrue, self).__init__(default=default, required=required, **action_kwargs)
+
+
 # functional sugar for the above
-def env_default(envvar):
+def env_default(envvar, store_true=False):
     def wrapper(**kwargs):
+        if store_true:
+            return EnvStoreTrue(envvar, **kwargs)
         return EnvDefault(envvar, **kwargs)
 
     return wrapper
+
+
+def _convert_value_to_bool(v: Union[str, bool]) -> bool:
+    if isinstance(v, str):
+        return bool(strtobool(v))
+    return v

quollio_core/profilers/bigquery.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Any, Dict, List
+
+from quollio_core.profilers.lineage import gen_table_lineage_payload, parse_bigquery_table_lineage
+from quollio_core.repository import qdc
+from quollio_core.repository.bigquery import BigQueryClient, GCPLineageClient, get_entitiy_reference, get_search_request
+
+logger = logging.getLogger(__name__)
+
+
+def bigquery_table_lineage(
+    qdc_client: qdc.QDCExternalAPIClient,
+    tenant_id: str,
+    project_id: str,
+    regions: list,
+    org_id: str,
+    credentials: Any,
+):
+    lineage_client = GCPLineageClient(credentials)
+    bq_client = BigQueryClient(credentials)
+
+    datasets = bq_client.list_datasets(project_id)
+    all_tables = generate_table_list(datasets, bq_client)
+    lineage_links = generate_lineage_links(all_tables, lineage_client, project_id, regions)
+    lineage_links = parse_bigquery_table_lineage(lineage_links)
+
+    update_table_lineage_inputs = gen_table_lineage_payload(tenant_id=tenant_id, endpoint=org_id, tables=lineage_links)
+
+    req_count = 0
+    for update_table_lineage_input in update_table_lineage_inputs:
+        logger.info(
+            "Generating table lineage. downstream: %s -> %s-> %s",
+            update_table_lineage_input.downstream_database_name,
+            update_table_lineage_input.downstream_schema_name,
+            update_table_lineage_input.downstream_table_name,
+        )
+        status_code = qdc_client.update_lineage_by_id(
+            global_id=update_table_lineage_input.downstream_global_id,
+            payload=update_table_lineage_input.upstreams.as_dict(),
+        )
+        if status_code == 200:
+            req_count += 1
+    logger.info("Generating table lineage is finished. %s lineages are ingested.", req_count)
+
+
+def generate_table_list(datasets: List[str], bq_client: BigQueryClient) -> List[str]:
+    all_tables = []
+    for dataset in datasets:
+        all_tables.extend(
+            [
+                table
+                for table in bq_client.list_tables(dataset.dataset_id)
+                if table.table_type in ["TABLE", "VIEW", "MATERIALIZED_VIEW"]
+            ]
+        )
+
+    all_table_names = []
+    for table in all_tables:
+        all_table_names.append(f"{table.project}.{table.dataset_id}.{table.table_id}")
+
+    return all_table_names
+
+
+def generate_lineage_links(
+    all_tables: List[str], lineage_client: GCPLineageClient, project_id: str, regions: List[str]
+) -> Dict[str, List[str]]:
+    lineage_links = {}
+    for table in all_tables:
+        downstream = get_entitiy_reference()
+        downstream.fully_qualified_name = f"bigquery:{table}"
+
+        for region in regions:
+            request = get_search_request(downstream_table=downstream, project_id=project_id, region=region)
+            response = lineage_client.get_links(request=request)
+            for lineage in response:
+                target_table = str(lineage.target.fully_qualified_name).replace("bigquery:", "")
+                if target_table not in lineage_links:
+                    lineage_links[target_table] = []
+                lineage_links[target_table].append(str(lineage.source.fully_qualified_name).replace("bigquery:", ""))
+
+    return lineage_links

quollio_core/profilers/databricks.py

@@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
 
 def databricks_table_level_lineage(
     conn: databricks.DatabricksConnectionConfig,
+    endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     dbt_table_name: str = "quollio_lineage_table_level",
@@ -31,7 +32,7 @@ def databricks_table_level_lineage(
         tables = parse_databricks_table_lineage(results)
         update_table_lineage_inputs = gen_table_lineage_payload(
             tenant_id=tenant_id,
-            endpoint=conn.host,
+            endpoint=endpoint,
             tables=tables,
         )
 
@@ -55,6 +56,7 @@ def databricks_table_level_lineage(
 
 def databricks_column_level_lineage(
     conn: databricks.DatabricksConnectionConfig,
+    endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     dbt_table_name: str = "quollio_lineage_column_level",
@@ -72,7 +74,7 @@ def databricks_column_level_lineage(
 
     update_column_lineage_inputs = gen_column_lineage_payload(
         tenant_id=tenant_id,
-        endpoint=conn.host,
+        endpoint=endpoint,
         columns=results,
     )
 
@@ -99,7 +101,7 @@ def databricks_column_level_lineage(
 
 
 def _get_monitoring_tables(
-    conn: databricks.DatabricksConnectionConfig, monitoring_table_id: str = "_profile_metrics"
+    conn: databricks.DatabricksConnectionConfig, monitoring_table_suffix: str = "_profile_metrics"
 ) -> List[Dict[str, str]]:
     tables = []
     query = f"""
@@ -110,7 +112,9 @@ def _get_monitoring_tables(
             CONCAT(table_catalog, '.', table_schema, '.', table_name) AS table_fqdn
         FROM
             system.information_schema.tables
-        WHERE table_name LIKE "%{monitoring_table_id}"
+        WHERE
+            table_name LIKE "%{monitoring_table_suffix}"
+            AND table_name NOT LIKE ('quollio_%')
         """
     with databricks.DatabricksQueryExecutor(config=conn) as databricks_executor:
         tables = databricks_executor.get_query_results(query)
@@ -123,9 +127,9 @@ def _get_monitoring_tables(
 
 
 def _get_column_stats(
-    conn: databricks.DatabricksConnectionConfig, monitoring_table_id: str = "_profile_metrics"
+    conn: databricks.DatabricksConnectionConfig, monitoring_table_suffix: str = "_profile_metrics"
 ) -> List[Dict[str, str]]:
-    tables = _get_monitoring_tables(conn, monitoring_table_id)
+    tables = _get_monitoring_tables(conn, monitoring_table_suffix)
     if not tables:
         return []
     stats = []
@@ -153,6 +157,8 @@ def _get_column_stats(
                     MAX(t.window) AS LATEST
                 FROM
                     {monitoring_table} t
+                WHERE
+                    t.column_name not in (':table')
                 GROUP BY
                     t.COLUMN_NAME,
                     t.DATA_TYPE,
@@ -176,13 +182,14 @@ def _get_column_stats(
 
 def databricks_column_stats(
     conn: databricks.DatabricksConnectionConfig,
+    endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
-    monitoring_table_id: str = "_profile_metrics",
+    monitoring_table_suffix: str = "_profile_metrics",
 ) -> None:
-    table_stats = _get_column_stats(conn, monitoring_table_id)
+    table_stats = _get_column_stats(conn, monitoring_table_suffix)
     for table in table_stats:
-        stats = gen_table_stats_payload(tenant_id, conn.host, table)
+        stats = gen_table_stats_payload(tenant_id=tenant_id, endpoint=endpoint, stats=table)
         for stat in stats:
             status_code = qdc_client.update_stats_by_id(
                 global_id=stat.global_id,

quollio_core/profilers/lineage.py

@@ -153,3 +153,17 @@ def parse_databricks_table_lineage(results: List) -> List[Dict[str, Dict]]:
         payload["UPSTREAM_TABLES"] = json.loads(result["UPSTREAM_TABLES"])
         payloads.append(payload)
     return payloads
+
+
+def parse_bigquery_table_lineage(tables: Dict) -> List[Dict[str, Dict]]:
+    payloads = list()
+    for downstream, upstream in tables.items():
+        payload = {
+            "DOWNSTREAM_TABLE_NAME": "",
+            "UPSTREAM_TABLES": [],
+        }
+        payload["DOWNSTREAM_TABLE_NAME"] = downstream
+        for upstream_table in upstream:
+            payload["UPSTREAM_TABLES"].append({"upstream_object_name": upstream_table})
+        payloads.append(payload)
+    return payloads

quollio_core/redshift.py

@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
 
 def build_view(
     conn: redshift.RedshiftConnectionConfig,
-    skip_heavy: bool,
+    aggregate_all: bool = False,
     target_tables: str = "",
     log_level: str = "info",
 ) -> None:
@@ -29,9 +29,9 @@ def build_view(
     project_path = f"{current_dir}/dbt_projects/redshift"
     template_path = f"{current_dir}/dbt_projects/redshift/profiles"
     template_name = "profiles_template.yml"
-    options = '{{"query_user": {query_user}, "skip_heavy": {skip_heavy}, "target_database": {database}}}'.format(
+    options = '{{"query_user": {query_user}, "aggregate_all": {aggregate_all}, "target_database": {database}}}'.format(
         query_user=conn.query_user,
-        skip_heavy=skip_heavy,
+        aggregate_all=aggregate_all,
         database=conn.database,
     )
 
@@ -210,12 +210,12 @@ if __name__ == "__main__":
         help="Target schema name where the views are built by dbt",
     )
     parser.add_argument(
-        "--skip_heavy",
+        "--aggregate_all",
         type=bool,
-        action=env_default("REDSHIFT_SKIP_HEAVY"),
-        default=True,
+        action=env_default("REDSHIFT_AGGREGATE_ALL", store_true=True),
+        default=False,
         required=False,
-        help="Skip heavy queries when building views by dbt",
+        help="Aggregate all stats values. False by default.",
     )
     parser.add_argument(
         "--target_tables",
@@ -283,7 +283,7 @@ if __name__ == "__main__":
     if "build_view" in args.commands:
         build_view(
             conn=conn,
-            skip_heavy=args.skip_heavy,
+            aggregate_all=args.aggregate_all,
             target_tables=args.target_tables,
             log_level=args.log_level,
         )

quollio_core/repository/bigquery.py

@@ -0,0 +1,61 @@
+import logging
+
+from google.cloud.bigquery import Client
+from google.cloud.datacatalog_lineage_v1 import EntityReference, LineageClient, SearchLinksRequest
+from google.oauth2.service_account import Credentials
+from googleapiclient.discovery import build
+
+logger = logging.getLogger(__name__)
+
+
+class BigQueryClient:
+    def __init__(self, credentials: Credentials) -> None:
+        self.client = self.__initialze(credentials=credentials)
+
+    def __initialze(self, credentials: Credentials) -> Client:
+        client = Client(credentials=credentials)
+        return client
+
+    def list_datasets(self, project_id) -> list:
+        datasets = list(self.client.list_datasets(project_id))
+        logger.debug("Found %s datasets in project %s", len(datasets), project_id)
+        return datasets
+
+    def list_tables(self, dataset_id) -> list:
+        tables = list(self.client.list_tables(dataset_id))
+        logger.debug("Found %s tables in dataset %s", len(tables), dataset_id)
+        return list(self.client.list_tables(dataset_id))
+
+
+class GCPLineageClient:
+    def __init__(self, credentials: Credentials) -> None:
+        self.client = self.__initialze(credentials=credentials)
+
+    def __initialze(self, credentials: Credentials) -> LineageClient:
+        client = LineageClient(credentials=credentials)
+        return client
+
+    def get_links(self, request: SearchLinksRequest) -> list:
+        response = self.client.search_links(request)
+        return response.links
+
+
+def get_entitiy_reference() -> EntityReference:
+    return EntityReference()
+
+
+def get_search_request(downstream_table: EntityReference, project_id: str, region: str) -> SearchLinksRequest:
+    return SearchLinksRequest(target=downstream_table, parent=f"projects/{project_id}/locations/{region.lower()}")
+
+
+def get_credentials(credentials_json: dict) -> Credentials:
+    return Credentials.from_service_account_info(credentials_json)
+
+
+def get_org_id(credentials_json: dict) -> str:
+    credentials = get_credentials(credentials_json)
+    crm_service = build("cloudresourcemanager", "v1", credentials=credentials)
+    project_id = credentials_json["project_id"]
+    project = crm_service.projects().get(projectId=project_id).execute()
+    org_id = project["parent"]["id"]
+    return org_id

quollio_core/snowflake.py

@@ -72,6 +72,7 @@ def load_lineage(
     conn: snowflake.SnowflakeConnectionConfig,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    enable_column_lineage: bool = False,
 ) -> None:
     logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
 
@@ -82,12 +83,17 @@ def load_lineage(
         tenant_id=tenant_id,
     )
 
-    logger.info("Generate Snowflake column to column lineage.")
-    snowflake_column_to_column_lineage(
-        conn=conn,
-        qdc_client=qdc_client,
-        tenant_id=tenant_id,
-    )
+    if enable_column_lineage:
+        logger.info(
+            f"enable_column_lineage is set to {enable_column_lineage}.Generate Snowflake column to column lineage."
+        )
+        snowflake_column_to_column_lineage(
+            conn=conn,
+            qdc_client=qdc_client,
+            tenant_id=tenant_id,
+        )
+    else:
+        logger.info("Skip column lineage ingestion. Set enable_column_lineage to True if you ingest column lineage.")
 
     logger.info("Lineage data is successfully loaded.")
 
@@ -264,6 +270,14 @@ if __name__ == "__main__":
         required=False,
         help="The client secrete that is created on Quollio console to let clients access Quollio External API",
     )
+    parser.add_argument(
+        "--enable_column_lineage",
+        type=bool,
+        action=env_default("ENABLE_COLUMN_LINEAGE", store_true=True),
+        default=False,
+        required=False,
+        help="Whether to ingest column lineage into QDIC or not. Default value is False",
+    )
     args = parser.parse_args()
     conn = snowflake.SnowflakeConnectionConfig(
         account_id=args.account_id,
@@ -296,6 +310,7 @@ if __name__ == "__main__":
             conn=conn,
             qdc_client=qdc_client,
             tenant_id=args.tenant_id,
+            enable_column_lineage=args.enable_column_lineage,
         )
     if "load_stats" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

{quollio_core-0.4.6.dist-info → quollio_core-0.4.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.6
+Version: 0.4.8
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>
@@ -31,6 +31,11 @@ Requires-Dist: snowflake-connector-python==3.5.0
 Requires-Dist: databricks-sdk==0.17.0
 Requires-Dist: databricks-sql-connector==2.9.5
 Requires-Dist: sqlglot==20.8.0
+Requires-Dist: google-cloud==0.34.0
+Requires-Dist: google-cloud-bigquery==3.22.0
+Requires-Dist: google-cloud-datacatalog==3.19.0
+Requires-Dist: google-cloud-datacatalog-lineage==0.3.6
+Requires-Dist: google-api-python-client==2.131.0
 Requires-Dist: black>=22.3.0 ; extra == "test"
 Requires-Dist: coverage>=7.3.2 ; extra == "test"
 Requires-Dist: isort>=5.10.1 ; extra == "test"
@@ -74,7 +79,7 @@ To see available commands and options, please run the following command. (ex: Sn
 コマンドやオプションの詳細については、下記のコマンドを実行してください。(例: Snowflake)
 
 ```
-$ python3 -m quollio_core.snowflake -h
+$ python -m quollio_core.snowflake -h
 ```
 
 Then run commands with the options provided.

{quollio_core-0.4.6.dist-info → quollio_core-0.4.8.dist-info}/RECORD

@@ -1,7 +1,8 @@
-quollio_core/__init__.py,sha256=ruo7YFCZ8vY6Xsa1V1XTfT-ulLkKuB1As4n1BJcylfs,83
-quollio_core/bricks.py,sha256=PCHyh_I6M4PBRpLDtc5DTr7rpharllu-vcSAhySM4xg,8001
-quollio_core/redshift.py,sha256=wap7QmV-YuHZAomIrHXytGUuxhQ5MFEb38QDY3XrThQ,10167
-quollio_core/snowflake.py,sha256=8IMbdTjCDBIiS_GF8APWRTVWNj6EM3ZT8MRN12T-1v0,10266
+quollio_core/__init__.py,sha256=bdxhTj5XUDW12XHez-8E9LL3JCLZ2EQpEibqkSXg8xQ,83
+quollio_core/bigquery.py,sha256=2DrUMo4evcH4BHiUtnY48IjmsdAsQMoPGtNx8SRoyzQ,3528
+quollio_core/bricks.py,sha256=BVwh9clJMXe_YXbd78ku6Y9470cYYE3CflcuCRzrY3I,9689
+quollio_core/redshift.py,sha256=wVqtNnv1fmZ5QwYq4A2JVLHH3VrCYPEZBx39NoCqRIM,10220
+quollio_core/snowflake.py,sha256=WRtnXxc01AaV4oTa382MTppYvmOn4mOY7jFDNDnzxDA,10922
 quollio_core/dbt_projects/databricks/.gitignore,sha256=1jJAyXSzJ3YUm0nx3i7wUSE4RjQMX3ad6F8O88UbtzI,29
 quollio_core/dbt_projects/databricks/README.md,sha256=ZpRQyhFAODAiS8dc1Kb_ndkul4cu4o4udN_EMa49CU4,440
 quollio_core/dbt_projects/databricks/dbt_project.yml,sha256=3sH98RNk7TnphvI3yEdXDstb92kW5BNxr-cT0tXhwzk,480
@@ -12,7 +13,7 @@ quollio_core/dbt_projects/databricks/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5
 quollio_core/dbt_projects/databricks/models/quollio_lineage_column_level.sql,sha256=mZ4mDCEZTwiSgCUr-w2QGze2-NQapt45EyQNQkCOI5I,2171
 quollio_core/dbt_projects/databricks/models/quollio_lineage_column_level.yml,sha256=tidAK_FMhYYuPTxFoactwcXYQPSMZwQTxWrGBly4-1o,450
 quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.sql,sha256=K63J7n7NIM2Jc7c4IF21JcW8AYOm9HxBNDiveUE4kzU,1558
-quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.yml,sha256=ZGjz6C2bguDJxJyA7LhCHbuyZSRPEaRMXln9rxcotuo,344
+quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.yml,sha256=0nQhdU5gBlHLAJHwsPotgCVrm6tReBdEdczvnxnvmpo,343
 quollio_core/dbt_projects/databricks/models/sources.yml,sha256=JXU-8lNsKm8dxIjmWos1vbTsWiea-9-pXnntik63ZpA,2231
 quollio_core/dbt_projects/databricks/profiles/profiles_template.yml,sha256=Dw1RuTrE04yvGIaPQL7uc6pgSWloKHhu0KrduzJ1Z6M,353
 quollio_core/dbt_projects/databricks/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,14 +24,14 @@ quollio_core/dbt_projects/redshift/package-lock.yml,sha256=Gef3zDCLF41j_FL-_h3sI
 quollio_core/dbt_projects/redshift/packages.yml,sha256=p9Bl2C44gdC6iYTUkz_15yq3xahSJf2IA3WOXLF_ahA,61
 quollio_core/dbt_projects/redshift/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/redshift/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql,sha256=3tRQeXXdjn3aSZ94DgMN6A6yMlTCE8aMKXeIdcZVDIM,3998
+quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql,sha256=rSKZJHpdU7vy0HvrI10nmnjFMkB5OI3oRmHfAvkjlyk,5686
 quollio_core/dbt_projects/redshift/models/quollio_lineage_table_level.sql,sha256=AVPcNXfVYHwyutJzg61QT_VF9umfoC4i8C2HecAU4d4,2042
 quollio_core/dbt_projects/redshift/models/quollio_lineage_table_level.yml,sha256=UcrXpUTT3ihBHKPljvjw8xHz-ND60PfvMJaXqGKOEic,236
 quollio_core/dbt_projects/redshift/models/quollio_lineage_view_level.sql,sha256=A0CTgQwlz8InabA0cHuygV2GMZGYuAa7Zd5DIUOYzQI,1289
 quollio_core/dbt_projects/redshift/models/quollio_lineage_view_level.yml,sha256=7Npwo3svL9715HpNU2MKzRI014Da4tIStLzAHmd0UaU,235
 quollio_core/dbt_projects/redshift/models/quollio_sqllineage_sources.sql,sha256=e0A_Wqv_OcC8gG_yzTbI59vT-4vCI3JiAzFlmkvLnMk,1049
 quollio_core/dbt_projects/redshift/models/quollio_sqllineage_sources.yml,sha256=qgazupx3ca4P8R0loY5F9hyCz2fmAcWqZ6iOySo_NoY,377
-quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql,sha256=lH8xPmAzSW-6wi_g1y_LFVhtFgHzBvTweVX-MKeJzUQ,302
+quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql,sha256=BzvP9gKMFItmwqEQ4bDgtS-Invxhhe6L73Qe1ucxfHo,284
 quollio_core/dbt_projects/redshift/models/quollio_stats_columns.yml,sha256=V_BESPk6IqE52ExT26-78As9l9AlWW86-Geb5PIhThU,67
 quollio_core/dbt_projects/redshift/models/quollio_stats_profiling_columns.sql,sha256=IPmHf51Er2jE9cMQHybT4adRxwwi2CEmgrBSv1Oeduc,1592
 quollio_core/dbt_projects/redshift/models/quollio_stats_profiling_columns.yml,sha256=s-p9F44TdwoFYlQN-b9gHzcFYOMqhqDGA9ORS_M4lhs,523
@@ -45,38 +46,40 @@ quollio_core/dbt_projects/snowflake/package-lock.yml,sha256=Gef3zDCLF41j_FL-_h3s
 quollio_core/dbt_projects/snowflake/packages.yml,sha256=p9Bl2C44gdC6iYTUkz_15yq3xahSJf2IA3WOXLF_ahA,61
 quollio_core/dbt_projects/snowflake/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=T4nFL76AbuQHBiLSAvNchoJnRjb1IRj4nToOyTNvLvw,2782
+quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=BC4fPMw2qEc39WeJbimalHeIj-klcDk1EF-zbcYfk0w,3704
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.sql,sha256=Cxt2U2aXNG_LUm63jwTyxUkapkrB7_uHmesx1PTcMJM,4721
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.yml,sha256=a2uNIAh-xw51eu-GmHVuAnGnTbwK7h8-DjDeQtK3KaQ,711
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_table_level.sql,sha256=Q_7vY1N1Hi1LFv5CxkkdR3gQw8fTDnoKECTLSK4gd3o,5112
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_table_level.yml,sha256=QXlMBIkHo1Y-ANveKVx1FwyoYTMRXKgE2Z-PNouhQTw,325
 quollio_core/dbt_projects/snowflake/models/quollio_sqllineage_sources.sql,sha256=gd6JhQO13xBIvOoeXcce1I7amNGytwE8pwUApXehwqM,1520
 quollio_core/dbt_projects/snowflake/models/quollio_sqllineage_sources.yml,sha256=qgazupx3ca4P8R0loY5F9hyCz2fmAcWqZ6iOySo_NoY,377
-quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql,sha256=lH8xPmAzSW-6wi_g1y_LFVhtFgHzBvTweVX-MKeJzUQ,302
+quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql,sha256=BzvP9gKMFItmwqEQ4bDgtS-Invxhhe6L73Qe1ucxfHo,284
 quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.yml,sha256=V_BESPk6IqE52ExT26-78As9l9AlWW86-Geb5PIhThU,67
-quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql,sha256=kt2aFimIPkgKI_UQTjvfRlAjrdSbO8z6C_749pnXrnE,1382
+quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql,sha256=ex5ax-KJoM_P1QspkolOUOQg9BazTdZO1Jllp08PQo8,2265
 quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.yml,sha256=W39VAmFnnX6RBoW7B_4CConC1lm0Jm9o50Jsz9bYZzY,538
 quollio_core/dbt_projects/snowflake/models/sources.yml,sha256=vGSV33cNj4UUyPUcYS-JFgc3r8KvSLfiA7qhbDCUU9s,10975
 quollio_core/dbt_projects/snowflake/profiles/profiles_template.yml,sha256=gcZsgdGP461QuUM9jLbBKdadT8cHTXgNarq_azOOMhk,379
 quollio_core/dbt_projects/snowflake/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/snapshots/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/helper/core.py,sha256=-3vCDlKExWPHJmWuZQNpYnvPP55uoGwRpTtnFvsDxIo,1127
-quollio_core/helper/env_default.py,sha256=YIL9hfrPs1ViL1AXohnbWEjVBUDXbVVakH0ZoSZWOlc,1202
+quollio_core/helper/core.py,sha256=wbu4FWI7YiFEttXGSuj3tMyAhtPAFlHOjDpWJGNXOHA,1202
+quollio_core/helper/env_default.py,sha256=H6gbSGUPrEDZr4YDrL49hbOpw6RntI4U82kX1q6vUnI,2148
 quollio_core/profilers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/profilers/databricks.py,sha256=oX1N_FmXvdMpCbZj4xRyGwcbsMQr_C6VOMGZrlBV44E,7257
-quollio_core/profilers/lineage.py,sha256=HrTjXxrchETRmHEb5tSFzzHdb6z2KMw-DTnUSeKxmr0,6379
+quollio_core/profilers/bigquery.py,sha256=e1Y8cZR-LxI9mSsYb0DurQyy0eCjM_kAKLfvl4IuQLE,3262
+quollio_core/profilers/databricks.py,sha256=aNUIk48GuwSDEnmNzhZ6yv6TqAc1dBJt9e81NiTGBjo,7496
+quollio_core/profilers/lineage.py,sha256=4FyxIuPBrUFihqZryqTQBcfB0Z7634lKl_WwkD82vzE,6865
 quollio_core/profilers/redshift.py,sha256=obdHVIsOM1bwHGdvYKalsJcTXwLK02kAKQMSBzSvsDo,7862
 quollio_core/profilers/snowflake.py,sha256=C1LC19ZaUMwNoXjsbnez0xANydJYs8oNRt6tixWKDq8,9090
 quollio_core/profilers/sqllineage.py,sha256=oCyl4tpXL5bkfguXAzTHSB9kZBL3tQK_rfcJ4XQMrLo,5177
 quollio_core/profilers/stats.py,sha256=PG1NbbUSpc1JuEYvBzD66rd24tp0C13_Y5Y7vRjYG1c,4720
 quollio_core/repository/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+quollio_core/repository/bigquery.py,sha256=KMJTeF4OUxtaJt0ymoJ4tkrMKq8yLyMYaMxNvU5yd_Y,2271
 quollio_core/repository/databricks.py,sha256=m68tja5N-QxH3VqEq-mOJKBeR2qldSgj_L9iIxvWwm0,1945
 quollio_core/repository/dbt.py,sha256=HXqW_xa4xYPh9CnKkg4L1gwG3SGjj2BAYoWgzWMFU4U,770
 quollio_core/repository/qdc.py,sha256=VCmzAUvjLemw1os5TaPtfBFkMCOMuPeftjZmUPhFj2Y,4702
 quollio_core/repository/redshift.py,sha256=UVHIpYzDQ2AbBTAGa8DgmEenG0NZsHfYroR1MmEPQGA,2991
 quollio_core/repository/snowflake.py,sha256=1YVMDfb9euJKvikv1pk_IxVF6SVsiemSvZ-WMTSbY7E,1874
-quollio_core-0.4.6.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.4.6.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-quollio_core-0.4.6.dist-info/METADATA,sha256=iXDj06XhRtzAmJ6rTQGYWxBbHiof6TOzTcU-202wguc,6571
-quollio_core-0.4.6.dist-info/RECORD,,
+quollio_core-0.4.8.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.4.8.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+quollio_core-0.4.8.dist-info/METADATA,sha256=BRWWUVTFdakMWC4muSecvVCYOkNYM8u5BZipXVUFqJE,6803
+quollio_core-0.4.8.dist-info/RECORD,,