quollio-core 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.3"
+__version__ = "0.4.5"
 __author__ = "Quollio Technologies, Inc"

quollio_core/bricks.py

@@ -0,0 +1,237 @@
+import argparse
+import logging
+import os
+
+from quollio_core.helper.core import setup_dbt_profile
+from quollio_core.helper.env_default import env_default
+from quollio_core.profilers.databricks import (
+    databricks_column_level_lineage,
+    databricks_column_stats,
+    databricks_table_level_lineage,
+)
+from quollio_core.repository import databricks as db
+from quollio_core.repository import dbt, qdc
+
+logger = logging.getLogger(__name__)
+
+
+def build_view(
+    conn: db.DatabricksConnectionConfig,
+    target_tables: str,
+    log_level: str = "info",
+) -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+
+    logger.info("Build profiler views using dbt")
+    # set parameters
+    dbt_client = dbt.DBTClient()
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    project_path = f"{current_dir}/dbt_projects/databricks"
+    template_path = f"{current_dir}/dbt_projects/databricks/profiles"
+    template_name = "profiles_template.yml"
+
+    # build views using dbt
+    setup_dbt_profile(connections_json=conn.as_dict(), template_path=template_path, template_name=template_name)
+    # FIXME: when executing some of the commands, directory changes due to the library bug.
+    # https://github.com/dbt-labs/dbt-core/issues/8997
+    dbt_client.invoke(
+        cmd="deps",
+        project_dir=project_path,
+        profile_dir=template_path,
+        options=["--no-use-colors", "--log-level", log_level],
+    )
+
+    run_options = ["--no-use-colors", "--log-level", log_level, "--select", target_tables]
+    dbt_client.invoke(
+        cmd="run",
+        project_dir=project_path,
+        profile_dir=template_path,
+        options=run_options,
+    )
+    return
+
+
+def load_lineage(
+    conn: db.DatabricksConnectionConfig,
+    qdc_client: qdc.QDCExternalAPIClient,
+    tenant_id: str,
+) -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+
+    logger.info("Generate Databricks table to table lineage.")
+    databricks_table_level_lineage(
+        conn=conn, qdc_client=qdc_client, tenant_id=tenant_id, dbt_table_name="quollio_lineage_table_level"
+    )
+
+    logger.info("Generate Databricks column to column lineage.")
+    databricks_column_level_lineage(
+        conn=conn, qdc_client=qdc_client, tenant_id=tenant_id, dbt_table_name="quollio_lineage_column_level"
+    )
+
+    logger.info("Lineage data is successfully loaded.")
+    return
+
+
+def load_column_stats(
+    conn: db.DatabricksConnectionConfig,
+    qdc_client: qdc.QDCExternalAPIClient,
+    tenant_id: str,
+) -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+
+    logger.info("Generate Databricks column stats.")
+    databricks_column_stats(
+        conn=conn,
+        qdc_client=qdc_client,
+        tenant_id=tenant_id,
+    )
+
+    logger.info("Column stats are successfully loaded.")
+    return
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="Quollio Intelligence Agent for Databricks",
+        description="Build views and load lineage and stats to Quollio from Databricks using dbt.",
+        epilog="Copyright (c) 2024 Quollio Technologies, Inc.",
+    )
+    parser.add_argument(
+        "commands",
+        choices=["build_view", "load_lineage", "load_stats"],
+        type=str,
+        nargs="+",
+        help="""
+        The command to execute.
+        'build_view': Build views using dbt,
+        'load_lineage': Load lineage data from created views to Quollio,
+        'load_stats': Load stats from created views to Quollio,
+        'load_sqllineage': Load lineage data from sql parse result(alpha),
+        """,
+    )
+    parser.add_argument(
+        "--host", type=str, action=env_default("DATABRICKS_HOST"), required=False, help="Host for Databricks workspace"
+    )
+    parser.add_argument(
+        "--http_path",
+        type=str,
+        action=env_default("DATABRICKS_HTTP_PATH"),
+        required=False,
+        help="HTTP path for a Databricks compute resource (i.e warehouse)",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        action=env_default("DATABRICKS_PORT"),
+        required=False,
+        help="Port for Databricks compute resource",
+    )
+    parser.add_argument(
+        "--databricks_client_secret",
+        type=str,
+        action=env_default("DATABRICKS_CLIENT_SECRET"),
+        required=False,
+        help="Secret for the service principal",
+    )
+    parser.add_argument(
+        "--databricks_client_id",
+        type=str,
+        action=env_default("DATABRICKS_CLIENT_ID"),
+        required=False,
+        help="Client id for the service principal",
+    )
+    parser.add_argument(
+        "--catalog",
+        type=str,
+        required=False,
+        action=env_default("DATABRICKS_TARGET_CATALOG"),
+        help="Target database name where the views are built by dbt",
+    )
+    parser.add_argument(
+        "--schema",
+        type=str,
+        action=env_default("DATABRICKS_TARGET_SCHEMA"),
+        required=False,
+        help="Target schema name where the views are built by dbt",
+    )
+    parser.add_argument(
+        "--log_level",
+        type=str,
+        choices=["debug", "info", "warn", "error", "none"],
+        action=env_default("LOG_LEVEL"),
+        required=False,
+        help="The log level for dbt commands. Default value is info",
+    )
+    parser.add_argument(
+        "--api_url",
+        type=str,
+        action=env_default("QDC_API_URL"),
+        required=False,
+        help="The base URL of Quollio External API",
+    )
+    parser.add_argument(
+        "--client_id",
+        type=str,
+        action=env_default("QDC_CLIENT_ID"),
+        required=False,
+        help="The client id that is created on Quollio console to let clients access Quollio External API",
+    )
+    parser.add_argument(
+        "--client_secret",
+        type=str,
+        action=env_default("QDC_CLIENT_SECRET"),
+        required=False,
+        help="The client secrete that is created on Quollio console to let clients access Quollio External API",
+    )
+    parser.add_argument(
+        "--tenant_id",
+        type=str,
+        action=env_default("TENANT_ID"),
+        required=False,
+        help="The tenant id (company id) where the lineage and stats are loaded",
+    )
+    parser.add_argument(
+        "--target_tables",
+        type=str,
+        nargs="*",
+        choices=["quollio_lineage_table_level", "quollio_lineage_view_level"],
+        action=env_default("DATABRICKS_TARGET_TABLES"),
+        required=False,
+        help="Target tables you want to create with dbt module. \
+              You need to specify this parameter if you want to specify tables, not all ones. \
+              Please specify table name with blank delimiter like tableA tableB \
+              if you want to create two or more tables",
+    )
+
+    args = parser.parse_args()
+
+    conn = db.DatabricksConnectionConfig(
+        host=args.host,
+        http_path=args.http_path,
+        client_id=args.databricks_client_id,
+        client_secret=args.databricks_client_secret,
+        catalog=args.catalog,
+        schema=args.schema,
+    )
+
+    if len(args.commands) == 0:
+        raise ValueError("No command is provided")
+
+    if "build_view" in args.commands:
+        build_view(
+            conn=conn,
+            target_tables=args.target_tables,
+            log_level=args.log_level,
+        )
+
+    if "load_lineage" in args.commands:
+        qdc_client = qdc.QDCExternalAPIClient(
+            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+        )
+        load_lineage(conn=conn, qdc_client=qdc_client, tenant_id=args.tenant_id)
+
+    if "load_stats" in args.commands:
+        qdc_client = qdc.QDCExternalAPIClient(
+            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+        )
+        databricks_column_stats(conn=conn, qdc_client=qdc_client, tenant_id=args.tenant_id)

quollio_core/dbt_projects/databricks/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

quollio_core/dbt_projects/databricks/README.md

@@ -0,0 +1,5 @@
+###  Quollio Intelligence Agent Support For Databricks
+Notable Files:
+1. [quollio_lineage_table_level.sql](models/quollio_lineage_table_level.sql) - Generates table lineage data from Databricks system tables.
+2. [quollio_lineage_column_level.sql](models/quollio_lineage_table_level.sql) - Generates column lineage data from Databricks system tables.
+3. [sources.yml](models/sources.yml) - Refrences sources in the Databricks system catalog.

quollio_core/dbt_projects/databricks/dbt_project.yml

@@ -0,0 +1,21 @@
+name: 'quollio_intelligence_databricks'
+version: '1.0.0'
+config-version: 2
+
+profile: 'quollio_intelligence_databricks'
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  +dbt-osmosis: "{model}.yml"
+  # Databricks automatically enables grants on SQL endpoints
+  # https://docs.getdbt.com/reference/resource-configs/grants

quollio_core/dbt_projects/databricks/models/quollio_lineage_column_level.sql

@@ -0,0 +1,73 @@
+-- Gets  full table lineage from Databricks
+WITH columns_lineage_history AS (
+  SELECT
+    -- The databricks columns table does not have a full table name, create with CONCAT()
+    source_table_full_name AS upstream_table,
+    target_table_full_name as downstream_table,
+    source_column_name as upstream_column,
+    target_column_name as downstream_column,
+    event_time,
+    RANK() OVER (
+      PARTITION BY target_table_full_name
+      ORDER BY
+        event_time DESC
+    ) AS rank
+  FROM
+    {{ source('access','column_lineage')  }}
+  WHERE
+    source_table_full_name IS NOT NULL
+    AND target_table_full_name IS NOT NULL
+    AND source_table_full_name NOT LIKE "%quollio%"
+    AND target_table_full_name NOT LIKE "%quollio%"
+),
+-- Gets list of existing columns in catalogs
+existing_columns (
+  SELECT
+    CONCAT(table_catalog, '.', table_schema, '.', table_name) AS table_full_name,
+    column_name
+  FROM
+   {{ source('inf_sch','columns')  }}
+),
+
+-- Checks if the downstream tables exists and group operations.
+downstream_column_exists (
+  SELECT
+    upstream_table AS UPSTREAM_TABLE_NAME,
+    upstream_column AS UPSTREAM_COLUMN_NAME,
+    downstream_table AS DOWNSTREAM_TABLE_NAME,
+    downstream_column AS DOWNSTREAM_COLUMN_NAME,
+    event_time
+  FROM
+    columns_lineage_history clh
+    INNER JOIN existing_columns ec ON clh.downstream_table = ec.table_full_name
+    AND clh.downstream_column = ec.column_name
+  WHERE
+    rank = 1
+  GROUP BY UPSTREAM_TABLE, UPSTREAM_COLUMN, DOWNSTREAM_TABLE, DOWNSTREAM_COLUMN, EVENT_TIME
+),
+
+-- Aggregates the column lineage
+aggregated_column_lineage AS (
+  SELECT
+      downstream_table_name,
+      downstream_column_name,
+      collect_set(
+          named_struct(
+              'upstream_table_name', upstream_table_name,
+              'upstream_column_name', upstream_column_name
+          )
+      ) AS upstream_columns
+  FROM
+    downstream_column_exists
+  GROUP BY
+      downstream_table_name,
+      downstream_column_name
+)
+
+SELECT
+  downstream_table_name AS DOWNSTREAM_TABLE_NAME,
+  downstream_column_name AS DOWNSTREAM_COLUMN_NAME,
+  to_json(upstream_columns) AS UPSTREAM_COLUMNS
+FROM
+  aggregated_column_lineage
+  

quollio_core/dbt_projects/databricks/models/quollio_lineage_column_level.yml

@@ -0,0 +1,14 @@
+version: 2
+
+model:
+  - name: quollio_lineage_column_level
+    columns: 
+      - name: UPSTREAM_COLUMNS
+        description: 'String column with all upstream columns in JSON format'
+        type: string
+      - name: DOWNSTREAM_TABLE_NAME
+        description: 'Full downstream table name in <catalog>.<schema>.<table> format'
+        type: string
+      - name: DOWNSTREAM_COLUMN_NAME
+        description: 'Downstream column name'
+        type: string

quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.sql

@@ -0,0 +1,63 @@
+-- Gets  full table lineage from Databricks
+WITH table_lineage_history AS (
+  SELECT
+    source_table_full_name as upstream_table,
+    target_table_full_name as downstream_table,
+    target_type,
+    event_time,
+    RANK() OVER (
+      PARTITION BY target_table_full_name
+      ORDER BY
+        event_time DESC
+    ) AS rank
+  FROM
+    {{ source('access','table_lineage')  }}
+  WHERE
+    source_table_full_name IS NOT NULL
+    AND target_table_full_name IS NOT NULL
+    AND source_table_full_name NOT LIKE "%quollio%"
+    AND target_table_full_name NOT LIKE "%quollio%"
+),
+-- Gets list of existing tables in catalogs
+existing_tables (
+  SELECT
+    CONCAT(table_catalog, '.', table_schema, '.', table_name) AS table_full_name
+  FROM
+   {{ source('inf_sch','tables')  }}
+),
+
+-- Checks if the downstream tables exists and group operations.
+downstream_table_exists (
+  SELECT
+    upstream_table,
+    downstream_table,
+    target_type,
+    event_time
+  FROM
+    table_lineage_history tlh
+    INNER JOIN existing_tables et ON tlh.downstream_table = et.table_full_name
+  WHERE
+    rank = 1
+  GROUP BY upstream_table, downstream_table, target_type, event_time
+),
+
+aggregated_table_lineage AS (
+  SELECT
+      downstream_table,
+      collect_set(
+          named_struct(
+              'upstream_object_name', upstream_table
+          )
+      ) AS upstream_tables
+  FROM
+    downstream_table_exists
+  GROUP BY
+      downstream_table
+)
+SELECT
+  downstream_table as DOWNSTREAM_TABLE_NAME,
+  to_json(upstream_tables) as UPSTREAM_TABLES
+
+FROM
+  aggregated_table_lineage
+  

quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.yml

@@ -0,0 +1,11 @@
+version: 2
+
+model:
+  - name: quollio_lineage_column_level
+    columns: 
+      - name: UPSTREAM_TABLES
+        description: 'String column with all upstream tables in JSON format'
+        type: string
+      - name: DOWNSTREAM_TABLE_NAME
+        description: 'Full downstream table name in <catalog>.<schema>.<table> format'
+        type: string

quollio_core/dbt_projects/databricks/models/sources.yml

@@ -0,0 +1,84 @@
+version: 2
+
+sources: 
+  - name: access
+    database: system
+    schema: access
+    tables: 
+      - name: table_lineage
+        description: Describes table level lineage
+        columns:
+          - name: source_table_full_name
+            description: ''
+            type: string
+          - name: target_table_full_name
+            description: ''
+            type: string
+          - name: target_type
+            description: ''
+            type: string
+          - name: event_time
+            description: ''
+            type: timestamp
+
+      - name: column_lineage
+        description: Describes column level lineage
+        columns:
+          - name: source_table_full_name
+            description: ''
+            type: string
+          - name: target_table_full_name
+            description: ''
+            type: string
+          - name: event_time
+            description: ''
+            type: timestamp
+          - name: source_column_name
+            description: ''
+            type: string
+          - name: target_column_name
+            description: ''
+            type: string
+  
+  - name: inf_sch
+    database: system
+    schema: information_schema
+    tables: 
+      - name: tables
+        description: Lists existing tables (i.e., not deleted).
+        columns:
+          - name: table_catalog
+            description: ''
+            type: string
+          - name: table_schema
+            description: ''
+            type: string
+          - name: table_name
+            description: ''
+            type: string
+
+      - name: views
+        description: Lists existing views (i.e., not deleted). Views are treated as tables.
+        columns:
+          - name: table_catalog
+            description: ''
+            type: string
+          - name: table_schema
+            description: ''
+            type: string
+          - name: table_name
+            description: ''
+            type: string
+
+      - name: columns
+        description: ''
+        columns:
+          - name: table_catalog
+            description: ''
+            type: string
+          - name: table_schema
+            description: ''
+            type: string
+          - name: table_name
+            description: ''
+            type: string

quollio_core/dbt_projects/databricks/package-lock.yml

@@ -0,0 +1,14 @@
+packages:
+- package: dbt-labs/dbt_utils
+  version: 1.1.1
+- package: dbt-labs/spark_utils
+  version: 0.3.0
+- package: dbt-labs/codegen
+  version: 0.12.1
+- package: dbt-labs/dbt_external_tables
+  version: 0.8.7
+- package: dbt-labs/dbt_project_evaluator
+  version: 0.8.1
+- package: brooklyn-data/dbt_artifacts
+  version: 2.6.2
+sha1_hash: cbb324267dbf6c6fb7de11b162e4fbafd1e32a9c

quollio_core/dbt_projects/databricks/packages.yml

@@ -0,0 +1,13 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: [">=0.0.0", "<2.0.0"]
+  - package: dbt-labs/spark_utils
+    version: [">=0.0.0", "<1.0.0"]
+  - package: dbt-labs/codegen
+    version: [">=0.0.0", "<1.0.0"]
+  - package: dbt-labs/dbt_external_tables
+    version: [">=0.0.0", "<1.0.0"]
+  - package: dbt-labs/dbt_project_evaluator
+    version: [">=0.0.0", "<1.0.0"]
+  - package: brooklyn-data/dbt_artifacts
+    version: [">=2.0.0", "<3.0.0"]

quollio_core/dbt_projects/databricks/profiles/profiles_template.yml

@@ -0,0 +1,14 @@
+quollio_intelligence_databricks:
+  target: project
+  outputs:
+    project:
+      type: databricks
+      host: {{ host }}
+      http_path: {{ http_path }}
+      catalog: {{ catalog }}
+      schema: {{ schema }}
+      auth_type: oauth
+      client_id: {{ client_id }}
+      client_secret: {{ client_secret }}
+      databricks_port: {{ databricks_port }}
+

quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql

@@ -2,6 +2,7 @@
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
 {%- set chunk = config.get('chunk') %}
+{%- set grant_config = config.get('grants') %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
 -- `BEGIN` happens here:
@@ -22,29 +23,57 @@ SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE table_name no
 {%- for i in range(0, records|length, chunk) -%}
   {%- set build_sql %}
   {%- for record in records[i: i+chunk] -%}
-    {%- if not loop.first %}UNION{% endif %}
+    {%- if not loop.first -%}UNION{% endif %}
     SELECT
-      DISTINCT
-      '{{record[0]}}'::varchar as db_name
-      , '{{record[1]}}'::varchar as schema_name
-      , '{{record[2]}}'::varchar as table_name
-      , '{{record[3]}}'::varchar as column_name
-      , {% if var("skip_heavy") == false and record[5] == true %}cast(max("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS max_value
-      , {% if var("skip_heavy") == false and record[5] == true %}cast(min("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS min_value
-      -- requires full table scan
-      , {% if var("skip_heavy") == false %}cast(SUM(NVL2("{{record[3]}}", 0, 1)) as integer){% else %}null::integer{% endif %} AS null_count
-      , APPROXIMATE COUNT(DISTINCT "{{record[3]}}") AS cardinality
-      -- requires full table scan
-      , {% if var("skip_heavy") == false and record[5] == true %}cast(avg("{{record[3]}}")as varchar){% else %}null::varchar{% endif %} AS avg_value
-      , {% if var("skip_heavy") == false and record[5] == true %}(SELECT cast(median("{{record[3]}}") as varchar) FROM {{record[2]}}){% else %}null::varchar{% endif %} AS median_value
-      -- requires full table scan
-      , {% if var("skip_heavy") == false and record[4] == false %}
-          (SELECT cast("{{record[3]}}" as varchar) FROM (
-            SELECT "{{record[3]}}", ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS row_num FROM {{record[2]}} GROUP BY "{{record[3]}}"
-          ) WHERE row_num = 1)
-        {% else %}null::varchar{% endif %} AS mode_value
-      , {% if record[5] == true %}cast(STDDEV_SAMP("{{record[3]}}") as integer){% else %}null::integer{% endif %} AS stddev_value
-    FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }}
+      main.db_name
+      , main.schema_name
+      , main.table_name
+      , main.column_name
+      , main.max_value
+      , main.min_value
+      , main.null_count
+      , main.cardinality
+      , main.avg_value
+      , main.median_value
+      , mode.mode_value
+      , main.stddev_value
+    FROM
+      (
+      SELECT
+        DISTINCT
+        '{{record[0]}}'::varchar as db_name
+        , '{{record[1]}}'::varchar as schema_name
+        , '{{record[2]}}'::varchar as table_name
+        , '{{record[3]}}'::varchar as column_name
+        , {% if var("skip_heavy") == false and record[5] == true %}cast(max("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS max_value
+        , {% if var("skip_heavy") == false and record[5] == true %}cast(min("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS min_value
+        -- requires full table scan
+        , {% if var("skip_heavy") == false %}cast(SUM(NVL2("{{record[3]}}", 0, 1)) as integer){% else %}null::integer{% endif %} AS null_count
+        , APPROXIMATE COUNT(DISTINCT "{{record[3]}}") AS cardinality
+        -- requires full table scan
+        , {% if var("skip_heavy") == false and record[5] == true %}cast(avg("{{record[3]}}")as varchar){% else %}null::varchar{% endif %} AS avg_value
+        , {% if var("skip_heavy") == false and record[5] == true %}cast(median("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS median_value
+        -- requires full table scan
+        , {% if record[5] == true %}cast(STDDEV_SAMP("{{record[3]}}") as integer){% else %}null::integer{% endif %} AS stddev_value
+      FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }}
+    ) main, (
+      {%- if var("skip_heavy") == false and record[4] == false %}
+        SELECT
+          cast("{{record[3]}}" as varchar) mode_value
+        FROM (
+           SELECT
+            DISTINCT
+            "{{record[3]}}"
+            , ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS row_num
+          FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }}
+          GROUP BY
+            "{{record[3]}}"
+        )
+        WHERE
+          row_num = 1
+      {% else %}
+        SELECT null as mode_value {%- endif -%}
+    ) mode
   {% endfor -%}
   {%- endset %}
   -- create a view with a index as suffix
@@ -54,6 +83,9 @@ SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE table_name no
   {% call statement("main") %}
     {{ get_replace_view_sql(target_relation, build_sql) }}
   {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+  {%- do apply_grants(target_relation, grant_config, should_revoke) %}
   {%- set target_relations = target_relations.append(target_relation) %}
 {%- endfor -%}
 

quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql

@@ -1,7 +1,7 @@
 {{
     config(
         materialized='divided_view',
-        chunk=1000
+        chunk=20
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core/dbt_projects/redshift/package-lock.yml

@@ -1,4 +1,4 @@
 packages:
 - package: dbt-labs/dbt_utils
   version: 1.1.1
-sha1_hash: 23451c207c1d4dd71b5925f12a5cd66f2ebb2b3b
+sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353

quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -2,6 +2,7 @@
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
 {%- set chunk = config.get('chunk') %}
+{%- set grant_config = config.get('grants') %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
 -- `BEGIN` happens here:
@@ -46,6 +47,9 @@ SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE NOT startswit
   {% call statement("main") %}
     {{ get_create_view_as_sql(target_relation, build_sql) }}
   {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+  {%- do apply_grants(target_relation, grant_config, should_revoke) %}
   {%- set target_relations = target_relations.append(target_relation) %}
 {%- endfor -%}
 

quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql

@@ -1,7 +1,7 @@
 {{
     config(
         materialized='divided_view',
-        chunk=1000
+        chunk=20
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core/helper/env_default.py

@@ -16,7 +16,10 @@ class EnvDefault(argparse.Action):
     def __init__(self, envvar, required=True, default=None, **kwargs):
         # override values if envvar exists
         if envvar in os.environ:
-            default = os.environ[envvar]
+            if kwargs.get("nargs", None) is None:
+                default = os.environ[envvar]
+            else:
+                default = os.environ[envvar].split(" ")
         if required and default:
             required = False
         super(EnvDefault, self).__init__(default=default, required=required, **kwargs)

quollio_core/profilers/databricks.py

@@ -0,0 +1,196 @@
+import logging
+from typing import Dict, List
+
+from quollio_core.profilers.lineage import (
+    gen_column_lineage_payload,
+    gen_table_lineage_payload,
+    parse_databricks_table_lineage,
+)
+from quollio_core.profilers.stats import gen_table_stats_payload
+from quollio_core.repository import databricks, qdc
+
+logger = logging.getLogger(__name__)
+
+
+def databricks_table_level_lineage(
+    conn: databricks.DatabricksConnectionConfig,
+    qdc_client: qdc.QDCExternalAPIClient,
+    tenant_id: str,
+    dbt_table_name: str = "quollio_lineage_table_level",
+) -> None:
+    logging.basicConfig(level=logging.info, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+    with databricks.DatabricksQueryExecutor(config=conn) as databricks_executor:
+        results = databricks_executor.get_query_results(
+            query=f"""
+            SELECT
+                DOWNSTREAM_TABLE_NAME,
+                UPSTREAM_TABLES
+            FROM {conn.catalog}.{conn.schema}.{dbt_table_name}
+            """
+        )
+        tables = parse_databricks_table_lineage(results)
+        update_table_lineage_inputs = gen_table_lineage_payload(
+            tenant_id=tenant_id,
+            endpoint=conn.host,
+            tables=tables,
+        )
+
+        req_count = 0
+        for update_table_lineage_input in update_table_lineage_inputs:
+            logger.info(
+                "Generating table lineage. downstream: %s -> %s-> %s",
+                update_table_lineage_input.downstream_database_name,
+                update_table_lineage_input.downstream_schema_name,
+                update_table_lineage_input.downstream_table_name,
+            )
+            status_code = qdc_client.update_lineage_by_id(
+                global_id=update_table_lineage_input.downstream_global_id,
+                payload=update_table_lineage_input.upstreams.as_dict(),
+            )
+            if status_code == 200:
+                req_count += 1
+        logger.info("Generating table lineage is finished. %s lineages are ingested.", req_count)
+    return
+
+
+def databricks_column_level_lineage(
+    conn: databricks.DatabricksConnectionConfig,
+    qdc_client: qdc.QDCExternalAPIClient,
+    tenant_id: str,
+    dbt_table_name: str = "quollio_lineage_column_level",
+) -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+    with databricks.DatabricksQueryExecutor(config=conn) as databricks_executor:
+        results = databricks_executor.get_query_results(
+            query=f"""
+            SELECT
+                *
+            FROM
+                {conn.catalog}.{conn.schema}.{dbt_table_name}
+            """
+        )
+
+    update_column_lineage_inputs = gen_column_lineage_payload(
+        tenant_id=tenant_id,
+        endpoint=conn.host,
+        columns=results,
+    )
+
+    req_count = 0
+    for update_column_lineage_input in update_column_lineage_inputs:
+        logger.info(
+            "Generating column lineage. downstream: %s -> %s -> %s -> %s",
+            update_column_lineage_input.downstream_database_name,
+            update_column_lineage_input.downstream_schema_name,
+            update_column_lineage_input.downstream_table_name,
+            update_column_lineage_input.downstream_column_name,
+        )
+        status_code = qdc_client.update_lineage_by_id(
+            global_id=update_column_lineage_input.downstream_global_id,
+            payload=update_column_lineage_input.upstreams.as_dict(),
+        )
+        if status_code == 200:
+            req_count += 1
+    logger.info(
+        "Generating column lineage is finished. %s lineages are ingested.",
+        req_count,
+    )
+    return
+
+
+def _get_monitoring_tables(
+    conn: databricks.DatabricksConnectionConfig, monitoring_table_id: str = "_profile_metrics"
+) -> List[Dict[str, str]]:
+    tables = []
+    query = f"""
+        SELECT
+            table_catalog,
+            table_schema,
+            table_name,
+            CONCAT(table_catalog, '.', table_schema, '.', table_name) AS table_fqdn
+        FROM
+            system.information_schema.tables
+        WHERE table_name LIKE "%{monitoring_table_id}"
+        """
+    with databricks.DatabricksQueryExecutor(config=conn) as databricks_executor:
+        tables = databricks_executor.get_query_results(query)
+    if len(tables) > 0:
+        logger.info("Found %s monitoring tables.", len(tables))
+        return tables
+    else:
+        logger.info("No monitoring tables found.")
+        return []
+
+
+def _get_column_stats(
+    conn: databricks.DatabricksConnectionConfig, monitoring_table_id: str = "_profile_metrics"
+) -> List[Dict[str, str]]:
+    tables = _get_monitoring_tables(conn, monitoring_table_id)
+    if not tables:
+        return []
+    stats = []
+    for table in tables:
+        monitored_table = table["table_fqdn"].removesuffix("_profile_metrics")
+        monitored_table = monitored_table.split(".")
+        if len(monitored_table) != 3:
+            raise ValueError(f"Invalid table name: {table['table_fqdn']}")
+        with databricks.DatabricksQueryExecutor(config=conn) as databricks_executor:
+            query = """
+                WITH MaxCounts AS (
+                    SELECT
+                        t.COLUMN_NAME,
+                        MAX(item.count) AS max_count,
+                        MAX(t.window) AS latest
+                    FROM
+                        {monitoring_table} t
+                    LATERAL VIEW EXPLODE(t.frequent_items) AS item
+                    GROUP BY t.COLUMN_NAME
+                )
+                SELECT
+                    "{monitored_table_catalog}" as DB_NAME,
+                    "{monitored_table_schema}" as SCHEMA_NAME,
+                    "{monitored_table_name}" as TABLE_NAME,
+                    t.COLUMN_NAME,
+                    t.DATA_TYPE,
+                    t.distinct_count as CARDINALITY,
+                    t.MAX as MAX_VALUE,
+                    t.MIN as MIN_VALUE,
+                    t.AVG as AVG_VALUE,
+                    t.MEDIAN as MEDIAN_VALUE,
+                    t.STDDEV as STDDEV_VALUE,
+                    t.NUM_NULLS as NULL_COUNT,
+                    item.item AS MODE_VALUE
+                FROM
+                    {monitoring_table} t
+                JOIN MaxCounts mc ON t.COLUMN_NAME = mc.COLUMN_NAME
+                LATERAL VIEW EXPLODE(t.frequent_items) AS item
+                WHERE
+                    item.count = mc.max_count
+                    AND t.window = mc.latest
+                """.format(
+                monitoring_table=table["table_fqdn"],
+                monitored_table_catalog=monitored_table[0],
+                monitored_table_schema=monitored_table[1],
+                monitored_table_name=monitored_table[2],
+            )
+            stats.append(databricks_executor.get_query_results(query))
+    return stats
+
+
+def databricks_column_stats(
+    conn: databricks.DatabricksConnectionConfig,
+    qdc_client: qdc.QDCExternalAPIClient,
+    tenant_id: str,
+    monitoring_table_id: str = "_profile_metrics",
+) -> None:
+    table_stats = _get_column_stats(conn, monitoring_table_id)
+    for table in table_stats:
+        stats = gen_table_stats_payload(tenant_id, conn.host, table)
+        for stat in stats:
+            status_code = qdc_client.update_stats_by_id(
+                global_id=stat.global_id,
+                payload=stat.body.as_dict(),
+            )
+            if status_code == 200:
+                logger.info("Stats for %s is successfully ingested.", stat.global_id)
+    return

quollio_core/profilers/lineage.py

@@ -141,3 +141,15 @@ def parse_snowflake_results(results: List[Dict[str, str]]):
         payload["UPSTREAM_TABLES"] = json.loads(result["UPSTREAM_TABLES"])
         payloads.append(payload)
     return payloads
+
+
+def parse_databricks_table_lineage(results: List) -> List[Dict[str, Dict]]:
+    # Parses results from Quollio Databricks lineage table
+    # Returns tuple of downstream_table_name (0) and upstream_tables (1)
+    payloads = list()
+    for result in results:
+        payload = dict()
+        payload["DOWNSTREAM_TABLE_NAME"] = result["DOWNSTREAM_TABLE_NAME"]
+        payload["UPSTREAM_TABLES"] = json.loads(result["UPSTREAM_TABLES"])
+        payloads.append(payload)
+    return payloads

quollio_core/profilers/stats.py

@@ -77,7 +77,6 @@ def gen_table_stats_payload(tenant_id: str, endpoint: str, stats: List[Dict[str,
         table_global_id = new_global_id(
             tenant_id=tenant_id, cluster_id=endpoint, data_id=global_id_arg, data_type="column"
         )
-
         stats_request = StatsRequest(
             global_id=table_global_id,
             db=stat["DB_NAME"],

quollio_core/redshift.py

@@ -50,11 +50,10 @@ def build_view(
         options=["--no-use-colors", "--log-level", log_level, "--vars", options],
     )
     run_options = ["--no-use-colors", "--log-level", log_level, "--vars", options]
-    target_tables_list = target_tables.split()
-    if target_tables_list is not None:
-        if "quollio_stats_columns" in target_tables_list:
-            target_tables_list.append("quollio_stats_profiling_columns")
-        target_tables_str = " ".join(target_tables_list)
+    if target_tables is not None:
+        if "quollio_stats_columns" in target_tables:
+            target_tables.append("quollio_stats_profiling_columns")
+        target_tables_str = " ".join(target_tables)
         run_options.append("--select")
         run_options.append(target_tables_str)
 

quollio_core/repository/databricks.py

@@ -0,0 +1,62 @@
+import logging
+from dataclasses import asdict, dataclass
+from typing import Dict, List, Optional
+
+from databricks.sdk.core import Config, HeaderFactory, oauth_service_principal
+from databricks.sql.client import Connection, connect
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+
+
+@dataclass
+class DatabricksConnectionConfig:
+    host: str
+    http_path: str
+    client_id: str
+    client_secret: str
+    catalog: str
+    schema: str
+
+    def as_dict(self) -> Dict[str, str]:
+        return asdict(self)
+
+
+class DatabricksQueryExecutor:
+    def __init__(self, config: DatabricksConnectionConfig) -> None:
+        self.config = config
+        self.conn = self.__initialize()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.conn.close()
+
+    def __initialize(self) -> Connection:
+        conn = connect(
+            server_hostname=self.config.host,
+            http_path=self.config.http_path,
+            credentials_provider=self.credential_provider,
+        )
+        return conn
+
+    def get_query_results(self, query: str) -> List[Dict[str, str]]:
+        results_asdict: List[Dict[str, str]] = []
+        with self.conn.cursor() as cur:
+            try:
+                cur.execute(query)
+                result: List[Dict[str, str]] = cur.fetchall()
+            except Exception as e:
+                logging.error(query, exc_info=True)
+                logging.error("databricks get_query_results failed. %s", e)
+                raise
+
+            for row in result:
+                results_asdict.append(row.asDict())
+        return results_asdict
+
+    def credential_provider(self) -> Optional[HeaderFactory]:
+        config = Config(
+            host=f"https://{self.config.host}", client_id=self.config.client_id, client_secret=self.config.client_secret
+        )
+        return oauth_service_principal(config)

quollio_core/snowflake.py

@@ -50,11 +50,10 @@ def build_view(
         options=["--no-use-colors", "--log-level", log_level, "--vars", options],
     )
     run_options = ["--no-use-colors", "--log-level", log_level, "--vars", options]
-    target_tables_list = target_tables.split()
-    if target_tables_list is not None:
-        if "quollio_stats_columns" in target_tables_list:
-            target_tables_list.append("quollio_stats_profiling_columns")
-        target_tables_str = " ".join(target_tables_list)
+    if target_tables is not None:
+        if "quollio_stats_columns" in target_tables:
+            target_tables.append("quollio_stats_profiling_columns")
+        target_tables_str = " ".join(target_tables)
         run_options.append("--select")
         run_options.append(target_tables_str)
 

{quollio_core-0.4.3.dist-info → quollio_core-0.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.3
+Version: 0.4.5
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>
@@ -21,14 +21,18 @@ Requires-Dist: blake3==0.3.3
 Requires-Dist: dbt-core==1.7.10
 Requires-Dist: dbt-snowflake==1.7.0
 Requires-Dist: dbt-redshift==1.7.1
+Requires-Dist: dbt-databricks==1.7.1
 Requires-Dist: jinja2==3.1.3
 Requires-Dist: PyYAML==6.0.1
 Requires-Dist: requests==2.31.0
 Requires-Dist: pyjwt==2.8.0
 Requires-Dist: redshift-connector==2.0.915
 Requires-Dist: snowflake-connector-python==3.5.0
+Requires-Dist: databricks-sdk==0.17.0
+Requires-Dist: databricks-sql-connector==2.9.5
 Requires-Dist: sqlglot==20.8.0
 Requires-Dist: black>=22.3.0 ; extra == "test"
+Requires-Dist: coverage>=7.3.2 ; extra == "test"
 Requires-Dist: isort>=5.10.1 ; extra == "test"
 Requires-Dist: pyproject-flake8>=0.0.1-alpha.2 ; extra == "test"
 Requires-Dist: pytest>=5.2 ; extra == "test"

{quollio_core-0.4.3.dist-info → quollio_core-0.4.5.dist-info}/RECORD

@@ -1,20 +1,36 @@
-quollio_core/__init__.py,sha256=v7qeULEwxg6wNqWICrvNSrkxgZTWbgvhU4C6DUBvsVw,83
-quollio_core/redshift.py,sha256=81cFHWzTHgO7u6wpW32w1dsvqttVRxpkhj-n7VL0Blo,10234
-quollio_core/snowflake.py,sha256=VtslGFeCWpLotGk2RE9JTLRFP3rEm0JBG08Gd3oGChY,10333
+quollio_core/__init__.py,sha256=BXXaDg79qecIHTSRqA3Yh4FvgDbWzUjgYYWbwaMTRJY,83
+quollio_core/bricks.py,sha256=PCHyh_I6M4PBRpLDtc5DTr7rpharllu-vcSAhySM4xg,8001
+quollio_core/redshift.py,sha256=wap7QmV-YuHZAomIrHXytGUuxhQ5MFEb38QDY3XrThQ,10167
+quollio_core/snowflake.py,sha256=8IMbdTjCDBIiS_GF8APWRTVWNj6EM3ZT8MRN12T-1v0,10266
+quollio_core/dbt_projects/databricks/.gitignore,sha256=1jJAyXSzJ3YUm0nx3i7wUSE4RjQMX3ad6F8O88UbtzI,29
+quollio_core/dbt_projects/databricks/README.md,sha256=ZpRQyhFAODAiS8dc1Kb_ndkul4cu4o4udN_EMa49CU4,440
+quollio_core/dbt_projects/databricks/dbt_project.yml,sha256=3sH98RNk7TnphvI3yEdXDstb92kW5BNxr-cT0tXhwzk,480
+quollio_core/dbt_projects/databricks/package-lock.yml,sha256=0s8qBWevHon05NSj37p8NMa-rMVs_6McdKmddUVyGQ8,376
+quollio_core/dbt_projects/databricks/packages.yml,sha256=JtwWg3XK-nJp4tPv2QEER0kc5F2sPeVYqTkE54zMsIM,443
+quollio_core/dbt_projects/databricks/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+quollio_core/dbt_projects/databricks/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+quollio_core/dbt_projects/databricks/models/quollio_lineage_column_level.sql,sha256=mZ4mDCEZTwiSgCUr-w2QGze2-NQapt45EyQNQkCOI5I,2171
+quollio_core/dbt_projects/databricks/models/quollio_lineage_column_level.yml,sha256=tidAK_FMhYYuPTxFoactwcXYQPSMZwQTxWrGBly4-1o,450
+quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.sql,sha256=K63J7n7NIM2Jc7c4IF21JcW8AYOm9HxBNDiveUE4kzU,1558
+quollio_core/dbt_projects/databricks/models/quollio_lineage_table_level.yml,sha256=ZGjz6C2bguDJxJyA7LhCHbuyZSRPEaRMXln9rxcotuo,344
+quollio_core/dbt_projects/databricks/models/sources.yml,sha256=JXU-8lNsKm8dxIjmWos1vbTsWiea-9-pXnntik63ZpA,2231
+quollio_core/dbt_projects/databricks/profiles/profiles_template.yml,sha256=Dw1RuTrE04yvGIaPQL7uc6pgSWloKHhu0KrduzJ1Z6M,353
+quollio_core/dbt_projects/databricks/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+quollio_core/dbt_projects/databricks/snapshots/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/redshift/README.md,sha256=55nDkX5uQXWmawpQbgG1hbyn64j_CegDBQddQ2C85C8,571
 quollio_core/dbt_projects/redshift/dbt_project.yml,sha256=WVCmT-2usdGSm6EBM6MCdzEeEFwv9ANsyknreoNXgBc,405
-quollio_core/dbt_projects/redshift/package-lock.yml,sha256=1N71QqV5p07pG5JVP73nSkz-cndl6UtjYLfmF0KrVsk,109
+quollio_core/dbt_projects/redshift/package-lock.yml,sha256=Gef3zDCLF41j_FL-_h3sIZOUVj6j7nTTvxXrQPLcBP0,109
 quollio_core/dbt_projects/redshift/packages.yml,sha256=p9Bl2C44gdC6iYTUkz_15yq3xahSJf2IA3WOXLF_ahA,61
 quollio_core/dbt_projects/redshift/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/redshift/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql,sha256=hED0GRVxbiJaXggJDWqHKJDhTXhC2GruI5lqYH2iId8,3237
+quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql,sha256=3tRQeXXdjn3aSZ94DgMN6A6yMlTCE8aMKXeIdcZVDIM,3998
 quollio_core/dbt_projects/redshift/models/quollio_lineage_table_level.sql,sha256=AVPcNXfVYHwyutJzg61QT_VF9umfoC4i8C2HecAU4d4,2042
 quollio_core/dbt_projects/redshift/models/quollio_lineage_table_level.yml,sha256=UcrXpUTT3ihBHKPljvjw8xHz-ND60PfvMJaXqGKOEic,236
 quollio_core/dbt_projects/redshift/models/quollio_lineage_view_level.sql,sha256=A0CTgQwlz8InabA0cHuygV2GMZGYuAa7Zd5DIUOYzQI,1289
 quollio_core/dbt_projects/redshift/models/quollio_lineage_view_level.yml,sha256=7Npwo3svL9715HpNU2MKzRI014Da4tIStLzAHmd0UaU,235
 quollio_core/dbt_projects/redshift/models/quollio_sqllineage_sources.sql,sha256=e0A_Wqv_OcC8gG_yzTbI59vT-4vCI3JiAzFlmkvLnMk,1049
 quollio_core/dbt_projects/redshift/models/quollio_sqllineage_sources.yml,sha256=qgazupx3ca4P8R0loY5F9hyCz2fmAcWqZ6iOySo_NoY,377
-quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql,sha256=lT5ApMHu_-8_k4TsJmR_5nMtEck1IoZiPRyeejGudsw,304
+quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql,sha256=lH8xPmAzSW-6wi_g1y_LFVhtFgHzBvTweVX-MKeJzUQ,302
 quollio_core/dbt_projects/redshift/models/quollio_stats_columns.yml,sha256=V_BESPk6IqE52ExT26-78As9l9AlWW86-Geb5PIhThU,67
 quollio_core/dbt_projects/redshift/models/quollio_stats_profiling_columns.sql,sha256=IPmHf51Er2jE9cMQHybT4adRxwwi2CEmgrBSv1Oeduc,1592
 quollio_core/dbt_projects/redshift/models/quollio_stats_profiling_columns.yml,sha256=s-p9F44TdwoFYlQN-b9gHzcFYOMqhqDGA9ORS_M4lhs,523
@@ -22,20 +38,21 @@ quollio_core/dbt_projects/redshift/models/sources.yml,sha256=NOSoR4ces2XivuenuG7
 quollio_core/dbt_projects/redshift/profiles/profiles_template.yml,sha256=8nS-IE25cLo6uhHdtiZG9YX0cd2fhUG0pPrFUJWs2AY,291
 quollio_core/dbt_projects/redshift/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/redshift/snapshots/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+quollio_core/dbt_projects/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/README.md,sha256=55nDkX5uQXWmawpQbgG1hbyn64j_CegDBQddQ2C85C8,571
 quollio_core/dbt_projects/snowflake/dbt_project.yml,sha256=LN5NDOyakQjIK99IogQX4Whh_1zmqUfD2gqDU9JR3As,407
 quollio_core/dbt_projects/snowflake/package-lock.yml,sha256=Gef3zDCLF41j_FL-_h3sIZOUVj6j7nTTvxXrQPLcBP0,109
 quollio_core/dbt_projects/snowflake/packages.yml,sha256=p9Bl2C44gdC6iYTUkz_15yq3xahSJf2IA3WOXLF_ahA,61
 quollio_core/dbt_projects/snowflake/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=WnAeuTNUUsr47be2vPz9ZB580dPvN4KuGMnojBn1I1Y,2527
+quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=T4nFL76AbuQHBiLSAvNchoJnRjb1IRj4nToOyTNvLvw,2782
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.sql,sha256=Cxt2U2aXNG_LUm63jwTyxUkapkrB7_uHmesx1PTcMJM,4721
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.yml,sha256=a2uNIAh-xw51eu-GmHVuAnGnTbwK7h8-DjDeQtK3KaQ,711
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_table_level.sql,sha256=Q_7vY1N1Hi1LFv5CxkkdR3gQw8fTDnoKECTLSK4gd3o,5112
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_table_level.yml,sha256=QXlMBIkHo1Y-ANveKVx1FwyoYTMRXKgE2Z-PNouhQTw,325
 quollio_core/dbt_projects/snowflake/models/quollio_sqllineage_sources.sql,sha256=gd6JhQO13xBIvOoeXcce1I7amNGytwE8pwUApXehwqM,1520
 quollio_core/dbt_projects/snowflake/models/quollio_sqllineage_sources.yml,sha256=qgazupx3ca4P8R0loY5F9hyCz2fmAcWqZ6iOySo_NoY,377
-quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql,sha256=lT5ApMHu_-8_k4TsJmR_5nMtEck1IoZiPRyeejGudsw,304
+quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql,sha256=lH8xPmAzSW-6wi_g1y_LFVhtFgHzBvTweVX-MKeJzUQ,302
 quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.yml,sha256=V_BESPk6IqE52ExT26-78As9l9AlWW86-Geb5PIhThU,67
 quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql,sha256=kt2aFimIPkgKI_UQTjvfRlAjrdSbO8z6C_749pnXrnE,1382
 quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.yml,sha256=W39VAmFnnX6RBoW7B_4CConC1lm0Jm9o50Jsz9bYZzY,538
@@ -45,19 +62,21 @@ quollio_core/dbt_projects/snowflake/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JC
 quollio_core/dbt_projects/snowflake/snapshots/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/helper/core.py,sha256=-3vCDlKExWPHJmWuZQNpYnvPP55uoGwRpTtnFvsDxIo,1127
-quollio_core/helper/env_default.py,sha256=XsBpbMNeOVqvHTDoNctk_vGuMdLInMsQpnZ5u5xzhSU,1074
+quollio_core/helper/env_default.py,sha256=YIL9hfrPs1ViL1AXohnbWEjVBUDXbVVakH0ZoSZWOlc,1202
 quollio_core/profilers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/profilers/lineage.py,sha256=N2_-sM0iNACl4ztcQ9NQdUwkqPWReBKbICPZe30qeNo,5892
+quollio_core/profilers/databricks.py,sha256=skTTlqogJGauZkN7c9uVSYalAACIB43yblGc1jEIM1U,7501
+quollio_core/profilers/lineage.py,sha256=HrTjXxrchETRmHEb5tSFzzHdb6z2KMw-DTnUSeKxmr0,6379
 quollio_core/profilers/redshift.py,sha256=obdHVIsOM1bwHGdvYKalsJcTXwLK02kAKQMSBzSvsDo,7862
 quollio_core/profilers/snowflake.py,sha256=C1LC19ZaUMwNoXjsbnez0xANydJYs8oNRt6tixWKDq8,9090
 quollio_core/profilers/sqllineage.py,sha256=oCyl4tpXL5bkfguXAzTHSB9kZBL3tQK_rfcJ4XQMrLo,5177
-quollio_core/profilers/stats.py,sha256=0AbxlJ_QtkdxN4W1yS14NAB3N3LkhB59MWX-d1jmQPE,4721
+quollio_core/profilers/stats.py,sha256=PG1NbbUSpc1JuEYvBzD66rd24tp0C13_Y5Y7vRjYG1c,4720
 quollio_core/repository/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+quollio_core/repository/databricks.py,sha256=m68tja5N-QxH3VqEq-mOJKBeR2qldSgj_L9iIxvWwm0,1945
 quollio_core/repository/dbt.py,sha256=HXqW_xa4xYPh9CnKkg4L1gwG3SGjj2BAYoWgzWMFU4U,770
 quollio_core/repository/qdc.py,sha256=VCmzAUvjLemw1os5TaPtfBFkMCOMuPeftjZmUPhFj2Y,4702
 quollio_core/repository/redshift.py,sha256=UVHIpYzDQ2AbBTAGa8DgmEenG0NZsHfYroR1MmEPQGA,2991
 quollio_core/repository/snowflake.py,sha256=1YVMDfb9euJKvikv1pk_IxVF6SVsiemSvZ-WMTSbY7E,1874
-quollio_core-0.4.3.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.4.3.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-quollio_core-0.4.3.dist-info/METADATA,sha256=ePyyL6UzINeA26hX-tOQ-uAKyyHmJNKIwe3nOwJT3iE,6400
-quollio_core-0.4.3.dist-info/RECORD,,
+quollio_core-0.4.5.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.4.5.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+quollio_core-0.4.5.dist-info/METADATA,sha256=fbpCG8MiXchuHMyatF_kiThLXWSP0gMoMV_ffVDU4MA,6571
+quollio_core-0.4.5.dist-info/RECORD,,