quollio-core 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.5.0"
+__version__ = "0.6.0"
 __author__ = "Quollio Technologies, Inc"

quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.sql

@@ -95,6 +95,23 @@ UNION
         {{ source('account_usage', 'TABLES') }}
     WHERE
         DELETED IS NULL
+		AND (
+			{% if var('target_databases_method') == 'ALLOWLIST' %}
+				{% if var('target_databases') %}
+					TABLE_CATALOG LIKE ANY ({{ var('target_databases')|join(",") }})
+				{% else %}
+					1=0  -- If no databases specified in allowlist, deny all
+				{% endif %}
+			{% elif var('target_databases_method') == 'DENYLIST' %}
+				{% if var('target_databases') %}
+					NOT (TABLE_CATALOG LIKE ANY ({{ var('target_databases')|join(",") }}))
+				{% else %}
+					1=1  -- If no databases specified in denylist, include all
+				{% endif %}
+			{% else %}
+				1=1  -- Default case: allow all databases
+			{% endif %}
+		)
 ), exists_upstream_column_lineage AS (
 	SELECT
 	    downstream_table_name

quollio_core/dbt_projects/snowflake/models/quollio_lineage_table_level.sql

@@ -49,6 +49,23 @@ WITH table_lineage_history AS (
         {{ source('account_usage', 'TABLES') }}
     WHERE
         DELETED IS NULL
+        AND (
+            {% if var('target_databases_method') == 'ALLOWLIST' %}
+                {% if var('target_databases') %}
+                    TABLE_CATALOG LIKE ANY ({{ var('target_databases')|join(",") }})
+                {% else %}
+                    1=0  -- If no databases specified in allowlist, deny all
+                {% endif %}
+            {% elif var('target_databases_method') == 'DENYLIST' %}
+                {% if var('target_databases') %}
+                    NOT (TABLE_CATALOG LIKE ANY ({{ var('target_databases')|join(",") }}))
+                {% else %}
+                    1=1  -- If no databases specified in denylist, include all
+                {% endif %}
+            {% else %}
+                1=1  -- Default case: allow all databases
+            {% endif %}
+        )
 ), upstream_exists_table AS (
     SELECT
     	downstream_table_name AS "DOWNSTREAM_TABLE_NAME"

quollio_core/dbt_projects/snowflake/models/quollio_sqllineage_sources.sql

@@ -48,3 +48,21 @@ on
     lst.query_id = qt.query_id
 where
     qt.query_id is not null
+    AND (
+        {% if var('target_databases_method') == 'ALLOWLIST' %}
+            {% if var('target_databases') %}
+                database_name LIKE ANY ({{ var('target_databases')|join(",") }})
+            {% else %}
+                1=0  -- If no databases specified in allowlist, deny all
+            {% endif %}
+        {% elif var('target_databases_method') == 'DENYLIST' %}
+            {% if var('target_databases') %}
+                NOT (database_name LIKE ANY ({{ var('target_databases')|join(",") }}))
+            {% else %}
+                1=1  -- If no databases specified in denylist, include all
+            {% endif %}
+        {% else %}
+            1=1  -- Default case: allow all databases
+        {% endif %}
+    )
+

quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql

@@ -28,7 +28,7 @@ WITH columns AS (
     FROM
       {{ source('account_usage', 'GRANTS_TO_ROLES') }}
     WHERE
-      granted_on in ('TABLE', 'MATERIALIZED VIEW')
+      granted_on in ('TABLE', 'VIEW', 'MATERIALIZED VIEW')
       AND grantee_name = '{{ var("query_role") }}'
       AND privilege in ('SELECT', 'OWNERSHIP')
       AND deleted_on IS NULL
@@ -92,5 +92,21 @@ WITH columns AS (
            else false END AS is_calculable
   FROM
     implicit_columns_removed
-)
+  WHERE
+      {% if var('target_databases_method') == 'ALLOWLIST' %}
+          {% if var('target_databases') %}
+              TABLE_CATALOG LIKE ANY ({{ var('target_databases')|join(",") }})
+          {% else %}
+              1=0  -- If no databases specified in allowlist, deny all
+          {% endif %}
+      {% elif var('target_databases_method') == 'DENYLIST' %}
+          {% if var('target_databases') %}
+              NOT (TABLE_CATALOG LIKE ANY ({{ var('target_databases')|join(",") }}))
+          {% else %}
+              1=1  -- If no databases specified in denylist, include all
+          {% endif %}
+      {% else %}
+          1=1  -- Default case: allow all databases
+      {% endif %}
+  )
 select * from final

quollio_core/dbt_projects/snowflake/profiles/profiles_template.yml

@@ -8,6 +8,11 @@ quollio_intelligence_snowflake:
       schema: {{ account_schema }}
       type: snowflake
       user: {{ account_user }}
-      password: {{ account_password }}
       warehouse: {{ account_warehouse }}
       threads: {{ threads }}
+      {% if private_key is defined %}
+      private_key: |
+        {{ private_key | indent(8) }}
+      {% else %}
+      password: {{ account_password }}
+      {% endif %}

quollio_core/helper/core.py

@@ -8,6 +8,7 @@ from jinja2 import Environment, FileSystemLoader
 def new_global_id(tenant_id: str, cluster_id: str, data_id: str, data_type: str) -> str:
     prefix = ""
     data_types = {
+        "data_source": "dsrc-",
         "schema": "schm-",
         "table": "tbl-",
         "column": "clmn-",

quollio_core/models/avroasset.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from dataclasses_avroschema import AvroModel
+
+
+@dataclass
+class AvroAsset(AvroModel):
+    "AvroAsset"
+
+    id: str
+    object_type: str
+    parents: List[str]
+    name: str
+    stats_max: Optional[str] = None
+    stats_min: Optional[str] = None
+    stats_mean: Optional[str] = None
+    stats_median: Optional[str] = None
+    stats_mode: Optional[str] = None
+    stats_stddev: Optional[str] = None
+    stats_number_of_null: Optional[str] = None
+    stats_number_of_unique: Optional[str] = None
+    upstream: Optional[List[str]] = None

quollio_core/models/qdc.py

@@ -0,0 +1,36 @@
+from dataclasses import asdict, dataclass
+from typing import Dict
+
+
+@dataclass
+class GetImportURLRequest:
+    service_name: str
+    source_name: str
+    file_name: str
+    override_logical_name: str
+    update_mode: str
+
+    def as_dict(self) -> Dict[str, str]:
+        return asdict(self)
+
+
+@dataclass
+class DataSourceMetadataResponseBody:
+    user_id: str
+    job_key: str
+    service_name: str
+    source_name: str
+    source_type: str
+    override_logical_name: str
+
+    def as_dict(self) -> Dict[str, str]:
+        return asdict(self)
+
+
+@dataclass
+class GetImportURLResponse:
+    location: str
+    datasource_metadata_response_body: DataSourceMetadataResponseBody
+
+    def as_dict(self) -> Dict[str, str]:
+        return asdict(self)

quollio_core/profilers/lineage.py

@@ -3,6 +3,7 @@ from dataclasses import asdict, dataclass
 from typing import Dict, List, Tuple, Union
 
 from quollio_core.helper.core import new_global_id
+from quollio_core.models.avroasset import AvroAsset
 
 
 @dataclass
@@ -23,6 +24,101 @@ class LineageInputs:
     upstreams: LineageInput
 
 
+def gen_table_avro_lineage_payload(
+    tenant_id: str,
+    endpoint: str,
+    tables: List[Dict[str, Union[Dict[str, str], str]]],
+    existing_global_ids: Dict[str, bool],
+) -> List[Dict[str, str]]:
+    payload = list()
+    for table in tables:
+        downstream_table_fqn = table["DOWNSTREAM_TABLE_NAME"].split(".")
+        if len(downstream_table_fqn) != 3:
+            continue
+        else:
+            global_id_arg = "{db}{schema}{table}".format(
+                db=downstream_table_fqn[0], schema=downstream_table_fqn[1], table=downstream_table_fqn[2]
+            )
+            downstream_table_global_id = new_global_id(
+                tenant_id=tenant_id, cluster_id=endpoint, data_id=global_id_arg, data_type="table"
+            )
+            if existing_global_ids.get(downstream_table_global_id) is not True:
+                continue
+            upstreams = list()
+            for upstream_table in table["UPSTREAM_TABLES"]:
+                upstream_table_fqn = upstream_table["upstream_object_name"].split(".")
+                if len(upstream_table_fqn) != 3:
+                    continue
+                else:
+                    upstream_global_id_arg = "{db}{schema}{table}".format(
+                        db=upstream_table_fqn[0], schema=upstream_table_fqn[1], table=upstream_table_fqn[2]
+                    )
+                    upstream_table_global_id = new_global_id(
+                        tenant_id=tenant_id, cluster_id=endpoint, data_id=upstream_global_id_arg, data_type="table"
+                    )
+                    upstreams.append(upstream_table_global_id)
+
+            avro_assets = AvroAsset(
+                id=downstream_table_global_id,
+                object_type="table",
+                parents=[downstream_table_fqn[0], downstream_table_fqn[1]],
+                name=downstream_table_fqn[2],
+                upstream=upstreams,
+            )
+            payload.append(avro_assets.to_dict())
+    return payload
+
+
+def gen_column_avro_lineage_payload(
+    tenant_id: str, endpoint: str, columns: List[Dict[str, str]], existing_global_ids: Dict[str, bool]
+) -> List[Dict[str, str]]:
+    payload = list()
+    for column in columns:
+        downstream_table_fqn = column["DOWNSTREAM_TABLE_NAME"].split(".")
+        if len(downstream_table_fqn) != 3:
+            continue
+        else:
+            global_id_arg = "{db}{schema}{table}{column}".format(
+                db=downstream_table_fqn[0],
+                schema=downstream_table_fqn[1],
+                table=downstream_table_fqn[2],
+                column=column["DOWNSTREAM_COLUMN_NAME"],
+            )
+            downstream_column_global_id = new_global_id(
+                tenant_id=tenant_id, cluster_id=endpoint, data_id=global_id_arg, data_type="column"
+            )
+            if existing_global_ids.get(downstream_column_global_id) is not True:
+                continue
+            upstream_columns: List[Dict[str, str]] = json.loads(column["UPSTREAM_COLUMNS"])
+            upstreams = list()
+            for upstream_column in upstream_columns:
+                upstream_table_fqn = upstream_column["upstream_table_name"].split(".")
+                if len(upstream_table_fqn) != 3:
+                    continue
+                elif not upstream_column.get("upstream_column_name"):
+                    continue
+                else:
+                    upstream_global_id_arg = "{db}{schema}{table}{column}".format(
+                        db=upstream_table_fqn[0],
+                        schema=upstream_table_fqn[1],
+                        table=upstream_table_fqn[2],
+                        column=upstream_column["upstream_column_name"],
+                    )
+                    upstream_column_global_id = new_global_id(
+                        tenant_id=tenant_id, cluster_id=endpoint, data_id=upstream_global_id_arg, data_type="column"
+                    )
+                    upstreams.append(upstream_column_global_id)
+            avro_assets = AvroAsset(
+                id=downstream_column_global_id,
+                object_type="column",
+                parents=[downstream_table_fqn[0], downstream_table_fqn[1], downstream_table_fqn[2]],
+                name=column["DOWNSTREAM_COLUMN_NAME"],
+                upstream=upstreams,
+            )
+            payload.append(avro_assets.to_dict())
+    return payload
+
+
 def gen_table_lineage_payload(
     tenant_id: str, endpoint: str, tables: List[Dict[str, Union[Dict[str, str], str]]]
 ) -> List[LineageInputs]:

quollio_core/profilers/qdc.py

@@ -0,0 +1,27 @@
+import logging
+from io import BytesIO
+from typing import Dict
+
+import fastavro
+
+from quollio_core.helper.core import new_global_id
+from quollio_core.models.avroasset import AvroAsset
+from quollio_core.repository import qdc
+
+logger = logging.getLogger(__name__)
+
+
+def gen_existing_global_id_dict(avro_content: bytes) -> Dict[str, bool]:
+    byte_io = BytesIO(avro_content)
+    avro_schema = AvroAsset.avro_schema_to_python()
+    reader = fastavro.reader(byte_io, avro_schema)
+    records = {record["id"]: True for record in reader}
+    return records
+
+
+def get_avro_file_content(tenant_id: str, account_id: str, qdc_client: qdc.QDCExternalAPIClient) -> bytes:
+    datasource_id = new_global_id(tenant_id=tenant_id, cluster_id=account_id, data_id="", data_type="data_source")
+    logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+    res = qdc_client.get_export_url(datasource_id=datasource_id)
+    file_content = qdc_client.download_file(res).content
+    return file_content

quollio_core/profilers/snowflake.py

@@ -1,13 +1,20 @@
+import io
 import logging
-from typing import List
+import os
+from typing import Dict, List
 
+from fastavro import writer
+
+from quollio_core.helper.core import new_global_id
+from quollio_core.models.avroasset import AvroAsset
+from quollio_core.models.qdc import GetImportURLRequest
 from quollio_core.profilers.lineage import (
-    gen_column_lineage_payload,
-    gen_table_lineage_payload,
+    gen_column_avro_lineage_payload,
+    gen_table_avro_lineage_payload,
     parse_snowflake_results,
 )
 from quollio_core.profilers.sqllineage import SQLLineage
-from quollio_core.profilers.stats import gen_table_stats_payload, get_is_target_stats_items, render_sql_for_stats
+from quollio_core.profilers.stats import gen_table_stats_avro_payload, get_is_target_stats_items, render_sql_for_stats
 from quollio_core.repository import qdc, snowflake
 
 logger = logging.getLogger(__name__)
@@ -17,6 +24,7 @@ def snowflake_table_to_table_lineage(
     conn: snowflake.SnowflakeConnectionConfig,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    existing_global_ids: Dict[str, bool],
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
         results, err = sf_executor.get_query_results(
@@ -38,28 +46,41 @@ def snowflake_table_to_table_lineage(
             )
             return
         parsed_results = parse_snowflake_results(results=results)
-        update_table_lineage_inputs = gen_table_lineage_payload(
+        update_table_lineage_inputs = gen_table_avro_lineage_payload(
             tenant_id=tenant_id,
             endpoint=conn.account_id,
             tables=parsed_results,
+            existing_global_ids=existing_global_ids,
+        )
+        stack_name = os.getenv("CF_STACK")
+        import_req = GetImportURLRequest(
+            service_name="snowflake",
+            source_name=stack_name,
+            file_name="{name}.avro".format(name=stack_name),
+            override_logical_name="false",
+            update_mode="partial",
+        )
+        datasource_id = new_global_id(
+            tenant_id=tenant_id, cluster_id=conn.account_id, data_id="", data_type="data_source"
         )
+        logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+        import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+        if import_res is None:
+            logger.error("get_import_url failed. Please retry `load_lineage` again")
+            return
+        logger.debug("ImportResponse: {res}".format(res=import_res))
 
-        req_count = 0
-        for update_table_lineage_input in update_table_lineage_inputs:
-            logger.info(
-                "Generating table lineage. downstream: {db} -> {schema} -> {table}".format(
-                    db=update_table_lineage_input.downstream_database_name,
-                    schema=update_table_lineage_input.downstream_schema_name,
-                    table=update_table_lineage_input.downstream_table_name,
-                )
-            )
-            status_code = qdc_client.update_lineage_by_id(
-                global_id=update_table_lineage_input.downstream_global_id,
-                payload=update_table_lineage_input.upstreams.as_dict(),
-            )
-            if status_code == 200:
-                req_count += 1
-        logger.info(f"Generating table lineage is finished. {req_count} lineages are ingested.")
+        avro_schema = AvroAsset.avro_schema_to_python()
+
+        buffer = io.BytesIO()
+        writer(buffer, avro_schema, update_table_lineage_inputs)
+        res = qdc_client.upload_file(
+            url=import_res.location,
+            metadata=import_res.datasource_metadata_response_body,
+            buffer=buffer.getbuffer().tobytes(),
+        )
+        if res == 200:
+            logger.info("Upload table lineage is finished.")
     return
 
 
@@ -67,6 +88,7 @@ def snowflake_column_to_column_lineage(
     conn: snowflake.SnowflakeConnectionConfig,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    existing_global_ids: Dict[str, bool],
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
         results, err = sf_executor.get_query_results(
@@ -87,29 +109,39 @@ def snowflake_column_to_column_lineage(
                 "No lineage data in ACCOUNT_USAGE.SNOWFLAKE. Please check the data in `QUOLLIO_LINEAGE_COLUMN_LEVEL`."
             )
             return
-        update_column_lineage_inputs = gen_column_lineage_payload(
-            tenant_id=tenant_id,
-            endpoint=conn.account_id,
-            columns=results,
+        update_column_lineage_inputs = gen_column_avro_lineage_payload(
+            tenant_id=tenant_id, endpoint=conn.account_id, columns=results, existing_global_ids=existing_global_ids
         )
 
-        req_count = 0
-        for update_column_lineage_input in update_column_lineage_inputs:
-            logger.info(
-                "Generating column lineage. downstream: {db} -> {schema} -> {table} -> {column}".format(
-                    db=update_column_lineage_input.downstream_database_name,
-                    schema=update_column_lineage_input.downstream_schema_name,
-                    table=update_column_lineage_input.downstream_table_name,
-                    column=update_column_lineage_input.downstream_column_name,
-                )
-            )
-            status_code = qdc_client.update_lineage_by_id(
-                global_id=update_column_lineage_input.downstream_global_id,
-                payload=update_column_lineage_input.upstreams.as_dict(),
-            )
-            if status_code == 200:
-                req_count += 1
-        logger.info(f"Generating column lineage is finished. {req_count} lineages are ingested.")
+        stack_name = os.getenv("CF_STACK")
+        import_req = GetImportURLRequest(
+            service_name="snowflake",
+            source_name=stack_name,
+            file_name="{name}.avro".format(name=stack_name),
+            override_logical_name="false",
+            update_mode="partial",
+        )
+        datasource_id = new_global_id(
+            tenant_id=tenant_id, cluster_id=conn.account_id, data_id="", data_type="data_source"
+        )
+        logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+        import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+        if import_res is None:
+            logger.error("get_import_url failed. Please retry load_lineage again")
+            return
+        logger.debug("ImportResponse: {res}".format(res=import_res))
+
+        avro_schema = AvroAsset.avro_schema_to_python()
+
+        buffer = io.BytesIO()
+        writer(buffer, avro_schema, update_column_lineage_inputs)
+        res = qdc_client.upload_file(
+            url=import_res.location,
+            metadata=import_res.datasource_metadata_response_body,
+            buffer=buffer.getbuffer().tobytes(),
+        )
+        if res == 200:
+            logger.info("Upload column lineage is finished.")
     return
 
 
@@ -177,6 +209,7 @@ def snowflake_table_stats(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     stats_items: List[str],
+    existing_global_ids: Dict[str, bool],
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
         get_stats_view_query = _gen_get_stats_views_query(
@@ -193,8 +226,8 @@ and fix it or grant usage permission to both `{conn.account_database}` and `{con
 and select permissions to views begins with `QUOLLIO_STATS_COLUMNS_`."
             )
             return
-        req_count = 0
         is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
+        update_stats_inputs = list()
         for stats_view in stats_views:
             table_fqn = '"{catalog}"."{schema}"."{table}"'.format(
                 catalog=stats_view["TABLE_CATALOG"], schema=stats_view["TABLE_SCHEMA"], table=stats_view["TABLE_NAME"]
@@ -210,23 +243,43 @@ and select permissions to views begins with `QUOLLIO_STATS_COLUMNS_`."
 or user has select permission to it."
                 )
                 continue
-            payloads = gen_table_stats_payload(tenant_id=tenant_id, endpoint=conn.account_id, stats=stats_result)
-            for payload in payloads:
-                logger.info(
-                    "Generating table stats. asset: {db} -> {schema} -> {table} -> {column}".format(
-                        db=payload.db,
-                        schema=payload.schema,
-                        table=payload.table,
-                        column=payload.column,
-                    )
-                )
-                status_code = qdc_client.update_stats_by_id(
-                    global_id=payload.global_id,
-                    payload=payload.body.get_column_stats(),
-                )
-                if status_code == 200:
-                    req_count += 1
-        logger.info(f"Generating table stats is finished. {req_count} stats are ingested.")
+            payloads = gen_table_stats_avro_payload(
+                tenant_id=tenant_id,
+                endpoint=conn.account_id,
+                stats=stats_result,
+                existing_global_ids=existing_global_ids,
+            )
+            update_stats_inputs += payloads
+
+        stack_name = os.getenv("CF_STACK")
+        import_req = GetImportURLRequest(
+            service_name="snowflake",
+            source_name=stack_name,
+            file_name="{name}.avro".format(name=stack_name),
+            override_logical_name="false",
+            update_mode="partial",
+        )
+        datasource_id = new_global_id(
+            tenant_id=tenant_id, cluster_id=conn.account_id, data_id="", data_type="data_source"
+        )
+        logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+        import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+        if import_res is None:
+            logger.error("get_import_url failed. Please retry load_stats again")
+            return
+        logger.debug("ImportResponse: {res}".format(res=import_res))
+
+        avro_schema = AvroAsset.avro_schema_to_python()
+
+        buffer = io.BytesIO()
+        writer(buffer, avro_schema, update_stats_inputs)
+        res = qdc_client.upload_file(
+            url=import_res.location,
+            metadata=import_res.datasource_metadata_response_body,
+            buffer=buffer.getbuffer().tobytes(),
+        )
+        if res == 200:
+            logger.info("Generating table stats is finished.")
     return
 
 

quollio_core/profilers/stats.py

@@ -6,6 +6,7 @@ from typing import Dict, List, Tuple, Union
 from jinja2 import Template
 
 from quollio_core.helper.core import new_global_id
+from quollio_core.models.avroasset import AvroAsset
 
 logger = logging.getLogger(__name__)
 
@@ -70,6 +71,41 @@ def convert_value_type(obj, cast_str: bool = False):
     return obj
 
 
+def gen_table_stats_avro_payload(
+    tenant_id: str, endpoint: str, stats: List[Dict[str, str]], existing_global_ids: Dict[str, bool]
+) -> List[Dict[str, str]]:
+    payloads = list()
+    for stat in stats:
+        db_name = stat.get("DB_NAME", stat.get("db_name"))
+        schema_name = stat.get("SCHEMA_NAME", stat.get("schema_name"))
+        table_name = stat.get("TABLE_NAME", stat.get("table_name"))
+        column_name = stat.get("COLUMN_NAME", stat.get("column_name"))
+        global_id_arg = "{db}{schema}{table}{column}".format(
+            db=db_name, schema=schema_name, table=table_name, column=column_name
+        )
+        column_global_id = new_global_id(
+            tenant_id=tenant_id, cluster_id=endpoint, data_id=global_id_arg, data_type="column"
+        )
+        if existing_global_ids.get(column_global_id) is not True:
+            continue
+        avro_assets = AvroAsset(
+            id=column_global_id,
+            object_type="column",
+            parents=[db_name, schema_name, table_name],
+            name=column_name,
+            stats_max=convert_value_type(stat.get("MAX_VALUE", stat.get("max_value")), True),
+            stats_min=convert_value_type(stat.get("MIN_VALUE", stat.get("min_value")), True),
+            stats_mean=convert_value_type(stat.get("AVG_VALUE", stat.get("avg_value")), True),
+            stats_median=convert_value_type(stat.get("MEDIAN_VALUE", stat.get("median_value")), True),
+            stats_mode=convert_value_type(stat.get("MODE_VALUE", stat.get("mode_value")), True),
+            stats_stddev=convert_value_type(stat.get("STDDEV_VALUE", stat.get("stddev_value")), True),
+            stats_number_of_null=convert_value_type(stat.get("NULL_COUNT", stat.get("null_count")), True),
+            stats_number_of_unique=convert_value_type(stat.get("CARDINALITY", stat.get("cardinality")), True),
+        )
+        payloads.append(avro_assets.to_dict())
+    return payloads
+
+
 def gen_table_stats_payload(tenant_id: str, endpoint: str, stats: List[Dict[str, str]]) -> List[StatsRequest]:
     payloads = list()
     for stat in stats:

quollio_core/profilers/teradata/lineage.py

@@ -17,11 +17,15 @@ def load_lineage(
     tenant_id: str = None,
     qdc_client: qdc.QDCExternalAPIClient = None,
     page_size: int = None,
+    system_database: str = None,
 ) -> None:
     page_size = page_size or int(os.environ.get("TERADATA_PAGE_SIZE", 1000))
     offset = 0
     all_lineage_results = []
 
+    # Use system_database from config if not provided
+    system_database = system_database or conn_config.system_database
+
     with teradata_repo.new_teradata_client(conn_config) as conn:
         while True:
             query = f"""
@@ -30,10 +34,10 @@ def load_lineage(
                     TRIM(a.SqlTextInfo) AS SqlTextInfo,
                     a.SqlRowNo,
                     TRIM(d.DatabaseName) AS DefaultDatabase
-                FROM DBC.QryLogSQLV a
-                JOIN DBC.QryLogV b
+                FROM {system_database}.QryLogSQLV a
+                JOIN {system_database}.QryLogV b
                     ON a.QueryID = b.QueryID
-                JOIN DBC.DatabasesV d
+                JOIN {system_database}.DatabasesV d
                     ON b.DefaultDatabase = d.DatabaseName
                 WHERE
                     UPPER(TRIM(SqlTextInfo)) LIKE 'CREATE TABLE%'