quollio-core 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.6.4"
+__version__ = "0.7.0"
 __author__ = "Quollio Technologies, Inc"

quollio_core/bigquery.py

@@ -6,6 +6,7 @@ from google.auth.credentials import Credentials
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log_utils import configure_logging, error_handling_decorator, logger
 from quollio_core.profilers.bigquery import bigquery_table_lineage, bigquery_table_stats
+from quollio_core.profilers.qdc import gen_existing_global_id_dict, get_avro_file_content
 from quollio_core.repository import qdc
 from quollio_core.repository.bigquery import BigQueryClient, get_credentials, get_org_id
 
@@ -30,8 +31,15 @@ def load_lineage(
     org_id: str,
     credentials: Credentials,
     qdc_client: qdc.QDCExternalAPIClient,
+    enable_multi_projects: str,
 ) -> None:
     logger.info("Loading lineage data.")
+    file_content = get_avro_file_content(
+        tenant_id=tenant_id,
+        account_id=org_id,
+        qdc_client=qdc_client,
+    )
+    existing_global_ids = gen_existing_global_id_dict(avro_content=file_content)
     bigquery_table_lineage(
         qdc_client=qdc_client,
         tenant_id=tenant_id,
@@ -39,6 +47,8 @@ def load_lineage(
         regions=regions,
         credentials=credentials,
         org_id=org_id,
+        existing_global_ids=existing_global_ids,
+        enable_multi_projects=enable_multi_projects,
     )
     logger.info("Lineage data loaded successfully.")
 
@@ -146,6 +156,17 @@ if __name__ == "__main__":
         help="Comma-separated list of dataplex stats tables - <project_id>.<dataset_id>.<table_id>",
     )
 
+    parser.add_argument(
+        "--enable_multi_projects",
+        type=str,
+        choices=["ENABLED", "DISABLED"],
+        action=env_default("ENABLE_MULTI_PROJECTS"),
+        default="DISABLED",
+        required=False,
+        help="Whether to enable multi-projects support. If set to 'true', \
+        the script will load lineage and stats from all projects accessible by the credentials. Default is 'false'.",
+    )
+
     args = parser.parse_args()
 
     # Validate that dataplex_stats_tables is provided if load_stats is in commands
@@ -170,6 +191,7 @@ if __name__ == "__main__":
             org_id=org_id,
             credentials=credentials,
             qdc_client=qdc_client,
+            enable_multi_projects=args.enable_multi_projects,
         )
 
     if "load_stats" in args.commands:

quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -2,9 +2,10 @@
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
 {%- set grant_config = config.get('grants') %}
+{%- set max_columns_per_view = config.get('max_columns_per_view', 100) %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
--- `BEGIN` happens here:
+-- BEGIN happens here:
 {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
 -- fetch target_tables
@@ -41,45 +42,60 @@
 
 -- create view for each table
 {%- for stats_target_table in stats_target_tables -%}
-  -- build sql for column value aggregation.
-  {%- set sql_for_column_stats %}
-  {% set columns_json = fromjson(stats_target_table[3]) %}
-  {%- for col_name, attr in columns_json.items() -%}
-    {%- if not loop.first %}UNION{% endif %}
-    SELECT
-      DISTINCT
-      '{{stats_target_table[0]}}' as db_name
-      , '{{stats_target_table[1]}}' as schema_name
-      , '{{stats_target_table[2]}}' as table_name
-      , '{{col_name}}' as column_name
-      , {% if attr["IS_CALCULABLE"] == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
-      , {% if attr["IS_CALCULABLE"] == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
-      , COUNT_IF("{{col_name}}" IS NULL) AS null_count
-      , {% if attr["CAN_APPROX_COUNT"] == True %}APPROX_COUNT_DISTINCT("{{col_name}}"){% else %}NULL{% endif %} AS cardinality
-      , {% if attr["IS_CALCULABLE"] == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
-      , {% if attr["IS_CALCULABLE"] == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
-      , {% if attr["IS_CALCULABLE"] == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
-      , {% if attr["IS_CALCULABLE"] == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
-    FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
-  {% endfor -%}
-  {%- endset %}
+  {%- set columns_json = fromjson(stats_target_table[3]) %}
+  {%- set column_list = columns_json.keys() | list %}
+  {%- set chunk_count = ((column_list | length) / max_columns_per_view) | round(0, 'ceil') | int %}
+  
+  {%- for chunk_index in range(chunk_count) %}
+    {%- set start_idx = chunk_index * max_columns_per_view %}
+    {%- set end_idx = start_idx + max_columns_per_view %}
+    {%- set chunk_columns = column_list[start_idx:end_idx] %}
+    
+    -- build sql for column value aggregation.
+    {%- set sql_for_column_stats %}
+    {%- for col_name in chunk_columns -%}
+      {%- set attr = columns_json[col_name] %}
+      {%- if not loop.first %}UNION{% endif %}
+      SELECT
+        DISTINCT
+        '{{stats_target_table[0]}}' as db_name
+        , '{{stats_target_table[1]}}' as schema_name
+        , '{{stats_target_table[2]}}' as table_name
+        , '{{col_name}}' as column_name
+        , {% if attr["IS_CALCULABLE"] == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
+        , {% if attr["IS_CALCULABLE"] == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
+        , COUNT_IF("{{col_name}}" IS NULL) AS null_count
+        , {% if attr["CAN_APPROX_COUNT"] == True %}APPROX_COUNT_DISTINCT("{{col_name}}"){% else %}NULL{% endif %} AS cardinality
+        , {% if attr["IS_CALCULABLE"] == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
+        , {% if attr["IS_CALCULABLE"] == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
+        , {% if attr["IS_CALCULABLE"] == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
+        , {% if attr["IS_CALCULABLE"] == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
+      FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
+    {% endfor -%}
+    {%- endset %}
 
-  -- create a view with a index as suffix
-  {%- set stats_view_identifier = "\"%s_%s_%s_%s\"" | format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) | upper %}
-  {%- set schema_name = "\"%s\""|format(schema) %}
-  {%- set db_name = "\"%s\""|format(database) %}
-  {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema_name, database=db_name, type='view') %}
-  {% call statement("main") %}
-    {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
-  {% endcall %}
-  {%- set full_refresh_mode = (should_full_refresh()) -%}
-  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
-  {%- do apply_grants(target_relation, grant_config, should_revoke) %}
-  {%- set target_relations = target_relations.append(target_relation) %}
+    -- create a view with a index as suffix and chunk indicator
+    {%- set chunk_suffix = "" if chunk_count == 1 else "_PART" ~ (chunk_index + 1) %}
+    {%- set stats_view_identifier = "\"%s_%s_%s_%s%s\"" | format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2], chunk_suffix) | upper %}
+    {%- set schema_name = "\"%s\""|format(schema) %}
+    {%- set db_name = "\"%s\""|format(database) %}
+    {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema_name, database=db_name, type='view') %}
+    
+    {{ log("Creating view " ~ stats_view_identifier ~ " with " ~ chunk_columns | length ~ " columns (chunk " ~ (chunk_index + 1) ~ " of " ~ chunk_count ~ ")", info=True) }}
+    
+    {% call statement("main") %}
+      {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
+    {% endcall %}
+    
+    {%- set full_refresh_mode = (should_full_refresh()) -%}
+    {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
+    {%- do apply_grants(target_relation, grant_config, should_revoke) %}
+    {%- set target_relations = target_relations.append(target_relation) %}
+  {%- endfor %}
 {%- endfor -%}
 
 {{ run_hooks(post_hooks, inside_transaction=True) }}
--- `COMMIT` happens here:
+-- COMMIT happens here:
 {{ adapter.commit() }}
 {{ run_hooks(post_hooks, inside_transaction=False) }}
 

quollio_core/profilers/bigquery.py

@@ -1,15 +1,25 @@
+import io
+import os
 from typing import Dict, List
 
+from fastavro import writer
 from google.auth.credentials import Credentials
 
+from quollio_core.helper.core import new_global_id
 from quollio_core.helper.log_utils import error_handling_decorator, logger
-from quollio_core.profilers.lineage import gen_table_lineage_payload, parse_bigquery_table_lineage
+from quollio_core.models.avroasset import AvroAsset
+from quollio_core.models.qdc import GetImportURLRequest
+from quollio_core.profilers.lineage import (
+    gen_table_avro_lineage_payload,
+    gen_table_lineage_payload,
+    parse_bigquery_table_lineage,
+)
 from quollio_core.profilers.stats import gen_table_stats_payload
 from quollio_core.repository import qdc
 from quollio_core.repository.bigquery import BigQueryClient, GCPLineageClient, get_entitiy_reference, get_search_request
+from quollio_core.repository.cloud_resource_manager import CloudResourceManagerClient
 
 
-@error_handling_decorator
 def bigquery_table_lineage(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
@@ -17,34 +27,84 @@ def bigquery_table_lineage(
     regions: list,
     org_id: str,
     credentials: Credentials,
+    existing_global_ids: Dict[str, bool],
+    enable_multi_projects: str = "DISABLED",
 ) -> None:
     lineage_client = GCPLineageClient(credentials)
-    bq_client = BigQueryClient(credentials, project_id)
-
-    datasets = bq_client.list_dataset_ids()
-    all_tables = generate_table_list(bq_client, datasets)
-    lineage_links = generate_lineage_links(all_tables, lineage_client, project_id, regions)
-    lineage_links = parse_bigquery_table_lineage(lineage_links)
-    logger.debug("The following resources will be ingested. %s", lineage_links)
-
-    update_table_lineage_inputs = gen_table_lineage_payload(tenant_id=tenant_id, endpoint=org_id, tables=lineage_links)
-
-    req_count = 0
-    for update_table_lineage_input in update_table_lineage_inputs:
-        logger.info(
-            "Generating table lineage. downstream: %s -> %s -> %s. upstream: %s",
-            update_table_lineage_input.downstream_database_name,
-            update_table_lineage_input.downstream_schema_name,
-            update_table_lineage_input.downstream_table_name,
-            update_table_lineage_input.upstreams.as_dict(),
+    crm_client = CloudResourceManagerClient(credentials)
+
+    target_project_ids = []
+    if enable_multi_projects == "ENABLED":
+        try:
+            target_projects = crm_client.list_projects()
+        except Exception as e:
+            raise Exception(f"ListProjects by cloud resource manager failed. Err. {str(e)}")
+
+        for target_project in target_projects["projects"]:
+            if target_project is None:
+                logger.warning("projects.Projects returns None. Proceed to loop project value")
+                continue
+
+            target_project_id = target_project.get("projectId", "")
+            if target_project_id == "":
+                logger.warning("projects.Projects is empty string. Proceed to loop project value")
+                continue
+
+            target_project_ids.append(target_project_id)
+    else:
+        target_project_ids.append(project_id)
+
+    update_table_lineage_inputs = []
+    for target_project_id in target_project_ids:
+        bq_client = BigQueryClient(credentials, target_project_id)
+        datasets = bq_client.list_dataset_ids()
+        all_tables = generate_table_list(bq_client, datasets)
+        lineage_links = generate_lineage_links(all_tables, lineage_client, target_project_id, regions)
+        lineage_links = parse_bigquery_table_lineage(lineage_links)
+        logger.debug("The following resources will be ingested. %s", lineage_links)
+
+        update_table_lineage_input = gen_table_lineage_payload(
+            tenant_id=tenant_id, endpoint=org_id, tables=lineage_links
         )
-        status_code = qdc_client.update_lineage_by_id(
-            global_id=update_table_lineage_input.downstream_global_id,
-            payload=update_table_lineage_input.upstreams.as_dict(),
+        update_table_lineage_input = gen_table_avro_lineage_payload(
+            tenant_id=tenant_id,
+            endpoint=org_id,
+            tables=lineage_links,
+            existing_global_ids=existing_global_ids,
         )
-        if status_code == 200:
-            req_count += 1
-    logger.info("Generating table lineage is finished. %s lineages are ingested.", req_count)
+        update_table_lineage_inputs.extend(update_table_lineage_input)
+
+    stack_name = os.getenv("CF_STACK")
+    import_req = GetImportURLRequest(
+        service_name="bigquery",
+        source_name=stack_name,
+        file_name="{name}.avro".format(name=stack_name),
+        override_logical_name="false",
+        update_mode="partial",
+    )
+    datasource_id = new_global_id(tenant_id=tenant_id, cluster_id=org_id, data_id="", data_type="data_source")
+    logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+
+    import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+    if import_res is None:
+        logger.error("get_import_url failed. Please retry `load_lineage` again")
+        return
+    logger.debug("ImportResponse: {res}".format(res=import_res))
+
+    avro_schema = AvroAsset.avro_schema_to_python()
+
+    buffer = io.BytesIO()
+    writer(buffer, avro_schema, update_table_lineage_inputs)
+
+    res = qdc_client.upload_file(
+        url=import_res.location,
+        metadata=import_res.datasource_metadata_response_body,
+        buffer=buffer.getbuffer().tobytes(),
+    )
+
+    if res == 200:
+        logger.info("Upload table lineage is finished.")
+    return
 
 
 @error_handling_decorator

quollio_core/profilers/redshift.py

@@ -1,10 +1,17 @@
+import io
 import logging
-from typing import List
+import os
+from typing import Dict, List
 
-from quollio_core.profilers.lineage import gen_table_lineage_payload, gen_table_lineage_payload_inputs
+from fastavro import writer
+
+from quollio_core.helper.core import new_global_id
+from quollio_core.models.avroasset import AvroAsset
+from quollio_core.models.qdc import GetImportURLRequest
+from quollio_core.profilers.lineage import gen_table_avro_lineage_payload, gen_table_lineage_payload_inputs
 from quollio_core.profilers.sqllineage import SQLLineage
 from quollio_core.profilers.stats import (
-    gen_table_stats_payload_from_tuple,
+    gen_table_stats_avro_payload_from_tuple,
     get_is_target_stats_items,
     render_sql_for_stats,
 )
@@ -18,6 +25,7 @@ def redshift_table_level_lineage(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     dbt_table_name: str,
+    existing_global_ids: Dict[str, bool],
 ) -> None:
     with redshift.RedshiftQueryExecutor(config=conn) as redshift_executor:
         results = redshift_executor.get_query_results(
@@ -34,28 +42,39 @@ def redshift_table_level_lineage(
         )
         lineage_payload_inputs = gen_table_lineage_payload_inputs(input_data=results)
 
-        update_table_lineage_inputs = gen_table_lineage_payload(
+        update_table_lineage_inputs = gen_table_avro_lineage_payload(
             tenant_id=tenant_id,
             endpoint=conn.host,
             tables=lineage_payload_inputs,
+            existing_global_ids=existing_global_ids,
+        )
+        stack_name = os.getenv("CF_STACK")
+        import_req = GetImportURLRequest(
+            service_name="redshift",
+            source_name=stack_name,
+            file_name="{name}.avro".format(name=stack_name),
+            override_logical_name="false",
+            update_mode="partial",
         )
+        datasource_id = new_global_id(tenant_id=tenant_id, cluster_id=conn.host, data_id="", data_type="data_source")
+        logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+        import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+        if import_res is None:
+            logger.error("get_import_url failed. Please retry `load_lineage` again")
+            return
+        logger.debug("ImportResponse: {res}".format(res=import_res))
 
-        req_count = 0
-        for update_table_lineage_input in update_table_lineage_inputs:
-            logger.info(
-                "Generating table lineage. downstream: {db} -> {schema} -> {table}".format(
-                    db=update_table_lineage_input.downstream_database_name,
-                    schema=update_table_lineage_input.downstream_schema_name,
-                    table=update_table_lineage_input.downstream_table_name,
-                )
-            )
-            status_code = qdc_client.update_lineage_by_id(
-                global_id=update_table_lineage_input.downstream_global_id,
-                payload=update_table_lineage_input.upstreams.as_dict(),
-            )
-            if status_code == 200:
-                req_count += 1
-        logger.info(f"Generating table lineage is finished. {req_count} lineages are ingested.")
+        avro_schema = AvroAsset.avro_schema_to_python()
+
+        buffer = io.BytesIO()
+        writer(buffer, avro_schema, update_table_lineage_inputs)
+        res = qdc_client.upload_file(
+            url=import_res.location,
+            metadata=import_res.datasource_metadata_response_body,
+            buffer=buffer.getbuffer().tobytes(),
+        )
+        if res == 200:
+            logger.info("Upload table lineage is finished.")
     return
 
 
@@ -82,6 +101,7 @@ def redshift_table_stats(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     stats_items: List[str],
+    existing_global_ids: Dict[str, bool],
 ) -> None:
     is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
     with redshift.RedshiftQueryExecutor(config=conn) as redshift_executor:
@@ -92,7 +112,7 @@ def redshift_table_stats(
         stats_views = redshift_executor.get_query_results(query=stats_query)
         logger.info("Found %s for table statistics.", len(stats_views))
 
-        req_count = 0
+        update_stats_inputs = list()
         for stats_view in stats_views:
             table_fqn = "{catalog}.{schema}.{table}".format(
                 catalog=stats_view[0], schema=stats_view[1], table=stats_view[2]
@@ -100,23 +120,39 @@ def redshift_table_stats(
             stats_query = render_sql_for_stats(is_aggregate_items=is_aggregate_items, table_fqn=table_fqn)
             logger.debug(f"The following sql will be fetched to retrieve stats values. {stats_query}")
             stats_result = redshift_executor.get_query_results(query=stats_query)
-            payloads = gen_table_stats_payload_from_tuple(tenant_id=tenant_id, endpoint=conn.host, stats=stats_result)
-            for payload in payloads:
-                logger.info(
-                    "Generating table stats. asset: {db} -> {schema} -> {table} -> {column}".format(
-                        db=payload.db,
-                        schema=payload.schema,
-                        table=payload.table,
-                        column=payload.column,
-                    )
-                )
-                status_code = qdc_client.update_stats_by_id(
-                    global_id=payload.global_id,
-                    payload=payload.body.get_column_stats(),
-                )
-                if status_code == 200:
-                    req_count += 1
-    logger.info(f"Generating table stats is finished. {req_count} stats are ingested.")
+            payload = gen_table_stats_avro_payload_from_tuple(
+                tenant_id=tenant_id, endpoint=conn.host, stats=stats_result, existing_global_ids=existing_global_ids
+            )
+            update_stats_inputs += payload
+
+        stack_name = os.getenv("CF_STACK")
+        import_req = GetImportURLRequest(
+            service_name="redshift",
+            source_name=stack_name,
+            file_name="{name}.avro".format(name=stack_name),
+            override_logical_name="false",
+            update_mode="partial",
+        )
+        datasource_id = new_global_id(tenant_id=tenant_id, cluster_id=conn.host, data_id="", data_type="data_source")
+        logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+        import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+        if import_res is None:
+            logger.error("get_import_url failed. Please retry load_stats again")
+            return
+        logger.debug("ImportResponse: {res}".format(res=import_res))
+
+        avro_schema = AvroAsset.avro_schema_to_python()
+
+        buffer = io.BytesIO()
+        writer(buffer, avro_schema, update_stats_inputs)
+        res = qdc_client.upload_file(
+            url=import_res.location,
+            metadata=import_res.datasource_metadata_response_body,
+            buffer=buffer.getbuffer().tobytes(),
+        )
+        if res == 200:
+            logger.info("Generating table stats is finished.")
+
     return
 
 

quollio_core/profilers/stats.py

@@ -147,6 +147,43 @@ def gen_table_stats_payload(tenant_id: str, endpoint: str, stats: List[Dict[str,
     return payloads
 
 
+def gen_table_stats_avro_payload_from_tuple(
+    tenant_id: str, endpoint: str, stats: Tuple[List[str]], existing_global_ids: Dict[str, bool]
+) -> List[Dict[str, str]]:
+    payloads = list()
+    for stat in stats:
+        db_name, schema_name, table_name, column_name = stat[:4]
+
+        global_id_arg = "{db}{schema}{table}{column}".format(
+            db=db_name, schema=schema_name, table=table_name, column=column_name
+        )
+        table_global_id = new_global_id(
+            tenant_id=tenant_id, cluster_id=endpoint, data_id=global_id_arg, data_type="column"
+        )
+
+        if existing_global_ids.get(table_global_id) is not True:
+            continue
+
+        avro_assets = AvroAsset(
+            id=table_global_id,
+            object_type="column",
+            parents=[db_name, schema_name, table_name],
+            name=column_name,
+            stats_max=convert_value_type(stat[4], True),
+            stats_min=convert_value_type(stat[5], True),
+            stats_mean=convert_value_type(stat[8], True),
+            stats_median=convert_value_type(stat[9], True),
+            stats_mode=convert_value_type(stat[10], True),
+            stats_stddev=convert_value_type(stat[11], True),
+            stats_number_of_null=convert_value_type(stat[6], True),
+            stats_number_of_unique=convert_value_type(stat[7], True),
+        )
+
+        payloads.append(avro_assets.to_dict())
+
+    return payloads
+
+
 def gen_table_stats_payload_from_tuple(
     tenant_id: str, endpoint: str, stats: Tuple[List[str]]
 ) -> List[Dict[str, Union[str, List[Dict[str, str]]]]]:

quollio_core/redshift.py

@@ -3,16 +3,17 @@ import logging
 import os
 import shutil
 
-from quollio_core.helper.core import setup_dbt_profile
+from quollio_core.helper.core import is_valid_domain, setup_dbt_profile
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log import set_log_level
+from quollio_core.profilers.qdc import gen_existing_global_id_dict, get_avro_file_content
 from quollio_core.profilers.redshift import (
     redshift_table_level_lineage,
     redshift_table_level_sqllineage,
     redshift_table_stats,
 )
 from quollio_core.profilers.stats import get_column_stats_items
-from quollio_core.repository import dbt, qdc, redshift
+from quollio_core.repository import dbt, qdc, redshift, ssm
 
 logger = logging.getLogger(__name__)
 
@@ -84,11 +85,20 @@ def load_lineage(
     tenant_id: str,
 ) -> None:
     logger.info("Generate redshift table to table lineage.")
+
+    file_content = get_avro_file_content(
+        tenant_id=tenant_id,
+        account_id=conn.host,
+        qdc_client=qdc_client,
+    )
+    existing_global_ids = gen_existing_global_id_dict(avro_content=file_content)
+
     redshift_table_level_lineage(
         conn=conn,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
         dbt_table_name="quollio_lineage_table_level",
+        existing_global_ids=existing_global_ids,
     )
 
     logger.info("Generate redshift view level lineage.")
@@ -97,6 +107,7 @@ def load_lineage(
         qdc_client=qdc_client,
         tenant_id=tenant_id,
         dbt_table_name="quollio_lineage_view_level",
+        existing_global_ids=existing_global_ids,
     )
 
     logger.info("Lineage data is successfully loaded.")
@@ -115,12 +126,20 @@ def load_stats(
     if stats_items is None:
         raise ValueError("No stats items are not selected. Please specify any value to `stats_items` param.")
 
+    file_content = get_avro_file_content(
+        tenant_id=tenant_id,
+        account_id=conn.host,
+        qdc_client=qdc_client,
+    )
+    existing_global_ids = gen_existing_global_id_dict(avro_content=file_content)
+
     logger.info("The following values will be aggregated. {stats_items}".format(stats_items=stats_items))
     redshift_table_stats(
         conn=conn,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
         stats_items=stats_items,
+        existing_global_ids=existing_global_ids,
     )
 
     logger.info("Stats data is successfully loaded.")
@@ -252,6 +271,7 @@ if __name__ == "__main__":
         type=str,
         choices=["debug", "info", "warn", "error", "none"],
         action=env_default("LOG_LEVEL"),
+        default="info",
         required=False,
         help="The log level for dbt commands. Default value is info",
     )
@@ -285,6 +305,16 @@ if __name__ == "__main__":
         required=False,
         help="The client secrete that is created on Quollio console to let clients access Quollio External API",
     )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
 
     stats_items = get_column_stats_items()
     parser.add_argument(
@@ -323,11 +353,22 @@ if __name__ == "__main__":
             log_level=args.log_level,
             dbt_macro_source=args.dbt_macro_source,
         )
+
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com`")
+
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
             client_id=args.client_id,
             client_secret=args.client_secret,
-            base_url=args.api_url,
+            base_url=api_url,
         )
         load_lineage(
             conn=conn,
@@ -338,7 +379,7 @@ if __name__ == "__main__":
         qdc_client = qdc.QDCExternalAPIClient(
             client_id=args.client_id,
             client_secret=args.client_secret,
-            base_url=args.api_url,
+            base_url=api_url,
         )
         load_stats(
             conn=conn,
@@ -348,7 +389,7 @@ if __name__ == "__main__":
         )
     if "load_sqllineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
-            base_url=args.api_url,
+            base_url=api_url,
             client_id=args.client_id,
             client_secret=args.client_secret,
         )

quollio_core/repository/cloud_resource_manager.py

@@ -0,0 +1,25 @@
+from google.oauth2.service_account import Credentials
+from googleapiclient.discovery import build
+
+
+class CloudResourceManagerClient:
+    """Client to interact with the Cloud Resource Manager API."""
+
+    def __init__(self, credentials: Credentials) -> None:
+        """Initialize the Cloud Resource Manager client with provided credentials."""
+        self.client = self.__initialize(credentials=credentials)
+
+    def __initialize(self, credentials: Credentials):
+        return build("cloudresourcemanager", "v1", credentials=credentials)
+
+    def list_projects(self):
+        """List all projects accessible with the current credentials."""
+        request = self.client.projects().list()
+        response = request.execute()
+        return response
+
+    def get_project(self, project_id: str):
+        """Get a specific project by project ID."""
+        request = self.client.projects().get(projectId=project_id)
+        response = request.execute()
+        return response

quollio_core/snowflake.py

@@ -312,6 +312,7 @@ if __name__ == "__main__":
         type=str,
         choices=["debug", "info", "warn", "error", "none"],
         action=env_default("LOG_LEVEL"),
+        default="info",
         required=False,
         help="The log level for dbt commands. Default value is info",
     )

{quollio_core-0.6.4.dist-info → quollio_core-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quollio-core
-Version: 0.6.4
+Version: 0.7.0
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>

{quollio_core-0.6.4.dist-info → quollio_core-0.7.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-quollio_core/__init__.py,sha256=Cb-B2rl0ckOxRzDpNBNZdhUUNRb6SBPYnZxyLrdrXj0,83
-quollio_core/bigquery.py,sha256=6Oq4DVGpa3X21Es_nbrsb8pK3vaxwb9Egnvq3huo95k,5894
+quollio_core/__init__.py,sha256=ZtRODvd0lC2-vqcXyfKDya8bb6FyyFWBsGNz7TSlO-s,83
+quollio_core/bigquery.py,sha256=nxMWldMr14HzOeyzYM_fRrfHQ7jbm2HyGzc1r46mlco,6821
 quollio_core/bricks.py,sha256=8h3kbI2b6lGH2s-56jE_Q5-R5-nIsQYMfvtRrkFOzoU,10784
-quollio_core/redshift.py,sha256=KcdljY95xYf9JYrsaMOBoP_XxQQ8wFVE5ue_XEMVSFc,11504
-quollio_core/snowflake.py,sha256=fMkZ7OlbimgLg4TxVASTCpXSH2MPeawveAFi0xhKqnw,17060
+quollio_core/redshift.py,sha256=Yi_udcgfen2PoCkDTIhemeCFbDVUU1rYWro9CyjHCZA,13192
+quollio_core/snowflake.py,sha256=aOQ8tLSbHJEx_TUGaZLjix5KWgLiRp3A1tRx5qgUtRI,17084
 quollio_core/teradata.py,sha256=H2VUcJvr8W-M2wvm3710Gf1ENb-BSscrDRKNm8gdHJE,8227
 quollio_core/dbt_projects/databricks/.gitignore,sha256=1jJAyXSzJ3YUm0nx3i7wUSE4RjQMX3ad6F8O88UbtzI,29
 quollio_core/dbt_projects/databricks/README.md,sha256=ZpRQyhFAODAiS8dc1Kb_ndkul4cu4o4udN_EMa49CU4,440
@@ -47,7 +47,7 @@ quollio_core/dbt_projects/snowflake/packages_hub.yml,sha256=p9Bl2C44gdC6iYTUkz_1
 quollio_core/dbt_projects/snowflake/packages_local.yml,sha256=ryyJSXv83gYFu48xmzG5Z1l746jGCUBE6hs7pUNwuXE,43
 quollio_core/dbt_projects/snowflake/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=oLf_rQ0tV_I_nhyEaMZ6rpmzIoLgMHor1bruBDyJaqU,3992
+quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=WjN_baGb_FuDX831rR5G7Payw5t91VaoQobEkTJR3jQ,4880
 quollio_core/dbt_projects/snowflake/macros/materialization/get_imported_databases.sql,sha256=WAFl9CyM-G07O7vZD2MkA1hncpjV_gSMGc2V8nRJRPk,1435
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.sql,sha256=Zhj0EXF1K8S-OkFxz3IBHe2olXktYrvly0LwZBOAUXw,5333
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.yml,sha256=a2uNIAh-xw51eu-GmHVuAnGnTbwK7h8-DjDeQtK3KaQ,711
@@ -71,18 +71,19 @@ quollio_core/helper/log_utils.py,sha256=QontLKETHjSAbQniJ7YqS0RY2AYvFHSjrlPiGr31
 quollio_core/models/avroasset.py,sha256=YZHzOS62N0_sidneXI3IZ2MA8Bz1vFVgF6F9_UilC3s,603
 quollio_core/models/qdc.py,sha256=UObaUpvAQ4vOhI6jfwvNFrJ3--6AX2v9yl9_d3Juy7M,739
 quollio_core/profilers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/profilers/bigquery.py,sha256=mEr7CFQNgBFqWR8XfCOk8WTm5k5qZhLF8ODVWfskPRI,5797
+quollio_core/profilers/bigquery.py,sha256=rJ2eMZlUtEmsNCCkenC9DuL7--btcO2dF_NmIOawGJ4,7947
 quollio_core/profilers/databricks.py,sha256=ik4RiR_GOeU3S7s6C6Y9SGe1D_Y_f98BDWJVlEJXL4U,7868
 quollio_core/profilers/lineage.py,sha256=GMWue6lgiz7wFYnNpaHVFivprA-iqhbCHf63IsVB8Vk,11260
 quollio_core/profilers/qdc.py,sha256=P0STRfe5G4d-UI7RdVbYmAfP_tAn1HbFUjeirxWipz4,995
-quollio_core/profilers/redshift.py,sha256=p6ONDCkhndZAOcKAwEyQ5fsi-jsQrlwHHb7LTI_m1uk,6473
+quollio_core/profilers/redshift.py,sha256=f5F3jnFJP2SbmPtG-PNyVgkt4mtfxuUX45pIS0mrw60,7856
 quollio_core/profilers/snowflake.py,sha256=ewvULWIlcq2h0jOyRzUpedW0NS8QlkSgICS-dZDYl18,13027
 quollio_core/profilers/sqllineage.py,sha256=h0FT6CYb0A20zSc68GELZ7Q8bDbaHLQnZQHsXBEXBug,5261
-quollio_core/profilers/stats.py,sha256=PwMNyr7JwdrVbSSfVd-XsgCZV7sKoRjihaRhhZ88uyM,9328
+quollio_core/profilers/stats.py,sha256=Go1tR8IMpMZnHZJzYtAZTC89ZDkebUmGxljwy8h5KC0,10752
 quollio_core/profilers/teradata/lineage.py,sha256=2wNksBQD8vC6UTQwCglPsF53YMEVIkAb2CWTmpiTHDU,7368
 quollio_core/profilers/teradata/stats.py,sha256=OagvkTRFiWVbiLABwZwR3wQ7y36edwOViDetHsYiyxI,9277
 quollio_core/repository/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/repository/bigquery.py,sha256=3AyGcJNYGnUyMweyc6lGm4quwrOzd-ZBS2zNnFwafII,3990
+quollio_core/repository/cloud_resource_manager.py,sha256=tmHxjV3AmKwv3OJ-f40N-XQA1qmxZTSDBDS3YK69CIo,978
 quollio_core/repository/databricks.py,sha256=9Cgdv8qBnVaHqu3RA-IUBieAqb69moQ-KAAMVSf5Ds4,1877
 quollio_core/repository/dbt.py,sha256=cnLwJPywLi8VowVW7zfIBa9jxVwDWO7xzzNRn1vWiuw,659
 quollio_core/repository/qdc.py,sha256=Ni0rk9CX8ienqM_HYLuWLBKTwycvTENC7x7wGWhzjXs,8978
@@ -90,7 +91,7 @@ quollio_core/repository/redshift.py,sha256=p2ouEuYcDCjx1oBhc6H1ekQsvEqHGd3bFu3PW
 quollio_core/repository/snowflake.py,sha256=yCYXrYf4I5GL_ITNTXoggj0xNbQsdwxPSmsVvZYwUVU,3869
 quollio_core/repository/ssm.py,sha256=xpm1FzbBnIsBptuYPUNnPgkKU2AH3XxI-ZL0bEetvW0,2182
 quollio_core/repository/teradata.py,sha256=1AExxRBTswpSyF4OVyAUkoiZ0yVRfqt4T99FdllkTEI,3763
-quollio_core-0.6.4.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.6.4.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-quollio_core-0.6.4.dist-info/METADATA,sha256=KSBNAaAXnsuB5w8NomEzIaKHX02N5ue-33qdc2zSDqI,7023
-quollio_core-0.6.4.dist-info/RECORD,,
+quollio_core-0.7.0.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.7.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+quollio_core-0.7.0.dist-info/METADATA,sha256=xrt5HSRtSF0M-2Ef09CNsB2fo6hyMifPOySYhF5U1nA,7023
+quollio_core-0.7.0.dist-info/RECORD,,