quollio-core 0.6.5__tar.gz → 0.8.0__tar.gz

{quollio_core-0.6.5 → quollio_core-0.8.0}/PKG-INFO

@@ -1,14 +1,13 @@
 Metadata-Version: 2.4
 Name: quollio-core
-Version: 0.6.5
+Version: 0.8.0
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -29,7 +28,7 @@ Requires-Dist: PyYAML==6.0.1
 Requires-Dist: requests==2.31.0
 Requires-Dist: pyjwt==2.8.0
 Requires-Dist: redshift-connector==2.0.915
-Requires-Dist: snowflake-connector-python==3.5.0
+Requires-Dist: snowflake-connector-python==3.15.0
 Requires-Dist: databricks-sdk==0.17.0
 Requires-Dist: databricks-sql-connector==2.9.5
 Requires-Dist: sqlglot==20.8.0

{quollio_core-0.6.5 → quollio_core-0.8.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "flit_core.buildapi"
 [project]
 name = "quollio-core"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = {file = "LICENSE"}
 authors = [
   {name = "quollio-dev", email = "qt.dev@quollio.com"},
@@ -17,7 +17,6 @@ maintainers = [
 classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -40,7 +39,7 @@ dependencies = [
     ,"requests==2.31.0"
     ,"pyjwt==2.8.0"
     ,"redshift-connector==2.0.915"
-    ,"snowflake-connector-python==3.5.0"
+    ,"snowflake-connector-python==3.15.0"
     ,"databricks-sdk==0.17.0"
     ,"databricks-sql-connector==2.9.5"
     ,"sqlglot==20.8.0"

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.6.5"
+__version__ = "0.8.0"
 __author__ = "Quollio Technologies, Inc"

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/bigquery.py

@@ -3,10 +3,12 @@ import json
 
 from google.auth.credentials import Credentials
 
+from quollio_core.helper.core import is_valid_domain
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log_utils import configure_logging, error_handling_decorator, logger
 from quollio_core.profilers.bigquery import bigquery_table_lineage, bigquery_table_stats
-from quollio_core.repository import qdc
+from quollio_core.profilers.qdc import gen_existing_global_id_dict, get_avro_file_content
+from quollio_core.repository import qdc, ssm
 from quollio_core.repository.bigquery import BigQueryClient, get_credentials, get_org_id
 
 
@@ -30,8 +32,15 @@ def load_lineage(
     org_id: str,
     credentials: Credentials,
     qdc_client: qdc.QDCExternalAPIClient,
+    enable_multi_projects: str,
 ) -> None:
     logger.info("Loading lineage data.")
+    file_content = get_avro_file_content(
+        tenant_id=tenant_id,
+        account_id=org_id,
+        qdc_client=qdc_client,
+    )
+    existing_global_ids = gen_existing_global_id_dict(avro_content=file_content)
     bigquery_table_lineage(
         qdc_client=qdc_client,
         tenant_id=tenant_id,
@@ -39,6 +48,8 @@ def load_lineage(
         regions=regions,
         credentials=credentials,
         org_id=org_id,
+        existing_global_ids=existing_global_ids,
+        enable_multi_projects=enable_multi_projects,
     )
     logger.info("Lineage data loaded successfully.")
 
@@ -146,6 +157,27 @@ if __name__ == "__main__":
         help="Comma-separated list of dataplex stats tables - <project_id>.<dataset_id>.<table_id>",
     )
 
+    parser.add_argument(
+        "--enable_multi_projects",
+        type=str,
+        choices=["ENABLED", "DISABLED"],
+        action=env_default("ENABLE_MULTI_PROJECTS"),
+        default="DISABLED",
+        required=False,
+        help="Whether to enable multi-projects support. If set to 'true', \
+        the script will load lineage and stats from all projects accessible by the credentials. Default is 'false'.",
+    )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
+
     args = parser.parse_args()
 
     # Validate that dataplex_stats_tables is provided if load_stats is in commands
@@ -154,9 +186,21 @@ if __name__ == "__main__":
 
     configure_logging(args.log_level)
 
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
+
     credentials = initialize_credentials(args.credentials_json)
     org_id = initialize_org_id(args.credentials_json)
-    qdc_client = qdc.initialize_qdc_client(args.api_url, args.client_id, args.client_secret)
+    qdc_client = qdc.initialize_qdc_client(api_url, args.client_id, args.client_secret)
     bq_client = initialize_bq_client(credentials, args.project_id)
     if args.project_id is None:
         args.project_id = json.loads(args.credentials_json)["project_id"]
@@ -170,6 +214,7 @@ if __name__ == "__main__":
             org_id=org_id,
             credentials=credentials,
             qdc_client=qdc_client,
+            enable_multi_projects=args.enable_multi_projects,
         )
 
     if "load_stats" in args.commands:

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/bricks.py

@@ -3,7 +3,7 @@ import logging
 import os
 import shutil
 
-from quollio_core.helper.core import setup_dbt_profile, trim_prefix
+from quollio_core.helper.core import is_valid_domain, setup_dbt_profile, trim_prefix
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log import set_log_level
 from quollio_core.profilers.databricks import (
@@ -13,7 +13,7 @@ from quollio_core.profilers.databricks import (
 )
 from quollio_core.profilers.stats import get_column_stats_items
 from quollio_core.repository import databricks as db
-from quollio_core.repository import dbt, qdc
+from quollio_core.repository import dbt, qdc, ssm
 
 logger = logging.getLogger(__name__)
 
@@ -256,6 +256,16 @@ if __name__ == "__main__":
         required=False,
         help="Whether to ingest column lineage into QDIC or not. Default value is False",
     )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
 
     stats_items = get_column_stats_items()
     parser.add_argument(
@@ -294,9 +304,21 @@ if __name__ == "__main__":
             dbt_macro_source=args.dbt_macro_source,
         )
 
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
+
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
-            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+            base_url=api_url, client_id=args.client_id, client_secret=args.client_secret
         )
         load_lineage(
             conn=conn,
@@ -308,7 +330,7 @@ if __name__ == "__main__":
 
     if "load_stats" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
-            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+            base_url=api_url, client_id=args.client_id, client_secret=args.client_secret
         )
         databricks_column_stats(
             conn=conn,

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/profilers/bigquery.py

@@ -1,15 +1,25 @@
+import io
+import os
 from typing import Dict, List
 
+from fastavro import writer
 from google.auth.credentials import Credentials
 
+from quollio_core.helper.core import new_global_id
 from quollio_core.helper.log_utils import error_handling_decorator, logger
-from quollio_core.profilers.lineage import gen_table_lineage_payload, parse_bigquery_table_lineage
+from quollio_core.models.avroasset import AvroAsset
+from quollio_core.models.qdc import GetImportURLRequest
+from quollio_core.profilers.lineage import (
+    gen_table_avro_lineage_payload,
+    gen_table_lineage_payload,
+    parse_bigquery_table_lineage,
+)
 from quollio_core.profilers.stats import gen_table_stats_payload
 from quollio_core.repository import qdc
 from quollio_core.repository.bigquery import BigQueryClient, GCPLineageClient, get_entitiy_reference, get_search_request
+from quollio_core.repository.cloud_resource_manager import CloudResourceManagerClient
 
 
-@error_handling_decorator
 def bigquery_table_lineage(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
@@ -17,34 +27,84 @@ def bigquery_table_lineage(
     regions: list,
     org_id: str,
     credentials: Credentials,
+    existing_global_ids: Dict[str, bool],
+    enable_multi_projects: str = "DISABLED",
 ) -> None:
     lineage_client = GCPLineageClient(credentials)
-    bq_client = BigQueryClient(credentials, project_id)
-
-    datasets = bq_client.list_dataset_ids()
-    all_tables = generate_table_list(bq_client, datasets)
-    lineage_links = generate_lineage_links(all_tables, lineage_client, project_id, regions)
-    lineage_links = parse_bigquery_table_lineage(lineage_links)
-    logger.debug("The following resources will be ingested. %s", lineage_links)
-
-    update_table_lineage_inputs = gen_table_lineage_payload(tenant_id=tenant_id, endpoint=org_id, tables=lineage_links)
-
-    req_count = 0
-    for update_table_lineage_input in update_table_lineage_inputs:
-        logger.info(
-            "Generating table lineage. downstream: %s -> %s -> %s. upstream: %s",
-            update_table_lineage_input.downstream_database_name,
-            update_table_lineage_input.downstream_schema_name,
-            update_table_lineage_input.downstream_table_name,
-            update_table_lineage_input.upstreams.as_dict(),
+    crm_client = CloudResourceManagerClient(credentials)
+
+    target_project_ids = []
+    if enable_multi_projects == "ENABLED":
+        try:
+            target_projects = crm_client.list_projects()
+        except Exception as e:
+            raise Exception(f"ListProjects by cloud resource manager failed. Err. {str(e)}")
+
+        for target_project in target_projects["projects"]:
+            if target_project is None:
+                logger.warning("projects.Projects returns None. Proceed to loop project value")
+                continue
+
+            target_project_id = target_project.get("projectId", "")
+            if target_project_id == "":
+                logger.warning("projects.Projects is empty string. Proceed to loop project value")
+                continue
+
+            target_project_ids.append(target_project_id)
+    else:
+        target_project_ids.append(project_id)
+
+    update_table_lineage_inputs = []
+    for target_project_id in target_project_ids:
+        bq_client = BigQueryClient(credentials, target_project_id)
+        datasets = bq_client.list_dataset_ids()
+        all_tables = generate_table_list(bq_client, datasets)
+        lineage_links = generate_lineage_links(all_tables, lineage_client, target_project_id, regions)
+        lineage_links = parse_bigquery_table_lineage(lineage_links)
+        logger.debug("The following resources will be ingested. %s", lineage_links)
+
+        update_table_lineage_input = gen_table_lineage_payload(
+            tenant_id=tenant_id, endpoint=org_id, tables=lineage_links
         )
-        status_code = qdc_client.update_lineage_by_id(
-            global_id=update_table_lineage_input.downstream_global_id,
-            payload=update_table_lineage_input.upstreams.as_dict(),
+        update_table_lineage_input = gen_table_avro_lineage_payload(
+            tenant_id=tenant_id,
+            endpoint=org_id,
+            tables=lineage_links,
+            existing_global_ids=existing_global_ids,
         )
-        if status_code == 200:
-            req_count += 1
-    logger.info("Generating table lineage is finished. %s lineages are ingested.", req_count)
+        update_table_lineage_inputs.extend(update_table_lineage_input)
+
+    stack_name = os.getenv("CF_STACK")
+    import_req = GetImportURLRequest(
+        service_name="bigquery",
+        source_name=stack_name,
+        file_name="{name}.avro".format(name=stack_name),
+        override_logical_name="false",
+        update_mode="partial",
+    )
+    datasource_id = new_global_id(tenant_id=tenant_id, cluster_id=org_id, data_id="", data_type="data_source")
+    logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+
+    import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+    if import_res is None:
+        logger.error("get_import_url failed. Please retry `load_lineage` again")
+        return
+    logger.debug("ImportResponse: {res}".format(res=import_res))
+
+    avro_schema = AvroAsset.avro_schema_to_python()
+
+    buffer = io.BytesIO()
+    writer(buffer, avro_schema, update_table_lineage_inputs)
+
+    res = qdc_client.upload_file(
+        url=import_res.location,
+        metadata=import_res.datasource_metadata_response_body,
+        buffer=buffer.getbuffer().tobytes(),
+    )
+
+    if res == 200:
+        logger.info("Upload table lineage is finished.")
+    return
 
 
 @error_handling_decorator

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/profilers/redshift.py

@@ -1,10 +1,17 @@
+import io
 import logging
-from typing import List
+import os
+from typing import Dict, List
 
-from quollio_core.profilers.lineage import gen_table_lineage_payload, gen_table_lineage_payload_inputs
+from fastavro import writer
+
+from quollio_core.helper.core import new_global_id
+from quollio_core.models.avroasset import AvroAsset
+from quollio_core.models.qdc import GetImportURLRequest
+from quollio_core.profilers.lineage import gen_table_avro_lineage_payload, gen_table_lineage_payload_inputs
 from quollio_core.profilers.sqllineage import SQLLineage
 from quollio_core.profilers.stats import (
-    gen_table_stats_payload_from_tuple,
+    gen_table_stats_avro_payload_from_tuple,
     get_is_target_stats_items,
     render_sql_for_stats,
 )
@@ -18,6 +25,7 @@ def redshift_table_level_lineage(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     dbt_table_name: str,
+    existing_global_ids: Dict[str, bool],
 ) -> None:
     with redshift.RedshiftQueryExecutor(config=conn) as redshift_executor:
         results = redshift_executor.get_query_results(
@@ -34,28 +42,39 @@ def redshift_table_level_lineage(
         )
         lineage_payload_inputs = gen_table_lineage_payload_inputs(input_data=results)
 
-        update_table_lineage_inputs = gen_table_lineage_payload(
+        update_table_lineage_inputs = gen_table_avro_lineage_payload(
             tenant_id=tenant_id,
             endpoint=conn.host,
             tables=lineage_payload_inputs,
+            existing_global_ids=existing_global_ids,
+        )
+        stack_name = os.getenv("CF_STACK")
+        import_req = GetImportURLRequest(
+            service_name="redshift",
+            source_name=stack_name,
+            file_name="{name}.avro".format(name=stack_name),
+            override_logical_name="false",
+            update_mode="partial",
         )
+        datasource_id = new_global_id(tenant_id=tenant_id, cluster_id=conn.host, data_id="", data_type="data_source")
+        logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+        import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+        if import_res is None:
+            logger.error("get_import_url failed. Please retry `load_lineage` again")
+            return
+        logger.debug("ImportResponse: {res}".format(res=import_res))
 
-        req_count = 0
-        for update_table_lineage_input in update_table_lineage_inputs:
-            logger.info(
-                "Generating table lineage. downstream: {db} -> {schema} -> {table}".format(
-                    db=update_table_lineage_input.downstream_database_name,
-                    schema=update_table_lineage_input.downstream_schema_name,
-                    table=update_table_lineage_input.downstream_table_name,
-                )
-            )
-            status_code = qdc_client.update_lineage_by_id(
-                global_id=update_table_lineage_input.downstream_global_id,
-                payload=update_table_lineage_input.upstreams.as_dict(),
-            )
-            if status_code == 200:
-                req_count += 1
-        logger.info(f"Generating table lineage is finished. {req_count} lineages are ingested.")
+        avro_schema = AvroAsset.avro_schema_to_python()
+
+        buffer = io.BytesIO()
+        writer(buffer, avro_schema, update_table_lineage_inputs)
+        res = qdc_client.upload_file(
+            url=import_res.location,
+            metadata=import_res.datasource_metadata_response_body,
+            buffer=buffer.getbuffer().tobytes(),
+        )
+        if res == 200:
+            logger.info("Upload table lineage is finished.")
     return
 
 
@@ -82,6 +101,7 @@ def redshift_table_stats(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     stats_items: List[str],
+    existing_global_ids: Dict[str, bool],
 ) -> None:
     is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
     with redshift.RedshiftQueryExecutor(config=conn) as redshift_executor:
@@ -92,7 +112,7 @@ def redshift_table_stats(
         stats_views = redshift_executor.get_query_results(query=stats_query)
         logger.info("Found %s for table statistics.", len(stats_views))
 
-        req_count = 0
+        update_stats_inputs = list()
         for stats_view in stats_views:
             table_fqn = "{catalog}.{schema}.{table}".format(
                 catalog=stats_view[0], schema=stats_view[1], table=stats_view[2]
@@ -100,23 +120,39 @@ def redshift_table_stats(
             stats_query = render_sql_for_stats(is_aggregate_items=is_aggregate_items, table_fqn=table_fqn)
             logger.debug(f"The following sql will be fetched to retrieve stats values. {stats_query}")
             stats_result = redshift_executor.get_query_results(query=stats_query)
-            payloads = gen_table_stats_payload_from_tuple(tenant_id=tenant_id, endpoint=conn.host, stats=stats_result)
-            for payload in payloads:
-                logger.info(
-                    "Generating table stats. asset: {db} -> {schema} -> {table} -> {column}".format(
-                        db=payload.db,
-                        schema=payload.schema,
-                        table=payload.table,
-                        column=payload.column,
-                    )
-                )
-                status_code = qdc_client.update_stats_by_id(
-                    global_id=payload.global_id,
-                    payload=payload.body.get_column_stats(),
-                )
-                if status_code == 200:
-                    req_count += 1
-    logger.info(f"Generating table stats is finished. {req_count} stats are ingested.")
+            payload = gen_table_stats_avro_payload_from_tuple(
+                tenant_id=tenant_id, endpoint=conn.host, stats=stats_result, existing_global_ids=existing_global_ids
+            )
+            update_stats_inputs += payload
+
+        stack_name = os.getenv("CF_STACK")
+        import_req = GetImportURLRequest(
+            service_name="redshift",
+            source_name=stack_name,
+            file_name="{name}.avro".format(name=stack_name),
+            override_logical_name="false",
+            update_mode="partial",
+        )
+        datasource_id = new_global_id(tenant_id=tenant_id, cluster_id=conn.host, data_id="", data_type="data_source")
+        logger.debug("Datasource id: {dsrc_id}".format(dsrc_id=datasource_id))
+        import_res = qdc_client.get_import_url(datasource_id=datasource_id, payload=import_req)
+        if import_res is None:
+            logger.error("get_import_url failed. Please retry load_stats again")
+            return
+        logger.debug("ImportResponse: {res}".format(res=import_res))
+
+        avro_schema = AvroAsset.avro_schema_to_python()
+
+        buffer = io.BytesIO()
+        writer(buffer, avro_schema, update_stats_inputs)
+        res = qdc_client.upload_file(
+            url=import_res.location,
+            metadata=import_res.datasource_metadata_response_body,
+            buffer=buffer.getbuffer().tobytes(),
+        )
+        if res == 200:
+            logger.info("Generating table stats is finished.")
+
     return
 
 

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/profilers/stats.py

@@ -147,6 +147,43 @@ def gen_table_stats_payload(tenant_id: str, endpoint: str, stats: List[Dict[str,
     return payloads
 
 
+def gen_table_stats_avro_payload_from_tuple(
+    tenant_id: str, endpoint: str, stats: Tuple[List[str]], existing_global_ids: Dict[str, bool]
+) -> List[Dict[str, str]]:
+    payloads = list()
+    for stat in stats:
+        db_name, schema_name, table_name, column_name = stat[:4]
+
+        global_id_arg = "{db}{schema}{table}{column}".format(
+            db=db_name, schema=schema_name, table=table_name, column=column_name
+        )
+        table_global_id = new_global_id(
+            tenant_id=tenant_id, cluster_id=endpoint, data_id=global_id_arg, data_type="column"
+        )
+
+        if existing_global_ids.get(table_global_id) is not True:
+            continue
+
+        avro_assets = AvroAsset(
+            id=table_global_id,
+            object_type="column",
+            parents=[db_name, schema_name, table_name],
+            name=column_name,
+            stats_max=convert_value_type(stat[4], True),
+            stats_min=convert_value_type(stat[5], True),
+            stats_mean=convert_value_type(stat[8], True),
+            stats_median=convert_value_type(stat[9], True),
+            stats_mode=convert_value_type(stat[10], True),
+            stats_stddev=convert_value_type(stat[11], True),
+            stats_number_of_null=convert_value_type(stat[6], True),
+            stats_number_of_unique=convert_value_type(stat[7], True),
+        )
+
+        payloads.append(avro_assets.to_dict())
+
+    return payloads
+
+
 def gen_table_stats_payload_from_tuple(
     tenant_id: str, endpoint: str, stats: Tuple[List[str]]
 ) -> List[Dict[str, Union[str, List[Dict[str, str]]]]]:

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/redshift.py

@@ -3,16 +3,17 @@ import logging
 import os
 import shutil
 
-from quollio_core.helper.core import setup_dbt_profile
+from quollio_core.helper.core import is_valid_domain, setup_dbt_profile
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log import set_log_level
+from quollio_core.profilers.qdc import gen_existing_global_id_dict, get_avro_file_content
 from quollio_core.profilers.redshift import (
     redshift_table_level_lineage,
     redshift_table_level_sqllineage,
     redshift_table_stats,
 )
 from quollio_core.profilers.stats import get_column_stats_items
-from quollio_core.repository import dbt, qdc, redshift
+from quollio_core.repository import dbt, qdc, redshift, ssm
 
 logger = logging.getLogger(__name__)
 
@@ -84,11 +85,20 @@ def load_lineage(
     tenant_id: str,
 ) -> None:
     logger.info("Generate redshift table to table lineage.")
+
+    file_content = get_avro_file_content(
+        tenant_id=tenant_id,
+        account_id=conn.host,
+        qdc_client=qdc_client,
+    )
+    existing_global_ids = gen_existing_global_id_dict(avro_content=file_content)
+
     redshift_table_level_lineage(
         conn=conn,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
         dbt_table_name="quollio_lineage_table_level",
+        existing_global_ids=existing_global_ids,
     )
 
     logger.info("Generate redshift view level lineage.")
@@ -97,6 +107,7 @@ def load_lineage(
         qdc_client=qdc_client,
         tenant_id=tenant_id,
         dbt_table_name="quollio_lineage_view_level",
+        existing_global_ids=existing_global_ids,
     )
 
     logger.info("Lineage data is successfully loaded.")
@@ -115,12 +126,20 @@ def load_stats(
     if stats_items is None:
         raise ValueError("No stats items are not selected. Please specify any value to `stats_items` param.")
 
+    file_content = get_avro_file_content(
+        tenant_id=tenant_id,
+        account_id=conn.host,
+        qdc_client=qdc_client,
+    )
+    existing_global_ids = gen_existing_global_id_dict(avro_content=file_content)
+
     logger.info("The following values will be aggregated. {stats_items}".format(stats_items=stats_items))
     redshift_table_stats(
         conn=conn,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
         stats_items=stats_items,
+        existing_global_ids=existing_global_ids,
     )
 
     logger.info("Stats data is successfully loaded.")
@@ -252,6 +271,7 @@ if __name__ == "__main__":
         type=str,
         choices=["debug", "info", "warn", "error", "none"],
         action=env_default("LOG_LEVEL"),
+        default="info",
         required=False,
         help="The log level for dbt commands. Default value is info",
     )
@@ -285,6 +305,16 @@ if __name__ == "__main__":
         required=False,
         help="The client secrete that is created on Quollio console to let clients access Quollio External API",
     )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
 
     stats_items = get_column_stats_items()
     parser.add_argument(
@@ -323,11 +353,24 @@ if __name__ == "__main__":
             log_level=args.log_level,
             dbt_macro_source=args.dbt_macro_source,
         )
+
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
+
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
             client_id=args.client_id,
             client_secret=args.client_secret,
-            base_url=args.api_url,
+            base_url=api_url,
         )
         load_lineage(
             conn=conn,
@@ -338,7 +381,7 @@ if __name__ == "__main__":
         qdc_client = qdc.QDCExternalAPIClient(
             client_id=args.client_id,
             client_secret=args.client_secret,
-            base_url=args.api_url,
+            base_url=api_url,
         )
         load_stats(
             conn=conn,
@@ -348,7 +391,7 @@ if __name__ == "__main__":
         )
     if "load_sqllineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
-            base_url=args.api_url,
+            base_url=api_url,
             client_id=args.client_id,
             client_secret=args.client_secret,
         )

quollio_core-0.8.0/quollio_core/repository/cloud_resource_manager.py

@@ -0,0 +1,25 @@
+from google.oauth2.service_account import Credentials
+from googleapiclient.discovery import build
+
+
+class CloudResourceManagerClient:
+    """Client to interact with the Cloud Resource Manager API."""
+
+    def __init__(self, credentials: Credentials) -> None:
+        """Initialize the Cloud Resource Manager client with provided credentials."""
+        self.client = self.__initialize(credentials=credentials)
+
+    def __initialize(self, credentials: Credentials):
+        return build("cloudresourcemanager", "v1", credentials=credentials)
+
+    def list_projects(self):
+        """List all projects accessible with the current credentials."""
+        request = self.client.projects().list()
+        response = request.execute()
+        return response
+
+    def get_project(self, project_id: str):
+        """Get a specific project by project ID."""
+        request = self.client.projects().get(projectId=project_id)
+        response = request.execute()
+        return response

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/snowflake.py

@@ -312,6 +312,7 @@ if __name__ == "__main__":
         type=str,
         choices=["debug", "info", "warn", "error", "none"],
         action=env_default("LOG_LEVEL"),
+        default="info",
         required=False,
         help="The log level for dbt commands. Default value is info",
     )
@@ -442,6 +443,7 @@ if __name__ == "__main__":
         )
     api_url = args.api_url
     if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
         api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
         if err is not None:
             logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
@@ -449,7 +451,8 @@ if __name__ == "__main__":
 
     is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
     if not is_domain_valid:
-        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com`")
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
 
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

{quollio_core-0.6.5 → quollio_core-0.8.0}/quollio_core/teradata.py

@@ -1,12 +1,13 @@
 import argparse
 import json
 
+from quollio_core.helper.core import is_valid_domain
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log_utils import configure_logging, error_handling_decorator, logger
 from quollio_core.profilers.stats import get_column_stats_items
 from quollio_core.profilers.teradata.lineage import load_lineage
 from quollio_core.profilers.teradata.stats import load_stats
-from quollio_core.repository import qdc
+from quollio_core.repository import qdc, ssm
 from quollio_core.repository import teradata as teradata_repo
 
 DEFAULT_SYSTEM_DATABASES = [
@@ -197,6 +198,16 @@ def main() -> None:
         help="Name of the Teradata system database.\
               Default is DBC",
     )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
 
     args = parser.parse_args()
 
@@ -217,7 +228,19 @@ def main() -> None:
         additional_params = {}
 
     logger.info("Initializing QDC client")
-    qdc_client = qdc.initialize_qdc_client(args.api_url, args.client_id, args.client_secret)
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+
+    logger.debug("API URL: %s", api_url)
+    qdc_client = qdc.initialize_qdc_client(api_url, args.client_id, args.client_secret)
 
     logger.info("Initializing Teradata client")
     config = teradata_repo.TeradataConfig.from_dict(