quollio-core 0.7.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.7.0"
+__version__ = "0.8.1"
 __author__ = "Quollio Technologies, Inc"

quollio_core/bigquery.py

@@ -3,11 +3,12 @@ import json
 
 from google.auth.credentials import Credentials
 
+from quollio_core.helper.core import is_valid_domain
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log_utils import configure_logging, error_handling_decorator, logger
 from quollio_core.profilers.bigquery import bigquery_table_lineage, bigquery_table_stats
 from quollio_core.profilers.qdc import gen_existing_global_id_dict, get_avro_file_content
-from quollio_core.repository import qdc
+from quollio_core.repository import qdc, ssm
 from quollio_core.repository.bigquery import BigQueryClient, get_credentials, get_org_id
 
 
@@ -166,6 +167,16 @@ if __name__ == "__main__":
         help="Whether to enable multi-projects support. If set to 'true', \
         the script will load lineage and stats from all projects accessible by the credentials. Default is 'false'.",
     )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
 
     args = parser.parse_args()
 
@@ -175,9 +186,21 @@ if __name__ == "__main__":
 
     configure_logging(args.log_level)
 
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
+
     credentials = initialize_credentials(args.credentials_json)
     org_id = initialize_org_id(args.credentials_json)
-    qdc_client = qdc.initialize_qdc_client(args.api_url, args.client_id, args.client_secret)
+    qdc_client = qdc.initialize_qdc_client(api_url, args.client_id, args.client_secret)
     bq_client = initialize_bq_client(credentials, args.project_id)
     if args.project_id is None:
         args.project_id = json.loads(args.credentials_json)["project_id"]

quollio_core/bricks.py

@@ -3,7 +3,7 @@ import logging
 import os
 import shutil
 
-from quollio_core.helper.core import setup_dbt_profile, trim_prefix
+from quollio_core.helper.core import is_valid_domain, setup_dbt_profile, trim_prefix
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log import set_log_level
 from quollio_core.profilers.databricks import (
@@ -13,7 +13,7 @@ from quollio_core.profilers.databricks import (
 )
 from quollio_core.profilers.stats import get_column_stats_items
 from quollio_core.repository import databricks as db
-from quollio_core.repository import dbt, qdc
+from quollio_core.repository import dbt, qdc, ssm
 
 logger = logging.getLogger(__name__)
 
@@ -256,6 +256,16 @@ if __name__ == "__main__":
         required=False,
         help="Whether to ingest column lineage into QDIC or not. Default value is False",
     )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
 
     stats_items = get_column_stats_items()
     parser.add_argument(
@@ -294,9 +304,21 @@ if __name__ == "__main__":
             dbt_macro_source=args.dbt_macro_source,
         )
 
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
+
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
-            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+            base_url=api_url, client_id=args.client_id, client_secret=args.client_secret
         )
         load_lineage(
             conn=conn,
@@ -308,7 +330,7 @@ if __name__ == "__main__":
 
     if "load_stats" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(
-            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
+            base_url=api_url, client_id=args.client_id, client_secret=args.client_secret
         )
         databricks_column_stats(
             conn=conn,

quollio_core/dbt_projects/databricks/dbt_project.yml

@@ -2,6 +2,9 @@ name: 'quollio_intelligence_databricks'
 version: '1.0.0'
 config-version: 2
 
+flags:
+  send_anonymous_usage_stats: false
+
 profile: 'quollio_intelligence_databricks'
 
 model-paths: ["models"]

quollio_core/dbt_projects/redshift/dbt_project.yml

@@ -2,6 +2,9 @@ name: 'quollio_intelligence_redshift'
 version: '1.0.0'
 config-version: 2
 
+flags:
+  send_anonymous_usage_stats: false
+
 profile: 'quollio_intelligence_redshift'
 
 model-paths: ["models"]

quollio_core/dbt_projects/snowflake/dbt_project.yml

@@ -2,6 +2,9 @@ name: 'quollio_intelligence_snowflake'
 version: '1.0.0'
 config-version: 2
 
+flags:
+  send_anonymous_usage_stats: false
+
 profile: 'quollio_intelligence_snowflake'
 
 model-paths: ["models"]

quollio_core/redshift.py

@@ -356,13 +356,15 @@ if __name__ == "__main__":
 
     api_url = args.api_url
     if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
         api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
         if err is not None:
             logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
             raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
     is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
     if not is_domain_valid:
-        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com`")
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
 
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

quollio_core/snowflake.py

@@ -443,6 +443,7 @@ if __name__ == "__main__":
         )
     api_url = args.api_url
     if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
         api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
         if err is not None:
             logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
@@ -450,7 +451,8 @@ if __name__ == "__main__":
 
     is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
     if not is_domain_valid:
-        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com`")
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+    logger.debug("API URL: %s", api_url)
 
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

quollio_core/teradata.py

@@ -1,12 +1,13 @@
 import argparse
 import json
 
+from quollio_core.helper.core import is_valid_domain
 from quollio_core.helper.env_default import env_default
 from quollio_core.helper.log_utils import configure_logging, error_handling_decorator, logger
 from quollio_core.profilers.stats import get_column_stats_items
 from quollio_core.profilers.teradata.lineage import load_lineage
 from quollio_core.profilers.teradata.stats import load_stats
-from quollio_core.repository import qdc
+from quollio_core.repository import qdc, ssm
 from quollio_core.repository import teradata as teradata_repo
 
 DEFAULT_SYSTEM_DATABASES = [
@@ -197,6 +198,16 @@ def main() -> None:
         help="Name of the Teradata system database.\
               Default is DBC",
     )
+    parser.add_argument(
+        "--external_api_access",
+        type=str,
+        choices=["PUBLIC", "VPC_ENDPOINT"],
+        action=env_default("EXTERNAL_API_ACCESS"),
+        default="PUBLIC",
+        required=False,
+        help="Access method to Quollio API. Default 'PUBLIC'. Choose 'VPC_ENDPOINT'\
+             if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
+    )
 
     args = parser.parse_args()
 
@@ -217,7 +228,19 @@ def main() -> None:
         additional_params = {}
 
     logger.info("Initializing QDC client")
-    qdc_client = qdc.initialize_qdc_client(args.api_url, args.client_id, args.client_secret)
+    api_url = args.api_url
+    if args.external_api_access == "VPC_ENDPOINT":
+        logger.debug("Using VPC Endpoint for Quollio API access")
+        api_url, err = ssm.get_parameter_by_assume_role(args.api_url)
+        if err is not None:
+            logger.error("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+            raise Exception("Fail to ssm.get_parameter_by_assume_role. {err}".format(err=err))
+    is_domain_valid = is_valid_domain(domain=api_url, domain_type=args.external_api_access)
+    if not is_domain_valid:
+        raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com` or /api.")
+
+    logger.debug("API URL: %s", api_url)
+    qdc_client = qdc.initialize_qdc_client(api_url, args.client_id, args.client_secret)
 
     logger.info("Initializing Teradata client")
     config = teradata_repo.TeradataConfig.from_dict(

{quollio_core-0.7.0.dist-info → quollio_core-0.8.1.dist-info}/METADATA

@@ -1,14 +1,13 @@
 Metadata-Version: 2.4
 Name: quollio-core
-Version: 0.7.0
+Version: 0.8.1
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -19,19 +18,19 @@ Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 License-File: LICENSE
 Requires-Dist: blake3==0.3.3
-Requires-Dist: dbt-core==1.7.10
-Requires-Dist: dbt-snowflake==1.7.0
-Requires-Dist: dbt-redshift==1.7.1
-Requires-Dist: dbt-databricks==1.7.1
+Requires-Dist: dbt-core==1.8.8
+Requires-Dist: dbt-snowflake==1.8.4
+Requires-Dist: dbt-redshift==1.8.1
+Requires-Dist: dbt-databricks==1.8.7
 Requires-Dist: db-dtypes==1.2.0
 Requires-Dist: jinja2==3.1.3
 Requires-Dist: PyYAML==6.0.1
 Requires-Dist: requests==2.31.0
 Requires-Dist: pyjwt==2.8.0
 Requires-Dist: redshift-connector==2.0.915
-Requires-Dist: snowflake-connector-python==3.5.0
+Requires-Dist: snowflake-connector-python==3.15.0
 Requires-Dist: databricks-sdk==0.17.0
-Requires-Dist: databricks-sql-connector==2.9.5
+Requires-Dist: databricks-sql-connector==3.1.2
 Requires-Dist: sqlglot==20.8.0
 Requires-Dist: google-cloud==0.34.0
 Requires-Dist: google-cloud-bigquery==3.22.0

{quollio_core-0.7.0.dist-info → quollio_core-0.8.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
-quollio_core/__init__.py,sha256=ZtRODvd0lC2-vqcXyfKDya8bb6FyyFWBsGNz7TSlO-s,83
-quollio_core/bigquery.py,sha256=nxMWldMr14HzOeyzYM_fRrfHQ7jbm2HyGzc1r46mlco,6821
-quollio_core/bricks.py,sha256=8h3kbI2b6lGH2s-56jE_Q5-R5-nIsQYMfvtRrkFOzoU,10784
-quollio_core/redshift.py,sha256=Yi_udcgfen2PoCkDTIhemeCFbDVUU1rYWro9CyjHCZA,13192
-quollio_core/snowflake.py,sha256=aOQ8tLSbHJEx_TUGaZLjix5KWgLiRp3A1tRx5qgUtRI,17084
-quollio_core/teradata.py,sha256=H2VUcJvr8W-M2wvm3710Gf1ENb-BSscrDRKNm8gdHJE,8227
+quollio_core/__init__.py,sha256=4mRB-QhWs-b5bo_r0onUha6KY4qdr91uRHlV0CjjmCk,83
+quollio_core/bigquery.py,sha256=xtLMuURkf8-Pak72Pv0XWu-ljhdIoyHQGko3aqsfMuM,7968
+quollio_core/bricks.py,sha256=frndLVGMlds-HthOPx9W4ULm5u-SLrC_2gLlhch2jIc,11890
+quollio_core/redshift.py,sha256=_7c7No9hqIjaIY4l0lDSR6vFZR9dHZWC0SzTZY8eTt8,13308
+quollio_core/snowflake.py,sha256=NkEM1qh-0XStmxCGqosRB7lyhrvVEULViRVc4lfOhpY,17200
+quollio_core/teradata.py,sha256=KGJhel93BbISsjaEX2s_WlbGAKKY0RfhmYcBlLNYqyo,9374
 quollio_core/dbt_projects/databricks/.gitignore,sha256=1jJAyXSzJ3YUm0nx3i7wUSE4RjQMX3ad6F8O88UbtzI,29
 quollio_core/dbt_projects/databricks/README.md,sha256=ZpRQyhFAODAiS8dc1Kb_ndkul4cu4o4udN_EMa49CU4,440
-quollio_core/dbt_projects/databricks/dbt_project.yml,sha256=3sH98RNk7TnphvI3yEdXDstb92kW5BNxr-cT0tXhwzk,480
+quollio_core/dbt_projects/databricks/dbt_project.yml,sha256=I9zgXYnOE6fYXwRKXfnEn4HkxiwA7ktsKoRfD6yOyPk,524
 quollio_core/dbt_projects/databricks/packages_hub.yml,sha256=p9Bl2C44gdC6iYTUkz_15yq3xahSJf2IA3WOXLF_ahA,61
 quollio_core/dbt_projects/databricks/packages_local.yml,sha256=gMo7BkmigAqbPjfWhLnVfgMtV4OpjK89jfJ_aMzyKwk,42
 quollio_core/dbt_projects/databricks/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,7 +20,7 @@ quollio_core/dbt_projects/databricks/profiles/profiles_template.yml,sha256=Dw1Ru
 quollio_core/dbt_projects/databricks/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/databricks/snapshots/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/redshift/README.md,sha256=55nDkX5uQXWmawpQbgG1hbyn64j_CegDBQddQ2C85C8,571
-quollio_core/dbt_projects/redshift/dbt_project.yml,sha256=QhrNorlI8f2XjapyR-BZ5fYHGNKiPdk28hab9LObO1s,409
+quollio_core/dbt_projects/redshift/dbt_project.yml,sha256=NCtplpxSofNvLc_5uUJ9Mm13TNy6OkGGH8yJ53tokhA,453
 quollio_core/dbt_projects/redshift/packages_hub.yml,sha256=p9Bl2C44gdC6iYTUkz_15yq3xahSJf2IA3WOXLF_ahA,61
 quollio_core/dbt_projects/redshift/packages_local.yml,sha256=gMo7BkmigAqbPjfWhLnVfgMtV4OpjK89jfJ_aMzyKwk,42
 quollio_core/dbt_projects/redshift/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -42,7 +42,7 @@ quollio_core/dbt_projects/redshift/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCe
 quollio_core/dbt_projects/redshift/snapshots/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/seeds/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/README.md,sha256=55nDkX5uQXWmawpQbgG1hbyn64j_CegDBQddQ2C85C8,571
-quollio_core/dbt_projects/snowflake/dbt_project.yml,sha256=LN5NDOyakQjIK99IogQX4Whh_1zmqUfD2gqDU9JR3As,407
+quollio_core/dbt_projects/snowflake/dbt_project.yml,sha256=dpuO5EL2SQIXntMTuehusJIZGVG8jiTOypQkkT-r9fU,451
 quollio_core/dbt_projects/snowflake/packages_hub.yml,sha256=p9Bl2C44gdC6iYTUkz_15yq3xahSJf2IA3WOXLF_ahA,61
 quollio_core/dbt_projects/snowflake/packages_local.yml,sha256=ryyJSXv83gYFu48xmzG5Z1l746jGCUBE6hs7pUNwuXE,43
 quollio_core/dbt_projects/snowflake/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -91,7 +91,7 @@ quollio_core/repository/redshift.py,sha256=p2ouEuYcDCjx1oBhc6H1ekQsvEqHGd3bFu3PW
 quollio_core/repository/snowflake.py,sha256=yCYXrYf4I5GL_ITNTXoggj0xNbQsdwxPSmsVvZYwUVU,3869
 quollio_core/repository/ssm.py,sha256=xpm1FzbBnIsBptuYPUNnPgkKU2AH3XxI-ZL0bEetvW0,2182
 quollio_core/repository/teradata.py,sha256=1AExxRBTswpSyF4OVyAUkoiZ0yVRfqt4T99FdllkTEI,3763
-quollio_core-0.7.0.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.7.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-quollio_core-0.7.0.dist-info/METADATA,sha256=xrt5HSRtSF0M-2Ef09CNsB2fo6hyMifPOySYhF5U1nA,7023
-quollio_core-0.7.0.dist-info/RECORD,,
+quollio_core-0.8.1.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.8.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+quollio_core-0.8.1.dist-info/METADATA,sha256=BtSfgpm2HXGxl3ixImGyqfFnauX7lqal1pLss46K4PU,6973
+quollio_core-0.8.1.dist-info/RECORD,,