quollio-core 0.4.14__tar.gz → 0.4.16__tar.gz

{quollio_core-0.4.14 → quollio_core-0.4.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.14
+Version: 0.4.16
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>

{quollio_core-0.4.14 → quollio_core-0.4.16}/quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.14"
+__version__ = "0.4.16"
 __author__ = "Quollio Technologies, Inc"

{quollio_core-0.4.14 → quollio_core-0.4.16}/quollio_core/bricks.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 import os
+import shutil
 
 from quollio_core.helper.core import setup_dbt_profile, trim_prefix
 from quollio_core.helper.env_default import env_default
@@ -21,6 +22,7 @@ def build_view(
     conn: db.DatabricksConnectionConfig,
     target_tables: str = "",
     log_level: str = "info",
+    dbt_macro_source: str = "hub",
 ) -> None:
     logger.info("Build profiler views using dbt")
     # set parameters
@@ -30,6 +32,14 @@ def build_view(
     template_path = f"{current_dir}/dbt_projects/databricks/profiles"
     template_name = "profiles_template.yml"
 
+    new_package_file = f"{project_path}/packages.yml"
+    if dbt_macro_source == "local":
+        shutil.copyfile(f"{project_path}/packages_local.yml", new_package_file)
+        logger.info("Will install dbt macro defined in packages_local.yml")
+    else:
+        shutil.copyfile(f"{project_path}/packages_hub.yml", new_package_file)
+        logger.info("Will install dbt macro defined in packages_hub.yml")
+
     # build views using dbt
     setup_dbt_profile(connections_json=conn.as_dict(), template_path=template_path, template_name=template_name)
     # FIXME: when executing some of the commands, directory changes due to the library bug.
@@ -38,7 +48,7 @@ def build_view(
         cmd="deps",
         project_dir=project_path,
         profile_dir=template_path,
-        options=["--no-use-colors", "--log-level", log_level],
+        options=["--no-use-colors", "--log-level", log_level, "--source", dbt_macro_source],
     )
 
     run_options = ["--no-use-colors", "--log-level", log_level]
@@ -180,6 +190,15 @@ if __name__ == "__main__":
         required=False,
         help="The log level for dbt commands. Default value is info",
     )
+    parser.add_argument(
+        "--dbt_macro_source",
+        type=str,
+        choices=["hub", "local"],
+        action=env_default("DBT_MACRO_SOURCE"),
+        default="hub",
+        required=False,
+        help="The dbt macro source",
+    )
     parser.add_argument(
         "--api_url",
         type=str,
@@ -272,6 +291,7 @@ if __name__ == "__main__":
             conn=conn,
             target_tables=args.target_tables,
             log_level=args.log_level,
+            dbt_macro_source=args.dbt_macro_source,
         )
 
     if "load_lineage" in args.commands:

quollio_core-0.4.16/quollio_core/dbt_projects/databricks/packages_local.yml

@@ -0,0 +1,2 @@
+packages:
+  - local: /root/deps/dbt-utils

quollio_core-0.4.16/quollio_core/dbt_projects/redshift/packages_local.yml

@@ -0,0 +1,2 @@
+packages:
+  - local: /root/deps/dbt-utils

{quollio_core-0.4.14 → quollio_core-0.4.16}/quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql

@@ -30,7 +30,7 @@ WITH columns AS (
     WHERE
       granted_on in ('TABLE', 'MATERIALIZED VIEW')
       AND grantee_name = '{{ var("query_role") }}'
-      AND privilege in ('SELECT', 'OWNERSHIP', 'REFERENCES')
+      AND privilege in ('SELECT', 'OWNERSHIP')
       AND deleted_on IS NULL
     GROUP BY
       table_catalog

quollio_core-0.4.16/quollio_core/dbt_projects/snowflake/packages_hub.yml

@@ -0,0 +1,3 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: 1.1.1

quollio_core-0.4.16/quollio_core/dbt_projects/snowflake/packages_local.yml

@@ -0,0 +1,2 @@
+packages:
+  - local: /root/deps/dbt-utils/

{quollio_core-0.4.14 → quollio_core-0.4.16}/quollio_core/profilers/bigquery.py

@@ -25,16 +25,18 @@ def bigquery_table_lineage(
     all_tables = generate_table_list(bq_client, datasets)
     lineage_links = generate_lineage_links(all_tables, lineage_client, project_id, regions)
     lineage_links = parse_bigquery_table_lineage(lineage_links)
+    logger.debug("The following resources will be ingested. %s", lineage_links)
 
     update_table_lineage_inputs = gen_table_lineage_payload(tenant_id=tenant_id, endpoint=org_id, tables=lineage_links)
 
     req_count = 0
     for update_table_lineage_input in update_table_lineage_inputs:
         logger.info(
-            "Generating table lineage. downstream: %s -> %s -> %s",
+            "Generating table lineage. downstream: %s -> %s -> %s. upstream: %s",
             update_table_lineage_input.downstream_database_name,
             update_table_lineage_input.downstream_schema_name,
             update_table_lineage_input.downstream_table_name,
+            update_table_lineage_input.upstreams.as_dict(),
         )
         status_code = qdc_client.update_lineage_by_id(
             global_id=update_table_lineage_input.downstream_global_id,

{quollio_core-0.4.14 → quollio_core-0.4.16}/quollio_core/redshift.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 import os
+import shutil
 
 from quollio_core.helper.core import setup_dbt_profile
 from quollio_core.helper.env_default import env_default
@@ -21,6 +22,7 @@ def build_view(
     aggregate_all: bool = False,
     target_tables: str = "",
     log_level: str = "info",
+    dbt_macro_source: str = "hub",
 ) -> None:
     logger.info("Build profiler views using dbt")
     # set parameters
@@ -34,6 +36,13 @@ def build_view(
         aggregate_all=aggregate_all,
         database=conn.database,
     )
+    new_package_file = f"{project_path}/packages.yml"
+    if dbt_macro_source == "local":
+        shutil.copyfile(f"{project_path}/packages_local.yml", new_package_file)
+        logger.info("Will install dbt macro defined in packages_local.yml")
+    else:
+        shutil.copyfile(f"{project_path}/packages_hub.yml", new_package_file)
+        logger.info("Will install dbt macro defined in packages_hub.yml")
 
     # build views using dbt
     setup_dbt_profile(
@@ -47,7 +56,7 @@ def build_view(
         cmd="deps",
         project_dir=project_path,
         profile_dir=template_path,
-        options=["--no-use-colors", "--log-level", log_level, "--vars", options],
+        options=["--no-use-colors", "--log-level", log_level, "--vars", options, "--source", dbt_macro_source],
     )
     run_options = ["--no-use-colors", "--log-level", log_level, "--vars", options]
     if target_tables is not None:
@@ -246,6 +255,15 @@ if __name__ == "__main__":
         required=False,
         help="The log level for dbt commands. Default value is info",
     )
+    parser.add_argument(
+        "--dbt_macro_source",
+        type=str,
+        choices=["hub", "local"],
+        action=env_default("DBT_MACRO_SOURCE"),
+        default="hub",
+        required=False,
+        help="The dbt macro source",
+    )
     parser.add_argument(
         "--api_url",
         type=str,
@@ -303,6 +321,7 @@ if __name__ == "__main__":
             aggregate_all=args.aggregate_all,
             target_tables=args.target_tables,
             log_level=args.log_level,
+            dbt_macro_source=args.dbt_macro_source,
         )
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

{quollio_core-0.4.14 → quollio_core-0.4.16}/quollio_core/snowflake.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 import os
+import shutil
 
 from quollio_core.helper.core import setup_dbt_profile
 from quollio_core.helper.env_default import env_default
@@ -22,6 +23,7 @@ def build_view(
     stats_sample_method: str,
     target_tables: str = "",
     log_level: str = "info",
+    dbt_macro_source: str = "hub",
 ) -> None:
     logger.info("Build profiler views using dbt")
     # set parameters
@@ -34,6 +36,13 @@ def build_view(
         query_role=conn.account_query_role,
         sample_method=stats_sample_method,
     )
+    new_package_file = f"{project_path}/packages.yml"
+    if dbt_macro_source == "local":
+        shutil.copyfile(f"{project_path}/packages_local.yml", new_package_file)
+        logger.info("Will install dbt macro defined in packages_local.yml")
+    else:
+        shutil.copyfile(f"{project_path}/packages_hub.yml", new_package_file)
+        logger.info("Will install dbt macro defined in packages_hub.yml")
 
     # build views using dbt
     setup_dbt_profile(
@@ -47,7 +56,7 @@ def build_view(
         cmd="deps",
         project_dir=project_path,
         profile_dir=template_path,
-        options=["--no-use-colors", "--log-level", log_level, "--vars", options],
+        options=["--no-use-colors", "--log-level", log_level, "--vars", options, "--source", dbt_macro_source],
     )
     run_options = ["--no-use-colors", "--log-level", log_level, "--vars", options]
     if target_tables is not None:
@@ -251,6 +260,15 @@ if __name__ == "__main__":
         required=False,
         help="The log level for dbt commands. Default value is info",
     )
+    parser.add_argument(
+        "--dbt_macro_source",
+        type=str,
+        choices=["hub", "local"],
+        action=env_default("DBT_MACRO_SOURCE"),
+        default="hub",
+        required=False,
+        help="The dbt macro source",
+    )
     parser.add_argument(
         "--api_url",
         type=str,
@@ -316,6 +334,7 @@ if __name__ == "__main__":
             stats_sample_method=args.sample_method,
             target_tables=args.target_tables,
             log_level=args.log_level,
+            dbt_macro_source=args.dbt_macro_source,
         )
     if "load_lineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

quollio_core-0.4.14/quollio_core/dbt_projects/databricks/package-lock.yml

@@ -1,14 +0,0 @@
-packages:
-- package: dbt-labs/dbt_utils
-  version: 1.1.1
-- package: dbt-labs/spark_utils
-  version: 0.3.0
-- package: dbt-labs/codegen
-  version: 0.12.1
-- package: dbt-labs/dbt_external_tables
-  version: 0.8.7
-- package: dbt-labs/dbt_project_evaluator
-  version: 0.8.1
-- package: brooklyn-data/dbt_artifacts
-  version: 2.6.2
-sha1_hash: cbb324267dbf6c6fb7de11b162e4fbafd1e32a9c

quollio_core-0.4.14/quollio_core/dbt_projects/databricks/packages.yml

@@ -1,13 +0,0 @@
-packages:
-  - package: dbt-labs/dbt_utils
-    version: [">=0.0.0", "<2.0.0"]
-  - package: dbt-labs/spark_utils
-    version: [">=0.0.0", "<1.0.0"]
-  - package: dbt-labs/codegen
-    version: [">=0.0.0", "<1.0.0"]
-  - package: dbt-labs/dbt_external_tables
-    version: [">=0.0.0", "<1.0.0"]
-  - package: dbt-labs/dbt_project_evaluator
-    version: [">=0.0.0", "<1.0.0"]
-  - package: brooklyn-data/dbt_artifacts
-    version: [">=2.0.0", "<3.0.0"]

quollio_core-0.4.14/quollio_core/dbt_projects/redshift/package-lock.yml

@@ -1,4 +0,0 @@
-packages:
-- package: dbt-labs/dbt_utils
-  version: 1.1.1
-sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353

quollio_core-0.4.14/quollio_core/dbt_projects/snowflake/package-lock.yml

@@ -1,4 +0,0 @@
-packages:
-- package: dbt-labs/dbt_utils
-  version: 1.1.1
-sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353