quollio-core 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.6.0"
+__version__ = "0.6.2"
 __author__ = "Quollio Technologies, Inc"

quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -13,7 +13,7 @@
       TABLE_CATALOG
       , TABLE_SCHEMA
       , TABLE_NAME
-      , OBJECT_AGG(COLUMN_NAME, IS_CALCULABLE) AS COLUMNS_OBJ
+      , OBJECT_AGG(COLUMN_NAME, OBJECT_CONSTRUCT('IS_CALCULABLE', IS_CALCULABLE, 'CAN_APPROX_COUNT', CAN_APPROX_COUNT))
     FROM
       {{ ref('quollio_stats_profiling_columns') }}
     WHERE NOT startswith(table_name, 'QUOLLIO_')
@@ -44,7 +44,7 @@
   -- build sql for column value aggregation.
   {%- set sql_for_column_stats %}
   {% set columns_json = fromjson(stats_target_table[3]) %}
-  {%- for col_name, is_calclable in columns_json.items() -%}
+  {%- for col_name, attr in columns_json.items() -%}
     {%- if not loop.first %}UNION{% endif %}
     SELECT
       DISTINCT
@@ -52,14 +52,14 @@
       , '{{stats_target_table[1]}}' as schema_name
       , '{{stats_target_table[2]}}' as table_name
       , '{{col_name}}' as column_name
-      , {% if is_calclable == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
-      , {% if is_calclable == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
+      , {% if attr["is_calclable"] == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
+      , {% if attr["is_calclable"] == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
       , COUNT_IF("{{col_name}}" IS NULL) AS null_count
-      , APPROX_COUNT_DISTINCT("{{col_name}}") AS cardinality
-      , {% if is_calclable == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
-      , {% if is_calclable == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
-      , {% if is_calclable == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
-      , {% if is_calclable == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
+      , {% if attr["can_approx_count"] == True %}APPROX_COUNT_DISTINCT("{{col_name}}"){% else %}NULL{% endif %} AS cardinality
+      , {% if attr["is_calclable"] == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
+      , {% if attr["is_calclable"] == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
+      , {% if attr["is_calclable"] == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
+      , {% if attr["is_calclable"] == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
     FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
   {% endfor -%}
   {%- endset %}

quollio_core/dbt_projects/snowflake/macros/materialization/get_imported_databases.sql

@@ -0,0 +1,20 @@
+{% macro get_imported_databases() %}
+    {%- set query %}
+        SELECT database_name
+        FROM snowflake.account_usage.databases
+        WHERE type = 'IMPORTED DATABASE'
+        AND database_owner IS NOT NULL
+        AND deleted is null
+    {%- endset %}
+
+    {%- set results = run_query(query) -%}
+    {%- if execute %}
+        {%- set all_databases = results.rows | map(attribute=0) | list %}
+        {{ log("Extracted Databases: " ~ all_databases, info=True) }}
+    {{ return(all_databases) }}
+    {%- else %}
+    {%- set all_databases = [] %}
+    {%- endif %}
+
+    {{ return(all_databases) }}
+{% endmacro %}

quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql

@@ -1,3 +1,7 @@
+{% if var('target_imported_databases') == 'ENABLED' %}
+  {% set imported_databases = get_imported_databases() %}
+{% endif %}
+
 WITH columns AS (
     SELECT
       table_catalog
@@ -20,7 +24,37 @@ WITH columns AS (
       table_catalog
       , table_schema
       , table_name
-), accessible_tables AS (
+),
+{% if var('target_imported_databases') == 'ENABLED' %}
+  {% if imported_databases | length > 0 %}
+  imported_tables AS (
+    {% for database in imported_databases %}
+        SELECT
+          c.table_catalog
+          , c.table_schema
+          , c.table_name
+          , c.column_name
+          , c.data_type
+        FROM
+          {{ database }}.INFORMATION_SCHEMA.COLUMNS c
+        INNER JOIN
+          {{ database }}.INFORMATION_SCHEMA.TABLE_PRIVILEGES tp
+        ON
+          c.table_catalog = tp.table_catalog
+          AND c.table_schema = tp.table_schema
+          AND c.table_name = tp.table_name
+        WHERE
+          c.TABLE_SCHEMA != 'INFORMATION_SCHEMA'
+          AND tp.grantee = '{{ var("query_role") }}'
+          AND tp.PRIVILEGE_TYPE = 'SELECT'
+        {% if not loop.last %}
+        UNION ALL
+        {% endif %}
+    {% endfor %}
+  ),
+  {% endif %}
+{% endif %}
+accessible_tables AS (
     SELECT
       table_catalog
       , table_schema
@@ -69,6 +103,17 @@ WITH columns AS (
     c.table_catalog = a.table_catalog
     AND c.table_schema = a.table_schema
     AND c.table_name = a.name
+  {% if var('target_imported_databases') == 'ENABLED' %}
+    UNION
+    SELECT
+      table_catalog
+      , table_schema
+      , table_name
+      , column_name
+      , data_type
+    FROM
+      imported_tables
+  {% endif %}
   MINUS
   SELECT
     table_catalog
@@ -90,6 +135,9 @@ WITH columns AS (
                              'TINYINT', 'BYTEINT', 'FLOAT')
                              THEN true
            else false END AS is_calculable
+    , case when data_type not in('GEOGRAPHY')
+                             THEN true
+           else false END AS can_approx_count
   FROM
     implicit_columns_removed
   WHERE

quollio_core/snowflake.py

@@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
 def build_view(
     conn: snowflake.SnowflakeConnectionConfig,
     stats_sample_method: str,
+    target_imported_databases: str,
     target_tables: str = "",
     log_level: str = "info",
     dbt_macro_source: str = "hub",
@@ -38,11 +39,13 @@ def build_view(
 
     options = '{{"query_role": "{query_role}", "sample_method": "{sample_method}",\
           "target_databases_method": "{target_databases_method}",\
-              "target_databases": {target_databases}}}'.format(
+              "target_databases": {target_databases},\
+              "target_imported_databases": "{target_imported_databases}"}}'.format(
         query_role=conn.account_query_role,
         sample_method=stats_sample_method,
         target_databases_method=target_databases_method,
         target_databases=target_databases,
+        target_imported_databases=target_imported_databases,
     )
 
     new_package_file = f"{project_path}/packages.yml"
@@ -253,6 +256,7 @@ if __name__ == "__main__":
             "quollio_lineage_column_level",
             "quollio_lineage_table_level",
             "quollio_stats_columns",
+            "quollio_stats_profiling_columns",
         ],
         action=env_default("SNOWFLAKE_TARGET_TABLES"),
         required=False,
@@ -280,6 +284,14 @@ if __name__ == "__main__":
             please specify database names with blank space as delimiter\
             wildcards (%) are supported "DATABASE%" ',
     )
+    parser.add_argument(
+        "--target_imported_databases",
+        type=str,
+        choices=["ENABLED", "DISABLED"],
+        action=env_default("SNOWFLAKE_TARGET_IMPORT_DATABASES"),
+        required=False,
+        help="Whether to get stats for import databases or not(ENABLED or DISABLED).",
+    )
     parser.add_argument(
         "--sample_method",
         type=str,
@@ -352,10 +364,10 @@ if __name__ == "__main__":
              if you use API Gateway VPC Endpoint, DefaultValue is set to PUBLIC.",
     )
     parser.add_argument(
-        "--auth_method",
+        "--auth_type",
         type=str,
         choices=["PASSWORD", "KEYPAIR"],
-        action=env_default("SNOWFLAKE_AUTH_METHOD"),
+        action=env_default("SNOWFLAKE_AUTH_TYPE"),
         default="PASSWORD",
         required=False,
         help="Authentication method to use (PASSWORD or KEYPAIR)",
@@ -395,7 +407,7 @@ if __name__ == "__main__":
     }
 
     # Add authentication specific parameters based on method
-    if args.auth_method == "KEYPAIR":
+    if args.auth_type == "KEYPAIR":
         if not args.private_key:
             raise ValueError("private_key is required when using keypair authentication")
         auth_params["private_key"] = args.private_key
@@ -413,7 +425,6 @@ if __name__ == "__main__":
         raise ValueError("No command is provided")
 
     if "build_view" in args.commands:
-
         if args.target_databases:
             target_databases = ["'" + db + "'" for db in args.target_databases[0].split(",")]
         else:
@@ -422,6 +433,7 @@ if __name__ == "__main__":
         build_view(
             conn=conn,
             stats_sample_method=args.sample_method,
+            target_imported_databases=args.target_imported_databases,
             target_tables=args.target_tables,
             log_level=args.log_level,
             dbt_macro_source=args.dbt_macro_source,

{quollio_core-0.6.0.dist-info → quollio_core-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quollio-core
-Version: 0.6.0
+Version: 0.6.2
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>

{quollio_core-0.6.0.dist-info → quollio_core-0.6.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
-quollio_core/__init__.py,sha256=NPy7jZcpkwallRslS02ytWJBnm2veGYFEKEpqBXAxGU,83
+quollio_core/__init__.py,sha256=zfic0veOATHjsRLRe6G0yoxZmkc0nGBkaT56mSAHuyk,83
 quollio_core/bigquery.py,sha256=6Oq4DVGpa3X21Es_nbrsb8pK3vaxwb9Egnvq3huo95k,5894
 quollio_core/bricks.py,sha256=8h3kbI2b6lGH2s-56jE_Q5-R5-nIsQYMfvtRrkFOzoU,10784
 quollio_core/redshift.py,sha256=KcdljY95xYf9JYrsaMOBoP_XxQQ8wFVE5ue_XEMVSFc,11504
-quollio_core/snowflake.py,sha256=APdGGikFl1oF0yuSLYvGE39FM9-cwyrYdtMYXLzrrxk,16473
+quollio_core/snowflake.py,sha256=fMkZ7OlbimgLg4TxVASTCpXSH2MPeawveAFi0xhKqnw,17060
 quollio_core/teradata.py,sha256=H2VUcJvr8W-M2wvm3710Gf1ENb-BSscrDRKNm8gdHJE,8227
 quollio_core/dbt_projects/databricks/.gitignore,sha256=1jJAyXSzJ3YUm0nx3i7wUSE4RjQMX3ad6F8O88UbtzI,29
 quollio_core/dbt_projects/databricks/README.md,sha256=ZpRQyhFAODAiS8dc1Kb_ndkul4cu4o4udN_EMa49CU4,440
@@ -47,7 +47,8 @@ quollio_core/dbt_projects/snowflake/packages_hub.yml,sha256=p9Bl2C44gdC6iYTUkz_1
 quollio_core/dbt_projects/snowflake/packages_local.yml,sha256=ryyJSXv83gYFu48xmzG5Z1l746jGCUBE6hs7pUNwuXE,43
 quollio_core/dbt_projects/snowflake/analyses/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/dbt_projects/snowflake/macros/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=6CM9L_SkTkfUVWOYj0APHqmS_uBOdsL8RdN4fwUXz1I,3822
+quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql,sha256=LO0FHBBtGrbuBuuJ_wbpNYxWuY6q_pifGtYIDQvf3Mg,3986
+quollio_core/dbt_projects/snowflake/macros/materialization/get_imported_databases.sql,sha256=RKpaCRSGXCDhDlN__ZXuSJ8IYo8qI4qwFnihr5m2Qiw,617
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.sql,sha256=Zhj0EXF1K8S-OkFxz3IBHe2olXktYrvly0LwZBOAUXw,5333
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_column_level.yml,sha256=a2uNIAh-xw51eu-GmHVuAnGnTbwK7h8-DjDeQtK3KaQ,711
 quollio_core/dbt_projects/snowflake/models/quollio_lineage_table_level.sql,sha256=lZ28A4E6s37-oBx8JbtT3ItXK6musdqr25eyaGn7kDk,5916
@@ -56,7 +57,7 @@ quollio_core/dbt_projects/snowflake/models/quollio_sqllineage_sources.sql,sha256
 quollio_core/dbt_projects/snowflake/models/quollio_sqllineage_sources.yml,sha256=qgazupx3ca4P8R0loY5F9hyCz2fmAcWqZ6iOySo_NoY,377
 quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql,sha256=BzvP9gKMFItmwqEQ4bDgtS-Invxhhe6L73Qe1ucxfHo,284
 quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.yml,sha256=V_BESPk6IqE52ExT26-78As9l9AlWW86-Geb5PIhThU,67
-quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql,sha256=ubMEzZNHq55zCncg7HbzdMKMSdqHnwbJmVKYpet8Otc,2968
+quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql,sha256=xJKK7ipP4LZ1I7OezJQ-FzQHhG6h0-SGQjJwhzt12aU,4327
 quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.yml,sha256=W39VAmFnnX6RBoW7B_4CConC1lm0Jm9o50Jsz9bYZzY,538
 quollio_core/dbt_projects/snowflake/models/sources.yml,sha256=vGSV33cNj4UUyPUcYS-JFgc3r8KvSLfiA7qhbDCUU9s,10975
 quollio_core/dbt_projects/snowflake/profiles/profiles_template.yml,sha256=6yIlaFSKxWxqnX-LboO1xVC-7DtPYcAKrRjn3rPGNfk,511
@@ -89,7 +90,7 @@ quollio_core/repository/redshift.py,sha256=p2ouEuYcDCjx1oBhc6H1ekQsvEqHGd3bFu3PW
 quollio_core/repository/snowflake.py,sha256=yCYXrYf4I5GL_ITNTXoggj0xNbQsdwxPSmsVvZYwUVU,3869
 quollio_core/repository/ssm.py,sha256=xpm1FzbBnIsBptuYPUNnPgkKU2AH3XxI-ZL0bEetvW0,2182
 quollio_core/repository/teradata.py,sha256=1AExxRBTswpSyF4OVyAUkoiZ0yVRfqt4T99FdllkTEI,3763
-quollio_core-0.6.0.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.6.0.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
-quollio_core-0.6.0.dist-info/METADATA,sha256=nd30OVTKTt-47gYplIxH5HETLCGHrimEd5FUjilA8_c,7023
-quollio_core-0.6.0.dist-info/RECORD,,
+quollio_core-0.6.2.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.6.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+quollio_core-0.6.2.dist-info/METADATA,sha256=eJGFeexxJLeeSV0k9ulVz3jCGOk51kCV7ZOOn_1X_lM,7023
+quollio_core-0.6.2.dist-info/RECORD,,

{quollio_core-0.6.0.dist-info → quollio_core-0.6.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.11.0
+Generator: flit 3.12.0
 Root-Is-Purelib: true
 Tag: py3-none-any