quollio-core 0.4.16__tar.gz → 0.4.19__tar.gz

{quollio_core-0.4.16 → quollio_core-0.4.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.16
+Version: 0.4.19
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>

{quollio_core-0.4.16 → quollio_core-0.4.19}/quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.16"
+__version__ = "0.4.19"
 __author__ = "Quollio Technologies, Inc"

{quollio_core-0.4.16 → quollio_core-0.4.19}/quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -65,8 +65,10 @@
   {%- endset %}
 
   -- create a view with a index as suffix
-  {%- set stats_view_identifier = "%s_%s_%s_%s"|format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) %}
-  {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema, database=database, type='view') %}
+  {%- set stats_view_identifier = "\"%s_%s_%s_%s\"" | format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) | upper %}
+  {%- set schema_name = "\"%s\""|format(schema) %}
+  {%- set db_name = "\"%s\""|format(database) %}
+  {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema_name, database=db_name, type='view') %}
   {% call statement("main") %}
     {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
   {% endcall %}

{quollio_core-0.4.16 → quollio_core-0.4.19}/quollio_core/profilers/snowflake.py

@@ -19,7 +19,7 @@ def snowflake_table_to_table_lineage(
     tenant_id: str,
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        results = sf_executor.get_query_results(
+        results, err = sf_executor.get_query_results(
             query="""
             SELECT
                 *
@@ -30,6 +30,13 @@ def snowflake_table_to_table_lineage(
                 schema=conn.account_schema,
             )
         )
+        if err is not None:
+            handle_error(err=err)
+        if len(results) == 0:
+            logger.warning(
+                "No lineage data in ACCOUNT_USAGE.SNOWFLAKE. Please check the data in `QUOLLIO_LINEAGE_TABLE_LEVEL`."
+            )
+            return
         parsed_results = parse_snowflake_results(results=results)
         update_table_lineage_inputs = gen_table_lineage_payload(
             tenant_id=tenant_id,
@@ -62,7 +69,7 @@ def snowflake_column_to_column_lineage(
     tenant_id: str,
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        results = sf_executor.get_query_results(
+        results, err = sf_executor.get_query_results(
             query="""
             SELECT
                 *
@@ -73,6 +80,13 @@ def snowflake_column_to_column_lineage(
                 schema=conn.account_schema,
             )
         )
+        if err is not None:
+            handle_error(err=err)
+        if len(results) == 0:
+            logger.warning(
+                "No lineage data in ACCOUNT_USAGE.SNOWFLAKE. Please check the data in `QUOLLIO_LINEAGE_COLUMN_LEVEL`."
+            )
+            return
         update_column_lineage_inputs = gen_column_lineage_payload(
             tenant_id=tenant_id,
             endpoint=conn.account_id,
@@ -105,7 +119,7 @@ def snowflake_table_level_sqllineage(
     tenant_id: str,
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        results = sf_executor.get_query_results(
+        results, err = sf_executor.get_query_results(
             query="""
             SELECT
                 database_name
@@ -118,6 +132,13 @@ def snowflake_table_level_sqllineage(
                 schema=conn.account_schema,
             )
         )
+        if err is not None:
+            handle_error(err=err)
+        if len(results) == 0:
+            logger.warning(
+                "No lineage data in ACCOUNT_USAGE.SNOWFLAKE. Please check the data in `QUOLLIO_SQLLINEAGE_SOURCES`."
+            )
+            return
         update_table_lineage_inputs_list = list()
         sql_lineage = SQLLineage()
         for result in results:
@@ -158,21 +179,37 @@ def snowflake_table_stats(
     stats_items: List[str],
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        stats_query = _gen_get_stats_views_query(
+        get_stats_view_query = _gen_get_stats_views_query(
             db=conn.account_database,
             schema=conn.account_schema,
         )
-        stats_views = sf_executor.get_query_results(query=stats_query)
-
+        stats_views, err = sf_executor.get_query_results(query=get_stats_view_query)
+        if err is not None:
+            handle_error(err=err)
+        if len(stats_views) == 0:
+            logger.warning(
+                f"No target table for stats aggregation. Please see the error message above \
+and fix it or grant usage permission to both `{conn.account_database}` and `{conn.account_schema}` \
+and select permissions to views begins with `QUOLLIO_STATS_COLUMNS_`."
+            )
+            return
         req_count = 0
         is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
         for stats_view in stats_views:
-            table_fqn = "{catalog}.{schema}.{table}".format(
+            table_fqn = '"{catalog}"."{schema}"."{table}"'.format(
                 catalog=stats_view["TABLE_CATALOG"], schema=stats_view["TABLE_SCHEMA"], table=stats_view["TABLE_NAME"]
             )
             stats_query = render_sql_for_stats(is_aggregate_items=is_aggregate_items, table_fqn=table_fqn)
             logger.debug(f"The following sql will be fetched to retrieve stats values. {stats_query}")
-            stats_result = sf_executor.get_query_results(query=stats_query)
+            stats_result, err = sf_executor.get_query_results(query=stats_query)
+            if err is not None:
+                handle_error(err=err, force_skip=True)
+            if len(stats_result) == 0:
+                logger.warning(
+                    f"No stats value. Please query {table_fqn} to check the value exists in it \
+or user has select permission to it."
+                )
+                continue
             payloads = gen_table_stats_payload(tenant_id=tenant_id, endpoint=conn.account_id, stats=stats_result)
             for payload in payloads:
                 logger.info(
@@ -209,3 +246,34 @@ def _gen_get_stats_views_query(db: str, schema: str) -> str:
         db=db, schema=schema
     )
     return query
+
+
+def handle_error(err: Exception, force_skip: bool = False):
+    if err.errno == 2037:
+        logger.warning(
+            "snowflake get_query_results failed. The table you query exists but user doesn't have permission to select.\
+Please check a user has select or ownership permissions. ErrorNo: {0} SQLState: {1} Message: {2} SfqID: {3}".format(
+                err.errno, err.sqlstate, err.msg, err.sfqid
+            )
+        )
+        return
+    elif err.errno == 2003:
+        logger.warning(
+            "snowflake get_query_results failed. User doesn't have select permission to the object \
+or the object you query doesn't exist.\
+Please check a user has select or ownership permissions and whether the object exists or not. \
+ErrorNo: {0} SQLState: {1} Message: {2} SfqID: {3}".format(
+                err.errno, err.sqlstate, err.msg, err.sfqid
+            )
+        )
+        return
+    else:
+        logger.error(
+            "snowflake get_query_results failed.\
+Please check ErrNo and message. ErrorNo: {0} SQLState: {1} Message: {2} SfqID: {3}".format(
+                err.errno, err.sqlstate, err.msg, err.sfqid
+            )
+        )
+        if not force_skip:
+            raise Exception
+        return

{quollio_core-0.4.16 → quollio_core-0.4.19}/quollio_core/repository/qdc.py

@@ -25,6 +25,10 @@ class QDCExternalAPIClient:
         Tried to find a package for oauth0 client credentials flow,
         but any of them contains bugs or lacks of features to handle the token refresh when it's expired
         """
+        is_domain_valid = is_valid_domain(domain=self.base_url)
+        if not is_domain_valid:
+            raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com`")
+
         url = f"{self.base_url}/oauth2/token"
         creds = f"{self.client_id}:{self.client_secret}"
         encoded_creds = base64.b64encode(creds.encode()).decode()
@@ -104,3 +108,7 @@ class QDCExternalAPIClient:
 
 def initialize_qdc_client(api_url: str, client_id: str, client_secret: str) -> QDCExternalAPIClient:
     return QDCExternalAPIClient(base_url=api_url, client_id=client_id, client_secret=client_secret)
+
+
+def is_valid_domain(domain: str) -> bool:
+    return domain.endswith(".com")

{quollio_core-0.4.16 → quollio_core-0.4.19}/quollio_core/repository/snowflake.py

@@ -1,6 +1,6 @@
 import logging
 from dataclasses import asdict, dataclass
-from typing import Dict, List
+from typing import Dict, List, Tuple
 
 from snowflake.connector import DictCursor, connect, errors
 from snowflake.connector.connection import SnowflakeConnection
@@ -46,16 +46,13 @@ class SnowflakeQueryExecutor:
         )
         return conn
 
-    def get_query_results(self, query: str) -> List[Dict[str, str]]:
+    def get_query_results(self, query: str) -> Tuple[List[Dict[str, str]], Exception]:
         with self.conn.cursor(DictCursor) as cur:
             try:
                 cur.execute(query)
                 result: List[Dict[str, str]] = cur.fetchall()
-                return result
+                return (result, None)
             except errors.ProgrammingError as e:
-                logger.error(query)
-                logger.error(
-                    "snowflake get_query_results failed. {0} ({1}): {2} ({3})".format(
-                        e.errno, e.sqlstate, e.msg, e.sfqid
-                    )
-                )
+                return ([], e)
+            except Exception as e:
+                return ([], e)

{quollio_core-0.4.16 → quollio_core-0.4.19}/quollio_core/snowflake.py

@@ -103,7 +103,7 @@ def load_lineage(
     else:
         logger.info("Skip column lineage ingestion. Set enable_column_lineage to True if you ingest column lineage.")
 
-    logger.info("Lineage data is successfully loaded.")
+    logger.info("Lineage data is successfully finished.")
 
     return
 
@@ -127,7 +127,7 @@ def load_stats(
         stats_items=stats_items,
     )
 
-    logger.info("Stats data is successfully loaded.")
+    logger.info("Stats data is successfully finished.")
 
     return
 
@@ -144,7 +144,7 @@ def load_sqllineage(
         tenant_id=tenant_id,
     )
 
-    logger.info("sqllineage data is successfully loaded.")
+    logger.info("sqllineage data is successfully finished.")
 
     return