quollio-core 0.4.17__py3-none-any.whl → 0.4.19__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.17"
+__version__ = "0.4.19"
 __author__ = "Quollio Technologies, Inc"

quollio_core/profilers/snowflake.py

@@ -19,7 +19,7 @@ def snowflake_table_to_table_lineage(
     tenant_id: str,
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        results = sf_executor.get_query_results(
+        results, err = sf_executor.get_query_results(
             query="""
             SELECT
                 *
@@ -30,6 +30,13 @@ def snowflake_table_to_table_lineage(
                 schema=conn.account_schema,
             )
         )
+        if err is not None:
+            handle_error(err=err)
+        if len(results) == 0:
+            logger.warning(
+                "No lineage data in ACCOUNT_USAGE.SNOWFLAKE. Please check the data in `QUOLLIO_LINEAGE_TABLE_LEVEL`."
+            )
+            return
         parsed_results = parse_snowflake_results(results=results)
         update_table_lineage_inputs = gen_table_lineage_payload(
             tenant_id=tenant_id,
@@ -62,7 +69,7 @@ def snowflake_column_to_column_lineage(
     tenant_id: str,
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        results = sf_executor.get_query_results(
+        results, err = sf_executor.get_query_results(
             query="""
             SELECT
                 *
@@ -73,6 +80,13 @@ def snowflake_column_to_column_lineage(
                 schema=conn.account_schema,
             )
         )
+        if err is not None:
+            handle_error(err=err)
+        if len(results) == 0:
+            logger.warning(
+                "No lineage data in ACCOUNT_USAGE.SNOWFLAKE. Please check the data in `QUOLLIO_LINEAGE_COLUMN_LEVEL`."
+            )
+            return
         update_column_lineage_inputs = gen_column_lineage_payload(
             tenant_id=tenant_id,
             endpoint=conn.account_id,
@@ -105,7 +119,7 @@ def snowflake_table_level_sqllineage(
     tenant_id: str,
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        results = sf_executor.get_query_results(
+        results, err = sf_executor.get_query_results(
             query="""
             SELECT
                 database_name
@@ -118,6 +132,13 @@ def snowflake_table_level_sqllineage(
                 schema=conn.account_schema,
             )
         )
+        if err is not None:
+            handle_error(err=err)
+        if len(results) == 0:
+            logger.warning(
+                "No lineage data in ACCOUNT_USAGE.SNOWFLAKE. Please check the data in `QUOLLIO_SQLLINEAGE_SOURCES`."
+            )
+            return
         update_table_lineage_inputs_list = list()
         sql_lineage = SQLLineage()
         for result in results:
@@ -158,12 +179,20 @@ def snowflake_table_stats(
     stats_items: List[str],
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
-        stats_query = _gen_get_stats_views_query(
+        get_stats_view_query = _gen_get_stats_views_query(
             db=conn.account_database,
             schema=conn.account_schema,
         )
-        stats_views = sf_executor.get_query_results(query=stats_query)
-
+        stats_views, err = sf_executor.get_query_results(query=get_stats_view_query)
+        if err is not None:
+            handle_error(err=err)
+        if len(stats_views) == 0:
+            logger.warning(
+                f"No target table for stats aggregation. Please see the error message above \
+and fix it or grant usage permission to both `{conn.account_database}` and `{conn.account_schema}` \
+and select permissions to views begins with `QUOLLIO_STATS_COLUMNS_`."
+            )
+            return
         req_count = 0
         is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
         for stats_view in stats_views:
@@ -172,7 +201,15 @@ def snowflake_table_stats(
             )
             stats_query = render_sql_for_stats(is_aggregate_items=is_aggregate_items, table_fqn=table_fqn)
             logger.debug(f"The following sql will be fetched to retrieve stats values. {stats_query}")
-            stats_result = sf_executor.get_query_results(query=stats_query)
+            stats_result, err = sf_executor.get_query_results(query=stats_query)
+            if err is not None:
+                handle_error(err=err, force_skip=True)
+            if len(stats_result) == 0:
+                logger.warning(
+                    f"No stats value. Please query {table_fqn} to check the value exists in it \
+or user has select permission to it."
+                )
+                continue
             payloads = gen_table_stats_payload(tenant_id=tenant_id, endpoint=conn.account_id, stats=stats_result)
             for payload in payloads:
                 logger.info(
@@ -209,3 +246,34 @@ def _gen_get_stats_views_query(db: str, schema: str) -> str:
         db=db, schema=schema
     )
     return query
+
+
+def handle_error(err: Exception, force_skip: bool = False):
+    if err.errno == 2037:
+        logger.warning(
+            "snowflake get_query_results failed. The table you query exists but user doesn't have permission to select.\
+Please check a user has select or ownership permissions. ErrorNo: {0} SQLState: {1} Message: {2} SfqID: {3}".format(
+                err.errno, err.sqlstate, err.msg, err.sfqid
+            )
+        )
+        return
+    elif err.errno == 2003:
+        logger.warning(
+            "snowflake get_query_results failed. User doesn't have select permission to the object \
+or the object you query doesn't exist.\
+Please check a user has select or ownership permissions and whether the object exists or not. \
+ErrorNo: {0} SQLState: {1} Message: {2} SfqID: {3}".format(
+                err.errno, err.sqlstate, err.msg, err.sfqid
+            )
+        )
+        return
+    else:
+        logger.error(
+            "snowflake get_query_results failed.\
+Please check ErrNo and message. ErrorNo: {0} SQLState: {1} Message: {2} SfqID: {3}".format(
+                err.errno, err.sqlstate, err.msg, err.sfqid
+            )
+        )
+        if not force_skip:
+            raise Exception
+        return

quollio_core/repository/qdc.py

@@ -25,6 +25,10 @@ class QDCExternalAPIClient:
         Tried to find a package for oauth0 client credentials flow,
         but any of them contains bugs or lacks of features to handle the token refresh when it's expired
         """
+        is_domain_valid = is_valid_domain(domain=self.base_url)
+        if not is_domain_valid:
+            raise ValueError("The format of quollio API URL is invalid. The URL must end with `.com`")
+
         url = f"{self.base_url}/oauth2/token"
         creds = f"{self.client_id}:{self.client_secret}"
         encoded_creds = base64.b64encode(creds.encode()).decode()
@@ -104,3 +108,7 @@ class QDCExternalAPIClient:
 
 def initialize_qdc_client(api_url: str, client_id: str, client_secret: str) -> QDCExternalAPIClient:
     return QDCExternalAPIClient(base_url=api_url, client_id=client_id, client_secret=client_secret)
+
+
+def is_valid_domain(domain: str) -> bool:
+    return domain.endswith(".com")

quollio_core/repository/snowflake.py

@@ -1,6 +1,6 @@
 import logging
 from dataclasses import asdict, dataclass
-from typing import Dict, List
+from typing import Dict, List, Tuple
 
 from snowflake.connector import DictCursor, connect, errors
 from snowflake.connector.connection import SnowflakeConnection
@@ -46,16 +46,13 @@ class SnowflakeQueryExecutor:
         )
         return conn
 
-    def get_query_results(self, query: str) -> List[Dict[str, str]]:
+    def get_query_results(self, query: str) -> Tuple[List[Dict[str, str]], Exception]:
         with self.conn.cursor(DictCursor) as cur:
             try:
                 cur.execute(query)
                 result: List[Dict[str, str]] = cur.fetchall()
-                return result
+                return (result, None)
             except errors.ProgrammingError as e:
-                logger.error(query)
-                logger.error(
-                    "snowflake get_query_results failed. {0} ({1}): {2} ({3})".format(
-                        e.errno, e.sqlstate, e.msg, e.sfqid
-                    )
-                )
+                return ([], e)
+            except Exception as e:
+                return ([], e)

quollio_core/snowflake.py

@@ -103,7 +103,7 @@ def load_lineage(
     else:
         logger.info("Skip column lineage ingestion. Set enable_column_lineage to True if you ingest column lineage.")
 
-    logger.info("Lineage data is successfully loaded.")
+    logger.info("Lineage data is successfully finished.")
 
     return
 
@@ -127,7 +127,7 @@ def load_stats(
         stats_items=stats_items,
     )
 
-    logger.info("Stats data is successfully loaded.")
+    logger.info("Stats data is successfully finished.")
 
     return
 
@@ -144,7 +144,7 @@ def load_sqllineage(
         tenant_id=tenant_id,
     )
 
-    logger.info("sqllineage data is successfully loaded.")
+    logger.info("sqllineage data is successfully finished.")
 
     return
 

{quollio_core-0.4.17.dist-info → quollio_core-0.4.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.17
+Version: 0.4.19
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>

{quollio_core-0.4.17.dist-info → quollio_core-0.4.19.dist-info}/RECORD

@@ -1,8 +1,8 @@
-quollio_core/__init__.py,sha256=nStM2GZJp1JH52jZ8xIBUcsz2SAv0sq6Sgsjtpoik5c,84
+quollio_core/__init__.py,sha256=y0SJ_NkmDeB1AzUwy_oG9Ivh80sUSTMebFbwH83kHe8,84
 quollio_core/bigquery.py,sha256=6Oq4DVGpa3X21Es_nbrsb8pK3vaxwb9Egnvq3huo95k,5894
 quollio_core/bricks.py,sha256=8h3kbI2b6lGH2s-56jE_Q5-R5-nIsQYMfvtRrkFOzoU,10784
 quollio_core/redshift.py,sha256=KcdljY95xYf9JYrsaMOBoP_XxQQ8wFVE5ue_XEMVSFc,11504
-quollio_core/snowflake.py,sha256=_UFIkr9liNby-5F46PAxupWPOqLhayoggjdLenwdMRU,12210
+quollio_core/snowflake.py,sha256=D-d26OwCUIpXIqfxZnkv4Ei1GJ1mdw9z8YA8K0G-bSE,12216
 quollio_core/dbt_projects/databricks/.gitignore,sha256=1jJAyXSzJ3YUm0nx3i7wUSE4RjQMX3ad6F8O88UbtzI,29
 quollio_core/dbt_projects/databricks/README.md,sha256=ZpRQyhFAODAiS8dc1Kb_ndkul4cu4o4udN_EMa49CU4,440
 quollio_core/dbt_projects/databricks/dbt_project.yml,sha256=3sH98RNk7TnphvI3yEdXDstb92kW5BNxr-cT0tXhwzk,480
@@ -71,17 +71,17 @@ quollio_core/profilers/bigquery.py,sha256=mEr7CFQNgBFqWR8XfCOk8WTm5k5qZhLF8ODVWf
 quollio_core/profilers/databricks.py,sha256=ik4RiR_GOeU3S7s6C6Y9SGe1D_Y_f98BDWJVlEJXL4U,7868
 quollio_core/profilers/lineage.py,sha256=4FyxIuPBrUFihqZryqTQBcfB0Z7634lKl_WwkD82vzE,6865
 quollio_core/profilers/redshift.py,sha256=p6ONDCkhndZAOcKAwEyQ5fsi-jsQrlwHHb7LTI_m1uk,6473
-quollio_core/profilers/snowflake.py,sha256=WbfFTeTCyfG3Yi3sVieuLBJ7-ABuWmMl2o0Qv4mHnI8,8317
+quollio_core/profilers/snowflake.py,sha256=m9Ivv2LRwnrmgKS36a039AhrO27sR1EaOOdqNF26PhI,11156
 quollio_core/profilers/sqllineage.py,sha256=XkF7hwDWIGNtyEP5cv2wETBgMfdQxeHolv7qPIkntSQ,5066
 quollio_core/profilers/stats.py,sha256=OLQrdrh0y64jo9rmzvGlDdxy_c7gMz_GnlXPJzWkBjM,7343
 quollio_core/repository/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/repository/bigquery.py,sha256=3AyGcJNYGnUyMweyc6lGm4quwrOzd-ZBS2zNnFwafII,3990
 quollio_core/repository/databricks.py,sha256=9Cgdv8qBnVaHqu3RA-IUBieAqb69moQ-KAAMVSf5Ds4,1877
 quollio_core/repository/dbt.py,sha256=cnLwJPywLi8VowVW7zfIBa9jxVwDWO7xzzNRn1vWiuw,659
-quollio_core/repository/qdc.py,sha256=0Gz6jauYMzs2bRAD-eO4RSeCIRq8y-JxMWfiefUVULs,4623
+quollio_core/repository/qdc.py,sha256=_5ygUD6h-zs02f4rzj6evxXqD1JjSgtWc-oHPEofaig,4902
 quollio_core/repository/redshift.py,sha256=p2ouEuYcDCjx1oBhc6H1ekQsvEqHGd3bFu3PW0ngYBc,2880
-quollio_core/repository/snowflake.py,sha256=J9rHshfWdOSnjQWxwGEYPpAU2lY7Tu5UFB_BNakkAX0,1892
-quollio_core-0.4.17.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.4.17.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-quollio_core-0.4.17.dist-info/METADATA,sha256=jZtfiEZjkdmmLyNCuYTiXWTS9RIZzbNOBb_OxXRB3g0,6887
-quollio_core-0.4.17.dist-info/RECORD,,
+quollio_core/repository/snowflake.py,sha256=zL9-xi98AIftdW9MuKI-M3pZ1kQuuH-UiZH8HcJvmk4,1769
+quollio_core-0.4.19.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.4.19.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+quollio_core-0.4.19.dist-info/METADATA,sha256=cGPdKKcHhb6b3D9rz46w4FMygML4fUZ-ZQgtfI2DqLY,6887
+quollio_core-0.4.19.dist-info/RECORD,,