quollio-core 0.4.10__py3-none-any.whl → 0.4.11__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.10"
+__version__ = "0.4.11"
 __author__ = "Quollio Technologies, Inc"

quollio_core/profilers/databricks.py

@@ -138,42 +138,47 @@ def _get_column_stats(
             raise ValueError(f"Invalid table name: {table['table_fqdn']}")
         with databricks.DatabricksQueryExecutor(config=conn) as databricks_executor:
             query = """
-                SELECT
-                    "{monitored_table_catalog}" as DB_NAME,
-                    "{monitored_table_schema}" as SCHEMA_NAME,
-                    "{monitored_table_name}" as TABLE_NAME,
-                    t.COLUMN_NAME,
-                    t.DATA_TYPE,
-                    t.distinct_count as CARDINALITY,
-                    t.MAX as MAX_VALUE,
-                    t.MIN as MIN_VALUE,
-                    t.AVG as AVG_VALUE,
-                    t.MEDIAN as MEDIAN_VALUE,
-                    t.STDDEV as STDDEV_VALUE,
-                    t.NUM_NULLS as NULL_COUNT,
-                    t.frequent_items[0].item AS MODE_VALUE,
-                    MAX(t.window) AS LATEST
-                FROM
-                    {monitoring_table} t
-                WHERE
-                    t.column_name not in (':table')
-                GROUP BY
-                    t.COLUMN_NAME,
-                    t.DATA_TYPE,
-                    t.distinct_count,
-                    t.MAX,
-                    t.MIN,
-                    t.AVG,
-                    t.MEDIAN,
-                    t.STDDEV,
-                    t.NUM_NULLS,
-                    t.frequent_items
+                    WITH profile_record_history AS (
+                        SELECT
+                            COLUMN_NAME
+                            , distinct_count as CARDINALITY
+                            , MAX as MAX_VALUE
+                            , MIN as MIN_VALUE
+                            , AVG as AVG_VALUE
+                            , MEDIAN as MEDIAN_VALUE
+                            , STDDEV as STDDEV_VALUE
+                            , NUM_NULLS as NULL_COUNT
+                            , frequent_items[0].item AS MODE_VALUE
+                            , row_number() over(partition by column_name order by window desc) rownum
+                        FROM
+                            {monitoring_table}
+                        WHERE
+                            column_name not in (':table')
+                    )
+                    SELECT
+                        "{monitored_table_catalog}" as DB_NAME
+                        , "{monitored_table_schema}" as SCHEMA_NAME
+                        , "{monitored_table_name}" as TABLE_NAME
+                        , COLUMN_NAME
+                        , CARDINALITY
+                        , MAX_VALUE
+                        , MIN_VALUE
+                        , AVG_VALUE
+                        , MEDIAN_VALUE
+                        , STDDEV_VALUE
+                        , NULL_COUNT
+                        , MODE_VALUE
+                    FROM
+                        profile_record_history
+                    WHERE
+                        rownum = 1
                 """.format(
                 monitoring_table=table["table_fqdn"],
                 monitored_table_catalog=monitored_table[0],
                 monitored_table_schema=monitored_table[1],
                 monitored_table_name=monitored_table[2],
             )
+            logger.debug(f"The following sql will be fetched to retrieve stats values. {query}")
             stats.append(databricks_executor.get_query_results(query))
     return stats
 

{quollio_core-0.4.10.dist-info → quollio_core-0.4.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.10
+Version: 0.4.11
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>

{quollio_core-0.4.10.dist-info → quollio_core-0.4.11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-quollio_core/__init__.py,sha256=MhFw5RS_znNS5UBo8a0yTwpAdCcNqqmg_G1rdQ4mDLE,84
+quollio_core/__init__.py,sha256=_jIDStu07dkGu5ouTAUr-ImjBx7pmvosgMOUjUT25Pc,84
 quollio_core/bigquery.py,sha256=RguUznaY5YjROzJtXimoS8yCNH9jgGphpzd5v_JgSQM,3884
 quollio_core/bricks.py,sha256=Lehv-qsBSMNNE9BGVvidGOXJsxLSSsbNtmiEZH4lSUg,9458
 quollio_core/redshift.py,sha256=1d-mHnalB1jtiGPgzsGd3lRwLHCxaBJlUMEV2dh4f60,9882
@@ -67,7 +67,7 @@ quollio_core/helper/env_default.py,sha256=H6gbSGUPrEDZr4YDrL49hbOpw6RntI4U82kX1q
 quollio_core/helper/log.py,sha256=flxyZZ44G79l1TaUp3OT58uCHcnE5z_pCduwoeI6IUs,645
 quollio_core/profilers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 quollio_core/profilers/bigquery.py,sha256=e1Y8cZR-LxI9mSsYb0DurQyy0eCjM_kAKLfvl4IuQLE,3262
-quollio_core/profilers/databricks.py,sha256=DkT7cLl7XvAfZ_vkgiiXIt6yoKpov4MAbgpzq-i1m1g,7282
+quollio_core/profilers/databricks.py,sha256=pFWV6ecTefOS8evbnnXTcqR1jmbLsqn1V89ISyM2uqU,7743
 quollio_core/profilers/lineage.py,sha256=4FyxIuPBrUFihqZryqTQBcfB0Z7634lKl_WwkD82vzE,6865
 quollio_core/profilers/redshift.py,sha256=6_4amsBL4QW0ZajWhS-TW3f_cjKKa6TpXClMgBC-fZo,6440
 quollio_core/profilers/snowflake.py,sha256=nitlP5pmDm2RhLGO4f_WTzkw41EmOTY2uWN1HZkCHbI,8465
@@ -80,7 +80,7 @@ quollio_core/repository/dbt.py,sha256=cnLwJPywLi8VowVW7zfIBa9jxVwDWO7xzzNRn1vWiu
 quollio_core/repository/qdc.py,sha256=qEpMF6rKdic23dPJoDYmbIcyCKDuSFqbDF2_jqmqoZw,4369
 quollio_core/repository/redshift.py,sha256=p2ouEuYcDCjx1oBhc6H1ekQsvEqHGd3bFu3PW0ngYBc,2880
 quollio_core/repository/snowflake.py,sha256=J9rHshfWdOSnjQWxwGEYPpAU2lY7Tu5UFB_BNakkAX0,1892
-quollio_core-0.4.10.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.4.10.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-quollio_core-0.4.10.dist-info/METADATA,sha256=QeostnBm-yqx48bVtcEaB6C-zIy9dCFwkVnm4sDIhlA,6804
-quollio_core-0.4.10.dist-info/RECORD,,
+quollio_core-0.4.11.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.4.11.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+quollio_core-0.4.11.dist-info/METADATA,sha256=0tv9AKZYWKt9yPa9iaziHZyTzhj708v-Aq22MFEbYAs,6804
+quollio_core-0.4.11.dist-info/RECORD,,