dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/flavors/snowflake/profiling/project_secondary_profiling_query_snowflake.sql

@@ -0,0 +1,29 @@
+-- Get Freqs for selected columns
+WITH ranked_vals AS (
+  SELECT "{COL_NAME}",
+         COUNT(*) AS ct,
+         ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC, "{COL_NAME}") AS rn
+    FROM {DATA_SCHEMA}.{DATA_TABLE}
+   WHERE "{COL_NAME}" > ' '
+   GROUP BY "{COL_NAME}"
+),
+consol_vals AS (
+  SELECT COALESCE(CASE WHEN rn <= 10 THEN '| ' || "{COL_NAME}" || ' | ' || CAST(ct AS VARCHAR)
+                       ELSE NULL
+                  END, '| Other Values (' || CAST(COUNT(DISTINCT "{COL_NAME}")  as VARCHAR) || ') | '  || CAST(SUM(ct)  as VARCHAR) ) AS val,
+         MIN(rn) as min_rn
+    FROM ranked_vals
+   GROUP BY CASE WHEN rn <= 10 THEN '| ' || "{COL_NAME}" || ' | ' || CAST(ct AS VARCHAR)
+                 ELSE NULL
+            END
+)
+SELECT '{PROJECT_CODE}' as project_code,
+       '{DATA_SCHEMA}' as schema_name,
+       '{RUN_DATE}' as run_date,
+       '{DATA_TABLE}' as table_name,
+       '{COL_NAME}' as column_name,
+       REPLACE(LISTAGG(val, '^#^') WITHIN GROUP (ORDER BY min_rn), '^#^', CHR(10)) AS top_freq_values,
+       ( SELECT MD5(LISTAGG(DISTINCT NULLIF("{COL_NAME}", ''), '|')
+                           WITHIN GROUP (ORDER BY NULLIF("{COL_NAME}", ''))) as dvh
+           FROM {DATA_SCHEMA}.{DATA_TABLE} ) as distinct_value_hash
+  FROM consol_vals;

testgen/template/flavors/snowflake/setup_profiling_tools/create_functions_snowflake.sql

@@ -0,0 +1,69 @@
+CREATE OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fndk_isnum(strparm VARCHAR)
+RETURNS INTEGER
+LANGUAGE SQL
+IMMUTABLE
+AS
+$$
+SELECT CASE
+            WHEN REGEXP_LIKE(strparm::VARCHAR, '^\\s*[+-]?\\$?\\s*[0-9]+(,[0-9]{3})*(\\.[0-9]*)?[\\%]?\\s*$') THEN 1
+            ELSE 0
+        END
+$$;
+
+
+CREATE OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fndk_isdate(strparm VARCHAR)
+RETURNS INTEGER
+LANGUAGE SQL
+IMMUTABLE
+AS
+$$
+SELECT CASE
+             -- YYYY-MM-DD HH:MM:SS SSSSSS
+             WHEN TRY_TO_DATE(strparm, 'YYYY-MM-DD HH:MI:SS SSSSSS') IS NOT NULL THEN 1
+
+             -- YYYY-MM-DD HH:MM:SS
+             WHEN TRY_TO_DATE(strparm, 'YYYY-MM-DD HH:MI:SS') IS NOT NULL THEN 1
+
+             -- YYYYMMDDHHMMSSSSSS
+             WHEN TRY_TO_DATE(strparm, 'YYYYMMDDHHMISSSSSS') IS NOT NULL THEN 1
+
+             -- YYYYMMDDHHMMSS
+             WHEN TRY_TO_DATE(strparm, 'YYYYMMDDHHMISS') IS NOT NULL THEN 1
+
+             -- YYYYMMDD
+             WHEN LENGTH(strparm) = 8 AND TRY_TO_DATE(strparm, 'YYYYMMDD') IS NOT NULL THEN 1
+
+             -- YYYY-MON-DD HH:MM:SS SSSSSS
+             --WHEN TRY_TO_DATE(strparm, 'YYYY-MON-DD HH:MI:SS SSSSSS') IS NOT NULL THEN 1
+
+              -- YYYY-MON-DD HH:MM:SS
+             --WHEN TRY_TO_DATE(strparm, 'YYYY-MON-DD HH:MI:SS') IS NOT NULL THEN 1
+
+              -- Exclude anything else long
+              WHEN LENGTH(strparm) > 11 THEN 0
+
+             -- YYYY-MON-DD
+             WHEN TRY_TO_DATE(strparm, 'YYYY-MON-DD') IS NOT NULL THEN 1
+
+             -- YYYY-MM-DD
+             WHEN TRY_TO_DATE(strparm, 'YYYY-MM-DD') IS NOT NULL THEN 1
+
+             -- MM/DD/YYYY
+             WHEN TRY_TO_DATE(strparm, 'MM/DD/YYYY') IS NOT NULL THEN 1
+
+             -- MM/DD/YY
+             WHEN TRY_TO_DATE(strparm, 'MM/DD/YY') IS NOT NULL THEN 1
+
+            --MM-DD-YYYY
+            WHEN TRY_TO_DATE(strparm, 'MM-DD-YYYY') IS NOT NULL THEN 1
+
+             --MM-DD-YY
+             WHEN TRY_TO_DATE(strparm, 'MM-DD-YY') IS NOT NULL THEN 1
+
+             --DD-MMM-YYYY
+             WHEN TRY_TO_DATE(strparm, 'DD-MON-YYYY') IS NOT NULL THEN 1
+
+
+              ELSE 0
+              END
+$$;

testgen/template/flavors/snowflake/setup_profiling_tools/create_qc_schema_snowflake.sql

@@ -0,0 +1 @@
+CREATE SCHEMA IF NOT exists {DATA_QC_SCHEMA};

testgen/template/flavors/snowflake/setup_profiling_tools/grant_execute_privileges_snowflake.sql

@@ -0,0 +1,6 @@
+
+CREATE ROLE IF NOT EXISTS dk_qc_role;
+GRANT ALL PRIVILEGES ON SCHEMA {DATA_QC_SCHEMA} TO ROLE dk_qc_role;
+GRANT USAGE ON FUNCTION {DATA_QC_SCHEMA}.fndk_isnum(VARCHAR) TO dk_qc_role;
+GRANT USAGE ON FUNCTION {DATA_QC_SCHEMA}.fndk_isdate(VARCHAR) TO dk_qc_role;
+GRANT ROLE dk_qc_role TO USER {DB_USER};

testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml

@@ -0,0 +1,219 @@
+---
+strTemplate01_sampling: "SELECT "
+strTemplate01_else: "SELECT "
+strTemplate02_all: |
+  {CONNECTION_ID} as connection_id,
+  '{PROJECT_CODE}' as project_code,
+  '{TABLE_GROUPS_ID}' as table_groups_id,
+  '{DATA_SCHEMA}' AS schema_name,
+  '{RUN_DATE}' AS run_date,
+  '{DATA_TABLE}' AS table_name,
+  {COL_POS} AS position,
+  '{COL_NAME_SANITIZED}' AS column_name,
+  '{COL_TYPE}' AS column_type,
+  '{COL_GEN_TYPE}' AS general_type,
+  COUNT(*) AS record_ct,
+  COUNT("{COL_NAME}") AS value_ct,
+  COUNT(DISTINCT "{COL_NAME}") AS distinct_value_ct,
+  SUM(NVL2("{COL_NAME}", 0, 1)) AS null_value_ct,
+strTemplate03_ADN: MIN(LENGTH("{COL_NAME}"))  AS min_length,
+  MAX(LENGTH("{COL_NAME}"))  AS max_length,
+  AVG(CAST(NULLIF(LENGTH("{COL_NAME}"), 0) AS REAL)) AS avg_length,
+strTemplate03_else:  NULL as min_length,
+  NULL as max_length,
+  NULL as avg_length,
+strTemplate04_A:  SUM(CASE
+        WHEN REGEXP_LIKE(TRIM("{COL_NAME}") , '^0(\.0*)?$') = TRUE THEN 1 ELSE 0
+      END)  AS zero_value_ct,
+strTemplate04_N:  CAST(SUM( 1 - ABS(SIGN("{COL_NAME}")) ) AS BIGINT) AS zero_value_ct,
+strTemplate04_else: NULL as zero_value_ct,
+strTemplate05_A:  COUNT(DISTINCT UPPER(TRANSLATE("{COL_NAME}", ' '',.-', ''))) as distinct_std_value_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" = '' THEN 1
+                             ELSE 0
+      END)  AS zero_length_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" BETWEEN ' !' AND '!' THEN 1
+                                              ELSE 0
+       END )  AS lead_space_ct,
+  SUM( CASE WHEN "{COL_NAME}" LIKE '"%"' OR "{COL_NAME}" LIKE '''%''' THEN 1 ELSE 0 END ) as quoted_value_ct,
+  SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '[0-9]') = TRUE THEN 1 ELSE 0 END ) as includes_digit_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" IN ('.', '?') THEN 1
+         WHEN REGEXP_LIKE(LOWER("{COL_NAME}") ,'(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)') = TRUE THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('blank','error','missing','tbd',
+                                    'n/a','#na','none','null','unknown')           THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('(blank)','(error)','(missing)','(tbd)',
+                                    '(n/a)','(#na)','(none)','(null)','(unknown)') THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('[blank]','[error]','[missing]','[tbd]',
+                                    '[n/a]','[#na]','[none]','[null]','[unknown]') THEN 1
+                                                                                   ELSE 0
+       END )   AS filled_value_ct,
+  SUBSTRING(MIN(NULLIF("{COL_NAME}", '')), 1, 100)  AS min_text,
+  SUBSTRING(MAX(NULLIF("{COL_NAME}", '')), 1, 100)  AS max_text,
+  SUM(fndk_isnum(SUBSTRING("{COL_NAME}", 1, 31)))  AS numeric_ct,
+  SUM(fndk_isdate(SUBSTRING("{COL_NAME}", 1, 26)))  AS date_ct,
+  CASE
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '^[0-9]{1,5}[a-zA-Z]?\\s\\w{1,5}\\.?\\s?\\w*\\s?\\w*\\s[a-zA-Z]{1,6}\\.?\\s?[0-9]{0,5}[A-Z]{0,1}$') = TRUE
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.8 THEN 'STREET_ADDR'
+    WHEN CAST(SUM(CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA')
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.9 THEN 'STATE_USA'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '^([\\+]1 |1-|)[\\+]?[(]?[0-9]{3}[)][ ]?[-\\s\\.]?[0-9]{3}[-\\s\.]?[0-9]{4,6}$') = TRUE
+                     OR REGEXP_LIKE("{COL_NAME}" , '^([\\+]1 |1-|)[2-9][01][0-9][-| ]?[0-9]{3}[-| ]?[0-9]{4}$') = TRUE
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.9 THEN 'PHONE_USA'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$') = TRUE
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.9 THEN 'EMAIL'
+    WHEN CAST(SUM( CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.9 THEN 'ZIP_USA'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '^[\\w\\s\-]+\\.(txt|csv|tsv|dat|doc|pdf|xlsx)$') = TRUE
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.9 THEN 'FILE_NAME'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '^([0-9]{4}[- ]){3}[0-9]{4}$') = TRUE
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.8 THEN 'CREDIT_CARD'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '^([^,|\t]{1,20}[,|\t]){2,}[^,|\t]{0,20}([,|\t]{0,1}[^,|\t]{0,20})*$') = TRUE
+                    AND REGEXP_LIKE("{COL_NAME}" , '\\s(and|but|or|yet)\\s') != TRUE
+         THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.8 THEN 'DELIMITED_DATA'
+    WHEN CAST(SUM ( CASE WHEN REGEXP_LIKE("{COL_NAME}" , '^[0-8][0-9]{2}-[0-9]{2}-[0-9]{4}$')  = TRUE
+                     AND SUBSTRING("{COL_NAME}", 1, 3) NOT BETWEEN '734' AND '749'
+                     AND SUBSTRING("{COL_NAME}", 1, 3) <> '666' THEN 1 END) AS REAL)/CAST(COUNT("{COL_NAME}") AS REAL) > 0.9 THEN 'SSN'
+  END as std_pattern_match,
+strTemplate05_else: NULL as distinct_std_value_ct,
+  NULL as zero_length_ct,
+  NULL as lead_space_ct,
+  NULL as quoted_value_ct,
+  NULL as includes_digit_ct,
+  NULL as filled_value_ct,
+  NULL as min_text,
+  NULL as max_text,
+  NULL as numeric_ct,
+  NULL as date_ct,
+  NULL as std_pattern_match,
+strTemplate06_A_patterns: (SELECT SUBSTRING(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1, 1000) AS concat_pats
+        FROM ( SELECT CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
+                   COUNT(*) AS ct
+              FROM (  SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
+                        "{COL_NAME}", '[a-z]', 'a'),
+                                    '[A-Z]', 'A'),
+                                    '[0-9]', 'N') AS pattern
+                       FROM {DATA_SCHEMA}.{DATA_TABLE}
+                      WHERE "{COL_NAME}" > ' ' AND (SELECT MAX(LENGTH("{COL_NAME}"))
+                                                    FROM {DATA_SCHEMA}.{DATA_TABLE}) BETWEEN 3 and {PARM_MAX_PATTERN_LENGTH}) p
+            GROUP BY pattern
+            HAVING pattern > ' '
+            ORDER BY COUNT(*) DESC LIMIT 5) as ps) AS top_patterns,
+strTemplate06_else: NULL as top_patterns,
+strTemplate07_A_freq:  (  SELECT SUBSTRING(LISTAGG(val, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1, 1000) as concat_vals
+  FROM (
+          SELECT CAST(COUNT(*) as VARCHAR(10)) || ' | ' || "{COL_NAME}" as val, COUNT(*) as ct
+          FROM {DATA_SCHEMA}.{DATA_TABLE}
+          WHERE "{COL_NAME}" > ' '
+          GROUP BY "{COL_NAME}"
+          HAVING "{COL_NAME}" > ' '
+          ORDER BY COUNT(*), "{COL_NAME}" DESC
+          LIMIT 10
+        ) ps
+  )  AS top_freq_values,
+strTemplate07_else: NULL as top_freq_values,
+strTemplate08_N: MIN("{COL_NAME}")  AS min_value,
+  MIN(CASE WHEN "{COL_NAME}" > 0 THEN "{COL_NAME}" ELSE NULL END)  AS min_value_over_0,
+  MAX("{COL_NAME}")  AS max_value,
+  AVG(CAST("{COL_NAME}" AS REAL))  AS avg_value,
+  STDDEV(CAST("{COL_NAME}" AS REAL))  AS stdev_value,
+  MIN(pct_25) as percentile_25,
+  MIN(pct_50) as percentile_50,
+  MIN(pct_75) as percentile_75,
+strTemplate08_else: NULL as min_value,
+  NULL as min_value_over_0,
+  NULL as max_value,
+  NULL as avg_value,
+  NULL as stdev_value,
+  NULL as percentile_25,
+  NULL as percentile_50,
+  NULL as percentile_75,
+strTemplate10_N_dec: SUM(ROUND(MOD("{COL_NAME}", 1), 5)) as fractional_sum,
+strTemplate10_else: NULL as fractional_sum,
+strTemplate11_D: CASE
+         WHEN MIN("{COL_NAME}") IS NULL THEN NULL
+         ELSE GREATEST(MIN("{COL_NAME}"), '0001-01-01')
+       END as min_date,
+  MAX("{COL_NAME}") as max_date,
+  SUM(CASE
+        WHEN DATE_DIFF('MONTH', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}') > 12 THEN 1
+                                                            ELSE 0
+      END) AS before_1yr_date_ct,
+  SUM(CASE
+        WHEN DATE_DIFF('MONTH', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}') > 60 THEN 1
+                                                            ELSE 0
+      END) AS before_5yr_date_ct,
+  SUM(CASE
+        WHEN DATE_DIFF('MONTH', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}') > 240 THEN 1
+                                                              ELSE 0
+      END) AS before_20yr_date_ct,
+  SUM(CASE
+        WHEN DATE_DIFF('DAY', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}') BETWEEN 0 AND 365 THEN 1
+                                                                         ELSE 0
+      END) AS within_1yr_date_ct,
+  SUM(CASE
+        WHEN DATE_DIFF('DAY', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}') BETWEEN 0 AND 30 THEN 1
+                                                                        ELSE 0
+      END) AS within_1mo_date_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" > '{RUN_DATE}' THEN 1 ELSE 0
+      END) AS future_date_ct,
+  COUNT(DISTINCT DATE_DIFF('day', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}' ) ) as date_days_present,
+  COUNT(DISTINCT DATE_DIFF('week', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}' ) ) as date_weeks_present,
+  COUNT(DISTINCT DATE_DIFF('month', TIMESTAMP "{COL_NAME}", TIMESTAMP '{RUN_DATE}' ) ) as date_months_present,
+
+strTemplate11_else:  NULL as min_date,
+  NULL as max_date,
+  NULL as before_1yr_date_ct,
+  NULL as before_5yr_date_ct,
+  NULL as before_20yr_date_ct,
+  NULL as within_1yr_date_ct,
+  NULL as within_1mo_date_ct,
+  NULL as future_date_ct,
+  NULL as date_days_present,
+  NULL as date_weeks_present,
+  NULL as date_months_present,
+
+strTemplate12_B: SUM(CAST("{COL_NAME}" AS INTEGER)) AS boolean_true_ct,
+
+strTemplate12_else: NULL as boolean_true_ct,
+
+strTemplate13_ALL: NULL AS datatype_suggestion,
+strTemplate14_A_do_patterns: ( SELECT COUNT(DISTINCT REGEXP_REPLACE( REGEXP_REPLACE( REGEXP_REPLACE(
+                          "{COL_NAME}", '[a-z]', 'a'),
+                                      '[A-Z]', 'A'),
+                                      '[0-9]', 'N')
+                ) AS pattern_ct
+     FROM {DATA_SCHEMA}.{DATA_TABLE}
+    WHERE "{COL_NAME}" > ' ' ) AS distinct_pattern_ct,
+  SUM(CAST(SIGN(REGEXP_COUNT(TRIM("{COL_NAME}"), ' ')) AS BIGINT)) AS embedded_space_ct,
+  AVG(CAST(REGEXP_COUNT(TRIM("{COL_NAME}"), ' ') AS REAL))  AS avg_embedded_spaces,
+
+strTemplate14_A_no_patterns:  NULL as distinct_pattern_ct,
+  SUM(CAST(SIGN(REGEXP_COUNT(TRIM("{COL_NAME}"), ' ')) AS BIGINT)) AS embedded_space_ct,
+  AVG(CAST(REGEXP_COUNT(TRIM("{COL_NAME}"), ' ') AS REAL))  AS avg_embedded_spaces,
+
+strTemplate14_else:  NULL as distinct_pattern_ct,
+  NULL as embedded_space_ct,
+  NULL as avg_embedded_spaces,
+
+strTemplate15_ALL:  NULL as functional_data_type,
+  NULL as functional_table_type,
+
+strTemplate16_ALL: " '{PROFILE_RUN_ID}' as profile_run_id"
+
+strTemplate98_sampling: ' FROM {DATA_SCHEMA}.{DATA_TABLE} '
+
+strTemplate98_else: ' FROM {DATA_SCHEMA}.{DATA_TABLE}'
+
+strTemplate99_N: |
+  , (SELECT
+             APPROX_PERCENTILE("{COL_NAME}", 0.25)  AS pct_25,
+             APPROX_PERCENTILE("{COL_NAME}", 0.50)  AS pct_50,
+             APPROX_PERCENTILE("{COL_NAME}", 0.75)  AS pct_75
+        FROM {DATA_SCHEMA}.{DATA_TABLE} LIMIT 1) pctile
+
+strTemplate99_else: ' '
+
+strTemplate100_sampling: 'WHERE RAND() <= 1.0 / {PROFILE_SAMPLE_RATIO}'

testgen/template/flavors/trino/setup_profiling_tools/create_functions_trino.sql

@@ -0,0 +1,92 @@
+
+-- The following functions are inline functions
+-- INLINE FUNCTION TO CHECK FOR A NUMBER
+
+WITH FUNCTION num_check(a varchar)
+    RETURNS integer
+    RETURN
+    CASE WHEN regexp_like(a, '^[0-9]+(\.[0-9]+)?$')  = TRUE THEN 1
+    WHEN regexp_like(a, '\$[0-9]+(\.[0-9]+)?$') = TRUE THEN 1
+    WHEN regexp_like(a, '^[0-9]+(\.[0-9]+)?\$') = TRUE THEN 1
+    ELSE 0
+END
+SELECT num_check('1234567'), num_check('$45.945843'), num_check('0.123$');
+
+
+-- INLINE FUNCTION TO CHECK FOR A DATE
+
+WITH FUNCTION date_check(a varchar)
+    RETURNS integer
+    RETURN
+    CASE WHEN REGEXP_LIKE(a, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])\s(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\s[0-9]{6})?$')
+    THEN CASE WHEN CAST(SUBSTRING(a, 1, 4) AS INT) BETWEEN 1800 AND 2200
+    AND( ( SUBSTRING(a, 6, 2) IN ('01', '03', '05', '07', '08', '10', '12')
+    AND CAST(SUBSTRING(a, 9, 2) AS INT) BETWEEN 1 AND 31)
+    OR (SUBSTRING(a, 6, 2) IN ('04', '06', '09') AND CAST(SUBSTRING(a, 9, 2) AS INT) BETWEEN 1 AND 30)
+    OR (SUBSTRING(a, 6, 2) = '02' AND CAST(SUBSTRING(a, 9, 2) AS INT)  BETWEEN 1 AND 29)
+    )
+    THEN 1
+    ELSE 0
+END
+WHEN REGEXP_LIKE(a, '^(\d{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])(2[0-3]|[01][0-9])([0-5][0-9])([0-5][0-9])([0-9]{6})$')
+             OR REGEXP_LIKE(a, '^(\d{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])$')
+         THEN CASE WHEN CAST(SUBSTRING(a, 1, 4) AS INT) BETWEEN 1800 AND 2200
+                        AND ( (SUBSTRING(a, 5, 2) IN ('01', '03', '05', '07', '08', '10', '12')
+                                    AND CAST(SUBSTRING(a, 7, 2) AS INT) BETWEEN 1 AND 31)
+                               OR (SUBSTRING(a, 5, 2) IN ('04', '06', '09') AND CAST(SUBSTRING(a, 7, 2) AS INT) BETWEEN 1 AND 30)
+                               OR (SUBSTRING(a, 5, 2) = '02' AND CAST(SUBSTRING(a, 7, 2) AS INT) BETWEEN 1 AND 29)
+                            )
+                    THEN 1
+                    ELSE 0
+END
+WHEN LENGTH(a) > 11 THEN 0
+        WHEN REGEXP_LIKE(REGEXP_REPLACE(UPPER(a), '(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)', '12'), '[12][09][0-9][0-9]-[0-1]?[0-9]-[0-3]?[0-9]')
+        THEN CASE WHEN CAST(SPLIT_PART(a, '-', 1) AS INT) BETWEEN 1800 AND 2200
+                        AND ( (UPPER(SPLIT_PART(a, '-', 2)) IN ('01', '03', '05', '07', '08',
+                                                           '1', '3', '5', '7', '8', '10', '12',
+                                                           'JAN', 'MAR', 'MAY', 'JUL', 'AUG',
+                                                           'OCT', 'DEC')
+                                AND CAST(SPLIT_PART(a, '-', 3) AS INT) BETWEEN 1 AND 31)
+                            OR (UPPER(SPLIT_PART(a, '-', 2)) IN ('04', '06', '09', '4', '6', '9', '11', 'APR', 'JUN', 'SEP', 'NOV')
+                                AND CAST(SPLIT_PART(a, '-', 3) AS INT) BETWEEN 1 AND 30)
+                            OR (UPPER(SPLIT_PART(a, '-', 2)) IN ('02', '2', 'FEB') AND CAST(SPLIT_PART(a, '-', 3) AS INT) BETWEEN 1 AND 29)
+                        )
+                    THEN 1
+                    ELSE 0
+END
+WHEN REGEXP_LIKE(REPLACE(a, '-', '/') , '^[0-1]?[0-9]/[0-3]?[0-9]/[12][09][0-9][0-9]$')
+             OR REGEXP_LIKE(REPLACE(a, '-', '/') , '^[0-1]?[0-9]/[0-3]?[0-9]/[0-9][0-9]$')
+        THEN CASE WHEN CAST(SPLIT_PART(REPLACE(a, '-', '/'), '/', 1) AS INT) BETWEEN 1 AND 12
+                        AND ( (CAST(SPLIT_PART(REPLACE(a, '-', '/'), '/', 1) AS INT) IN (1, 3, 5, 7, 8, 10, 12)
+                                    AND CAST(SPLIT_PART(REPLACE(a, '-', '/'), '/', 2) AS INT)  BETWEEN 1 AND 31)
+                              OR (CAST(SPLIT_PART(REPLACE(a, '-', '/'), '/', 1) AS INT) IN (4, 6, 9, 11)
+                                    AND CAST(SPLIT_PART(REPLACE(a, '-', '/'), '/', 2) AS INT)  BETWEEN 1 AND 30)
+                              OR (CAST(SPLIT_PART(REPLACE(a, '-', '/'), '/', 1) AS INT) = 2
+                                    AND CAST(SPLIT_PART(REPLACE(a, '-', '/'), '/', 2) AS INT)  BETWEEN 1 AND 29)
+                        )
+                        AND CAST(('20' || SUBSTRING(SPLIT_PART(REPLACE(a, '-', '/'), '/', 3), -2 )) AS INT) BETWEEN 1800 AND 2200
+                   THEN 1
+                   ELSE 0
+END
+WHEN REGEXP_LIKE(UPPER(a) , '[0-3]?[0-9]-(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)-[12][09][0-9][0-9]')
+        THEN CASE WHEN CAST(SPLIT_PART(a, '-', 3) AS INT) BETWEEN 1800 AND 2200
+                        AND ( (UPPER(SPLIT_PART(a, '-', 2)) IN ('JAN', 'MAR', 'MAY', 'JUL', 'AUG', 'OCT', 'DEC')
+                                    AND CAST(SPLIT_PART(a, '-', 1) AS INT) BETWEEN 1 AND 31)
+                              OR (UPPER(SPLIT_PART(a, '-', 2)) IN ('APR', 'JUN', 'SEP', 'NOV')
+                                    AND CAST(SPLIT_PART(a, '-', 1) AS INT) BETWEEN 1 AND 30)
+                              OR (UPPER(SPLIT_PART(a, '-', 2)) = 'FEB'
+                                    AND CAST(SPLIT_PART(a, '-', 1) AS INT) BETWEEN 1 AND 29)
+                        )
+                   THEN 1
+                   ELSE 0
+END
+ELSE 0
+END
+SELECT date_check('2002-02-30 12:01:35'),
+       date_check('2002-02-21 12:01:35 121324'),
+       date_check('20100314224518304596'),
+       date_check('20100230'),
+       date_check('201002301234'),
+       date_check('2010-03-30'), date_check('2010-MAR-30'),
+       date_check('05-21-22'), date_check('10/23/2023'),
+       date_check('10-SEP-2024');

testgen/template/flavors/trino/setup_profiling_tools/create_qc_schema_trino.sql

@@ -0,0 +1 @@
+CREATE SCHEMA IF NOT exists {DATA_QC_SCHEMA};

testgen/template/gen_funny_cat_tests/gen_test_constant.sql

@@ -0,0 +1,104 @@
+-- Then insert new tests where a locked test is not already present
+INSERT INTO test_definitions (project_code, table_groups_id, profile_run_id,
+                              test_type, test_suite, test_suite_id,
+                              schema_name, table_name, column_name, skip_errors,
+                              last_auto_gen_date, test_active,
+                              baseline_value, threshold_value, profiling_as_of_date)
+WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date
+                    FROM profile_results p
+                  INNER JOIN profiling_runs r
+                     ON (p.profile_run_id = r.id)
+                    INNER JOIN test_suites tg
+                       ON p.project_code = tg.project_code
+                      AND p.connection_id = tg.connection_id
+                   WHERE p.project_code = '{PROJECT_CODE}'
+                     AND r.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                     AND tg.test_suite = '{TEST_SUITE}'
+                     AND p.run_date::DATE <= '{AS_OF_DATE}'
+                  GROUP BY r.table_groups_id),
+     curprof AS (SELECT p.*
+                   FROM last_run lr
+                 INNER JOIN profile_results p
+                    ON (lr.table_groups_id = p.table_groups_id
+                    AND lr.last_run_date = p.run_date) ),
+     locked AS (SELECT schema_name, table_name, column_name, test_type
+                  FROM test_definitions
+				     WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                   AND test_suite = '{TEST_SUITE}'
+                   AND lock_refresh = 'Y'),
+     all_runs AS ( SELECT DISTINCT p.table_groups_id, p.schema_name, p.run_date,
+                          DENSE_RANK() OVER (PARTITION BY p.table_groups_id ORDER BY p.run_date DESC) as run_rank
+                     FROM profile_results p
+                   INNER JOIN test_suites ts
+                      ON p.connection_id = ts.connection_id
+                     AND p.project_code = ts.project_code
+                   WHERE p.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                     AND ts.test_suite = '{TEST_SUITE}'
+                     AND p.run_date::DATE <= '{AS_OF_DATE}'),
+     recent_runs AS (SELECT table_groups_id, schema_name, run_date, run_rank
+                       FROM all_runs
+                      WHERE run_rank <= 5),
+     rightcols as (SELECT p.schema_name, p.table_name, p.column_name,
+                          SUM(CASE WHEN distinct_value_ct = 1 THEN 0 ELSE 1 END) as always_one_val,
+                          COUNT(DISTINCT CASE
+                                           WHEN p.general_type = 'A' THEN min_text
+                                           WHEN p.general_type = 'N' THEN min_value::VARCHAR
+                                           WHEN p.general_type IN ('D','T') THEN min_date::VARCHAR
+                                           WHEN p.general_type = 'B'
+                                            AND boolean_true_ct = value_ct  THEN 'TRUE'
+                                           WHEN p.general_type = 'B'
+                                            AND p.boolean_true_ct = 0
+                                            AND p.distinct_value_ct = 1     THEN 'FALSE'
+                                         END ) as agg_distinct_val_ct
+                    FROM recent_runs rr
+                  INNER JOIN profile_results p
+                     ON (rr.table_groups_id = p.table_groups_id
+                    AND  rr.run_date = p.run_date)
+                  GROUP BY p.schema_name, p.table_name, p.column_name
+                  HAVING SUM(CASE WHEN distinct_value_ct = 1 THEN 0 ELSE 1 END) = 0
+                     AND SUM(CASE WHEN max_length < 100 THEN 0 ELSE 1 END) = 0
+                     AND COUNT(DISTINCT CASE
+                                           WHEN p.general_type = 'A' THEN min_text
+                                           WHEN p.general_type = 'N' THEN min_value::VARCHAR
+                                           WHEN p.general_type IN ('D','T') THEN min_date::VARCHAR
+                                           WHEN p.general_type = 'B'
+                                            AND boolean_true_ct = value_ct  THEN 'TRUE'
+                                           WHEN p.general_type = 'B'
+                                            AND p.boolean_true_ct = 0
+                                            AND p.distinct_value_ct = 1     THEN 'FALSE'
+                                         END ) = 1 ),
+newtests AS ( SELECT 'Constant'::VARCHAR AS test_type,
+                     '{TEST_SUITE}'::VARCHAR AS test_suite,
+                     '{TEST_SUITE_ID}'::UUID AS test_suite_id,
+                     c.profile_run_id,
+                     c.project_code,
+                     c.schema_name, c.table_name, c.column_name,
+                     c.run_date AS last_run_date,
+                   case when general_type='A' then fn_quote_literal_escape(min_text, '{SQL_FLAVOR}')::VARCHAR
+                        when general_type='D' then fn_quote_literal_escape(min_date :: VARCHAR, '{SQL_FLAVOR}')::VARCHAR
+                        when general_type='N' then min_value::VARCHAR
+                        when general_type='B' and boolean_true_ct = 0 then 'FALSE'::VARCHAR
+                        when general_type='B' and boolean_true_ct > 0 then 'TRUE'::VARCHAR
+                   end as baseline_value
+                FROM curprof c
+               INNER JOIN rightcols r
+                  ON (c.schema_name = r.schema_name
+                     AND c.table_name = r.table_name
+                     AND c.column_name = r.column_name)
+               LEFT JOIN generation_sets s
+                  ON ('Constant' = s.test_type
+                 AND  '{GENERATION_SET}' = s.generation_set)
+               WHERE (s.generation_set IS NOT NULL
+                  OR  '{GENERATION_SET}' = '')  )
+SELECT n.project_code, '{TABLE_GROUPS_ID}'::UUID as table_groups_id, n.profile_run_id,
+       n.test_type, n.test_suite, n.test_suite_id, n.schema_name, n.table_name, n.column_name,
+       0 as skip_errors, '{RUN_DATE}'::TIMESTAMP as auto_gen_date,
+       'Y' as test_active, COALESCE(baseline_value, '') as baseline_value,
+       '0' as threshold_value, '{AS_OF_DATE}'::TIMESTAMP
+  FROM newtests n
+LEFT JOIN locked l
+  ON (n.schema_name = l.schema_name
+ AND  n.table_name = l.table_name
+ AND  n.column_name = l.column_name
+ AND  n.test_type = l.test_type)
+ WHERE l.test_type IS NULL;

testgen/template/gen_funny_cat_tests/gen_test_distinct_value_ct.sql

@@ -0,0 +1,98 @@
+-- FIRST TYPE OF CONSTANT IS HANDLED IN SEPARATE SQL FILE gen_standard_tests.sql using generic parameters
+-- Second type:  constants with changing values (1 distinct value)
+INSERT INTO test_definitions (project_code, table_groups_id, profile_run_id, test_type, test_suite, test_suite_id,
+                              schema_name, table_name, column_name, skip_errors,
+                              last_auto_gen_date, test_active,
+                              baseline_value_ct, threshold_value, profiling_as_of_date)
+WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date
+                    FROM profile_results p
+                  INNER JOIN profiling_runs r
+                     ON (p.profile_run_id = r.id)
+                    INNER JOIN test_suites tg
+                       ON p.project_code = tg.project_code
+                      AND p.connection_id = tg.connection_id
+                   WHERE p.project_code = '{PROJECT_CODE}'
+                     AND r.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                     AND tg.test_suite = '{TEST_SUITE}'
+                     AND p.run_date::DATE <= '{AS_OF_DATE}'
+                  GROUP BY r.table_groups_id),
+     curprof AS (SELECT p.*
+                   FROM last_run lr
+                 INNER JOIN profile_results p
+                    ON (lr.table_groups_id = p.table_groups_id
+                    AND lr.last_run_date = p.run_date) ),
+     locked AS (SELECT schema_name, table_name, column_name, test_type
+                  FROM test_definitions
+             WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                   AND test_suite = '{TEST_SUITE}'
+                   AND lock_refresh = 'Y'),
+     all_runs AS ( SELECT DISTINCT p.table_groups_id, p.schema_name, p.run_date,
+                          DENSE_RANK() OVER (PARTITION BY p.table_groups_id ORDER BY p.run_date DESC) as run_rank
+                     FROM profile_results p
+                   INNER JOIN test_suites ts
+                      ON p.connection_id = ts.connection_id
+                     AND p.project_code = ts.project_code
+                   WHERE p.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                     AND ts.test_suite = '{TEST_SUITE}'
+                     AND p.run_date::DATE <= '{AS_OF_DATE}'),
+     recent_runs AS (SELECT table_groups_id, schema_name, run_date, run_rank
+                       FROM all_runs
+                      WHERE run_rank <= 5),
+     rightcols as (SELECT p.schema_name, p.table_name, p.column_name,
+                          SUM(CASE WHEN distinct_value_ct = 1 THEN 0 ELSE 1 END) as always_one_val,
+                          COUNT(DISTINCT CASE
+                                           WHEN p.general_type = 'A' THEN min_text
+                                           WHEN p.general_type = 'N' THEN min_value::VARCHAR
+                                           WHEN p.general_type IN ('D','T') THEN min_date::VARCHAR
+                                           WHEN p.general_type = 'B'
+                                            AND boolean_true_ct = value_ct  THEN 'TRUE'
+                                           WHEN p.general_type = 'B'
+                                            AND p.boolean_true_ct = 0
+                                            AND p.distinct_value_ct = 1     THEN 'FALSE'
+                                         END ) as agg_distinct_val_ct
+                    FROM recent_runs rr
+                  INNER JOIN profile_results p
+                     ON (rr.table_groups_id = p.table_groups_id
+                    AND  rr.run_date = p.run_date)
+                  GROUP BY p.schema_name, p.table_name, p.column_name
+                  HAVING SUM(CASE WHEN distinct_value_ct = 1 THEN 0 ELSE 1 END) = 0
+                     AND COUNT(DISTINCT CASE
+                                           WHEN p.general_type = 'A' THEN min_text
+                                           WHEN p.general_type = 'N' THEN min_value::VARCHAR
+                                           WHEN p.general_type IN ('D','T') THEN min_date::VARCHAR
+                                           WHEN p.general_type = 'B'
+                                            AND boolean_true_ct = value_ct  THEN 'TRUE'
+                                           WHEN p.general_type = 'B'
+                                            AND p.boolean_true_ct = 0
+                                            AND p.distinct_value_ct = 1     THEN 'FALSE'
+                                         END ) > 1 ),
+newtests AS ( SELECT 'Distinct_Value_Ct'::VARCHAR AS test_type,
+                     '{TEST_SUITE}'::VARCHAR AS test_suite,
+                     '{TEST_SUITE_ID}'::UUID AS test_suite_id,
+                     c.project_code, c.table_groups_id, c.profile_run_id,
+                     c.schema_name, c.table_name, c.column_name,
+                     c.run_date AS last_run_date,
+                     c.distinct_value_ct
+                FROM curprof c
+               INNER JOIN rightcols r
+                  ON (c.schema_name = r.schema_name
+                     AND c.table_name = r.table_name
+                     AND c.column_name = r.column_name)
+               LEFT JOIN generation_sets s
+                  ON ('Distinct_Value_Ct' = s.test_type
+                 AND  '{GENERATION_SET}' = s.generation_set)
+               WHERE (s.generation_set IS NOT NULL
+                  OR  '{GENERATION_SET}' = '')  )
+SELECT n.project_code, n.table_groups_id, n.profile_run_id,
+       n.test_type, n.test_suite, n.test_suite_id,
+       n.schema_name, n.table_name, n.column_name, 0 as skip_errors,
+       '{RUN_DATE}'::TIMESTAMP as last_auto_gen_date, 'Y' as test_active,
+       distinct_value_ct as baseline_value_ct, distinct_value_ct as threshold_value,
+       '{AS_OF_DATE}'::TIMESTAMP as profiling_as_of_date
+  FROM newtests n
+LEFT JOIN locked l
+  ON (n.schema_name = l.schema_name
+ AND  n.table_name = l.table_name
+ AND  n.column_name = l.column_name
+ AND  n.test_type = l.test_type)
+ WHERE l.test_type IS NULL;