dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml

@@ -0,0 +1,221 @@
+---
+strTemplate01_sampling: "SELECT "
+strTemplate01_else: "SELECT "
+strTemplate02_all: |
+  {CONNECTION_ID} as connection_id,
+  '{PROJECT_CODE}' as project_code,
+  '{TABLE_GROUPS_ID}' as table_groups_id,
+  '{DATA_SCHEMA}' AS schema_name,
+  '{RUN_DATE}' AS run_date,
+  '{DATA_TABLE}' AS table_name,
+  {COL_POS} AS position,
+  '{COL_NAME_SANITIZED}' AS column_name,
+  '{COL_TYPE}' AS column_type,
+  '{COL_GEN_TYPE}' AS general_type,
+  COUNT(*) AS record_ct,
+  COUNT("{COL_NAME}") AS value_ct,
+  COUNT(DISTINCT "{COL_NAME}") AS distinct_value_ct,
+  SUM(NVL2("{COL_NAME}", 0, 1)) AS null_value_ct,
+strTemplate03_ADN: MIN(LEN("{COL_NAME}"))  AS min_length,
+  MAX(LEN("{COL_NAME}"))  AS max_length,
+  AVG(NULLIF(LEN("{COL_NAME}"), 0)::FLOAT) AS avg_length,
+strTemplate03_else:  NULL as min_length,
+  NULL as max_length,
+  NULL as avg_length,
+strTemplate04_A:  SUM(CASE
+        WHEN TRIM("{COL_NAME}") ~ '^0(\.0*)?$' THEN 1 ELSE 0
+      END)  AS zero_value_ct,
+strTemplate04_N:  SUM( 1 - ABS(SIGN("{COL_NAME}")) )::BIGINT AS zero_value_ct,
+strTemplate04_else: NULL as zero_value_ct,
+strTemplate05_A:  COUNT(DISTINCT UPPER(TRANSLATE("{COL_NAME}", ' '',.-', ''))) as distinct_std_value_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" = '' THEN 1
+                             ELSE 0
+      END)  AS zero_length_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" BETWEEN ' !' AND '!' THEN 1
+                                              ELSE 0
+       END )  AS lead_space_ct,
+  SUM( CASE WHEN "{COL_NAME}" ILIKE '"%"' OR "{COL_NAME}" ILIKE '''%''' THEN 1 ELSE 0 END ) as quoted_value_ct,
+  SUM( CASE WHEN "{COL_NAME}" ~ '[0-9]' THEN 1 ELSE 0 END ) as includes_digit_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" IN ('.', '?', ' ') THEN 1
+         WHEN LOWER("{COL_NAME}") SIMILAR TO '(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)' THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('blank','error','missing','tbd',
+                                    'n/a','#na','none','null','unknown')           THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('(blank)','(error)','(missing)','(tbd)',
+                                    '(n/a)','(#na)','(none)','(null)','(unknown)') THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('[blank]','[error]','[missing]','[tbd]',
+                                    '[n/a]','[#na]','[none]','[null]','[unknown]') THEN 1
+                                                                                   ELSE 0
+       END )   AS filled_value_ct,
+  LEFT(MIN(NULLIF("{COL_NAME}", '')), 100)  AS min_text,
+  LEFT(MAX(NULLIF("{COL_NAME}", '')), 100)  AS max_text,
+  SUM({DATA_QC_SCHEMA}.fndk_isnum(LEFT("{COL_NAME}", 31)))  AS numeric_ct,
+  SUM({DATA_QC_SCHEMA}.fndk_isdate(LEFT("{COL_NAME}", 26)))  AS date_ct,
+  CASE
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[0-9]{1,5}[a-zA-Z]?\\s\\w{1,5}\\.?\\s?\\w*\\s?\\w*\\s[a-zA-Z]{1,6}\\.?\\s?[0-9]{0,5}[A-Z]{0,1}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'STREET_ADDR'
+    WHEN SUM(CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA')
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'STATE_USA'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^([\\+]1 |1-|)[\\+]?[(]?[0-9]{3}[)][ ]?[-\\s\\.]?[0-9]{3}[-\\s\.]?[0-9]{4,6}$'
+                     OR "{COL_NAME}" ~ '^([\\+]1 |1-|)[2-9][01][0-9][-| ]?[0-9]{3}[-| ]?[0-9]{4}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'PHONE_USA'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'EMAIL'
+    WHEN SUM( CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'ZIP_USA'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[\\w\\s\-]+\\.(txt|csv|tsv|dat|doc|pdf|xlsx)$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'FILE_NAME'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^([0-9]{4}[- ]){3}[0-9]{4}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'CREDIT_CARD'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^([^,|\t]{1,20}[,|\t]){2,}[^,|\t]{0,20}([,|\t]{0,1}[^,|\t]{0,20})*$'
+                    AND "{COL_NAME}" !~ '\\s(and|but|or|yet)\\s'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'DELIMITED_DATA'
+    WHEN SUM ( CASE WHEN "{COL_NAME}" ~ '^[0-8][0-9]{2}-[0-9]{2}-[0-9]{4}$'
+                     AND LEFT("{COL_NAME}", 3) NOT BETWEEN '734' AND '749'
+                     AND LEFT("{COL_NAME}", 3) <> '666' THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'SSN'
+  END as std_pattern_match,
+strTemplate05_else: NULL as distinct_std_value_ct,
+  NULL as zero_length_ct,
+  NULL as lead_space_ct,
+  NULL as quoted_value_ct,
+  NULL as includes_digit_ct,
+  NULL as filled_value_ct,
+  NULL as min_text,
+  NULL as max_text,
+  NULL as numeric_ct,
+  NULL as date_ct,
+  NULL as std_pattern_match,
+strTemplate06_A_patterns: (SELECT LEFT(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats
+        FROM ( SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
+                   COUNT(*) AS ct
+              FROM (  SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
+                        "{COL_NAME}", '[a-z]', 'a'),
+                                    '[A-Z]', 'A'),
+                                    '[0-9]', 'N') AS pattern
+                       FROM {DATA_SCHEMA}.{DATA_TABLE}
+                      WHERE "{COL_NAME}" > ' ' AND (SELECT MAX(LEN("{COL_NAME}"))
+                                                    FROM {DATA_SCHEMA}.{DATA_TABLE}) BETWEEN 3 and {PARM_MAX_PATTERN_LENGTH}) p
+            GROUP BY pattern
+            HAVING pattern > ' '
+            ORDER BY COUNT(*) DESC) as ps) AS top_patterns,
+strTemplate06_else: NULL as top_patterns,
+strTemplate07_A_freq:  (  SELECT LEFT(LISTAGG(val, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) as concat_vals
+  FROM (
+          SELECT TOP 10 CAST(COUNT(*) as VARCHAR(10)) || ' | ' || "{COL_NAME}" as val,
+          COUNT(*) as ct
+          FROM {DATA_SCHEMA}.{DATA_TABLE}
+          WHERE "{COL_NAME}" > ' '
+          GROUP BY "{COL_NAME}"
+          HAVING "{COL_NAME}" > ' '
+          ORDER BY COUNT(*), "{COL_NAME}" DESC
+        ) ps
+  )  AS top_freq_values,
+strTemplate07_else: NULL as top_freq_values,
+strTemplate08_N: MIN("{COL_NAME}")  AS min_value,
+  MIN(CASE WHEN "{COL_NAME}" > 0 THEN "{COL_NAME}" ELSE NULL END)  AS min_value_over_0,
+  MAX("{COL_NAME}")  AS max_value,
+  AVG(CAST("{COL_NAME}" AS FLOAT))  AS avg_value,
+  STDDEV(CAST("{COL_NAME}" AS FLOAT))  AS stdev_value,
+  MIN(pct_25) as percentile_25,
+  MIN(pct_50) as percentile_50,
+  MIN(pct_75) as percentile_75,
+strTemplate08_else: NULL as min_value,
+  NULL as min_value_over_0,
+  NULL as max_value,
+  NULL as avg_value,
+  NULL as stdev_value,
+  NULL as percentile_25,
+  NULL as percentile_50,
+  NULL as percentile_75,
+strTemplate10_N_dec: SUM(ROUND(MOD("{COL_NAME}", 1), 5)) as fractional_sum,
+
+strTemplate10_else: NULL as fractional_sum,
+
+strTemplate11_D: CASE
+         WHEN MIN("{COL_NAME}") IS NULL THEN NULL
+         ELSE GREATEST(MIN("{COL_NAME}"), '0001-01-01')
+       END as min_date,
+  MAX("{COL_NAME}") as max_date,
+  SUM(CASE
+        WHEN DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 12 THEN 1
+                                                            ELSE 0
+      END) AS before_1yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 60 THEN 1
+                                                            ELSE 0
+      END) AS before_5yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 240 THEN 1
+                                                              ELSE 0
+      END) AS before_20yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('DAY', "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 365 THEN 1
+                                                                         ELSE 0
+      END) AS within_1yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('DAY', "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 30 THEN 1
+                                                                        ELSE 0
+      END) AS within_1mo_date_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" > '{RUN_DATE}' THEN 1 ELSE 0
+      END) AS future_date_ct,
+  COUNT(DISTINCT DATEDIFF(day, "{COL_NAME}", '{RUN_DATE}' ) ) as date_days_present,
+  COUNT(DISTINCT DATEDIFF(week, "{COL_NAME}", '{RUN_DATE}' ) ) as date_weeks_present,
+  COUNT(DISTINCT DATEDIFF(month, "{COL_NAME}", '{RUN_DATE}' ) ) as date_months_present,
+
+strTemplate11_else:  NULL as min_date,
+  NULL as max_date,
+  NULL as before_1yr_date_ct,
+  NULL as before_5yr_date_ct,
+  NULL as before_20yr_date_ct,
+  NULL as within_1yr_date_ct,
+  NULL as within_1mo_date_ct,
+  NULL as future_date_ct,
+  NULL as date_days_present,
+  NULL as date_weeks_present,
+  NULL as date_months_present,
+
+strTemplate12_B: SUM(CAST("{COL_NAME}" AS INTEGER)) AS boolean_true_ct,
+
+strTemplate12_else: NULL as boolean_true_ct,
+
+strTemplate13_ALL: NULL AS datatype_suggestion,
+strTemplate14_A_do_patterns: ( SELECT COUNT(DISTINCT REGEXP_REPLACE( REGEXP_REPLACE( REGEXP_REPLACE(
+                          "{COL_NAME}", '[a-z]', 'a'),
+                                      '[A-Z]', 'A'),
+                                      '[0-9]', 'N')
+                ) AS pattern_ct
+     FROM {DATA_SCHEMA}.{DATA_TABLE}
+    WHERE "{COL_NAME}" > ' ' ) AS distinct_pattern_ct,
+  SUM(SIGN(REGEXP_COUNT(TRIM("{COL_NAME}"), ' '))::BIGINT) AS embedded_space_ct,
+  AVG(REGEXP_COUNT(TRIM("{COL_NAME}"), ' ')::FLOAT)  AS avg_embedded_spaces,
+
+strTemplate14_A_no_patterns:  NULL as distinct_pattern_ct,
+  SUM(SIGN(REGEXP_COUNT(TRIM("{COL_NAME}"), ' '))::BIGINT) AS embedded_space_ct,
+  AVG(REGEXP_COUNT(TRIM("{COL_NAME}"), ' ')::FLOAT)  AS avg_embedded_spaces,
+
+strTemplate14_else:  NULL as distinct_pattern_ct,
+  NULL as embedded_space_ct,
+  NULL as avg_embedded_spaces,
+
+strTemplate15_ALL:  NULL as functional_data_type,
+  NULL as functional_table_type,
+
+strTemplate16_ALL: " '{PROFILE_RUN_ID}' as profile_run_id"
+
+strTemplate98_sampling: ' FROM {DATA_SCHEMA}.{DATA_TABLE} '
+
+strTemplate98_else: ' FROM {DATA_SCHEMA}.{DATA_TABLE}'
+
+strTemplate99_N: |
+  , (SELECT
+             PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_25,
+             PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_50,
+             PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_75
+        FROM {DATA_SCHEMA}.{DATA_TABLE} LIMIT 1) pctile
+
+strTemplate99_else: ' '
+
+strTemplate100_sampling: 'WHERE RAND() <= 1.0 / {PROFILE_SAMPLE_RATIO}'

testgen/template/flavors/redshift/profiling/project_secondary_profiling_query_redshift.sql

@@ -0,0 +1,29 @@
+-- Get Freqs for selected columns
+WITH ranked_vals AS (
+  SELECT "{COL_NAME}",
+         COUNT(*) AS ct,
+         ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC, "{COL_NAME}") AS rn
+    FROM {DATA_SCHEMA}.{DATA_TABLE}
+   WHERE "{COL_NAME}" > ' '
+   GROUP BY "{COL_NAME}"
+),
+consol_vals AS (
+  SELECT COALESCE(CASE WHEN rn <= 10 THEN '| ' || "{COL_NAME}" || ' | ' || CAST(ct AS VARCHAR)
+                       ELSE NULL
+                  END, '| Other Values (' || CAST(COUNT(DISTINCT "{COL_NAME}")  as VARCHAR) || ') | '  || CAST(SUM(ct)  as VARCHAR) ) AS val,
+         MIN(rn) as min_rn
+    FROM ranked_vals
+   GROUP BY CASE WHEN rn <= 10 THEN '| ' || "{COL_NAME}" || ' | ' || CAST(ct AS VARCHAR)
+                 ELSE NULL
+            END
+)
+SELECT '{PROJECT_CODE}' as project_code,
+       '{DATA_SCHEMA}' as schema_name,
+       '{RUN_DATE}' as run_date,
+       '{DATA_TABLE}' as table_name,
+       '{COL_NAME}' as column_name,
+       REPLACE(LISTAGG(val, '^#^') WITHIN GROUP (ORDER BY min_rn), '^#^', CHR(10)) AS top_freq_values,
+       ( SELECT MD5(LISTAGG(DISTINCT "{COL_NAME}", '|')
+                           WITHIN GROUP (ORDER BY "{COL_NAME}")) as dvh
+           FROM {DATA_SCHEMA}.{DATA_TABLE} ) as distinct_value_hash
+  FROM consol_vals;

testgen/template/flavors/redshift/setup_profiling_tools/create_functions_redshift.sql

@@ -0,0 +1,115 @@
+CREATE OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fndk_isnum(VARCHAR)
+          RETURNS INTEGER
+          IMMUTABLE
+        AS
+        $$
+SELECT CASE
+            WHEN $1 ~ '^\\s*[+-]?\\$?\\s*[0-9]+(,[0-9]{3})*(\\.[0-9]*)?[\\%]?\\s*$'   THEN 1
+            ELSE 0
+        END;
+$$
+LANGUAGE sql;
+
+
+CREATE OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fndk_isdate(VARCHAR)
+          RETURNS INTEGER
+          IMMUTABLE
+        AS $$
+SELECT CASE
+          -- YYYY-MM-DD HH:MM:SS SSSSSS or YYYY-MM-DD HH:MM:SS
+          WHEN $1 ~
+               '^(\\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])\\s(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\s[0-9]{6})?$'
+                               THEN CASE
+                                        WHEN LEFT($1, 4):: INT BETWEEN 1800 AND 2200
+                                           AND (
+                                                   (SUBSTRING($1, 6, 2) IN ('01', '03', '05', '07', '08',
+                                                                            '10', '12')
+                                                      AND SUBSTRING($1, 9, 2):: INT BETWEEN 1 AND 31)
+                                                   OR (SUBSTRING($1, 6, 2) IN ('04', '06', '09')
+                                                   AND SUBSTRING($1, 9, 2):: INT BETWEEN 1 AND 30)
+                                                   OR (SUBSTRING($1, 6, 2) = '02'
+                                                   AND SUBSTRING($1, 9, 2):: INT :: INT BETWEEN 1 AND 29)
+                                                )
+                                           THEN 1
+                                           ELSE 0
+                                    END
+          -- YYYYMMDDHHMMSSSSSS or YYYYMMDD
+          WHEN $1 ~
+               '^(\\d{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])(2[0-3]|[01][0-9])([0-5][0-9])([0-5][0-9])([0-9]{6})$'
+            OR $1 ~ '^(\\d{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])(2[0-3]|[01][0-9])$'
+                               THEN CASE
+                                      WHEN LEFT($1, 4)::INT BETWEEN 1800 AND 2200
+                                       AND (
+                                             (SUBSTRING($1, 5, 2) IN ('01', '03', '05', '07', '08',
+                                                                      '10', '12')
+                                                AND SUBSTRING($1, 7, 2)::INT BETWEEN 1 AND 31)
+                                             OR (SUBSTRING($1, 5, 2) IN ('04', '06', '09')
+                                             AND SUBSTRING($1, 7, 2)::INT BETWEEN 1 AND 30)
+                                             OR (SUBSTRING($1, 5, 2) = '02'
+                                             AND SUBSTRING($1, 7, 2)::INT::INT BETWEEN 1 AND 29)
+                                            )
+                                         THEN 1
+                                         ELSE 0
+                                    END
+          -- Exclude anything else long
+          WHEN LENGTH($1) > 11 THEN 0
+          -- YYYY-MMM/MM-DD
+          WHEN REGEXP_REPLACE(UPPER($1), '(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)', '12')
+             ~ '[12][09][0-9][0-9]-[0-1]?[0-9]-[0-3]?[0-9]'
+                               THEN CASE
+             WHEN SPLIT_PART($1, '-', 1)::INT BETWEEN 1800 AND 2200
+                AND (
+                        (UPPER(SPLIT_PART($1, '-', 2)) IN ('01', '03', '05', '07', '08',
+                                                           '1', '3', '5', '7', '8', '10', '12',
+                                                           'JAN', 'MAR', 'MAY', 'JUL', 'AUG',
+                                                           'OCT', 'DEC')
+                           AND SPLIT_PART($1, '-', 3)::INT BETWEEN 1 AND 31)
+                        OR (UPPER(SPLIT_PART($1, '-', 2)) IN ('04', '06', '09', '4', '6', '9', '11',
+                                                              'APR', 'JUN', 'SEP', 'NOV')
+                        AND SPLIT_PART($1, '-', 3)::INT BETWEEN 1 AND 30)
+                        OR (UPPER(SPLIT_PART($1, '-', 2)) IN ('02', '2', 'FEB')
+                        AND SPLIT_PART($1, '-', 3)::INT BETWEEN 1 AND 29)
+                     )
+                THEN 1
+                ELSE 0
+                                    END
+          -- MM/-DD/-YY/YYYY
+          WHEN REPLACE($1, '-', '/') ~ '^[0-1]?[0-9]/[0-3]?[0-9]/[12][09][0-9][0-9]$'
+             OR REPLACE($1, '-', '/') ~ '^[0-1]?[0-9]/[0-3]?[0-9]/[0-9][0-9]$'
+                               THEN
+             CASE
+                WHEN SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT BETWEEN 1 AND 12
+                   AND (
+                           (SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT IN (1, 3, 5, 7, 8, 10, 12)
+                              AND SPLIT_PART(REPLACE($1, '-', '/'), '/', 2)::INT BETWEEN 1 AND 31)
+                           OR (SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT IN (4, 6, 9, 11)
+                           AND SPLIT_PART(REPLACE($1, '-', '/'), '/', 2)::INT BETWEEN 1 AND 30)
+                           OR (SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT = 2
+                           AND SPLIT_PART(REPLACE($1, '-', '/'), '/', 2)::INT BETWEEN 1 AND 29)
+                        )
+                   AND
+                     ('20' + RIGHT(SPLIT_PART(REPLACE($1, '-', '/'), '/', 3), 2))::INT BETWEEN 1800 AND 2200
+                   THEN 1
+                   ELSE 0
+             END
+          -- DD-MMM-YYYY
+          WHEN UPPER($1) ~ '[0-3]?[0-9]-(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)-[12][09][0-9][0-9]'
+                               THEN
+             CASE
+                WHEN SPLIT_PART($1, '-', 3)::INT BETWEEN 1800 AND 2200
+                   AND (
+                           (UPPER(SPLIT_PART($1, '-', 2)) IN ('JAN', 'MAR', 'MAY', 'JUL', 'AUG', 'OCT', 'DEC')
+                              AND SPLIT_PART($1, '-', 1)::INT BETWEEN 1 AND 31)
+                           OR (UPPER(SPLIT_PART($1, '-', 2)) IN ('APR', 'JUN', 'SEP', 'NOV')
+                           AND SPLIT_PART($1, '-', 1)::INT BETWEEN 1 AND 30)
+                           OR (UPPER(SPLIT_PART($1, '-', 2)) = 'FEB'
+                           AND SPLIT_PART($1, '-', 1)::INT BETWEEN 1 AND 29)
+                        )
+                   THEN 1
+                   ELSE 0
+             END
+                               ELSE 0
+       END
+          AS isdate;
+        $$
+          LANGUAGE sql;

testgen/template/flavors/redshift/setup_profiling_tools/create_qc_schema_redshift.sql

@@ -0,0 +1 @@
+CREATE SCHEMA IF NOT exists {DATA_QC_SCHEMA};

testgen/template/flavors/redshift/setup_profiling_tools/grant_execute_privileges_redshift.sql

@@ -0,0 +1,2 @@
+GRANT ALL PRIVILEGES ON SCHEMA {DATA_QC_SCHEMA} TO {DB_USER};
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA {DATA_QC_SCHEMA} TO {DB_USER};

testgen/template/flavors/snowflake/profiling/project_ddf_query_snowflake.sql

@@ -0,0 +1,38 @@
+SELECT '{PROJECT_CODE}'            as project_code,
+       CURRENT_TIMESTAMP                    as refresh_timestamp,
+       c.table_schema,
+       c.table_name,
+       c.column_name,
+       CASE
+           WHEN c.data_type ILIKE 'timestamp%' THEN lower(c.data_type)
+           WHEN c.data_type ILIKE 'date' THEN lower(c.data_type)
+           WHEN c.data_type ILIKE 'boolean' THEN 'boolean'
+           WHEN c.data_type = 'TEXT'
+               THEN 'varchar(' || CAST(c.character_maximum_length AS VARCHAR) || ')'
+           WHEN c.data_type ILIKE 'char%' THEN 'char(' || CAST(c.character_maximum_length AS VARCHAR) || ')'
+           WHEN c.data_type = 'NUMBER' AND c.numeric_precision = 38 AND c.numeric_scale = 0 THEN 'bigint'
+           WHEN c.data_type ILIKE 'num%' THEN 'numeric(' || CAST(c.numeric_precision AS VARCHAR) || ',' ||
+                                             CAST(c.numeric_scale AS VARCHAR) || ')'
+           ELSE c.data_type
+       END AS data_type,
+       c.character_maximum_length,
+       c.ordinal_position,
+       CASE
+           WHEN c.data_type ILIKE '%char%' OR c.data_type = 'TEXT'
+               THEN 'A'
+           WHEN c.data_type ILIKE 'boolean'
+               THEN 'B'
+           WHEN c.data_type ILIKE 'date'
+               OR c.data_type ILIKE 'timestamp%'
+               THEN 'D'
+           WHEN c.data_type = 'time without time zone'
+               THEN 'T'
+           WHEN lower(c.data_type) IN ('bigint', 'double precision', 'integer', 'smallint', 'real', 'float')
+               OR c.data_type ILIKE 'num%'
+               THEN 'N'
+           ELSE
+               'X' END          AS general_type,
+       numeric_scale > 0        as is_decimal
+FROM information_schema.columns c
+WHERE c.table_schema = '{DATA_SCHEMA}' {TABLE_CRITERIA}
+ORDER BY c.table_schema, c.table_name, c.ordinal_position;

testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml

@@ -0,0 +1,220 @@
+---
+strTemplate01_sampling: "SELECT "
+strTemplate01_else: "SELECT "
+strTemplate02_all: |
+  {CONNECTION_ID} as connection_id,
+  '{PROJECT_CODE}' as project_code,
+  '{TABLE_GROUPS_ID}' as table_groups_id,
+  '{DATA_SCHEMA}' AS schema_name,
+  '{RUN_DATE}' AS run_date,
+  '{DATA_TABLE}' AS table_name,
+  {COL_POS} AS position,
+  '{COL_NAME_SANITIZED}' AS column_name,
+  '{COL_TYPE}' AS column_type,
+  '{COL_GEN_TYPE}' AS general_type,
+  COUNT(*) AS record_ct,
+  COUNT("{COL_NAME}") AS value_ct,
+  COUNT(DISTINCT "{COL_NAME}") AS distinct_value_ct,
+  SUM(NVL2("{COL_NAME}", 0, 1)) AS null_value_ct,
+strTemplate03_ADN: MIN(LEN("{COL_NAME}"))  AS min_length,
+  MAX(LEN("{COL_NAME}"))  AS max_length,
+  AVG(NULLIF(LEN("{COL_NAME}"), 0)::FLOAT) AS avg_length,
+strTemplate03_else:  NULL as min_length,
+  NULL as max_length,
+  NULL as avg_length,
+strTemplate04_A:  SUM(CASE
+        WHEN REGEXP_LIKE(TRIM("{COL_NAME}"::VARCHAR), '^0(\.0*)?$') THEN 1 ELSE 0
+      END)  AS zero_value_ct,
+strTemplate04_N:  SUM( 1 - ABS(SIGN("{COL_NAME}")) )::BIGINT AS zero_value_ct,
+strTemplate04_else: NULL as zero_value_ct,
+strTemplate05_A:  COUNT(DISTINCT UPPER(TRANSLATE("{COL_NAME}", ' '',.-', ''))) as distinct_std_value_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" = '' THEN 1
+                             ELSE 0
+      END)  AS zero_length_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" BETWEEN ' !' AND '!' THEN 1
+                                              ELSE 0
+       END )  AS lead_space_ct,
+  SUM( CASE WHEN "{COL_NAME}"::VARCHAR ILIKE '"%"' OR "{COL_NAME}"::VARCHAR ILIKE '''%''' THEN 1 ELSE 0 END ) as quoted_value_ct,
+  SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '.*[0-9].*') THEN 1 ELSE 0 END ) as includes_digit_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" IN ('.', '?', ' ') THEN 1
+         WHEN LOWER("{COL_NAME}"::VARCHAR) REGEXP '-{2,}' OR LOWER("{COL_NAME}"::VARCHAR) REGEXP '0{2,}' OR LOWER("{COL_NAME}"::VARCHAR) REGEXP '9{2,}'
+              OR LOWER("{COL_NAME}"::VARCHAR) REGEXP 'x{2,}' OR LOWER("{COL_NAME}"::VARCHAR) REGEXP 'z{2,}' THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('blank','error','missing','tbd',
+                                    'n/a','#na','none','null','unknown')           THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('(blank)','(error)','(missing)','(tbd)',
+                                    '(n/a)','(#na)','(none)','(null)','(unknown)') THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('[blank]','[error]','[missing]','[tbd]',
+                                    '[n/a]','[#na]','[none]','[null]','[unknown]') THEN 1
+                                                                                   ELSE 0
+       END )   AS filled_value_ct,
+  LEFT(MIN(NULLIF("{COL_NAME}", '')), 100)  AS min_text,
+  LEFT(MAX(NULLIF("{COL_NAME}", '')), 100)  AS max_text,
+  SUM({DATA_QC_SCHEMA}.fndk_isnum(LEFT("{COL_NAME}", 31)))  AS numeric_ct,
+  SUM({DATA_QC_SCHEMA}.fndk_isdate(LEFT("{COL_NAME}", 26)))  AS date_ct,
+  CASE
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^[0-9]{1,5}[a-zA-Z]?\\s\\w{1,5}\\.?\\s?\\w*\\s?\\w*\\s[a-zA-Z]{1,6}\\.?\\s?[0-9]{0,5}[A-Z]{0,1}$')
+                        THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'STREET_ADDR'
+    WHEN CAST(SUM(CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA')
+                        THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'STATE_USA'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^([\\+]1 |1-|)[\\+]?[(]?[0-9]{3}[)][ ]?[-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4,6}$')
+      OR REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^([\+]1 |1-|)[2-9][01][0-9][-| ]?[0-9]{3}[-| ]?[0-9]{4}$')
+                        THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'PHONE_USA'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$')
+                        THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'EMAIL'
+    WHEN CAST(SUM( CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
+                        THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'ZIP_USA'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^[\\w\\s\-]+\\.(txt|csv|tsv|dat|doc|pdf|xlsx)$')
+                        THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'FILE_NAME'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^([0-9]{4}[- ]?){3}[0-9]{4}$')
+                        THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'CREDIT_CARD'
+    WHEN CAST(SUM( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^([^,|\t]{1,20}[,|\t]){2,}[^,|\t]{0,20}([,|\t]{0,1}[^,|\t]{0,20})*$')
+                     AND NOT REGEXP_LIKE("{COL_NAME}"::VARCHAR, '.*\\s(and|but|or|yet)\\s.*')
+                    THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'DELIMITED_DATA'
+    WHEN SUM ( CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^[0-8][0-9]{2}-[0-9]{2}-[0-9]{4}$')
+                     AND LEFT("{COL_NAME}", 3) NOT BETWEEN '734' AND '749'
+                     AND LEFT("{COL_NAME}", 3) <> '666' THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'SSN'
+  END as std_pattern_match,
+strTemplate05_else: NULL as distinct_std_value_ct,
+  NULL as zero_length_ct,
+  NULL as lead_space_ct,
+  NULL as quoted_value_ct,
+  NULL as includes_digit_ct,
+  NULL as filled_value_ct,
+  NULL as min_text,
+  NULL as max_text,
+  NULL as numeric_ct,
+  NULL as date_ct,
+  NULL as std_pattern_match,
+strTemplate06_A_patterns: ( SELECT LEFT(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats
+      FROM (
+            SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
+                   COUNT(*) AS ct
+              FROM (  SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
+                        "{COL_NAME}"::VARCHAR, '[a-z]', 'a'),
+                                    '[A-Z]', 'A'),
+                                    '[0-9]', 'N') AS pattern
+                       FROM {DATA_SCHEMA}.{DATA_TABLE}
+                      WHERE "{COL_NAME}" > ' ' AND (SELECT MAX(LEN("{COL_NAME}"))
+                                                      FROM {DATA_SCHEMA}.{DATA_TABLE}) BETWEEN 3 and {PARM_MAX_PATTERN_LENGTH}) p
+            GROUP BY pattern
+            HAVING pattern > ' '
+            ORDER BY COUNT(*) DESC) as ps) AS top_patterns,
+strTemplate06_else: NULL as top_patterns,
+strTemplate07_A_freq:  (  SELECT LEFT(LISTAGG(val, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) as concat_vals
+  FROM (
+          SELECT TOP 10 CAST(COUNT(*) as VARCHAR(10)) || ' | ' || "{COL_NAME}" as val,
+          COUNT(*) as ct
+          FROM {DATA_SCHEMA}.{DATA_TABLE}
+          WHERE "{COL_NAME}" > ' '
+          GROUP BY "{COL_NAME}"
+          HAVING "{COL_NAME}" > ' '
+          ORDER BY COUNT(*), "{COL_NAME}" DESC
+        ) ps
+  )  AS top_freq_values,
+strTemplate07_else: NULL as top_freq_values,
+strTemplate08_N: MIN("{COL_NAME}")  AS min_value,
+  MIN(CASE WHEN "{COL_NAME}" > 0 THEN "{COL_NAME}" ELSE NULL END)  AS min_value_over_0,
+  MAX("{COL_NAME}")  AS max_value,
+  AVG(CAST("{COL_NAME}" AS FLOAT))  AS avg_value,
+  STDDEV(CAST("{COL_NAME}" AS FLOAT))  AS stdev_value,
+  MIN(pct_25) as percentile_25,
+  MIN(pct_50) as percentile_50,
+  MIN(pct_75) as percentile_75,
+strTemplate08_else: NULL as min_value,
+  NULL as min_value_over_0,
+  NULL as max_value,
+  NULL as avg_value,
+  NULL as stdev_value,
+  NULL as percentile_25,
+  NULL as percentile_50,
+  NULL as percentile_75,
+strTemplate10_N_dec: SUM(ROUND(MOD("{COL_NAME}", 1), 5)) as fractional_sum,
+
+strTemplate10_else: NULL as fractional_sum,
+
+strTemplate11_D:  GREATEST(MIN("{COL_NAME}"), '0001-01-01') as min_date,
+  MAX("{COL_NAME}") as max_date,
+  SUM(CASE
+        WHEN DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 12 THEN 1
+                                                            ELSE 0
+      END) AS before_1yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 60 THEN 1
+                                                            ELSE 0
+      END) AS before_5yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 240 THEN 1
+                                                              ELSE 0
+      END) AS before_20yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('DAY', "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 365 THEN 1
+                                                                         ELSE 0
+      END) AS within_1yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF('DAY', "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 30 THEN 1
+                                                                        ELSE 0
+      END) AS within_1mo_date_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" > '{RUN_DATE}' THEN 1 ELSE 0
+      END) AS future_date_ct,
+  COUNT(DISTINCT DATEDIFF(day, "{COL_NAME}", '{RUN_DATE}' ) ) as date_days_present,
+  COUNT(DISTINCT DATEDIFF(week, "{COL_NAME}", '{RUN_DATE}' ) ) as date_weeks_present,
+  COUNT(DISTINCT DATEDIFF(month, "{COL_NAME}", '{RUN_DATE}' ) ) as date_months_present,
+
+strTemplate11_else:  NULL as min_date,
+  NULL as max_date,
+  NULL as before_1yr_date_ct,
+  NULL as before_5yr_date_ct,
+  NULL as before_20yr_date_ct,
+  NULL as within_1yr_date_ct,
+  NULL as within_1mo_date_ct,
+  NULL as future_date_ct,
+  NULL as date_days_present,
+  NULL as date_weeks_present,
+  NULL as date_months_present,
+
+strTemplate12_B: SUM(CAST("{COL_NAME}" AS INTEGER)) AS boolean_true_ct,
+
+strTemplate12_else: NULL as boolean_true_ct,
+
+strTemplate13_ALL: NULL AS datatype_suggestion,
+strTemplate14_A_do_patterns:  ( SELECT COUNT(DISTINCT REGEXP_REPLACE( REGEXP_REPLACE( REGEXP_REPLACE(
+                          "{COL_NAME}"::VARCHAR, '[a-z]', 'a'),
+                                      '[A-Z]', 'A'),
+                                      '[0-9]', 'N')
+                ) AS pattern_ct
+     FROM {DATA_SCHEMA}.{DATA_TABLE}
+    WHERE "{COL_NAME}" > ' ' )  AS distinct_pattern_ct,
+  SUM(SIGN(REGEXP_COUNT(TRIM("{COL_NAME}"::VARCHAR), ' '))::BIGINT) AS embedded_space_ct,
+  AVG(REGEXP_COUNT(TRIM("{COL_NAME}"::VARCHAR), ' ')::FLOAT)  AS avg_embedded_spaces,
+
+strTemplate14_A_no_patterns:  NULL as distinct_pattern_ct,
+  SUM(SIGN(REGEXP_COUNT(TRIM("{COL_NAME}"::VARCHAR), ' '))::BIGINT) AS embedded_space_ct,
+  AVG(REGEXP_COUNT(TRIM("{COL_NAME}"::VARCHAR), ' ')::FLOAT)  AS avg_embedded_spaces,
+
+strTemplate14_else:  NULL as distinct_pattern_ct,
+  NULL as embedded_space_ct,
+  NULL as avg_embedded_spaces,
+
+strTemplate15_ALL:  NULL as functional_data_type,
+  NULL as functional_table_type,
+
+strTemplate16_ALL:  " '{PROFILE_RUN_ID}' as profile_run_id "
+
+strTemplate98_sampling: ' FROM {DATA_SCHEMA}.{DATA_TABLE} SAMPLE ({SAMPLE_SIZE} rows)'
+
+strTemplate98_else: ' FROM {DATA_SCHEMA}.{DATA_TABLE}'
+
+strTemplate99_N: |
+      ,
+      (SELECT
+             PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_25,
+             PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_50,
+             PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_75
+        FROM {DATA_SCHEMA}.{DATA_TABLE} LIMIT 1) pctile
+strTemplate99_else: ;
+
+strTemplate100_sampling: ' '