dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/flavors/postgresql/profiling/project_ddf_query_postgresql.sql

@@ -0,0 +1,42 @@
+SELECT '{PROJECT_CODE}'            as project_code,
+       CURRENT_TIMESTAMP AT TIME ZONE 'UTC'                    as refresh_timestamp,
+       c.table_schema,
+       c.table_name,
+       c.column_name,
+       CASE
+           WHEN c.data_type = 'timestamp without time zone' THEN 'timestamp'
+           WHEN c.data_type = 'text'
+             OR (c.data_type = 'character varying' and c.character_maximum_length is NULL)  THEN 'varchar(65535)'
+           WHEN c.data_type = 'character varying'
+               THEN 'varchar(' || CAST(c.character_maximum_length AS VARCHAR) || ')'
+           WHEN c.data_type = 'character' THEN 'char(' || CAST(c.character_maximum_length AS VARCHAR) || ')'
+           WHEN c.data_type = 'numeric' THEN 'numeric'
+                                                || COALESCE( '(' || CAST(c.numeric_precision AS VARCHAR) || ','
+                                                                 || CAST(c.numeric_scale AS VARCHAR) || ')', '')
+           ELSE c.data_type
+       END AS data_type,
+       COALESCE(c.character_maximum_length, CASE WHEN c.data_type IN ('text', 'character varying') THEN 65535 END)
+          as character_maximum_length,
+       c.ordinal_position,
+       CASE
+           WHEN c.data_type ILIKE '%char%' or c.data_type = 'text'
+               THEN 'A'
+           WHEN c.data_type ILIKE 'boolean'
+               THEN 'B'
+           WHEN c.data_type ILIKE 'date'
+               OR c.data_type ILIKE 'timestamp%'
+               THEN 'D'
+           WHEN c.data_type ILIKE 'time without time zone'
+               THEN 'T'
+           WHEN LOWER(c.data_type) IN ('bigint', 'double precision', 'integer', 'smallint', 'real')
+               OR c.data_type ILIKE 'numeric%'
+               THEN 'N'
+           ELSE
+               'X' END          AS general_type,
+       CASE
+         WHEN c.data_type = 'numeric' THEN COALESCE(numeric_scale, 1) > 0
+         ELSE numeric_scale > 0
+       END as is_decimal
+FROM information_schema.columns c
+WHERE c.table_schema = '{DATA_SCHEMA}' {TABLE_CRITERIA}
+ORDER BY c.table_schema, c.table_name, c.ordinal_position

testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml

@@ -0,0 +1,225 @@
+---
+strTemplate01_sampling: "SELECT "
+strTemplate01_else: "SELECT "
+strTemplate02_all: |
+  {CONNECTION_ID} as connection_id,
+  '{PROJECT_CODE}' as project_code,
+  '{TABLE_GROUPS_ID}' as table_groups_id,
+  '{DATA_SCHEMA}' AS schema_name,
+  '{RUN_DATE}' AS run_date,
+  '{DATA_TABLE}' AS table_name,
+  {COL_POS} AS position,
+  '{COL_NAME_SANITIZED}' AS column_name,
+  '{COL_TYPE}' AS column_type,
+  '{COL_GEN_TYPE}' AS general_type,
+  COUNT(*) AS record_ct,
+  COUNT("{COL_NAME}") AS value_ct,
+  COUNT(DISTINCT "{COL_NAME}") AS distinct_value_ct,
+  SUM(CASE WHEN "{COL_NAME}" IS NULL THEN 1 ELSE 0 END) AS null_value_ct,
+strTemplate03_ADN: MIN(LENGTH(CAST("{COL_NAME}" AS TEXT)))  AS min_length,
+  MAX(LENGTH(CAST("{COL_NAME}" AS TEXT)))  AS max_length,
+  AVG(NULLIF(LENGTH(CAST("{COL_NAME}" AS TEXT)), 0)::FLOAT) AS avg_length,
+strTemplate03_else:  NULL as min_length,
+  NULL as max_length,
+  NULL as avg_length,
+strTemplate04_A:  SUM(CASE
+        WHEN TRIM("{COL_NAME}") ~ '^0(\.0*)?$' THEN 1 ELSE 0
+      END)  AS zero_value_ct,
+strTemplate04_N:  SUM( 1 - ABS(SIGN("{COL_NAME}")) )::BIGINT AS zero_value_ct,
+strTemplate04_else: NULL as zero_value_ct,
+strTemplate05_A:  COUNT(DISTINCT UPPER(TRANSLATE("{COL_NAME}", ' '',.-', ''))) as distinct_std_value_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" = '' THEN 1
+                             ELSE 0
+      END)  AS zero_length_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" BETWEEN ' !' AND '!' THEN 1
+                                              ELSE 0
+       END )  AS lead_space_ct,
+  SUM( CASE WHEN "{COL_NAME}" ILIKE '"%"' OR "{COL_NAME}" ILIKE '''%''' THEN 1 ELSE 0 END ) as quoted_value_ct,
+  SUM( CASE WHEN "{COL_NAME}" ~ '[0-9]' THEN 1 ELSE 0 END ) as includes_digit_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" IN ('.', '?', ' ') THEN 1
+         WHEN LOWER("{COL_NAME}") SIMILAR TO '(^.{2,}|-{2,}|0{2,}|9{2,}|x{2,}|z{2,}$)' THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('blank','error','missing','tbd',
+                                    'n/a','#na','none','null','unknown')           THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('(blank)','(error)','(missing)','(tbd)',
+                                    '(n/a)','(#na)','(none)','(null)','(unknown)') THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('[blank]','[error]','[missing]','[tbd]',
+                                    '[n/a]','[#na]','[none]','[null]','[unknown]') THEN 1
+                                                                                   ELSE 0
+       END )   AS filled_value_ct,
+  LEFT(MIN(NULLIF("{COL_NAME}", '')), 100)  AS min_text,
+  LEFT(MAX(NULLIF("{COL_NAME}", '')), 100)  AS max_text,
+  SUM({DATA_QC_SCHEMA}.fndk_isnum(LEFT("{COL_NAME}", 31)))  AS numeric_ct,
+  SUM({DATA_QC_SCHEMA}.fndk_isdate(LEFT("{COL_NAME}", 26)))  AS date_ct,
+  CASE
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[0-9]{1,5}[a-zA-Z]?\s\w{1,5}\.?\s?\w*\s?\w*\s[a-zA-Z]{1,6}\.?\s?[0-9]{0,5}[A-Z]{0,1}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'STREET_ADDR'
+    WHEN SUM(CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA')
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'STATE_USA'
+    WHEN SUM( CASE WHEN "{COL_NAME}" SIMILAR TO '^([\+]1 |1-|)[\+]?[(]?[0-9]{3}[)][ ]?[-\s\.]?[0-9]{3}[-\s\.]?[0-9]{4,6}$'
+                     OR "{COL_NAME}" SIMILAR TO '^([\+]1 |1-|)[2-9][01][0-9][-| ]?[0-9]{3}[-| ]?[0-9]{4}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'PHONE_USA'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'EMAIL'
+    WHEN SUM( CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'ZIP_USA'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^[\w\s\-]+(?<!\s)\.(txt|csv|tsv|dat|doc|pdf|xlsx)$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'FILE_NAME'
+    WHEN SUM( CASE WHEN "{COL_NAME}" SIMILAR TO '^([0-9]{4}[- ]){3}[0-9]{4}$'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'CREDIT_CARD'
+    WHEN SUM( CASE WHEN "{COL_NAME}" ~ '^([^,|\t]{1,20}[,|\t]){2,}[^,|\t]{0,20}([,|\t]{0,1}[^,|\t]{0,20})*$'
+                    AND "{COL_NAME}" !~ '\s(and|but|or|yet)\s'
+         THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'DELIMITED_DATA'
+    WHEN SUM ( CASE WHEN "{COL_NAME}" SIMILAR TO '^[0-8][0-9]{2}-[0-9]{2}-[0-9]{4}$'
+                     AND LEFT("{COL_NAME}", 3) NOT BETWEEN '734' AND '749'
+                     AND LEFT("{COL_NAME}", 3) <> '666' THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'SSN'
+  END as std_pattern_match,
+strTemplate05_else: NULL as distinct_std_value_ct,
+  NULL as zero_length_ct,
+  NULL as lead_space_ct,
+  NULL as quoted_value_ct,
+  NULL as includes_digit_ct,
+  NULL as filled_value_ct,
+  NULL as min_text,
+  NULL as max_text,
+  NULL as numeric_ct,
+  NULL as date_ct,
+  NULL as std_pattern_match,
+strTemplate06_A_patterns: ( SELECT LEFT(STRING_AGG(pattern, ' | ' ORDER BY ct DESC) , 1000) AS concat_pats
+      FROM (
+            SELECT CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
+                   COUNT(*) AS ct
+              FROM (  SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
+                        "{COL_NAME}", '[a-z]', 'a', 'g'),
+                                    '[A-Z]', 'A', 'g'),
+                                    '[0-9]', 'N', 'g') AS pattern
+                       FROM {DATA_SCHEMA}.{DATA_TABLE}
+                      WHERE "{COL_NAME}" > ' ' AND (SELECT MAX(LENGTH("{COL_NAME}"))
+                                                      FROM {DATA_SCHEMA}.{DATA_TABLE}) BETWEEN 3 and {PARM_MAX_PATTERN_LENGTH}) p
+            GROUP BY pattern
+            HAVING pattern > ' '
+            ORDER BY COUNT(*) DESC
+            LIMIT 5
+           ) ps) AS top_patterns,
+strTemplate06_else: NULL as top_patterns,
+strTemplate07_A_freq:  (  SELECT LEFT(STRING_AGG(val, ' | ' ORDER BY ct DESC), 1000) as concat_vals
+  FROM (
+          SELECT TOP 10 CAST(COUNT(*) as VARCHAR(10)) || ' | ' || "{COL_NAME}" as val,
+          COUNT(*) as ct
+          FROM {DATA_SCHEMA}.{DATA_TABLE}
+          WHERE "{COL_NAME}" > ' '
+          GROUP BY "{COL_NAME}"
+          HAVING "{COL_NAME}" > ' '
+          ORDER BY COUNT(*), "{COL_NAME}" DESC
+        ) ps
+  )  AS top_freq_values,
+strTemplate07_else: NULL as top_freq_values,
+strTemplate08_N: MIN("{COL_NAME}")  AS min_value,
+  MIN(CASE WHEN "{COL_NAME}" > 0 THEN "{COL_NAME}" ELSE NULL END)  AS min_value_over_0,
+  MAX("{COL_NAME}")  AS max_value,
+  AVG(CAST("{COL_NAME}" AS FLOAT))  AS avg_value,
+  STDDEV(CAST("{COL_NAME}" AS FLOAT))  AS stdev_value,
+  MIN(pct_25) as percentile_25,
+  MIN(pct_50) as percentile_50,
+  MIN(pct_75) as percentile_75,
+strTemplate08_else: NULL as min_value,
+  NULL as min_value_over_0,
+  NULL as max_value,
+  NULL as avg_value,
+  NULL as stdev_value,
+  NULL as percentile_25,
+  NULL as percentile_50,
+  NULL as percentile_75,
+strTemplate10_N_dec: SUM(ROUND(MOD("{COL_NAME}", 1), 5)) as fractional_sum,
+
+strTemplate10_else: NULL as fractional_sum,
+
+strTemplate11_D:  CASE
+         WHEN MIN("{COL_NAME}") IS NULL THEN NULL
+         ELSE GREATEST(MIN("{COL_NAME}"), '0001-01-01')
+       END as min_date,
+  MAX("{COL_NAME}") as max_date,
+  SUM(CASE
+        WHEN {DATA_QC_SCHEMA}.DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 12 THEN 1
+                                                            ELSE 0
+      END) AS before_1yr_date_ct,
+  SUM(CASE
+        WHEN {DATA_QC_SCHEMA}.DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 60 THEN 1
+                                                            ELSE 0
+      END) AS before_5yr_date_ct,
+  SUM(CASE
+          WHEN {DATA_QC_SCHEMA}.DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}') > 240 THEN 1
+                                                              ELSE 0
+        END) AS before_20yr_date_ct,
+  SUM(CASE
+        WHEN {DATA_QC_SCHEMA}.DATEDIFF('DAY', "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 365 THEN 1
+                                                                         ELSE 0
+      END) AS within_1yr_date_ct,
+  SUM(CASE
+        WHEN {DATA_QC_SCHEMA}.DATEDIFF('DAY', "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 30 THEN 1
+                                                                        ELSE 0
+      END) AS within_1mo_date_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" > '{RUN_DATE}' THEN 1 ELSE 0
+      END) AS future_date_ct,
+  COUNT(DISTINCT {DATA_QC_SCHEMA}.DATEDIFF('DAY', "{COL_NAME}", '{RUN_DATE}' ) ) as date_days_present,
+  COUNT(DISTINCT {DATA_QC_SCHEMA}.DATEDIFF('WEEK', "{COL_NAME}", '{RUN_DATE}' ) ) as date_weeks_present,
+  COUNT(DISTINCT {DATA_QC_SCHEMA}.DATEDIFF('MON', "{COL_NAME}", '{RUN_DATE}' ) ) as date_months_present,
+
+
+strTemplate11_else:  NULL as min_date,
+  NULL as max_date,
+  NULL as before_1yr_date_ct,
+  NULL as before_5yr_date_ct,
+  NULL as before_20yr_date_ct,
+  NULL as within_1yr_date_ct,
+  NULL as within_1mo_date_ct,
+  NULL as future_date_ct,
+  NULL as date_days_present,
+  NULL as date_weeks_present,
+  NULL as date_months_present,
+
+strTemplate12_B: SUM(CAST("{COL_NAME}" AS INTEGER)) AS boolean_true_ct,
+
+strTemplate12_else: NULL as boolean_true_ct,
+
+strTemplate13_ALL: NULL AS datatype_suggestion,
+strTemplate14_A_do_patterns: ( SELECT COUNT(DISTINCT REGEXP_REPLACE( REGEXP_REPLACE( REGEXP_REPLACE(
+                          "{COL_NAME}", '[a-z]', 'a', 'g'),
+                                      '[A-Z]', 'A', 'g'),
+                                      '[0-9]', 'N', 'g')
+                ) AS pattern_ct
+     FROM {DATA_SCHEMA}.{DATA_TABLE}
+    WHERE "{COL_NAME}" > ' ' ) AS distinct_pattern_ct,
+  SUM(SIGN(LENGTH(TRIM("{COL_NAME}")) - LENGTH(REGEXP_REPLACE(TRIM("{COL_NAME}"), ' ', '', 'g')))::BIGINT) AS embedded_space_ct,
+  AVG(LENGTH(TRIM("{COL_NAME}")) - LENGTH(REGEXP_REPLACE(TRIM("{COL_NAME}"), ' ', '', 'g'))::FLOAT)  AS avg_embedded_spaces,
+
+strTemplate14_A_no_patterns:  NULL as distinct_pattern_ct,
+  SUM(SIGN(LENGTH(TRIM("{COL_NAME}")) - LENGTH(REGEXP_REPLACE(TRIM("{COL_NAME}"), ' ', '', 'g')))::BIGINT) AS embedded_space_ct,
+  AVG(LENGTH(TRIM("{COL_NAME}")) - LENGTH(REGEXP_REPLACE(TRIM("{COL_NAME}"), ' ', '', 'g'))::FLOAT)  AS avg_embedded_spaces,
+
+strTemplate14_else:  NULL as distinct_pattern_ct,
+  NULL as embedded_space_ct,
+  NULL as avg_embedded_spaces,
+
+strTemplate15_ALL:  NULL as functional_data_type,
+  NULL as functional_table_type,
+
+strTemplate16_ALL: " '{PROFILE_RUN_ID}' as profile_run_id"
+
+strTemplate98_sampling: ' FROM {DATA_SCHEMA}.{DATA_TABLE} '
+
+strTemplate98_else: ' FROM {DATA_SCHEMA}.{DATA_TABLE} '
+
+strTemplate99_N: |
+  , (SELECT
+             PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY "{COL_NAME}") AS pct_25,
+             PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY "{COL_NAME}") AS pct_50,
+             PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY "{COL_NAME}") AS pct_75
+        FROM {DATA_SCHEMA}.{DATA_TABLE} LIMIT 1) pctile
+
+strTemplate99_else: ' '
+
+strTemplate100_sampling: 'WHERE RAND() <= 1.0 / {PROFILE_SAMPLE_RATIO}'

testgen/template/flavors/postgresql/profiling/project_secondary_profiling_query_postgresql.sql

@@ -0,0 +1,28 @@
+-- Get Freqs for selected columns
+WITH ranked_vals AS (
+  SELECT "{COL_NAME}",
+         COUNT(*) AS ct,
+         ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC, "{COL_NAME}") AS rn
+    FROM {DATA_SCHEMA}.{DATA_TABLE}
+   WHERE "{COL_NAME}" > ' '
+   GROUP BY "{COL_NAME}"
+),
+consol_vals AS (
+  SELECT COALESCE(CASE WHEN rn <= 10 THEN '| ' || "{COL_NAME}" || ' | ' || CAST(ct AS VARCHAR)
+                       ELSE NULL
+                  END, '| Other Values (' || CAST(COUNT(DISTINCT "{COL_NAME}")  as VARCHAR) || ') | '  || CAST(SUM(ct)  as VARCHAR) ) AS val,
+         MIN(rn) as min_rn
+    FROM ranked_vals
+   GROUP BY CASE WHEN rn <= 10 THEN '| ' || "{COL_NAME}" || ' | ' || CAST(ct AS VARCHAR)
+                 ELSE NULL
+            END
+)
+SELECT '{PROJECT_CODE}' as project_code,
+       '{DATA_SCHEMA}' as schema_name,
+       '{RUN_DATE}' as run_date,
+       '{DATA_TABLE}' as table_name,
+       '{COL_NAME}' as column_name,
+       REPLACE(STRING_AGG(val, '^#^' ORDER BY min_rn), '^#^', CHR(10)) AS top_freq_values,
+       ( SELECT MD5(STRING_AGG(DISTINCT "{COL_NAME}", '|' ORDER BY "{COL_NAME}")) as dvh
+           FROM {DATA_SCHEMA}.{DATA_TABLE} ) as distinct_value_hash
+  FROM consol_vals;

testgen/template/flavors/postgresql/setup_profiling_tools/create_functions_postgresql.sql

@@ -0,0 +1,157 @@
+CREATE OR REPLACE FUNCTION {DATA_QC_SCHEMA}.DATEDIFF(difftype character varying, firstdate timestamp without time zone, seconddate timestamp without time zone)
+RETURNS BIGINT AS $$
+    SELECT
+       CASE
+        WHEN UPPER(difftype) IN ('DAY', 'DD', 'D') THEN
+            DATE(seconddate) - DATE(firstdate)
+        WHEN UPPER(difftype) IN ('WEEK','WK', 'W') THEN
+            (DATE(seconddate) - DATE(firstdate)) / 7
+        WHEN UPPER(difftype) IN ('MON', 'MONTH', 'MM') THEN
+            (DATE_PART('year', seconddate) - DATE_PART('year', firstdate)) * 12 + (DATE_PART('month', seconddate) - DATE_PART('month', firstdate))
+        WHEN UPPER(difftype) IN ('QUARTER', 'QTR', 'Q') THEN
+            ((DATE_PART('year', seconddate) - DATE_PART('year', firstdate)) * 4) + (DATE_PART('quarter', seconddate) - DATE_PART('quarter', firstdate))
+        WHEN UPPER(difftype) IN ('YEAR', 'YY', 'Y') THEN
+            DATE_PART('year', seconddate) - DATE_PART('year', firstdate)
+        ELSE
+            NULL::BIGINT
+    END;
+$$ LANGUAGE sql IMMUTABLE STRICT;
+
+CREATE OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fn_charcount(instring character varying, searchstring character varying) returns bigint
+    language plpgsql
+as
+$$
+   BEGIN
+      RETURN (CHAR_LENGTH(instring) - CHAR_LENGTH(REPLACE(instring, searchstring, ''))) / CHAR_LENGTH(searchstring);
+   END;
+$$;
+
+
+CREATE OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fn_parsefreq(top_freq_values VARCHAR(1000), rowno INTEGER, colno INTEGER) returns VARCHAR(1000)
+    language plpgsql
+as
+$$
+   BEGIN
+      RETURN SPLIT_PART(SPLIT_PART(top_freq_values, CHR(10), rowno), '|', colno+1);
+   END;
+$$;
+
+
+CREATE
+OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fndk_isnum(VARCHAR)
+          RETURNS INTEGER
+          IMMUTABLE
+        AS
+        $$
+SELECT CASE
+            WHEN $1 ~ E'^\\s*[+-]?\\$?\\s*[0-9]+(,[0-9]{3})*(\\.[0-9]*)?[\\%]?\\s*$' THEN 1
+            ELSE 0
+        END;
+$$
+LANGUAGE sql;
+
+
+
+
+
+CREATE
+OR REPLACE FUNCTION {DATA_QC_SCHEMA}.fndk_isdate(VARCHAR)
+          RETURNS INTEGER
+          IMMUTABLE
+        AS $$
+SELECT CASE
+           -- YYYY-MM-DD HH:MM:SS SSSSSS or YYYY-MM-DD HH:MM:SS
+           WHEN $1 ~ '^(\\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])\\s(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\s[0-9]{6})?$'
+                             THEN CASE
+                                                         WHEN LEFT($1, 4):: INT BETWEEN 1800 AND 2200
+    AND (
+    ( SUBSTRING ($1, 6, 2) IN ('01', '03', '05', '07', '08',
+    '10', '12')
+    AND SUBSTRING ($1, 9, 2):: INT BETWEEN 1 AND 31 )
+    OR ( SUBSTRING ($1, 6, 2) IN ('04', '06', '09')
+    AND SUBSTRING ($1, 9, 2):: INT BETWEEN 1 AND 30 )
+    OR ( SUBSTRING ($1, 6, 2) = '02'
+    AND SUBSTRING ($1, 9, 2):: INT :: INT BETWEEN 1 AND 29)
+    )
+    THEN 1
+    ELSE 0
+END
+                        -- YYYYMMDDHHMMSSSSSS or YYYYMMDD
+WHEN $1 ~ '^(\\d{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])(2[0-3]|[01][0-9])([0-5][0-9])([0-5][0-9])([0-9]{6})$'
+  OR $1 ~ '^(\\d{4})(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])(2[0-3]|[01][0-9])$'
+                             THEN CASE
+                                   WHEN LEFT($1, 4)::INT BETWEEN 1800 AND 2200
+                                       AND (
+                                            ( SUBSTRING($1, 5, 2) IN ('01', '03', '05', '07', '08',
+                                                                                '10', '12')
+                                               AND SUBSTRING($1, 7, 2)::INT BETWEEN 1 AND 31 )
+                                           OR  ( SUBSTRING($1, 5, 2) IN ('04', '06', '09')
+                                                  AND SUBSTRING($1, 7, 2)::INT BETWEEN 1 AND 30 )
+                                           OR  ( SUBSTRING($1, 5, 2)  = '02'
+                                                  AND SUBSTRING($1, 7, 2)::INT::INT BETWEEN 1 AND 29)
+                                           )
+                                      THEN 1
+                                     ELSE 0
+END
+                        -- Exclude anything else long
+WHEN LENGTH($1) > 11 THEN 0
+                        -- YYYY-MMM/MM-DD
+                        WHEN REGEXP_REPLACE(UPPER($1), '(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)', '12', 'g')
+                               ~ '[12][09][0-9][0-9]-[0-1]?[0-9]-[0-3]?[0-9]'
+                            THEN CASE
+                                   WHEN SPLIT_PART($1, '-', 1)::INT BETWEEN 1800 AND 2200
+                                       AND (
+                                            ( UPPER(SPLIT_PART($1, '-', 2)) IN ('01', '03', '05', '07', '08',
+                                                                                '1', '3', '5', '7', '8', '10', '12',
+                                                                                'JAN', 'MAR', 'MAY', 'JUL', 'AUG',
+                                                                                'OCT', 'DEC')
+                                               AND SPLIT_PART($1, '-', 3)::INT BETWEEN 1 AND 31 )
+                                        OR  ( UPPER(SPLIT_PART($1, '-', 2)) IN ('04', '06', '09', '4', '6', '9', '11',
+                                                                                'APR', 'JUN', 'SEP', 'NOV')
+                                               AND SPLIT_PART($1, '-', 3)::INT BETWEEN 1 AND 30 )
+                                        OR  ( UPPER(SPLIT_PART($1, '-', 2)) IN ('02', '2', 'FEB')
+                                               AND SPLIT_PART($1, '-', 3)::INT BETWEEN 1 AND 29)
+                                           )
+                                      THEN 1
+                                     ELSE 0
+END
+                        -- MM/-DD/-YY/YYYY
+WHEN REPLACE($1, '-', '/') ~ '^[0-1]?[0-9]/[0-3]?[0-9]/[12][09][0-9][0-9]$'
+                          OR REPLACE($1, '-', '/') ~ '^[0-1]?[0-9]/[0-3]?[0-9]/[0-9][0-9]$'
+                             THEN
+                                  CASE
+                                    WHEN SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT BETWEEN 1 AND 12
+                                     AND (
+                                          ( SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT IN (1, 3, 5, 7, 8, 10, 12)
+                                             AND SPLIT_PART(REPLACE($1, '-', '/'), '/', 2)::INT BETWEEN 1 AND 31 )
+                                      OR  ( SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT IN (4, 6, 9, 11)
+                                             AND SPLIT_PART(REPLACE($1, '-', '/'), '/', 2)::INT BETWEEN 1 AND 30 )
+                                      OR  ( SPLIT_PART(REPLACE($1, '-', '/'), '/', 1)::INT = 2
+                                             AND SPLIT_PART(REPLACE($1, '-', '/'), '/', 2)::INT BETWEEN 1 AND 29)
+                                         )
+                                     AND
+                                         ('20' || RIGHT(SPLIT_PART(REPLACE($1, '-', '/'), '/', 3), 2))::INT BETWEEN 1800 AND 2200
+                                    THEN 1
+                                    ELSE 0
+END
+                        -- DD-MMM-YYYY
+WHEN UPPER($1) ~ '[0-3]?[0-9]-(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)-[12][09][0-9][0-9]'
+                          THEN
+                               CASE
+                                 WHEN SPLIT_PART($1, '-', 3)::INT BETWEEN 1800 AND 2200
+                                     AND (
+                                          ( UPPER(SPLIT_PART($1, '-', 2)) IN ('JAN', 'MAR', 'MAY', 'JUL', 'AUG', 'OCT', 'DEC')
+                                             AND SPLIT_PART($1, '-', 1)::INT BETWEEN 1 AND 31 )
+                                      OR  ( UPPER(SPLIT_PART($1, '-', 2)) IN ('APR', 'JUN', 'SEP', 'NOV')
+                                             AND SPLIT_PART($1, '-', 1)::INT BETWEEN 1 AND 30 )
+                                      OR  ( UPPER(SPLIT_PART($1, '-', 2)) = 'FEB'
+                                             AND SPLIT_PART($1, '-', 1)::INT BETWEEN 1 AND 29)
+                                         )
+                                   THEN 1
+                                   ELSE 0
+END
+ELSE 0
+END
+as isdate
+        $$
+          LANGUAGE sql;

testgen/template/flavors/postgresql/setup_profiling_tools/create_qc_schema_postgresql.sql

@@ -0,0 +1 @@
+CREATE SCHEMA IF NOT exists {DATA_QC_SCHEMA};

testgen/template/flavors/postgresql/setup_profiling_tools/grant_execute_privileges_postgresql.sql

@@ -0,0 +1,2 @@
+GRANT ALL PRIVILEGES ON SCHEMA {DATA_QC_SCHEMA} TO {DB_USER};
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA {DATA_QC_SCHEMA} TO {DB_USER};

testgen/template/flavors/redshift/profiling/project_ddf_query_redshift.sql

@@ -0,0 +1,38 @@
+SELECT '{PROJECT_CODE}'            as project_code,
+       CURRENT_TIMESTAMP AT TIME ZONE 'UTC'                    as refresh_timestamp,
+       c.table_schema,
+       c.table_name,
+       c.column_name,
+       CASE
+           WHEN c.data_type = 'timestamp without time zone' THEN 'timestamp'
+           WHEN c.data_type = 'character varying'
+               THEN 'varchar(' || CAST(c.character_maximum_length AS VARCHAR) || ')'
+           WHEN c.data_type = 'character' THEN 'char(' || CAST(c.character_maximum_length AS VARCHAR) || ')'
+           WHEN c.data_type = 'numeric' THEN 'numeric'
+                                                || COALESCE( '(' || CAST(c.numeric_precision AS VARCHAR) || ','
+                                                                 || CAST(c.numeric_scale AS VARCHAR) || ')', '')
+           ELSE c.data_type END AS data_type,
+       c.character_maximum_length,
+       c.ordinal_position,
+       CASE
+           WHEN c.data_type ILIKE '%char%'
+               THEN 'A'
+           WHEN c.data_type ILIKE 'boolean'
+               THEN 'B'
+           WHEN c.data_type ILIKE 'date'
+               OR c.data_type ILIKE 'timestamp%'
+               THEN 'D'
+           WHEN c.data_type ILIKE 'time without time zone'
+               THEN 'T'
+           WHEN LOWER(c.data_type) IN ('bigint', 'double precision', 'integer', 'smallint', 'real')
+               OR c.data_type ILIKE 'numeric%'
+               THEN 'N'
+           ELSE
+               'X' END          AS general_type,
+       CASE
+         WHEN c.data_type = 'numeric' THEN COALESCE(numeric_scale, 1) > 0
+         ELSE numeric_scale > 0
+       END as is_decimal
+FROM information_schema.columns c
+WHERE c.table_schema = '{DATA_SCHEMA}' {TABLE_CRITERIA}
+ORDER BY c.table_schema, c.table_name, c.ordinal_position