dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/flavors/generic/exec_query_tests/ex_window_match_same_generic.sql

@@ -0,0 +1,59 @@
+SELECT '{PROJECT_CODE}'   as project_code,
+       '{TEST_TYPE}'   as test_type,
+       '{TEST_DEFINITION_ID}' as test_definition_id,
+       '{TEST_SUITE}'  as test_suite,
+       '{TEST_RUN_ID}' as test_run_id,
+       '{RUN_DATE}'    as test_time,
+       '{START_TIME}'  as starttime,
+       CURRENT_TIMESTAMP       as endtime,
+       '{SCHEMA_NAME}' as schema_name,
+       '{TABLE_NAME}'  as table_name,
+       '{COLUMN_NAME_NO_QUOTES}' as column_names,
+       '{SKIP_ERRORS}' as threshold_value,
+       {SKIP_ERRORS} as skip_errors,
+       '{INPUT_PARAMETERS}' as input_parameters,
+       CASE WHEN COUNT (*) > {SKIP_ERRORS} THEN 0 ELSE 1 END as result_code,
+       CASE
+        WHEN COUNT(*) > 0 THEN
+               CONCAT(
+                      CONCAT( CAST(COUNT(*) AS VARCHAR), ' error(s) identified, ' ),
+                      CONCAT(
+                             CASE
+                               WHEN COUNT(*) > {SKIP_ERRORS} THEN 'exceeding limit of '
+                                                                        ELSE 'within limit of '
+                             END,
+                             '{SKIP_ERRORS}.'
+                             )
+                      )
+        ELSE 'No errors found.'
+       END AS result_message,
+       COUNT(*) as result_measure,
+       '{SUBSET_DISPLAY}' as subset_condition,
+       NULL as result_query
+  FROM (
+        (
+SELECT 'Prior Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= DATEADD("day", - {WINDOW_DAYS}, (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}))
+EXCEPT
+SELECT 'Prior Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= DATEADD("day",  - 2 * {WINDOW_DAYS}, (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}))
+  AND {WINDOW_DATE_COLUMN} <  DATEADD("day", - {WINDOW_DAYS}, (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}))
+)
+UNION ALL
+(
+SELECT 'Latest Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= DATEADD("day",  - 2 * {WINDOW_DAYS}, (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}))
+  AND {WINDOW_DATE_COLUMN} <  DATEADD("day", - {WINDOW_DAYS}, (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}))
+    EXCEPT
+SELECT 'Latest Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= DATEADD("day", - {WINDOW_DAYS}, (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}))
+)
+       ) test;

testgen/template/flavors/generic/profiling/contingency_counts.sql

@@ -0,0 +1,3 @@
+SELECT {CONTINGENCY_COLUMNS}, COUNT(*) as freq_ct
+  FROM {DATA_SCHEMA}.{DATA_TABLE}
+GROUP BY {CONTINGENCY_COLUMNS};

testgen/template/flavors/generic/validate_tests/ex_get_project_column_list_generic.sql

@@ -0,0 +1,3 @@
+select concat(concat(concat(table_schema, '.'), concat(table_name, '.')), column_name) as columns
+from information_schema.columns
+where table_schema in ({TEST_SCHEMAS});

testgen/template/flavors/mssql/exec_query_tests/ex_relative_entropy_mssql.sql

@@ -0,0 +1,53 @@
+-- Relative Entropy:  measured by Jensen-Shannon Divergence
+--   Smoothed and normalized version of KL divergence,
+--   with scores between 0 (identical) and 1 (maximally different),
+--   when using the base-2 logarithm.   Formula is:
+--   0.5 * kl_divergence(p, m) + 0.5 * kl_divergence(q, m)
+--   Log base 2 of x = LN(x)/LN(2)
+WITH latest_ver
+   AS ( SELECT {CONCAT_COLUMNS} as category,
+               CAST(COUNT(*) as FLOAT) / CAST(SUM(COUNT(*)) OVER () as FLOAT) AS pct_of_total
+          FROM {SCHEMA_NAME}.{TABLE_NAME} v1
+         WHERE {SUBSET_CONDITION}
+         GROUP BY {COLUMN_NAME_NO_QUOTES} ),
+older_ver
+   AS ( SELECT {CONCAT_MATCH_GROUPBY} as category,
+               CAST(COUNT(*) as FLOAT) / CAST(SUM(COUNT(*)) OVER () as FLOAT) AS pct_of_total
+          FROM {MATCH_SCHEMA_NAME}.{TABLE_NAME} v2
+         WHERE {MATCH_SUBSET_CONDITION}
+         GROUP BY {MATCH_GROUPBY_NAMES} ),
+dataset
+   AS ( SELECT COALESCE(l.category, o.category) AS category,
+               COALESCE(o.pct_of_total, 0.0000001)        AS old_pct,
+               COALESCE(l.pct_of_total, 0.0000001)        AS new_pct,
+               (COALESCE(o.pct_of_total, 0.0000001)
+                + COALESCE(l.pct_of_total, 0.0000001))/2.0 AS avg_pct
+          FROM latest_ver l
+          FULL JOIN older_ver o
+            ON (l.category = o.category) )
+SELECT '{PROJECT_CODE}' as project_code,
+       '{TEST_TYPE}' as test_type,
+       '{TEST_DEFINITION_ID}' as test_definition_id,
+       '{TEST_SUITE}' as test_suite,
+       '{TEST_RUN_ID}' as test_run_id,
+       '{RUN_DATE}' as test_time,
+       '{START_TIME}' as starttime,
+       CURRENT_TIMESTAMP as endtime,
+       '{SCHEMA_NAME}' as schema_name,
+       '{TABLE_NAME}' as table_name,
+       '{COLUMN_NAME_NO_QUOTES}' as column_names,
+--        '{GROUPBY_NAMES}' as column_names,
+       '{THRESHOLD_VALUE}' as threshold_value,
+       NULL as skip_errors,
+       '{INPUT_PARAMETERS}' as input_parameters,
+       CASE WHEN js_divergence > {THRESHOLD_VALUE} THEN 0 ELSE 1 END as result_code,
+       CONCAT('Divergence Level: ',
+              CONCAT(CAST(js_divergence AS VARCHAR),
+                     ', Threshold: {THRESHOLD_VALUE}.')) as result_message,
+       js_divergence as result_measure,
+       '{SUBSET_DISPLAY}' as subset_condition,
+       NULL as result_query
+  FROM (
+         SELECT 0.5 * ABS(SUM(new_pct * LOG(new_pct/avg_pct)/LOG(2)))
+                 + 0.5 * ABS(SUM(old_pct * LOG(old_pct/avg_pct)/LOG(2))) as js_divergence
+           FROM dataset ) rslt;

testgen/template/flavors/mssql/profiling/project_ddf_query_mssql.sql

@@ -0,0 +1,35 @@
+SELECT '{PROJECT_CODE}'            as project_code,
+       CURRENT_TIMESTAMP           as refresh_timestamp,
+       c.table_schema,
+       c.table_name,
+       c.column_name,
+       CASE
+           WHEN c.data_type = 'datetime' THEN 'datetime'
+           WHEN c.data_type = 'datetime2' THEN 'datetime'
+           WHEN c.data_type = 'varchar'
+               THEN 'varchar(' + CAST(c.character_maximum_length AS VARCHAR) + ')'
+           WHEN c.data_type = 'char' THEN 'char(' + CAST(c.character_maximum_length AS VARCHAR) + ')'
+           WHEN c.data_type = 'numeric' THEN 'numeric(' + CAST(c.numeric_precision AS VARCHAR) + ',' +
+                                             CAST(c.numeric_scale AS VARCHAR) + ')'
+           ELSE c.data_type END AS data_type,
+       c.character_maximum_length,
+       c.ordinal_position,
+       CASE
+           WHEN LOWER(c.data_type) LIKE '%char%'
+               THEN 'A'
+           WHEN c.data_type = 'bit'
+               THEN 'B'
+           WHEN c.data_type = 'date'
+               OR c.data_type LIKE 'datetime%'
+               THEN 'D'
+           WHEN c.data_type like 'time%'
+               THEN 'T'
+           WHEN c.data_type IN ('bigint', 'double precision', 'integer', 'smallint', 'real')
+               OR c.data_type LIKE 'numeric%'
+               THEN 'N'
+           ELSE
+               'X' END          AS general_type,
+       case when c.numeric_scale > 0 then 1 else 0 END       as is_decimal
+FROM information_schema.columns c
+WHERE c.table_schema = '{DATA_SCHEMA}' {TABLE_CRITERIA}
+ORDER BY c.table_schema, c.table_name, c.ordinal_position;

testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml

@@ -0,0 +1,246 @@
+---
+strTemplate01_sampling:  "SELECT TOP {SAMPLE_SIZE} "
+strTemplate01_else:  "SELECT "
+strTemplate02_all: |
+  {CONNECTION_ID} as connection_id,
+  '{PROJECT_CODE}' as project_code,
+  '{TABLE_GROUPS_ID}' as table_groups_id,
+  '{DATA_SCHEMA}' AS schema_name,
+  '{RUN_DATE}' AS run_date,
+  '{DATA_TABLE}' AS table_name,
+  {COL_POS} AS position,
+  '{COL_NAME_SANITIZED}' AS column_name,
+  '{COL_TYPE}' AS column_type,
+  '{COL_GEN_TYPE}' AS general_type,
+  COUNT(*) AS record_ct,
+  COUNT("{COL_NAME}") AS value_ct,
+  COUNT(DISTINCT "{COL_NAME}") AS distinct_value_ct,
+  SUM(CASE WHEN "{COL_NAME}" IS NULL THEN 1 ELSE 0 END) AS null_value_ct,
+strTemplate03_ADN: MIN(LEN("{COL_NAME}"))  AS min_length,
+  MAX(LEN("{COL_NAME}"))  AS max_length,
+  AVG(CAST(NULLIF(LEN("{COL_NAME}"), 0) AS FLOAT)) AS avg_length,
+strTemplate03_else:  NULL as min_length,
+  NULL as max_length,
+  NULL as avg_length,
+strTemplate04_A:  SUM(CASE
+        WHEN LTRIM(RTRIM("{COL_NAME}")) LIKE '0([.]0*)' THEN 1 ELSE 0
+      END)  AS zero_value_ct,
+strTemplate04_N:   CAST(SUM( 1 - ABS(SIGN("{COL_NAME}")))AS BIGINT ) AS zero_value_ct,
+strTemplate04_else: NULL as zero_value_ct,
+strTemplate05_A: COUNT(DISTINCT UPPER(REPLACE(TRANSLATE("{COL_NAME}",' '''',.-',REPLICATE(' ', LEN(' '''',.-'))),' ',''))) as distinct_std_value_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" = '' THEN 1
+                             ELSE 0
+      END)  AS zero_length_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" BETWEEN ' !' AND '!' THEN 1
+                                              ELSE 0
+       END )  AS lead_space_ct,
+  SUM( CASE WHEN "{COL_NAME}" LIKE '"%"' OR "{COL_NAME}" LIKE '''%''' THEN 1 ELSE 0 END ) as quoted_value_ct,
+  SUM( CASE WHEN "{COL_NAME}" LIKE '%[0-9]%' THEN 1 ELSE 0 END ) as includes_digit_ct,
+  SUM( CASE
+         WHEN "{COL_NAME}" IN ('.', '?') OR "{COL_NAME}" LIKE ' ' THEN 1
+         WHEN LEN("{COL_NAME}") > 1
+          AND ( LOWER("{COL_NAME}") LIKE '%..%' OR  LOWER("{COL_NAME}") LIKE '%--%'
+                 OR (LEN(REPLACE("{COL_NAME}", '0', ''))= 0 )
+                 OR (LEN(REPLACE("{COL_NAME}", '9', ''))= 0 )
+                 OR (LEN(REPLACE(LOWER("{COL_NAME}"), 'x', ''))= 0 )
+                 OR (LEN(REPLACE(LOWER("{COL_NAME}"), 'z', ''))= 0 )
+             )  THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('blank','error','missing','tbd',
+                                    'n/a','#na','none','null','unknown')           THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('(blank)','(error)','(missing)','(tbd)',
+                                    '(n/a)','(#na)','(none)','(null)','(unknown)') THEN 1
+         WHEN LOWER("{COL_NAME}") IN ('[blank]','[error]','[missing]','[tbd]',
+                                    '[n/a]','[#na]','[none]','[null]','[unknown]') THEN 1
+                                                                                   ELSE 0
+       END )   AS filled_value_ct,
+  LEFT(MIN(NULLIF("{COL_NAME}", '') COLLATE Latin1_General_BIN ), 100)  AS min_text,
+  LEFT(MAX(NULLIF("{COL_NAME}", '') COLLATE Latin1_General_BIN ), 100)  AS max_text,
+  SUM({DATA_QC_SCHEMA}.fndk_isnum(LEFT("{COL_NAME}", 31)))  AS numeric_ct,
+  SUM({DATA_QC_SCHEMA}.fndk_isdate(LEFT("{COL_NAME}", 26)))  AS date_ct,
+  CASE
+    WHEN CAST(SUM( CASE WHEN UPPER("{COL_NAME}") LIKE '[1-9]% [A-Z]% %'
+                           AND CHARINDEX(' ', "{COL_NAME}") BETWEEN 2 and 6  THEN 1
+                     END ) as FLOAT) /CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8  THEN 'STREET_ADDR'
+    WHEN CAST(SUM(CASE WHEN "{COL_NAME}" IN ('AL','AK','AS','AZ','AR','CA','CO','CT','DE','DC','FM','FL','GA','GU','HI','ID','IL','IN','IA','KS','KY','LA','ME','MH','MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','MP','OH','OK','OR','PW','PA','PR','RI','SC','SD','TN','TX','UT','VT','VI','VA','WA','WV','WI','WY','AE','AP','AA')
+         THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'STATE_USA'
+    WHEN CAST(SUM( CASE WHEN ("{COL_NAME}" LIKE '[+]1%[0-9][0-9][0-9]%[-. ][0-9][0-9][0-9]%[0-9][0-9][0-9][0-9,0-9,0-9,0-9,0-9,0-9]' AND "{COL_NAME}" NOT LIKE '%[^0-9+()-]%')
+                     OR ("{COL_NAME}" LIKE '[+]1%[0-9][0-9][0-9][- ][0-9][0-9][0-9][- ][0-9][0-9][0-9][0-9]' AND "{COL_NAME}" NOT LIKE '%[^0-9+-]%')
+         THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'PHONE_USA'
+    WHEN CAST(SUM( CASE WHEN "{COL_NAME}" LIKE '%[_a-zA-Z0-9.-]%@%[a-zA-Z0-9.-]%.[a-zA-Z][a-zA-Z]%'
+         THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'EMAIL'
+    WHEN CAST(SUM( CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
+         THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'ZIP_USA'
+    WHEN CAST(SUM( CASE WHEN "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS NOT LIKE ' %'
+                          AND "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '[a-z0-9 _-]%'
+                          AND ("{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '%.txt'
+                          OR "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '%.csv'
+                          OR "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '%.tsv'
+                          OR "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '%.dat'
+                          OR "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '%.doc'
+                          OR "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '%.pdf'
+                          OR "{COL_NAME}" COLLATE SQL_Latin1_General_CP1_CI_AS LIKE '%.xlsx')
+         THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'FILE_NAME'
+    WHEN CAST(SUM( CASE WHEN "{COL_NAME}" LIKE '[0-9][0-9][0-9][0-9][- ][0-9][0-9][0-9][0-9][- ][0-9][0-9][0-9][0-9][- ][0-9][0-9][0-9][0-9]'
+         THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'CREDIT_CARD'
+    WHEN CAST(SUM( CASE WHEN ( "{COL_NAME}" LIKE '%,%,%,%'
+                            OR "{COL_NAME}" LIKE '%|%|%|%'
+                            OR "{COL_NAME}" LIKE '%^%^%^%'
+                            OR "{COL_NAME}" LIKE '%' + CHAR(9) + '%' + CHAR(9) + '%' + CHAR(9) + '%' )
+                         AND NOT ( "{COL_NAME}" LIKE '% and %'
+                                OR "{COL_NAME}" LIKE '% but %'
+                                OR "{COL_NAME}" LIKE '% or %'
+                                OR "{COL_NAME}" LIKE '% yet %' )
+                         AND ISNULL(CAST(LEN("{COL_NAME}") - LEN(REPLACE("{COL_NAME}", ',', '')) as FLOAT)
+                              / CAST(NULLIF(LEN("{COL_NAME}") - LEN(REPLACE("{COL_NAME}", ' ', '')), 0) as FLOAT), 1) > 0.6
+         THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'DELIMITED_DATA'
+    WHEN CAST(SUM ( CASE WHEN "{COL_NAME}" LIKE '[0-8][0-9][0-9][- ][0-9][0-9][- ][0-9][0-9][0-9][0-9]'
+                     AND LEFT("{COL_NAME}", 3) NOT BETWEEN '734' AND '749'
+                     AND LEFT("{COL_NAME}", 3) <> '666' THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'SSN'
+  END as std_pattern_match,
+strTemplate05_else: NULL as distinct_std_value_ct,
+  NULL as zero_length_ct,
+  NULL as lead_space_ct,
+  NULL as quoted_value_ct,
+  NULL as includes_digit_ct,
+  NULL as filled_value_ct,
+  NULL as min_text,
+  NULL as max_text,
+  NULL as numeric_ct,
+  NULL as date_ct,
+  NULL as std_pattern_match,
+strTemplate06_A_patterns:  ( SELECT LEFT(STRING_AGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats
+     FROM (
+           SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) + ' | ' + pattern AS pattern,
+                  COUNT(*) AS ct
+             FROM (  SELECT TRANSLATE("{COL_NAME}"  COLLATE Latin1_General_BIN,
+                            'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789',
+                            'aaaaaaaaaaaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAAAAAAAAAAANNNNNNNNNN' )
+                     AS pattern
+                      FROM {DATA_SCHEMA}.{DATA_TABLE} WITH (NOLOCK)
+                     WHERE "{COL_NAME}" > ' ' AND ((SELECT MAX(LEN("{COL_NAME}"))
+                                                    FROM {DATA_SCHEMA}.{DATA_TABLE} WITH (NOLOCK)) BETWEEN 3 and {PARM_MAX_PATTERN_LENGTH})) p
+           GROUP BY pattern
+           HAVING pattern > ' '
+           ORDER BY COUNT(*) DESC
+          ) ps) AS top_patterns,
+strTemplate06_else: NULL as top_patterns,
+strTemplate07_A_freq:  (  SELECT LEFT(STRING_AGG(val, ' | ') WITHIN GROUP (ORDER BY ct DESC, val ASC), 1000) as concat_vals
+  FROM (
+          SELECT TOP 10 CAST(COUNT(*) as VARCHAR(10)) + ' | ' + "{COL_NAME}" as val,
+          COUNT(*) as ct
+          FROM {DATA_SCHEMA}.{DATA_TABLE} WITH (NOLOCK)
+          WHERE "{COL_NAME}" > ' '
+          GROUP BY "{COL_NAME}"
+          HAVING "{COL_NAME}" > ' '
+          ORDER BY COUNT(*) DESC
+        ) ps
+  )  AS top_freq_values,
+strTemplate07_else: NULL as top_freq_values,
+strTemplate08_N: MIN("{COL_NAME}")  AS min_value,
+  MIN(CASE WHEN "{COL_NAME}" > 0 THEN "{COL_NAME}" ELSE NULL END)  AS min_value_over_0,
+  MAX("{COL_NAME}")  AS max_value,
+  AVG(CAST("{COL_NAME}" AS FLOAT))  AS avg_value,
+  STDEV(CAST("{COL_NAME}" AS FLOAT))  AS stdev_value,
+  MIN(pct_25) as percentile_25,
+  MIN(pct_50) as percentile_50,
+  MIN(pct_75) as percentile_75,
+strTemplate08_else: NULL as min_value,
+  NULL as min_value_over_0,
+  NULL as max_value,
+  NULL as avg_value,
+  NULL as stdev_value,
+  NULL as percentile_25,
+  NULL as percentile_50,
+  NULL as percentile_75,
+strTemplate10_N_dec: SUM(ROUND(("{COL_NAME}" % 1), 5)) as fractional_sum,
+
+strTemplate10_else: NULL as fractional_sum,
+
+strTemplate11_D:  CASE
+         WHEN MIN("{COL_NAME}") IS NULL THEN NULL
+         ELSE CASE WHEN MIN("{COL_NAME}") >= CAST('0001-01-01' as date) THEN MIN("{COL_NAME}") ELSE CAST('0001-01-01' as date) END
+       END as min_date,
+  MAX("{COL_NAME}") as max_date,
+  SUM(CASE
+        WHEN DATEDIFF(month, "{COL_NAME}", '{RUN_DATE}') > 12 THEN 1
+                                                            ELSE 0
+      END) AS before_1yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF(month, "{COL_NAME}", '{RUN_DATE}') > 60 THEN 1
+                                                            ELSE 0
+      END) AS before_5yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF(month, "{COL_NAME}", '{RUN_DATE}') > 240 THEN 1
+                                                              ELSE 0
+      END) AS before_20yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF(day, "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 365 THEN 1
+                                                                         ELSE 0
+      END) AS within_1yr_date_ct,
+  SUM(CASE
+        WHEN DATEDIFF(day, "{COL_NAME}", '{RUN_DATE}') BETWEEN 0 AND 30 THEN 1
+                                                                        ELSE 0
+      END) AS within_1mo_date_ct,
+  SUM(CASE
+        WHEN "{COL_NAME}" > '{RUN_DATE}' THEN 1 ELSE 0
+      END) AS future_date_ct,
+  COUNT(DISTINCT DATEDIFF(day, "{COL_NAME}", '{RUN_DATE}' ) ) as date_days_present,
+  COUNT(DISTINCT DATEDIFF(week, "{COL_NAME}", '{RUN_DATE}' ) ) as date_weeks_present,
+  COUNT(DISTINCT DATEDIFF(month, "{COL_NAME}", '{RUN_DATE}' ) ) as date_months_present,
+
+strTemplate11_else:  NULL as min_date,
+  NULL as max_date,
+  NULL as before_1yr_date_ct,
+  NULL as before_5yr_date_ct,
+  NULL as before_20yr_date_ct,
+  NULL as within_1yr_date_ct,
+  NULL as within_1mo_date_ct,
+  NULL as future_date_ct,
+  NULL as date_days_present,
+  NULL as date_weeks_present,
+  NULL as date_months_present,
+
+strTemplate12_B: SUM(CAST("{COL_NAME}" AS INTEGER)) AS boolean_true_ct,
+
+strTemplate12_else: NULL as boolean_true_ct,
+
+strTemplate13_ALL: NULL AS datatype_suggestion,
+strTemplate14_A_do_patterns:  ( SELECT COUNT(DISTINCT TRANSLATE("{COL_NAME}"  COLLATE Latin1_General_BIN,
+                                'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789',
+                                'aaaaaaaaaaaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAAAAAAAAAAANNNNNNNNNN'
+                              )
+             ) AS pattern_ct
+  FROM {DATA_SCHEMA}.{DATA_TABLE} WITH (NOLOCK)
+ WHERE "{COL_NAME}" > ' ' )  AS distinct_pattern_ct,
+  SUM(CAST(SIGN(LEN(RTRIM(LTRIM("{COL_NAME}"))) - LEN(REPLACE(RTRIM(LTRIM("{COL_NAME}")),' ',''))) AS BIGINT)) AS embedded_space_ct,
+  AVG(CAST(LEN(RTRIM(LTRIM("{COL_NAME}"))) - LEN(REPLACE(RTRIM(LTRIM("{COL_NAME}")),' ','')) AS FLOAT))  AS avg_embedded_spaces,
+
+strTemplate14_A_no_patterns:  NULL as distinct_pattern_ct,
+  SUM(CAST(SIGN(LEN(RTRIM(LTRIM("{COL_NAME}"))) - LEN(REPLACE(RTRIM(LTRIM("{COL_NAME}")),' ',''))) AS BIGINT)) AS embedded_space_ct,
+  AVG(CAST(LEN(RTRIM(LTRIM("{COL_NAME}"))) - LEN(REPLACE(RTRIM(LTRIM("{COL_NAME}")),' ','')) AS FLOAT)) AS avg_embedded_spaces,
+
+strTemplate14_else:  NULL as distinct_pattern_ct,
+  NULL as embedded_space_ct,
+  NULL as avg_embedded_spaces,
+
+strTemplate15_ALL:  NULL as functional_data_type,
+  NULL as functional_table_type,
+
+strTemplate16_ALL: " '{PROFILE_RUN_ID}' as profile_run_id"
+
+strTemplate98_sampling: ' FROM {DATA_SCHEMA}.{DATA_TABLE} WITH (NOLOCK)'
+
+strTemplate98_else: ' FROM {DATA_SCHEMA}.{DATA_TABLE} WITH (NOLOCK)'
+
+strTemplate99_N: |
+     , (SELECT TOP 1
+             PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_25,
+             PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_50,
+             PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY "{COL_NAME}") OVER () AS pct_75
+        FROM {DATA_SCHEMA}.{DATA_TABLE} WITH (NOLOCK)) pctile
+
+strTemplate99_else: ' '
+
+strTemplate100_sampling: ' ORDER BY RAND()'

testgen/template/flavors/mssql/profiling/project_secondary_profiling_query_mssql.sql

@@ -0,0 +1,36 @@
+-- Get Freqs for selected columns
+WITH ranked_vals
+AS
+    (SELECT "{COL_NAME}",
+            COUNT(*) AS  ct,
+            ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn
+     FROM {DATA_SCHEMA}.{DATA_TABLE}
+     WHERE "{COL_NAME}" > ' '
+     GROUP BY "{COL_NAME}"
+    ),
+consol_vals
+AS (
+    SELECT COALESCE (CASE WHEN rn <= 10 THEN '| ' + "{COL_NAME}" + ' | ' + CAST (ct AS VARCHAR)
+                                        ELSE NULL
+                     END,
+                    '| Other Values (' + CAST ( CAST(COUNT (DISTINCT CAST ("{COL_NAME}" as VARCHAR)) AS VARCHAR ) + ') | '
+                    + CAST (SUM (ct) as VARCHAR) AS VARCHAR)) AS val,
+            MIN (rn) as min_rn
+    FROM ranked_vals
+    GROUP BY CASE WHEN rn <= 10 THEN '| ' + "{COL_NAME}" + ' | ' + CAST (ct AS VARCHAR) ELSE NULL
+             END
+    )
+SELECT '{PROJECT_CODE}' as project_code,
+       '{DATA_SCHEMA}'  as schema_name,
+       '{RUN_DATE}'     as run_date,
+       '{DATA_TABLE}'   as table_name,
+       '{COL_NAME}'     as column_name,
+       REPLACE(STRING_AGG(CONVERT(NVARCHAR(max), val), '^#^') WITHIN GROUP (ORDER BY min_rn), '^#^', CHAR(10)) AS top_freq_values,
+       (SELECT CONVERT(VARCHAR(40), HASHBYTES('MD5', STRING_AGG( NULLIF(dist_col_name,''),
+                       '|') WITHIN GROUP (ORDER BY dist_col_name)), 2)  as dvh
+        FROM (SELECT DISTINCT "{COL_NAME}" as dist_col_name
+              FROM {DATA_SCHEMA}.{DATA_TABLE}) a
+       ) as distinct_value_hash
+FROM consol_vals;
+
+-- Convert function has style = 2 : The characters 0x aren't added to the left of the converted result for style 2.

testgen/template/flavors/mssql/setup_profiling_tools/00_drop_existing_functions_mssql.sql

@@ -0,0 +1,8 @@
+-- Step 1: Drop both functions if they exist
+BEGIN
+    IF OBJECT_ID('{DATA_QC_SCHEMA}.fndk_isnum', 'FN') IS NOT NULL
+        DROP FUNCTION {DATA_QC_SCHEMA}.fndk_isnum;
+
+    IF OBJECT_ID('{DATA_QC_SCHEMA}.fndk_isdate', 'FN') IS NOT NULL
+        DROP FUNCTION {DATA_QC_SCHEMA}.fndk_isdate;
+END

testgen/template/flavors/mssql/setup_profiling_tools/01_create_functions_mssql.sql

@@ -0,0 +1,12 @@
+-- Step 2: Create isnum function
+CREATE FUNCTION {DATA_QC_SCHEMA}.fndk_isnum (@strparm VARCHAR(500))
+RETURNS INT
+AS
+BEGIN
+	IF TRY_CAST(NULLIF(@strparm, '') AS float) IS NOT NULL
+	BEGIN
+	    RETURN(1)
+	END
+
+	RETURN(0)
+END;

testgen/template/flavors/mssql/setup_profiling_tools/02_create_functions_mssql.sql

@@ -0,0 +1,54 @@
+-- Step 3: Create isdate function
+
+CREATE FUNCTION {DATA_QC_SCHEMA}.fndk_isdate(@strparm VARCHAR(500))
+    RETURNS INT
+AS
+BEGIN
+    DECLARE @ret INT
+
+    SET @ret =
+
+   CASE WHEN TRY_CAST(NULLIF(@strparm, '') AS float) IS NOT NULL
+                    AND LEFT(NULLIF(@strparm, ''),4) BETWEEN 1800 AND 2200 THEN
+                 CASE
+                        WHEN LEN((NULLIF(@strparm, ''))) > 11 THEN 0
+                        -- YYYYMMDD
+                        WHEN TRY_CONVERT(DATE, NULLIF(@strparm, ''), 112) IS NOT NULL THEN 1
+
+                        -- YYYY-MM-DD
+                        WHEN TRY_CONVERT(DATE, NULLIF(@strparm, ''), 23) IS NOT NULL THEN 1
+
+                        -- MM/DD/YYYY
+                        WHEN TRY_CONVERT(DATE, NULLIF(@strparm, ''), 101) IS NOT NULL THEN 1
+
+                        -- MM/DD/YY
+                        WHEN TRY_CONVERT(DATE, NULLIF(@strparm, ''), 1) IS NOT NULL THEN 1
+
+                        --MM-DD-YYYY
+                        WHEN TRY_CONVERT(DATE, NULLIF(@strparm, ''), 110) IS NOT NULL THEN 1
+
+                        --MM-DD-YY
+                        WHEN TRY_CONVERT(DATE, NULLIF(@strparm, ''), 10) IS NOT NULL THEN 1
+
+
+                   ELSE 0 END
+              --DD MMM YYYY
+               WHEN (TRY_CONVERT(DATE, NULLIF(@strparm, ''), 106) IS NOT NULL
+                   AND LEFT(NULLIF(@strparm, ''), 4)  BETWEEN 1800 AND 2200)
+                   THEN 1
+
+              -- YYYY-MM-DD HH:MM:SS SSSSSS
+               WHEN (TRY_CONVERT(DATETIME2, NULLIF(@strparm, ''), 121) IS NOT NULL
+                   AND LEFT(NULLIF(@strparm, ''), 4)  BETWEEN 1800 AND 2200)
+                   THEN 1
+
+              -- YYYY-MM-DD HH:MM:SS
+               WHEN (TRY_CONVERT(DATETIME2, NULLIF(@strparm, ''), 120) IS NOT NULL
+                   AND LEFT(NULLIF(@strparm, ''), 4)  BETWEEN 1800 AND 2200)
+                   THEN 1
+              ELSE 0
+              END
+	RETURN @ret
+
+END
+;

testgen/template/flavors/mssql/setup_profiling_tools/create_qc_schema_mssql.sql

@@ -0,0 +1,4 @@
+IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{DATA_QC_SCHEMA}')
+BEGIN
+   EXEC('CREATE SCHEMA {DATA_QC_SCHEMA}')
+END

testgen/template/flavors/mssql/setup_profiling_tools/grant_execute_privileges_mssql.sql

@@ -0,0 +1 @@
+GRANT EXECUTE ON SCHEMA::{DATA_QC_SCHEMA} TO {DB_USER};

testgen/template/flavors/postgresql/exec_query_tests/ex_window_match_no_drops_postgresql.sql

@@ -0,0 +1,46 @@
+SELECT '{PROJECT_CODE}'   as project_code,
+       '{TEST_TYPE}'   as test_type,
+       '{TEST_DEFINITION_ID}' as test_definition_id,
+       '{TEST_SUITE}'  as test_suite,
+       '{TEST_RUN_ID}' as test_run_id,
+       '{RUN_DATE}'    as test_time,
+       '{START_TIME}'  as starttime,
+       CURRENT_TIMESTAMP       as endtime,
+       '{SCHEMA_NAME}' as schema_name,
+       '{TABLE_NAME}'  as table_name,
+       '{COLUMN_NAME_NO_QUOTES}' as column_names,
+       '{SKIP_ERRORS}' as threshold_value,
+       {SKIP_ERRORS} as skip_errors,
+       '{INPUT_PARAMETERS}' as input_parameters,
+       CASE WHEN COUNT (*) > {SKIP_ERRORS} THEN 0 ELSE 1 END as result_code,
+       CASE
+        WHEN COUNT(*) > 0 THEN
+               CONCAT(
+                      CONCAT( CAST(COUNT(*) AS VARCHAR), ' error(s) identified, ' ),
+                      CONCAT(
+                             CASE
+                               WHEN COUNT(*) > {SKIP_ERRORS} THEN 'exceeding limit of '
+                                                                        ELSE 'within limit of '
+                             END,
+                             '{SKIP_ERRORS}.'
+                             )
+                      )
+        ELSE 'No errors found.'
+       END AS result_message,
+       COUNT(*) as result_measure,
+       '{SUBSET_DISPLAY}' as subset_condition,
+       NULL as result_query
+FROM (
+      SELECT {COLUMN_NAME_NO_QUOTES}
+        FROM {SCHEMA_NAME}.{TABLE_NAME}
+       WHERE {SUBSET_CONDITION}
+         AND {WINDOW_DATE_COLUMN} >= (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - 2 * {WINDOW_DAYS}
+         AND {WINDOW_DATE_COLUMN} < (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - {WINDOW_DAYS}
+      GROUP BY {COLUMN_NAME_NO_QUOTES}
+       EXCEPT
+      SELECT {COLUMN_NAME_NO_QUOTES}
+        FROM {SCHEMA_NAME}.{TABLE_NAME}
+       WHERE {SUBSET_CONDITION}
+         AND {WINDOW_DATE_COLUMN} >= (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - {WINDOW_DAYS}
+      GROUP BY {COLUMN_NAME_NO_QUOTES}
+     ) test;

testgen/template/flavors/postgresql/exec_query_tests/ex_window_match_same_postgresql.sql

@@ -0,0 +1,59 @@
+SELECT '{PROJECT_CODE}'   as project_code,
+       '{TEST_TYPE}'   as test_type,
+       '{TEST_DEFINITION_ID}' as test_definition_id,
+       '{TEST_SUITE}'  as test_suite,
+       '{TEST_RUN_ID}' as test_run_id,
+       '{RUN_DATE}'    as test_time,
+       '{START_TIME}'  as starttime,
+       CURRENT_TIMESTAMP       as endtime,
+       '{SCHEMA_NAME}' as schema_name,
+       '{TABLE_NAME}'  as table_name,
+       '{COLUMN_NAME_NO_QUOTES}' as column_names,
+       '{SKIP_ERRORS}' as threshold_value,
+       {SKIP_ERRORS} as skip_errors,
+       '{INPUT_PARAMETERS}' as input_parameters,
+       CASE WHEN COUNT (*) > {SKIP_ERRORS} THEN 0 ELSE 1 END as result_code,
+       CASE
+        WHEN COUNT(*) > 0 THEN
+               CONCAT(
+                      CONCAT( CAST(COUNT(*) AS VARCHAR), ' error(s) identified, ' ),
+                      CONCAT(
+                             CASE
+                               WHEN COUNT(*) > {SKIP_ERRORS} THEN 'exceeding limit of '
+                                                                        ELSE 'within limit of '
+                             END,
+                             '{SKIP_ERRORS}.'
+                             )
+                      )
+        ELSE 'No errors found.'
+       END AS result_message,
+       COUNT(*) as result_measure,
+       '{SUBSET_DISPLAY}' as subset_condition,
+       NULL as result_query
+  FROM (
+        (
+SELECT 'Prior Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - {WINDOW_DAYS}
+EXCEPT
+SELECT 'Prior Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - 2 * {WINDOW_DAYS}
+  AND {WINDOW_DATE_COLUMN} <  (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - {WINDOW_DAYS}
+)
+UNION ALL
+(
+SELECT 'Latest Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - 2 * {WINDOW_DAYS}
+  AND {WINDOW_DATE_COLUMN} < (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - {WINDOW_DAYS}
+    EXCEPT
+SELECT 'Latest Timeframe' as missing_from, {COLUMN_NAME}
+FROM {SCHEMA_NAME}.{TABLE_NAME}
+WHERE {SUBSET_CONDITION}
+  AND {WINDOW_DATE_COLUMN} >= (SELECT MAX({WINDOW_DATE_COLUMN}) FROM {SCHEMA_NAME}.{TABLE_NAME}) - {WINDOW_DAYS}
+)
+       ) test;