dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/dbsetup/060_create_standard_views.sql

@@ -0,0 +1,248 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+-- ==============================================================================
+-- |   Creates Standard Views:
+-- |      Runs on new or existing schema, so DROP VIEWS IF EXIST first
+-- ==============================================================================
+
+DROP VIEW IF EXISTS v_latest_profile_results CASCADE;
+
+CREATE VIEW v_latest_profile_results
+AS
+  WITH last_run AS ( SELECT table_groups_id,
+                            MAX(profiling_starttime) AS last_run_date
+                       FROM profiling_runs
+                      GROUP BY table_groups_id )
+SELECT r.*
+  FROM last_run lr
+INNER JOIN profiling_runs p
+   ON lr.table_groups_id = p.table_groups_id
+  AND lr.last_run_date = p.profiling_starttime
+INNER JOIN profile_results r
+   ON p.id = r.profile_run_id;
+
+
+DROP VIEW IF EXISTS v_latest_profile_anomalies;
+
+CREATE VIEW v_latest_profile_anomalies
+   AS
+WITH last_profile_date
+   AS (SELECT table_groups_id, MAX(profiling_starttime) as last_profile_run_date
+         FROM profiling_runs
+       GROUP BY table_groups_id)
+SELECT r.id, r.project_code, r.table_groups_id,
+       r.profile_run_id, pr.profiling_starttime as profile_run_date,
+       r.schema_name, r.table_name, r.column_name, r.column_type,
+       t.anomaly_name, t.anomaly_description, t.issue_likelihood,
+       r.detail,
+       t.suggested_action, r.disposition
+  FROM profile_anomaly_results r
+INNER JOIN profile_anomaly_types t
+   ON r.anomaly_id = t.id
+INNER JOIN profiling_runs pr
+   ON (r.profile_run_id = pr.id)
+INNER JOIN last_profile_date l
+   ON (pr.table_groups_id = l.table_groups_id
+  AND  pr.profiling_starttime = l.last_profile_run_date);
+
+
+DROP VIEW IF EXISTS v_inactive_anomalies;
+
+CREATE VIEW v_inactive_anomalies
+ AS
+SELECT DISTINCT anomaly_id, table_groups_id, schema_name, table_name, column_name, column_id
+  FROM profile_anomaly_results
+ WHERE disposition = 'Inactive';
+
+
+DROP VIEW IF EXISTS v_profiling_runs;
+
+CREATE VIEW v_profiling_runs
+ AS
+SELECT r.id as profiling_run_id,
+       r.project_code, cc.connection_name, r.connection_id, r.table_groups_id,
+       tg.table_groups_name,
+       tg.table_group_schema as schema_name,
+       r.profiling_starttime as start_time,
+       TO_CHAR(r.profiling_endtime - r.profiling_starttime, 'HH24:MI:SS') as duration,
+       r.status,
+       r.log_message,
+       r.table_ct,
+       r.column_ct,
+       r.anomaly_ct, r.anomaly_table_ct, r.anomaly_column_ct, process_id
+  FROM profiling_runs r
+INNER JOIN table_groups tg
+   ON r.table_groups_id = tg.id
+INNER JOIN connections cc
+   ON r.connection_id = cc.connection_id
+GROUP BY r.id, r.project_code, cc.connection_name, r.connection_id,
+         r.table_groups_id, tg.table_groups_name, tg.table_group_schema,
+         r.profiling_starttime, r.profiling_endtime, r.status;
+
+
+DROP VIEW IF EXISTS v_test_runs;
+
+CREATE VIEW v_test_runs
+ AS
+SELECT r.id as test_run_id,
+       r.project_code, p.project_name,
+       r.test_suite,
+       r.test_starttime,
+       TO_CHAR(r.test_endtime - r.test_starttime, 'HH24:MI:SS') as duration,
+       r.status, r.log_message,
+       COUNT(*) as test_ct,
+       SUM(result_code) as passed_ct,
+       COALESCE(SUM(CASE WHEN tr.result_status = 'Failed' THEN 1 END), 0) as failed_ct,
+       COALESCE(SUM(CASE WHEN tr.result_status = 'Warning' THEN 1 END), 0) as warning_ct,
+       r.process_id
+  FROM test_runs r
+INNER JOIN projects p
+   ON (r.project_code = p.project_code)
+INNER JOIN test_results tr
+   ON (r.id = tr.test_run_id)
+GROUP BY r.id, r.project_code, r.test_suite, r.test_starttime, r.test_endtime,
+         r.process_id, r.status, r.log_message, p.project_name;
+
+
+DROP VIEW IF EXISTS v_test_results;
+
+CREATE VIEW v_test_results
+AS
+SELECT p.project_name,
+       ts.test_suite,
+       tg.table_groups_name,
+       cn.connection_name, cn.project_host, cn.sql_flavor,
+       tt.dq_dimension,
+       r.schema_name, r.table_name, r.column_names,
+       r.test_time as test_date,
+       r.test_type,  tt.id as test_type_id, tt.test_name_short, tt.test_name_long,
+       r.test_description,
+       tt.measure_uom, tt.measure_uom_description,
+       c.test_operator,
+       r.threshold_value::NUMERIC(16, 5) as threshold_value,
+       r.result_measure::NUMERIC(16, 5),
+       r.result_status,
+       r.input_parameters,
+       r.result_message,
+       CASE WHEN result_code <> 1 THEN r.severity END as severity,
+       CASE
+         WHEN result_code <> 1 THEN r.disposition
+            ELSE 'Passed'
+       END as disposition,
+       r.result_code as passed_ct,
+       (1 - r.result_code)::INTEGER as exception_ct,
+       CASE
+         WHEN result_status = 'Warning'
+          AND result_message NOT ILIKE 'ERROR - TEST COLUMN MISSING%' THEN 1
+       END::INTEGER as warning_ct,
+       CASE
+         WHEN result_status = 'Failed'
+          AND result_message NOT ILIKE 'ERROR - TEST COLUMN MISSING%' THEN 1
+       END::INTEGER as failed_ct,
+       CASE
+         WHEN result_message ILIKE 'ERROR - TEST COLUMN MISSING%' THEN 1
+       END as execution_error_ct,
+       r.project_code,
+       r.table_groups_id,
+       r.id as test_result_id, c.id as connection_id,
+       r.test_suite_id,
+       r.test_definition_id as test_definition_id_runtime,
+       CASE
+         WHEN r.auto_gen = TRUE THEN d.id
+                                ELSE r.test_definition_id
+       END as test_definition_id_current,
+       r.test_run_id as test_run_id,
+       r.auto_gen
+  FROM test_results r
+INNER JOIN test_types tt
+   ON (r.test_type = tt.test_type)
+LEFT JOIN test_definitions d
+   ON (r.test_suite_id = d.test_suite_id
+  AND  r.table_name = d.table_name
+  AND  r.column_names = COALESCE(d.column_name, 'N/A')
+  AND  r.test_type = d.test_type
+  AND  r.auto_gen = TRUE
+  AND  d.last_auto_gen_date IS NOT NULL)
+INNER JOIN test_suites ts
+   ON (r.test_suite_id = ts.id)
+INNER JOIN projects p
+   ON (r.project_code = p.project_code)
+INNER JOIN table_groups tg
+   ON (r.table_groups_id = tg.id)
+INNER JOIN connections cn
+   ON (tg.connection_id = cn.connection_id)
+LEFT JOIN cat_test_conditions c
+   ON (cn.sql_flavor = c.sql_flavor
+  AND  r.test_type = c.test_type);
+
+
+DROP VIEW IF EXISTS v_queued_observability_results;
+
+CREATE VIEW v_queued_observability_results
+  AS
+SELECT
+       p.project_name,
+       cn.sql_flavor as component_tool,
+       ts.test_suite_schema as schema,
+       cn.connection_name,
+       cn.project_db,
+
+       CASE
+         WHEN tg.profile_use_sampling = 'Y' THEN tg.profile_sample_min_count
+       END as sample_min_count,
+       tg.id as group_id,
+       tg.profile_use_sampling = 'Y' as uses_sampling,
+       r.project_code,
+       CASE
+         WHEN tg.profile_use_sampling = 'Y' THEN tg.profile_sample_percent
+       END as sample_percentage,
+
+       tg.profiling_table_set,
+       tg.profiling_include_mask,
+       tg.profiling_exclude_mask,
+
+       COALESCE(ts.component_type, 'dataset') as component_type,
+       COALESCE(ts.component_key, tg.id::VARCHAR) as component_key,
+       COALESCE(ts.component_name, tg.table_groups_name) as component_name,
+
+       r.column_names,
+       r.table_name,
+       r.test_suite,
+       r.input_parameters,
+       r.test_definition_id,
+       tt.test_name_short as type,
+       CASE
+         WHEN c.test_operator IN ('>', '>=') THEN d.threshold_value
+       END as min_threshold,
+       CASE
+         WHEN c.test_operator IN ('<', '<=') THEN d.threshold_value
+       END as max_threshold,
+      tt.test_name_long as name,
+      tt.test_description as description,
+      r.test_time as start_time,
+      r.test_time as end_time,
+      r.result_message as result_message,
+      tt.dq_dimension,
+      r.result_status,
+      r.result_id,
+      r.result_measure as metric_value,
+      tt.measure_uom,
+      tt.measure_uom_description
+  FROM test_results r
+INNER JOIN test_types tt
+   ON (r.test_type = tt.test_type)
+INNER JOIN test_definitions d
+   ON (r.test_definition_id = d.id)
+INNER JOIN test_suites ts
+   ON (r.project_code = ts.project_code
+  AND  r.test_suite = ts.test_suite)
+INNER JOIN table_groups tg
+   ON (d.table_groups_id = tg.id)
+INNER JOIN connections cn
+   ON (tg.connection_id = cn.connection_id)
+INNER JOIN projects p
+   ON (r.project_code = p.project_code)
+INNER JOIN cat_test_conditions c
+   ON (cn.sql_flavor = c.sql_flavor
+  AND  d.test_type = c.test_type)
+WHERE r.observability_status = 'Queued';

testgen/template/dbsetup/070_create_default_users.sql

@@ -0,0 +1,17 @@
+-- ==============================================================================
+-- |   Create default roles and users if not present  |
+-- ==============================================================================
+
+-- testgen_execute_role: read-write to operational tables, otherwise read-only
+CREATE ROLE testgen_execute_role;
+
+--create user and assign role
+CREATE USER {TESTGEN_USER} WITH PASSWORD '{TESTGEN_PASSWORD}';
+GRANT testgen_execute_role TO {TESTGEN_USER};
+
+-- testgen_report_role:  Read-Only to all data
+CREATE ROLE testgen_report_role;
+
+--create user and assign role
+CREATE USER {TESTGEN_REPORT_USER} WITH PASSWORD '{TESTGEN_REPORT_PASSWORD}';
+GRANT testgen_report_role TO {TESTGEN_REPORT_USER};

testgen/template/dbsetup/075_grant_role_rights.sql

@@ -0,0 +1,43 @@
+-- ==============================================================================
+-- |   Assigns Standard Rights to Roles:
+-- |      Runs on schema create or upgrade.
+-- |      No new objects should be created in this script.
+-- ==============================================================================
+
+-- testgen_execute_role:
+--     read-write to test_results, test_suites, test_definitions
+--     read-only to all other tables
+GRANT USAGE ON SCHEMA {SCHEMA_NAME} TO testgen_execute_role;
+GRANT SELECT ON ALL TABLES IN SCHEMA {SCHEMA_NAME} TO testgen_execute_role;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA {SCHEMA_NAME} TO testgen_execute_role;
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA {SCHEMA_NAME} TO testgen_execute_role;
+GRANT SELECT, INSERT, DELETE, UPDATE ON
+    {SCHEMA_NAME}.test_results,
+    {SCHEMA_NAME}.test_suites,
+    {SCHEMA_NAME}.test_definitions,
+    {SCHEMA_NAME}.profiling_runs,
+    {SCHEMA_NAME}.profile_results,
+    {SCHEMA_NAME}.profile_pair_rules,
+    {SCHEMA_NAME}.profile_anomaly_results,
+    {SCHEMA_NAME}.stg_functional_table_updates,
+    {SCHEMA_NAME}.stg_secondary_profile_updates,
+    {SCHEMA_NAME}.test_runs,
+    {SCHEMA_NAME}.working_agg_cat_results,
+    {SCHEMA_NAME}.working_agg_cat_tests,
+    {SCHEMA_NAME}.functional_test_results,
+    {SCHEMA_NAME}.connections,
+    {SCHEMA_NAME}.table_groups,
+    {SCHEMA_NAME}.projects,
+    {SCHEMA_NAME}.data_table_chars,
+    {SCHEMA_NAME}.data_column_chars,
+    {SCHEMA_NAME}.auth_users
+    TO testgen_execute_role;
+
+
+
+-- testgen_report_role:
+--     read-only to all data
+GRANT USAGE ON SCHEMA {SCHEMA_NAME} TO testgen_report_role;
+GRANT SELECT ON ALL TABLES IN SCHEMA {SCHEMA_NAME} TO testgen_report_role;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA {SCHEMA_NAME} TO testgen_report_role;
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA {SCHEMA_NAME} TO testgen_report_role;

testgen/template/dbsetup/080_set_current_revision.sql

@@ -0,0 +1,5 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+UPDATE tg_revision
+   SET revision = {DB_REVISION}
+ WHERE component = 'metadata_db';

testgen/template/dbupgrade/0100_incremental_upgrade.sql

@@ -0,0 +1,5 @@
+-- ===========================================
+-- |      Initial App DB Schema Upgrade      |
+-- ===========================================
+
+SELECT 1;

testgen/template/dbupgrade/0101_incremental_upgrade.sql

@@ -0,0 +1,15 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1
+        FROM information_schema.columns
+        where table_schema = '{SCHEMA_NAME}'
+        and table_name='projects'
+        AND column_name='observability_api_url'
+    ) THEN
+		alter table {SCHEMA_NAME}.projects add column observability_api_url TEXT DEFAULT '';
+        update {SCHEMA_NAME}.projects set observability_api_url='{OBSERVABILITY_URL}' WHERE observability_api_url = '';
+    END IF;
+END $$;

testgen/template/dbupgrade/0102_incremental_upgrade.sql

@@ -0,0 +1,4 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+alter table {SCHEMA_NAME}.profiling_runs add column process_id INTEGER;
+alter table {SCHEMA_NAME}.test_runs add column process_id INTEGER;

testgen/template/dbupgrade/0103_incremental_upgrade.sql

@@ -0,0 +1,22 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+ALTER TABLE test_types
+ ADD COLUMN column_name_prompt TEXT;
+
+ALTER TABLE test_types
+ ADD COLUMN column_name_help TEXT;
+
+ALTER TABLE test_results
+ ADD COLUMN auto_gen BOOLEAN;
+
+UPDATE test_results
+   SET auto_gen = TRUE
+  FROM test_results r
+INNER JOIN test_definitions d
+   ON (r.project_code = d.project_code
+  AND  r.test_suite = d.test_suite
+  AND  r.table_name = d.table_name
+  AND  r.column_names = COALESCE(d.column_name, 'N/A')
+  AND  r.test_type = d.test_type)
+WHERE d.last_auto_gen_date IS NOT NULL
+  AND test_results.id = r.id;

testgen/template/dbupgrade/0104_incremental_upgrade.sql

@@ -0,0 +1,44 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+ALTER TABLE test_results
+  ADD COLUMN test_suite_id UUID;
+
+UPDATE test_results
+   SET test_suite_id = s.id
+  FROM test_results r
+INNER JOIN test_suites s
+   ON (r.table_groups_id = s.table_groups_id
+  AND  r.test_suite = s.test_suite)
+ WHERE test_results.id = r.id;
+
+CREATE INDEX ix_tr_ts_tctt
+   ON test_results(test_suite_id, table_name, column_names, test_type);
+
+ALTER TABLE test_definitions
+  ADD COLUMN test_suite_id UUID;
+
+UPDATE test_definitions
+   SET test_suite_id = s.id
+  FROM test_definitions d
+INNER JOIN test_suites s
+   ON (D.table_groups_id = s.table_groups_id
+  AND  d.test_suite = s.test_suite)
+ WHERE test_definitions.id = d.id;
+
+CREATE INDEX ix_td_ts_tc
+   ON test_definitions(test_suite_id, table_name, column_name, test_type);
+
+ALTER TABLE table_groups
+   ADD COLUMN data_source            VARCHAR(40),
+   ADD COLUMN source_system          VARCHAR(40),
+   ADD COLUMN data_location          VARCHAR(40),
+   ADD COLUMN source_process         VARCHAR(40),
+   ADD COLUMN business_domain        VARCHAR(40),
+   ADD COLUMN stakeholder_group      VARCHAR(40),
+   ADD COLUMN transform_level        VARCHAR(40);
+
+ALTER TABLE data_table_chars
+   RENAME COLUMN transformation_level to transform_level;
+
+ALTER TABLE data_column_chars
+   RENAME COLUMN transformation_level to transform_level;

testgen/template/dbupgrade/0105_incremental_upgrade.sql

@@ -0,0 +1 @@
+SELECT 1;

testgen/template/dbupgrade/0106_incremental_upgrade.sql

@@ -0,0 +1,5 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+ALTER TABLE connections ADD COLUMN connect_by_key BOOLEAN DEFAULT FALSE;
+ALTER TABLE connections ADD COLUMN private_key BYTEA;
+ALTER TABLE connections ADD COLUMN private_key_passphrase BYTEA;

testgen/template/dbupgrade/0107_incremental_upgrade.sql

@@ -0,0 +1,3 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+ALTER TABLE profile_results ADD COLUMN pii_flag VARCHAR(50);

testgen/template/dbupgrade_helpers/get_tg_revision.sql

@@ -0,0 +1,2 @@
+SELECT revision
+  FROM {SCHEMA_NAME}.tg_revision;

testgen/template/exec_cat_tests/ex_cat_build_agg_table_tests.sql

@@ -0,0 +1,116 @@
+-- Create one record per CAT query: all test sets against one table, split over max chars
+INSERT INTO working_agg_cat_tests
+ (project_code, test_run_id,
+  test_suite, schema_name, table_name, cat_sequence, test_count, test_time,
+  column_names, test_types, test_definition_ids,
+  test_actions, test_descriptions,
+  test_parms, test_measures, test_conditions)
+-- Test details from each test type
+WITH test_detail
+  AS (
+       SELECT '{TEST_SUITE}' as test_suite,
+              '{SCHEMA_NAME}' as schema_name, '{TABLE_NAME}' as table_name,
+              '{RUN_DATE}'::TIMESTAMP as test_time,
+              t.column_name, t.test_type, t.id::VARCHAR as test_definition_id,
+              t.test_action, t.test_description,
+
+              SUBSTRING(
+                        CASE WHEN t.baseline_ct > '' THEN ', Baseline_Ct=' || t.baseline_ct ELSE '' END
+                        || CASE WHEN t.baseline_unique_ct > '' THEN ', Baseline_Unique_Ct=' || t.baseline_unique_ct ELSE '' END
+                        || CASE WHEN t.baseline_value > '' THEN ', Baseline_Value=' || t.baseline_value ELSE '' END
+                        || CASE WHEN t.baseline_value_ct > '' THEN ', Baseline_Value_Ct=' || t.baseline_value_ct ELSE '' END
+                        || CASE WHEN t.baseline_sum > '' THEN ', Baseline_Sum=' || t.baseline_sum ELSE '' END
+                        || CASE WHEN t.baseline_avg > '' THEN ', Baseline_Avg=' || t.baseline_avg ELSE '' END
+                        || CASE WHEN t.baseline_sd > '' THEN ', Baseline_SD=' || t.baseline_sd ELSE '' END
+                        || CASE WHEN t.threshold_value > '' THEN ', Threshold_Value=' || t.threshold_value ELSE '' END,
+                        3, 999) || ' '
+               as parms,
+
+              -- Standard Measure start
+              'CAST(' ||
+                -- Nested parm replacements - part of query, not Python parms
+                REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
+                   c.measure,
+                        '{COLUMN_NAME}', COALESCE(fn_PrepColumnName(t.column_name), '')),
+                        '{DATA_QC_SCHEMA}', '{REPLACE_QC_SCHEMA}'),
+                        '{BASELINE_CT}', COALESCE(t.baseline_ct, '')),
+                        '{BASELINE_UNIQUE_CT}', COALESCE(t.baseline_unique_ct, '')),
+                        '{BASELINE_VALUE}', COALESCE(t.baseline_value, '') ),
+                        '{BASELINE_VALUE_CT}', COALESCE(t.baseline_value_ct, '') ),
+                        '{BASELINE_SUM}', COALESCE(t.baseline_sum, '') ),
+                        '{BASELINE_AVG}', COALESCE(t.baseline_avg, '') ),
+                        '{BASELINE_SD}', COALESCE(t.baseline_sd, '') ),
+                        '{CUSTOM_QUERY}', COALESCE(t.custom_query, '')),
+                        '{THRESHOLD_VALUE}', COALESCE(t.threshold_value, '') )
+                -- Standard measure end with pipe delimiter
+                || ' AS VARCHAR(1000) ) {CONCAT_OPERATOR} ''|'' ' as measure,
+
+              -- Standard CASE for condition starts
+              'CASE WHEN ' ||
+                -- Nested parm replacements - standard
+                REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
+                   c.measure || c.test_operator || c.test_condition,
+                        '{COLUMN_NAME}', COALESCE(fn_PrepColumnName(t.column_name), '')),
+                        '{DATA_QC_SCHEMA}', '{REPLACE_QC_SCHEMA}'),
+                        '{BASELINE_CT}', COALESCE(t.baseline_ct, '')),
+                        '{BASELINE_UNIQUE_CT}', COALESCE(t.baseline_unique_ct, '')),
+                        '{BASELINE_VALUE}', COALESCE(t.baseline_value, '') ),
+                        '{BASELINE_VALUE_CT}', COALESCE(t.baseline_value_ct, '') ),
+                        '{BASELINE_SUM}', COALESCE(t.baseline_sum, '') ),
+                        '{BASELINE_AVG}', COALESCE(t.baseline_avg, '') ),
+                        '{BASELINE_SD}', COALESCE(t.baseline_sd, '') ),
+                        '{CUSTOM_QUERY}', COALESCE(t.custom_query, '')),
+                        '{THRESHOLD_VALUE}', COALESCE(t.threshold_value, '') )
+                -- Standard case ends
+                || ' THEN ''0,'' ELSE ''1,'' END' as condition
+         FROM test_definitions t
+       INNER JOIN cat_test_conditions c
+          ON (t.test_type = c.test_type
+         AND  '{SQL_FLAVOR}' = c.sql_flavor)
+        WHERE t.project_code = '{PROJECT_CODE}'
+          AND t.test_suite = '{TEST_SUITE}'
+          AND t.schema_name = '{SCHEMA_NAME}'
+          AND t.table_name = '{TABLE_NAME}'
+          AND COALESCE(t.test_active, 'Y') = 'Y'
+      ),
+test_detail_split
+   AS ( SELECT test_suite, schema_name, table_name, test_time,
+               column_name, test_type, test_definition_id, test_action, test_description,
+               parms, measure, condition,
+               SUM(LENGTH(condition)) OVER (PARTITION BY t.schema_name, t.table_name
+                                      ORDER BY t.column_name ROWS UNBOUNDED PRECEDING ) as run_total_chars,
+               FLOOR( SUM(LENGTH(condition)) OVER (PARTITION BY t.schema_name, t.table_name
+                                             ORDER BY t.column_name ROWS UNBOUNDED PRECEDING )
+                  / {MAX_QUERY_CHARS} ) + 1 as query_split_no
+          FROM test_detail t )
+SELECT
+       '{PROJECT_CODE}' as project_code,
+       '{TEST_RUN_ID}' as test_run_id,
+       d.test_suite,
+       d.schema_name, d.table_name,
+       d.query_split_no as cat_sequence,
+       COUNT(*) as test_count,
+       '{RUN_DATE}'::TIMESTAMP as test_time,
+       STRING_AGG(COALESCE(d.column_name, 'N/A'), '~|~' ORDER BY d.column_name) as column_names,
+       STRING_AGG(d.test_type, ',' ORDER BY d.column_name) as test_types,
+       STRING_AGG(d.test_definition_id, ',' ORDER BY d.column_name) as test_definition_ids,
+       -- Pipe delimiter below, because commas may be embedded
+       STRING_AGG(d.test_action, '|' ORDER BY d.column_name) as test_actions,
+       STRING_AGG(d.test_description, '|' ORDER BY d.column_name) as test_descriptions,
+
+       -- Consolidated Parms
+       STRING_AGG( d.parms, '|' ORDER BY d.column_name) as parms,
+
+       -- Consolidated Measures
+       -- Encode Null as text to decode when freeing kittens
+        STRING_AGG( 'COALESCE(' || d.measure || ',''' || '<NULL>' || '|'')',
+                -- Use ++ as STRING_AGG delimiter -- replace with + later
+                '++' ORDER BY d.column_name) as measures,
+
+       -- Consolidated CASE statements
+       STRING_AGG( d.condition,
+                -- Use ++ as STRING_AGG delimiter -- replace with + later
+                '++' ORDER BY d.column_name) as conditions
+
+  FROM test_detail_split d
+GROUP BY d.test_suite, d.schema_name, d.table_name, test_time, d.query_split_no;

testgen/template/exec_cat_tests/ex_cat_get_distinct_tables.sql

@@ -0,0 +1,11 @@
+SELECT DISTINCT schema_name, table_name,
+                project_qc_schema as replace_qc_schema
+  FROM test_definitions t
+INNER JOIN test_types tt
+   ON t.test_type = tt.test_type
+INNER JOIN table_groups tg
+   ON (t.table_groups_id = tg.id)
+INNER JOIN connections c
+   ON (tg.connection_id = c.connection_id)
+  WHERE t.test_suite = '{TEST_SUITE}'
+    AND tt.run_type = 'CAT';

testgen/template/exec_cat_tests/ex_cat_results_parse.sql

@@ -0,0 +1,69 @@
+-- Parses aggregated results and inserts into test_results table
+WITH seq_digit  AS (
+                 SELECT 0 as d UNION ALL
+                 SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL
+                 SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL
+                 SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9 ),
+      seq_table_raw AS (
+                 SELECT CAST(a.d + (10 * b.d) + (100 * c.d) + (1000 * d.d) as INT) as nbr
+                 FROM seq_digit a CROSS JOIN seq_digit b CROSS JOIN seq_digit c CROSS JOIN seq_digit d
+                 ORDER BY nbr LIMIT 1000),
+      seq_table AS (
+                 SELECT nbr FROM seq_table_raw WHERE nbr > 0),
+      raw_results AS (
+                 SELECT t.project_code, t.test_run_id, t.test_suite, t.schema_name, t.table_name, t.cat_sequence, t.test_count,
+                        t.test_time, t.start_time, t.end_time, t.column_names, t.test_types, t.test_definition_ids,
+                        t.test_actions, t.test_descriptions,
+                        t.test_parms, t.test_measures, t.test_conditions,
+                        r.measure_results, r.test_results
+                   FROM working_agg_cat_tests t
+                 INNER JOIN working_agg_cat_results r
+                    ON (t.test_run_id = r.test_run_id
+                   AND  t.schema_name = r.schema_name
+                   AND  t.table_name = r.table_name
+                   AND  t.cat_sequence = r.cat_sequence)
+                  WHERE t.test_run_id = '{TEST_RUN_ID}'
+                    AND t.column_names > ''
+      ),
+      parsed_results AS (
+                 SELECT t.test_suite,
+                        t.schema_name,
+                        t.table_name,
+                        t.test_time,
+                        t.start_time,
+                        t.end_time,
+                        nbr                                                        AS test_number,
+                        SPLIT_PART(t.test_actions, '|,', s.nbr)                    AS test_action,
+                        SPLIT_PART(t.test_descriptions, '|', s.nbr)                AS test_description,
+                        SPLIT_PART(t.column_names, '~|~', s.nbr)                   AS column_name,
+                        SPLIT_PART(t.test_types, ',', s.nbr)                       AS test_type,
+                        SPLIT_PART(t.test_definition_ids, ',', s.nbr)              AS test_definition_id,
+                        SPLIT_PART(t.test_parms, '|', s.nbr)                       AS test_parms,
+                        SPLIT_PART(t.test_measures, '++', s.nbr)                   AS measure,
+                        TRIM(SPLIT_PART(t.test_conditions, '++', s.nbr))           AS condition,
+                        -- Restore encoded null value
+                        NULLIF(SPLIT_PART(t.measure_results, '|', s.nbr), '<NULL>') AS measure_result,
+                        SPLIT_PART(t.test_results, ',', s.nbr)                     AS test_result
+                   FROM raw_results t
+                        CROSS JOIN seq_table s
+      )
+INSERT INTO test_results
+         (project_code, test_run_id,
+          test_type, test_definition_id,
+          test_suite, test_time, starttime, endtime, schema_name, table_name, column_names,
+          skip_errors, input_parameters, result_code,
+          result_measure, test_action, subset_condition, result_query, test_description)
+SELECT '{PROJECT_CODE}' as project_code, '{TEST_RUN_ID}' as test_run_id,
+        r.test_type, r.test_definition_id::UUID, r.test_suite, r.test_time, r.start_time, r.end_time,
+        r.schema_name, r.table_name, r.column_name,
+        0 as skip_errors,
+        r.test_parms as input_parameters,
+        r.test_result::INT as result_code,
+        r.measure_result as result_measure,
+        r.test_action, NULL as subset_condition,
+        'SELECT ' || LEFT(REPLACE(r.condition, '{RUN_' || 'DATE}', '{RUN_DATE}'), LENGTH(REPLACE(r.condition, '{RUN_' || 'DATE}', '{RUN_DATE}')) - LENGTH(' THEN ''0,'' ELSE ''1,'' END'))  || ' THEN 0 ELSE 1 END'
+           || ' FROM ' || r.schema_name || '.' || r.table_name as result_query,
+        COALESCE(r.test_description, c.test_description) as test_description
+  FROM parsed_results r
+   INNER JOIN test_types c
+      ON r.test_type = c.test_type;

testgen/template/exec_cat_tests/ex_cat_retrieve_agg_test_parms.sql

@@ -0,0 +1,6 @@
+SELECT test_suite, schema_name, table_name, cat_sequence,
+       -- Replace list delimiters with concat operator
+        REPLACE(test_measures, '++', '{CONCAT_OPERATOR}') as test_measures,
+        REPLACE(test_conditions, '++', '{CONCAT_OPERATOR}') as test_conditions
+  FROM working_agg_cat_tests
+ WHERE test_run_id = '{TEST_RUN_ID}';

testgen/template/exec_cat_tests/ex_cat_test_query.sql

@@ -0,0 +1,8 @@
+SELECT '{TEST_RUN_ID}' as test_run_id,
+       '{TEST_SUITE}' as test_suite,
+       '{SCHEMA_NAME}' as schema_name,
+       '{TABLE_NAME}' as table_name,
+       '{CAT_SEQUENCE}' as cat_sequence,
+       {TEST_MEASURES} as measure_results,
+       {TEST_CONDITIONS} as test_results
+  FROM {SCHEMA_NAME}.{TABLE_NAME}