dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/quick_start/recreate_target_data_schema.sql

@@ -0,0 +1,167 @@
+create schema if not exists {PROJECT_SCHEMA};
+
+SET SEARCH_PATH TO {PROJECT_SCHEMA};
+
+DROP TABLE if exists d_ebike_customers CASCADE;
+
+CREATE TABLE d_ebike_customers (
+    customer_id INTEGER,
+    first_name VARCHAR(50) NOT NULL,
+    last_name VARCHAR(50) NOT NULL,
+    address VARCHAR(255) NOT NULL,
+    city VARCHAR(100),
+    state VARCHAR(50),
+    postal_code VARCHAR(10),
+    country VARCHAR(100),
+    customer_type VARCHAR(20),
+    avg_age INT,
+    income_level VARCHAR(10),
+    gender VARCHAR(10),
+    occupation VARCHAR(100),
+    marital_status VARCHAR(20),
+    customer_note TEXT,
+    sales_region VARCHAR(10),
+    sales_territory VARCHAR(10),
+    customer_decile INT,
+    phone_number VARCHAR(40),
+    tax_id VARCHAR(15),
+    credit_card VARCHAR(20),
+    last_contact DATE
+);
+
+DROP TABLE if exists tmp_d_ebike_customers CASCADE;
+
+CREATE TABLE tmp_d_ebike_customers (
+    customer_id INTEGER,
+    first_name VARCHAR(50) NOT NULL,
+    last_name VARCHAR(50) NOT NULL,
+    address VARCHAR(255) NOT NULL,
+    city VARCHAR(100),
+    state VARCHAR(50),
+    postal_code VARCHAR(10),
+    country VARCHAR(100),
+    customer_type VARCHAR(20),
+    avg_age INT,
+    income_level VARCHAR(10),
+    gender VARCHAR(10),
+    occupation VARCHAR(100),
+    marital_status VARCHAR(20),
+    customer_note TEXT,
+    sales_region VARCHAR(10),
+    sales_territory VARCHAR(10),
+    customer_decile INT,
+    phone_number VARCHAR(40),
+    tax_id VARCHAR(15),
+    credit_card VARCHAR(20),
+    last_contact DATE
+);
+
+DROP TABLE if exists d_ebike_products CASCADE;
+
+CREATE TABLE d_ebike_products (
+    product_id INTEGER,
+    product_name VARCHAR(255) NOT NULL,
+    product_description TEXT,
+    product_type VARCHAR(50), --CHECK (product_type IN ('Bicycle', 'E-bike', 'E-scooter')),
+    frame_size VARCHAR(50),
+    battery_life INT,  -- in hours; NULL for regular bicycles
+    max_speed DECIMAL(5,2),  -- in mph or km/h; NULL for regular bicycles
+    weight_capacity DECIMAL(10,2),  -- in lbs or kg
+    color VARCHAR(50),
+    wheel_size DECIMAL(5,2),  -- in inches or cm
+    gear_count INT,
+    country_of_origin VARCHAR(50),
+    price DECIMAL(10,2) NOT NULL,
+    max_discount DECIMAL(5,2)
+);
+
+DROP TABLE if exists tmp_d_ebike_products CASCADE;
+
+CREATE TABLE tmp_d_ebike_products (
+    product_id INTEGER,
+    product_name VARCHAR(255) NOT NULL,
+    product_description TEXT,
+    product_type VARCHAR(50), -- CHECK (product_type IN ('Bicycle', 'E-bike', 'E-scooter')),
+    frame_size VARCHAR(50),
+    battery_life INT,  -- in hours; NULL for regular bicycles
+    max_speed DECIMAL(5,2),  -- in mph or km/h; NULL for regular bicycles
+    weight_capacity DECIMAL(10,2),  -- in lbs or kg
+    color VARCHAR(50),
+    wheel_size DECIMAL(5,2),  -- in inches or cm
+    gear_count INT,
+    country_of_origin VARCHAR(50),
+    price DECIMAL(10,2) NOT NULL,
+    max_discount DECIMAL(5,2)
+);
+
+DROP TABLE IF EXISTS d_ebike_suppliers CASCADE;
+
+CREATE TABLE d_ebike_suppliers (
+    supplier_id INTEGER,
+    supplier_name VARCHAR(255) NOT NULL,
+    is_manufacturer BOOLEAN DEFAULT FALSE,
+    manufacturing_certifications TEXT,
+    contact_name VARCHAR(255),
+    contact_email VARCHAR(100),
+    address VARCHAR(255),
+    country VARCHAR(100),
+    region_code VARCHAR(50),
+    phone VARCHAR(30),
+    last_order DATE,
+    key_supplier VARCHAR(3),
+    supply_reliability VARCHAR(3)
+);
+
+DROP TABLE IF EXISTS tmp_d_ebike_suppliers CASCADE;
+
+CREATE TABLE tmp_d_ebike_suppliers (
+    supplier_id INTEGER,
+    supplier_name VARCHAR(255) NOT NULL,
+    is_manufacturer BOOLEAN DEFAULT FALSE,
+    manufacturing_certifications TEXT,
+    contact_name VARCHAR(255),
+    contact_email VARCHAR(100),
+    address VARCHAR(255),
+    country VARCHAR(100),
+    region_code VARCHAR(50),
+    phone VARCHAR(30),
+    last_order DATE,
+    key_supplier VARCHAR(3),
+    supply_reliability VARCHAR(3)
+);
+
+DROP TABLE IF EXISTS tmp_ebike_sales CASCADE;
+
+CREATE TABLE tmp_ebike_sales (
+    sale_id INTEGER,
+    sale_date DATE,
+    customer_id INT,
+    supplier_id INT,
+    product_id INT,
+    quantity_sold INT,
+    sale_price DECIMAL(10,2),
+    total_amount DECIMAL(10,2),
+    discount_amount DECIMAL(10,2) DEFAULT 0,
+    adjusted_total_amount DECIMAL(10,2),
+    warranty_end_date DATE,
+    next_maintenance_date DATE,
+    return_reason TEXT
+);
+
+DROP TABLE IF EXISTS f_ebike_sales CASCADE;
+
+CREATE TABLE f_ebike_sales (
+    sale_id INTEGER,
+    sale_date DATE,
+    customer_id INT,
+    supplier_id INT,
+    product_id INT,
+    quantity_sold INT,
+    sale_price DECIMAL(10,2),
+    total_amount DECIMAL(10,2),
+    discount_amount DECIMAL(10,2) DEFAULT 0,
+    adjusted_total_amount DECIMAL(10,2),
+    warranty_end_date DATE,
+    next_maintenance_date DATE,
+    return_reason TEXT
+);

testgen/template/quick_start/update_target_data.sql

@@ -0,0 +1,100 @@
+SET SEARCH_PATH TO {PROJECT_SCHEMA};
+
+TRUNCATE TABLE f_ebike_sales;
+
+INSERT INTO f_ebike_sales
+   (sale_id,
+   customer_id, supplier_id, product_id, quantity_sold, sale_price, total_amount,
+   discount_amount, adjusted_total_amount, return_reason,
+   sale_date, warranty_end_date, next_maintenance_date)
+SELECT t.sale_id,
+       t.customer_id, t.supplier_id, t.product_id, t.quantity_sold, t.sale_price, t.total_amount,
+       t.discount_amount, t.adjusted_total_amount, t.return_reason,
+       t.sale_date + (SELECT (CURRENT_DATE - MAX(sale_date))::integer - 1 FROM tmp_ebike_sales),
+       t.warranty_end_date + (SELECT (CURRENT_DATE - MAX(sale_date))::integer - 1 FROM tmp_ebike_sales),
+       t.next_maintenance_date + (SELECT (CURRENT_DATE - MAX(sale_date))::integer - 1 FROM tmp_ebike_sales)
+  FROM tmp_ebike_sales t
+LEFT JOIN f_ebike_sales fes ON t.sale_id = fes.sale_id
+ WHERE fes.sale_id IS NULL
+   AND t.sale_date <= '{MAX_DATE}';
+
+
+TRUNCATE TABLE d_ebike_customers;
+
+INSERT INTO d_ebike_customers
+(customer_id, first_name, last_name, address, city, state, postal_code, country, customer_type, avg_age, income_level,
+ gender, occupation, marital_status, customer_note, sales_region, sales_territory, customer_decile, phone_number,
+ tax_id, credit_card, last_contact)
+SELECT t.customer_id,
+       t.first_name,
+       t.last_name,
+       t.address,
+       t.city,
+       t.state,
+       t.postal_code,
+       t.country,
+       t.customer_type,
+       t.avg_age,
+       t.income_level,
+       t.gender,
+       t.occupation,
+       t.marital_status,
+       t.customer_note,
+       t.sales_region,
+       t.sales_territory,
+       t.customer_decile,
+       t.phone_number,
+       t.tax_id,
+       t.credit_card,
+       t.last_contact - (55 - (SELECT (CURRENT_DATE  - MAX(last_contact)) FROM tmp_d_ebike_customers))
+FROM tmp_d_ebike_customers t
+WHERE t.customer_id <= '{MAX_CUSTOMER_ID_SEQ}';
+
+UPDATE d_ebike_customers
+    SET last_contact =   CASE WHEN '{ITERATION_NUMBER}' = 1 AND
+            current_date - last_contact <= 60 THEN last_contact - (62 - (current_date - last_contact))
+           ELSE last_contact END;
+
+TRUNCATE TABLE d_ebike_products;
+
+INSERT INTO d_ebike_products
+(product_id, product_name, product_description, product_type, frame_size, battery_life, max_speed, weight_capacity,
+ color, wheel_size, gear_count, country_of_origin, price, max_discount)
+SELECT t.product_id,
+       t.product_name,
+       t.product_description,
+       t.product_type,
+       t.frame_size,
+       t.battery_life,
+       t.max_speed,
+       t.weight_capacity,
+       t.color,
+       t.wheel_size,
+       t.gear_count,
+       t.country_of_origin,
+       t.price,
+       t.max_discount
+FROM tmp_d_ebike_products t
+WHERE  product_id <= '{MAX_PRODUCT_ID_SEQ}';
+
+
+TRUNCATE TABLE d_ebike_suppliers;
+
+INSERT INTO d_ebike_suppliers
+(supplier_id, supplier_name, is_manufacturer, manufacturing_certifications, contact_name, contact_email, address,
+ country, region_code, phone, last_order, key_supplier, supply_reliability)
+SELECT t.supplier_id,
+       t.supplier_name,
+       t.is_manufacturer,
+       t.manufacturing_certifications,
+       t.contact_name,
+       t.contact_email,
+       t.address,
+       t.country,
+       t.region_code,
+       t.phone,
+       t.last_order,
+       t.key_supplier,
+       t.supply_reliability
+FROM tmp_d_ebike_suppliers t
+WHERE t.supplier_id <= '{MAX_SUPPLIER_ID_SEQ}';

testgen/template/updates/create_tmp_test_definition.sql

@@ -0,0 +1,19 @@
+DROP TABLE IF EXISTS tmp_test_definition CASCADE;
+
+CREATE TEMPORARY TABLE  tmp_test_definition
+(
+    project_code           varchar(30),
+    test_suite             varchar(200),
+    schema_name            varchar(100),
+    table_name             varchar(100),
+    column_name            varchar(500),
+    id                     uuid,
+    test_type              varchar(200),
+    test_description       varchar(1000),
+    test_action            varchar(100),
+    test_active            varchar(10),
+    lock_refresh           varchar(10),
+    severity               varchar(10),
+    test_parameter         varchar(100),
+    test_parameter_value   varchar(1000)
+);

testgen/template/updates/get_test_def_parms.sql

@@ -0,0 +1,38 @@
+SELECT td.project_code, td.test_suite,
+       td.schema_name, td.table_name, td.column_name,
+       td.id::VARCHAR(50), td.test_type,
+       CASE WHEN td.test_description IS NOT NULL THEN td.test_description ELSE tt.test_description END
+           as test_description,
+       td.test_action,
+       td.test_active,
+       td.lock_refresh,
+       td.severity,
+       tt.default_parm_columns  as test_parameters,
+       td.baseline_ct,
+       td.baseline_unique_ct,
+       td.baseline_value,
+       td.baseline_value_ct,
+       td.threshold_value,
+       td.baseline_sum,
+       td.baseline_avg,
+       td.baseline_sd,
+       td.subset_condition,
+       td.groupby_names,
+       td.having_condition,
+       td.window_date_column,
+       td.window_days,
+       td.match_schema_name,
+       td.match_table_name,
+       td.match_column_names,
+       td.match_subset_condition,
+       td.match_groupby_names,
+       td.match_having_condition,
+       td.custom_query
+
+FROM test_definitions td
+         INNER JOIN test_types tt
+                    ON (td.test_type = tt.test_type)
+WHERE project_code = '{PROJECT_CODE}'
+  AND test_suite = '{TEST_SUITE}'
+ORDER BY td.project_code, td.test_suite,
+         td.schema_name, td.table_name, td.column_name, td.test_type;

testgen/template/updates/populate_stg_test_definitions.sql

@@ -0,0 +1,184 @@
+DROP TABLE IF EXISTS tmp_stg_test_definitions CASCADE;
+
+CREATE TEMPORARY TABLE tmp_stg_test_definitions
+(
+    id                     uuid,
+    cat_test_id            bigint,
+    project_code           varchar(30),
+    table_groups_id        uuid,
+    profile_run_id         uuid,
+    test_type              varchar(200),
+    test_suite             varchar(200),
+    test_description       varchar(1000),
+    test_action            varchar(100),
+    schema_name            varchar(100),
+    table_name             varchar(100),
+    column_name            varchar(500),
+    skip_errors            integer,
+    baseline_ct            varchar(1000),
+    baseline_unique_ct     varchar(1000),
+    baseline_value         varchar(1000),
+    baseline_value_ct      varchar(1000),
+    threshold_value        varchar(1000),
+    baseline_sum           varchar(1000),
+    baseline_avg           varchar(1000),
+    baseline_sd            varchar(1000),
+    subset_condition       varchar(500),
+    groupby_names          varchar(200),
+    having_condition       varchar(500),
+    window_date_column     varchar(100),
+    window_days            integer,
+    match_schema_name      varchar(100),
+    match_table_name       varchar(100),
+    match_column_names     varchar(200),
+    match_subset_condition varchar(500),
+    match_groupby_names    varchar(200),
+    match_having_condition varchar(500),
+    test_mode              varchar(20),
+    custom_query           varchar(4000),
+    test_active            varchar(10),
+    severity               varchar(10),
+    watch_level            varchar(10),
+    check_result           varchar(500),
+    lock_refresh           varchar(10),
+    last_auto_gen_date     timestamp,
+    profiling_as_of_date   timestamp
+);
+
+
+INSERT INTO tmp_stg_test_definitions (project_code, test_suite, schema_name, table_name, column_name,
+                                  id, test_type, test_description, test_action, test_active,lock_refresh, severity,
+                                  baseline_ct,baseline_unique_ct,baseline_value,baseline_value_ct,threshold_value,
+                                  baseline_sum,baseline_avg,baseline_sd,subset_condition,groupby_names,having_condition,
+                                  window_date_column,window_days,match_schema_name,match_table_name,match_column_names,
+                                  match_subset_condition,match_groupby_names, match_having_condition   )
+SELECT project_code,
+       test_suite,
+       schema_name,
+       table_name,
+       column_name,
+       id,
+       test_type,
+       test_description,
+       test_action,
+       CASE WHEN lower(test_active) = 'none' THEN 'N' ELSE test_active END as test_active,
+       CASE WHEN lower(lock_refresh) = 'none' THEN 'N' ELSE lock_refresh END as lock_refresh,
+       CASE WHEN lower(severity) IN ('warning','fail','ignore') THEN initcap(severity) ELSE NULL END as severity,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'baseline_ct'            )  AS baseline_ct,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'baseline_unique_ct'     )  AS baseline_unique_ct,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'baseline_value'         )  AS baseline_value,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'baseline_value_ct'      )  AS baseline_value_ct,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'threshold_value'        )  AS threshold_value,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'baseline_sum'           )  AS baseline_sum,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'baseline_avg'           )  AS baseline_avg,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'baseline_sd'            )  AS baseline_sd,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'subset_condition'       )  AS subset_condition,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'groupby_names'          )  AS groupby_names,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'having_condition'       )  AS having_condition,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'window_date_column'     )  AS window_date_column,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'window_days'            ) :: integer AS window_days,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'match_schema_name'      )  AS match_schema_name,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'match_table_name'       )  AS match_table_name,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'match_column_names'     )  AS match_column_names,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'match_subset_condition' )  AS match_subset_condition,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'match_groupby_names'    )  AS match_groupby_names,
+        MAX(test_parameter_value) FILTER( WHERE test_parameter = 'match_having_condition' )  AS match_having_condition
+FROM tmp_test_definition
+GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
+ORDER BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12;
+
+
+--- UPDATE
+
+UPDATE test_definitions
+SET test_active            = CASE WHEN lower(c.test_active) = 'none' THEN 'N' ELSE c.test_active END,
+    lock_refresh           = CASE WHEN lower(c.lock_refresh) = 'none' THEN 'N' ELSE c.lock_refresh END,
+    severity               = CASE WHEN lower(c.severity) IN ('warning','fail','ignore') THEN initcap(c.severity) ELSE NULL END,
+    last_manual_update     = current_timestamp       ,
+    baseline_ct            = c.baseline_ct           ,
+    baseline_unique_ct     = c.baseline_unique_ct    ,
+    baseline_value         = c.baseline_value        ,
+    baseline_value_ct      = c.baseline_value_ct     ,
+    threshold_value        = c.threshold_value       ,
+    baseline_sum           = c.baseline_sum          ,
+    baseline_avg           = c.baseline_avg          ,
+    baseline_sd            = c.baseline_sd           ,
+    subset_condition       = c.subset_condition      ,
+    groupby_names          = c.groupby_names         ,
+    having_condition       = c.having_condition      ,
+    window_date_column     = c.window_date_column    ,
+    window_days            = c.window_days           ,
+    match_schema_name      = c.match_schema_name     ,
+    match_table_name       = c.match_table_name      ,
+    match_column_names     = c.match_column_names    ,
+    match_subset_condition = c.match_subset_condition,
+    match_groupby_names    = c.match_groupby_names
+    FROM ( SELECT b.* FROM tmp_stg_test_definitions b
+       LEFT JOIN test_definitions a
+       ON a.project_code = b.project_code
+           AND a.test_suite    = b.test_suite
+           AND a.schema_name   = b.schema_name
+           AND a.table_name    = b.table_name
+           AND a.column_name   = b.column_name
+           AND a.test_type     = b.test_type
+       WHERE (a.test_active != b.test_active)
+          OR (a.lock_refresh != b.lock_refresh)
+          OR (coalesce(a.severity,'') != coalesce(b.severity,''))
+          OR (a.baseline_ct            != b.baseline_ct           )
+          OR (a.baseline_unique_ct     != b.baseline_unique_ct    )
+          OR (a.baseline_value         != b.baseline_value        )
+          OR (a.baseline_value_ct      != b.baseline_value_ct     )
+          OR (a.threshold_value        != b.threshold_value       )
+          OR (a.baseline_sum           != b.baseline_sum          )
+          OR (a.baseline_avg           != b.baseline_avg          )
+          OR (a.baseline_sd            != b.baseline_sd           )
+          OR (a.subset_condition       != b.subset_condition      )
+          OR (a.groupby_names          != b.groupby_names         )
+          OR (a.having_condition       != b.having_condition      )
+          OR (a.window_date_column     != b.window_date_column    )
+          OR (a.window_days            != b.window_days           )
+          OR (a.match_schema_name      != b.match_schema_name     )
+          OR (a.match_table_name       != b.match_table_name      )
+          OR (a.match_column_names     != b.match_column_names    )
+          OR (a.match_subset_condition != b.match_subset_condition)
+          OR (a.match_groupby_names    != b.match_groupby_names   )
+       ) c
+WHERE test_definitions.project_code  = c.project_code
+  AND test_definitions.test_suite    = c.test_suite
+  AND test_definitions.schema_name   = c.schema_name
+  AND test_definitions.table_name    = c.table_name
+  AND test_definitions.column_name   = c.column_name
+  AND test_definitions.test_type     = c.test_type  ;
+
+
+-- INSERT
+INSERT INTO test_definitions (project_code, test_suite, schema_name, table_name, column_name,
+                                  test_type, test_action, test_active, lock_refresh, severity, last_manual_update,
+                                  baseline_ct,baseline_unique_ct,baseline_value,baseline_value_ct,threshold_value,
+                                  baseline_sum,baseline_avg,baseline_sd,subset_condition,groupby_names,having_condition,
+                                  window_date_column,window_days,match_schema_name,match_table_name,match_column_names,
+                                  match_subset_condition,match_groupby_names, match_having_condition   )
+SELECT a.project_code, a.test_suite, a.schema_name, a.table_name, a.column_name,
+       a.test_type,
+       CASE WHEN lower(a.test_action) = 'none' THEN NULL ELSE a.test_action END as test_action,
+       CASE WHEN lower(a.test_active) = 'none' THEN 'N' ELSE a.test_active END as test_active,
+       CASE WHEN lower(a.lock_refresh) = 'none' THEN 'N' ELSE a.lock_refresh END as lock_refresh,
+       CASE WHEN lower(a.severity) IN ('warning','fail','ignore') THEN initcap(a.severity) ELSE NULL END as severity,
+       current_timestamp  as last_manual_update,
+       a.baseline_ct,a.baseline_unique_ct,a.baseline_value,a.baseline_value_ct,a.threshold_value,
+       a.baseline_sum,a.baseline_avg,a.baseline_sd,a.subset_condition,a.groupby_names,a.having_condition,
+       a.window_date_column,a.window_days,a.match_schema_name,a.match_table_name,a.match_column_names,
+       a.match_subset_condition,a.match_groupby_names, a.match_having_condition
+FROM tmp_stg_test_definitions a
+         LEFT JOIN test_definitions b
+                   ON a.project_code = b.project_code
+                       AND a.test_suite    = b.test_suite
+                       AND a.schema_name   = b.schema_name
+                       AND a.table_name    = b.table_name
+                       AND a.column_name   = b.column_name
+                       AND a.test_type     = b.test_type
+WHERE a.id IS NULL AND b.id is NULL;
+
+DROP TABLE tmp_test_definition CASCADE;
+
+DROP TABLE tmp_stg_test_definitions CASCADE;

testgen/template/validate_tests/ex_disable_tests_test_definitions.sql

@@ -0,0 +1,5 @@
+update test_definitions
+set test_active = 'N'
+ where project_code = '{PROJECT_CODE}'
+   and test_suite = '{TEST_SUITE}'
+   and test_active = 'D';

testgen/template/validate_tests/ex_flag_tests_test_definitions.sql

@@ -0,0 +1,64 @@
+/*
+Mark Test inactive for Missing columns with update status
+*/
+with test_columns as
+         (SELECT DISTINCT schema_name || '.' || table_name || '.' || column_name AS columns
+  FROM ( SELECT cat_test_id,
+                project_code,
+                test_suite,
+                schema_name,
+                table_name,
+                UNNEST(STRING_TO_ARRAY(all_columns, '~|~')) AS column_name
+           FROM ( SELECT cat_test_id,
+                         project_code,
+                         test_suite,
+                         schema_name,
+                         table_name,
+                         CONCAT_WS('~|~', column_name,
+                                   groupby_names,
+                                   window_date_column) AS all_columns
+                    FROM test_definitions d
+                         INNER JOIN test_types t
+                                    ON d.test_type = t.test_type
+                   WHERE project_code = '{PROJECT_CODE}'
+                     AND test_suite = '{TEST_SUITE}'
+                     AND t.test_scope = 'column'
+
+                   UNION
+                  SELECT cat_test_id,
+                         project_code,
+                         test_suite,
+                         match_schema_name              AS schema_name,
+                         match_table_name               AS table_name,
+                         CONCAT_WS('~|~',
+                                   match_column_names,
+                                   match_groupby_names) AS all_columns
+                    FROM test_definitions d
+                         INNER JOIN test_types t
+                                    ON d.test_type = t.test_type
+                   WHERE project_code = '{PROJECT_CODE}'
+                     AND test_suite = '{TEST_SUITE}'
+                     AND t.test_scope = 'column') a ) b)
+update test_definitions
+set test_active            = '{FLAG}',
+    test_definition_status = 'Inactivated {RUN_DATE}: Missing Column'
+where cat_test_id in (select distinct cat_test_id
+                      from test_columns
+                      where lower(columns) in
+                            ({MISSING_COLUMNS}));
+
+
+/*
+Mark Test inactive for Missing table with update status
+*/
+with test_columns as
+         (select distinct cat_test_id, schema_name || '.' || table_name || '.' || column_name as columns
+          from test_definitions
+          where project_code = '{PROJECT_CODE}'
+            and test_suite = '{TEST_SUITE}'
+            and lower(schema_name || '.' || table_name) in ({MISSING_TABLES}))
+update test_definitions
+set test_active            = '{FLAG}',
+    test_definition_status = 'Inactivated {RUN_DATE}: Missing Table'
+where cat_test_id in (select distinct cat_test_id
+                      from test_columns);

testgen/template/validate_tests/ex_get_project_column_list_generic.sql

@@ -0,0 +1,3 @@
+select concat(concat(concat(table_schema, '.'), concat(table_name, '.')), column_name) as columns
+from information_schema.columns
+where table_schema in ({TEST_SCHEMAS});

testgen/template/validate_tests/ex_get_test_column_list_tg.sql

@@ -0,0 +1,65 @@
+SELECT DISTINCT schema_name || '.' || table_name || '.' || column_name AS columns
+  FROM ( SELECT cat_test_id,
+                project_code,
+                test_suite,
+                schema_name              AS schema_name,
+                table_name               AS table_name,
+                TRIM(UNNEST(STRING_TO_ARRAY(column_name, ','))) as column_name
+         FROM test_definitions d
+         INNER JOIN test_types t
+               ON d.test_type = t.test_type
+         WHERE project_code = '{PROJECT_CODE}'
+         AND test_suite = '{TEST_SUITE}'
+         AND t.test_scope IN ('column', 'referential')
+         UNION
+         SELECT cat_test_id,
+                project_code,
+                test_suite,
+                schema_name              AS schema_name,
+                table_name               AS table_name,
+                TRIM(UNNEST(STRING_TO_ARRAY(groupby_names, ','))) as column_name
+         FROM test_definitions d
+         INNER JOIN test_types t
+               ON d.test_type = t.test_type
+         WHERE project_code = '{PROJECT_CODE}'
+         AND test_suite = '{TEST_SUITE}'
+         AND t.test_scope IN ('column', 'referential')
+         UNION
+         SELECT cat_test_id,
+                project_code,
+                test_suite,
+                schema_name              AS schema_name,
+                table_name               AS table_name,
+                TRIM(UNNEST(STRING_TO_ARRAY(window_date_column, ','))) as column_name
+         FROM test_definitions d
+         INNER JOIN test_types t
+               ON d.test_type = t.test_type
+         WHERE project_code = '{PROJECT_CODE}'
+         AND test_suite = '{TEST_SUITE}'
+         AND t.test_scope IN ('column', 'referential')
+         UNION
+         SELECT cat_test_id,
+                project_code,
+                test_suite,
+                match_schema_name              AS schema_name,
+                match_table_name               AS table_name,
+                TRIM(UNNEST(STRING_TO_ARRAY(match_column_names, ','))) as column_name
+         FROM test_definitions d
+         INNER JOIN test_types t
+               ON d.test_type = t.test_type
+         WHERE project_code = '{PROJECT_CODE}'
+         AND test_suite = '{TEST_SUITE}'
+         AND t.test_scope = 'referential'
+         UNION
+         SELECT cat_test_id,
+                project_code,
+                test_suite,
+                match_schema_name              AS schema_name,
+                match_table_name               AS table_name,
+                TRIM(UNNEST(STRING_TO_ARRAY(match_groupby_names, ','))) as column_name
+         FROM test_definitions d
+         INNER JOIN test_types t
+               ON d.test_type = t.test_type
+         WHERE project_code = '{PROJECT_CODE}'
+         AND test_suite = '{TEST_SUITE}'
+         AND t.test_scope = 'referential' ) cols;

testgen/template/validate_tests/ex_write_test_val_errors.sql

@@ -0,0 +1,22 @@
+INSERT INTO test_results
+   (project_code, test_suite, test_type, test_definition_id,
+    schema_name, table_name, column_names, test_time, test_run_id,
+    input_parameters, result_code, result_message, result_measure)
+SELECT '{PROJECT_CODE}'   as project_code,
+       '{TEST_SUITE}'  as test_suite,
+       test_type,
+       id,
+       schema_name,
+       table_name,
+       column_name,
+       '{RUN_DATE}'    as test_time,
+       '{TEST_RUN_ID}' as test_run_id,
+    NULL as input_parameters,
+    0 as result_code,
+    -- TODO: show only missing columns referenced in this test
+    left('ERROR - TEST COLUMN MISSING: {MISSING_COLUMNS_NO_QUOTES}', 470) AS result_message,
+    NULL as result_measure
+  FROM test_definitions
+ WHERE test_active = '-1'
+   AND project_code = '{PROJECT_CODE}'
+   AND test_suite = '{TEST_SUITE}';