dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/gen_funny_cat_tests/gen_test_row_ct.sql

@@ -0,0 +1,57 @@
+-- Insert new tests where a locked test is not already present
+INSERT INTO test_definitions (project_code, table_groups_id, profile_run_id, test_type, test_suite, test_suite_id,
+                              schema_name, table_name,
+                              skip_errors, threshold_value,
+                              last_auto_gen_date, test_active, baseline_ct, profiling_as_of_date)
+WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date
+                    FROM profile_results p
+                  INNER JOIN profiling_runs r
+                     ON (p.profile_run_id = r.id)
+                    INNER JOIN test_suites tg
+                       ON p.project_code = tg.project_code
+                      AND p.connection_id = tg.connection_id
+                   WHERE p.project_code = '{PROJECT_CODE}'
+                     AND r.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                     AND tg.test_suite = '{TEST_SUITE}'
+                     AND p.run_date::DATE <= '{AS_OF_DATE}'
+                  GROUP BY r.table_groups_id),
+     curprof AS (SELECT p.*
+                   FROM last_run lr
+                 INNER JOIN profile_results p
+                    ON (lr.table_groups_id = p.table_groups_id
+                    AND lr.last_run_date = p.run_date) ),
+     locked AS (SELECT schema_name, table_name, column_name, test_type
+                  FROM test_definitions
+				     WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                   AND test_suite = '{TEST_SUITE}'
+                   AND lock_refresh = 'Y'),
+     newtests AS (SELECT project_code, table_groups_id, profile_run_id,
+                         'Row_Ct' AS test_type,
+                         '{TEST_SUITE}' AS test_suite,
+                         '{TEST_SUITE_ID}'::UUID AS test_suite_id,
+                         schema_name,
+                         table_name,
+                         MAX(record_ct) as record_ct
+                    FROM curprof c
+                  LEFT JOIN generation_sets s
+                     ON ('Row_Ct' = s.test_type
+                    AND  '{GENERATION_SET}' = s.generation_set)
+                   WHERE schema_name = '{DATA_SCHEMA}'
+                     AND functional_table_type LIKE '%cumulative%'
+                     AND (s.generation_set IS NOT NULL
+                      OR  '{GENERATION_SET}' = '')
+                  GROUP BY project_code, table_groups_id, profile_run_id,
+                           test_type, test_suite, schema_name, table_name )
+SELECT n.project_code, n.table_groups_id, n.profile_run_id,
+       n.test_type, n.test_suite, n.test_suite_id,
+       n.schema_name, n.table_name,
+       0 as skip_errors, record_ct AS threshold_value,
+       '{RUN_DATE}'::TIMESTAMP as last_auto_gen_date,
+       'Y' as test_active, record_ct as baseline_ct,
+       '{AS_OF_DATE}'::TIMESTAMP as profiling_as_of_date
+FROM newtests n
+LEFT JOIN locked l
+  ON (n.schema_name = l.schema_name
+ AND  n.table_name = l.table_name
+ AND  n.test_type = l.test_type)
+WHERE l.test_type IS NULL;

testgen/template/gen_funny_cat_tests/gen_test_row_ct_pct.sql

@@ -0,0 +1,59 @@
+-- Insert new tests where a locked test is not already present
+INSERT INTO test_definitions (project_code, table_groups_id, profile_run_id, test_type, test_suite, test_suite_id,
+                              schema_name, table_name, skip_errors,
+                              last_auto_gen_date, profiling_as_of_date, test_active,
+                              baseline_ct, threshold_value)
+WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date
+                    FROM profile_results p
+                  INNER JOIN profiling_runs r
+                     ON (p.profile_run_id = r.id)
+                    INNER JOIN test_suites tg
+                       ON p.project_code = tg.project_code
+                      AND p.connection_id = tg.connection_id
+                   WHERE p.project_code = '{PROJECT_CODE}'
+                     AND r.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                     AND tg.test_suite = '{TEST_SUITE}'
+                     AND p.run_date::DATE <= '{AS_OF_DATE}'
+                  GROUP BY r.table_groups_id),
+     curprof AS (SELECT p.*
+                   FROM last_run lr
+                 INNER JOIN profile_results p
+                    ON (lr.table_groups_id = p.table_groups_id
+                    AND lr.last_run_date = p.run_date) ),
+     locked AS (SELECT schema_name, table_name, column_name, test_type
+                  FROM test_definitions
+				     WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                   AND test_suite = '{TEST_SUITE}'
+                   AND lock_refresh = 'Y'),
+       newtests AS (
+                    SELECT project_code, table_groups_id, profile_run_id,
+                           'Row_Ct_Pct' AS test_type,
+                           '{TEST_SUITE}' AS test_suite,
+                           '{TEST_SUITE_ID}'::UUID AS test_suite_id,
+                           schema_name,
+                           table_name,
+                           MAX(record_ct) as record_ct
+                      FROM curprof
+                    LEFT JOIN generation_sets s
+                           ON ('Row_Ct_Pct' = s.test_type
+                          AND  '{GENERATION_SET}' = s.generation_set)
+                     WHERE schema_name = '{DATA_SCHEMA}'
+                       AND functional_table_type NOT ILIKE '%cumulative%'
+                       AND (s.generation_set IS NOT NULL
+                        OR  '{GENERATION_SET}' = '')
+                    GROUP BY project_code, table_groups_id, profile_run_id,
+                             test_type, test_suite, schema_name, table_name
+                    HAVING MAX(record_ct) >= 500)
+SELECT n.project_code, n.table_groups_id, n.profile_run_id,
+       n.test_type, n.test_suite, n.test_suite_id,
+       n.schema_name, n.table_name, 0 as skip_errors,
+       '{RUN_DATE}'::TIMESTAMP as last_auto_gen_date,
+       '{AS_OF_DATE}'::TIMESTAMP as profiling_as_of_date,
+       'Y' as test_active,
+       record_ct as baseline_ct, 0.5 AS threshold_value
+  FROM newtests n
+LEFT JOIN locked l
+  ON (n.schema_name = l.schema_name
+ AND  n.table_name = l.table_name
+ AND  n.test_type = l.test_type)
+WHERE l.test_type IS NULL;

testgen/template/generation/gen_delete_old_tests.sql

@@ -0,0 +1,5 @@
+DELETE FROM test_definitions
+ WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+   AND test_suite = '{TEST_SUITE}'
+   AND last_auto_gen_date IS NOT NULL
+   AND COALESCE(lock_refresh, 'N') <> 'Y';

testgen/template/generation/gen_insert_test_suite.sql

@@ -0,0 +1,5 @@
+INSERT INTO test_suites
+  (project_code, test_suite, connection_id, table_groups_id, test_suite_description,
+   component_type, component_key)
+VALUES ('{PROJECT_CODE}', '{TEST_SUITE}', {CONNECTION_ID}, '{TABLE_GROUPS_ID}', '{TEST_SUITE} Test Suite',
+        'dataset', '{TEST_SUITE}');

testgen/template/generation/gen_retrieve_or_insert_test_suite.sql

@@ -0,0 +1,58 @@
+WITH existing_rec
+ AS ( SELECT tg.project_code, tg.connection_id,
+             cc.sql_flavor,
+             cc.project_host,
+             cc.project_port,
+             cc.project_user,
+             cc.project_db,
+             tg.table_group_schema,
+             s.export_to_observability,
+             s.test_suite,
+             s.id as test_suite_id,
+             cc.url,
+             cc.connect_by_url,
+             CURRENT_TIMESTAMP AT TIME ZONE
+             'UTC' - CAST(tg.profiling_delay_days AS INTEGER) * INTERVAL '1 day' AS profiling_as_of_date
+        FROM table_groups tg
+             INNER JOIN connections cc
+                        ON (tg.connection_id = cc.connection_id)
+             INNER JOIN test_suites s
+                        ON (tg.id = s.table_groups_id
+                          AND '{TEST_SUITE}' = s.test_suite)
+       WHERE tg.id = '{TABLE_GROUPS_ID}' ),
+new_rec
+ AS ( INSERT INTO test_suites
+         (project_code, test_suite, connection_id, table_groups_id, test_suite_description,
+          component_type, component_key)
+      SELECT '{PROJECT_CODE}', '{TEST_SUITE}', {CONNECTION_ID}, '{TABLE_GROUPS_ID}', '{TEST_SUITE} Test Suite',
+              'dataset', '{TEST_SUITE}'
+      WHERE NOT EXISTS
+             (SELECT 1
+                FROM test_suites
+               WHERE table_groups_id = '{TABLE_GROUPS_ID}'
+                 AND test_suite = '{TEST_SUITE}')
+      RETURNING id as test_suite_id, test_suite, table_groups_id, export_to_observability )
+SELECT project_code, connection_id, sql_flavor,
+       project_host, project_port, project_user, project_db, table_group_schema,
+       export_to_observability, test_suite, test_suite_id, url, connect_by_url, profiling_as_of_date
+  FROM existing_rec
+ UNION ALL
+SELECT tg.project_code, tg.connection_id,
+       cc.sql_flavor,
+       cc.project_host,
+       cc.project_port,
+       cc.project_user,
+       cc.project_db,
+       tg.table_group_schema,
+       s.export_to_observability,
+       s.test_suite,
+       s.test_suite_id,
+       cc.url,
+       cc.connect_by_url,
+       CURRENT_TIMESTAMP AT TIME ZONE
+       'UTC' - CAST(tg.profiling_delay_days AS INTEGER) * INTERVAL '1 day' AS profiling_as_of_date
+  FROM new_rec s
+INNER JOIN table_groups tg
+   ON (s.table_groups_id = tg.id)
+INNER JOIN connections cc
+   ON (tg.connection_id = cc.connection_id);

testgen/template/generation/gen_standard_test_type_list.sql

@@ -0,0 +1,13 @@
+SELECT t.test_type,
+       t.selection_criteria,
+       t.default_parm_columns,
+       t.default_parm_values
+FROM test_types t
+LEFT JOIN generation_sets s
+  ON (t.test_type = s.test_type
+ AND  '{GENERATION_SET}' = s.generation_set)
+WHERE t.active = 'Y'
+  AND t.selection_criteria <> 'TEMPLATE' -- Also excludes NULL
+  AND (s.generation_set IS NOT NULL
+   OR  '{GENERATION_SET}' = '')
+ORDER BY test_type;

testgen/template/generation/gen_standard_tests.sql

@@ -0,0 +1,48 @@
+-- Insert new tests where a locked test is not already present
+INSERT INTO test_definitions (project_code, table_groups_id, profile_run_id, test_type, test_suite, test_suite_id,
+                              schema_name, table_name, column_name,
+                              skip_errors, test_active, last_auto_gen_date, profiling_as_of_date,
+                              {DEFAULT_PARM_COLUMNS} )
+WITH last_run AS (SELECT r.table_groups_id, MAX(run_date) AS last_run_date
+                    FROM profile_results p
+                  INNER JOIN profiling_runs r
+                     ON (p.profile_run_id = r.id)
+                    INNER JOIN test_suites tg
+                       ON p.project_code = tg.project_code
+                      AND p.connection_id = tg.connection_id
+                   WHERE p.project_code = '{PROJECT_CODE}'
+                     AND r.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                     AND tg.test_suite = '{TEST_SUITE}'
+                     AND p.run_date::DATE <= '{AS_OF_DATE}'
+                  GROUP BY r.table_groups_id),
+     curprof AS (SELECT p.*, datediff('MM', p.min_date, p.max_date)  as min_max_months, datediff('week', '1800-01-05'::DATE, p.max_date) - datediff('week', '1800-01-05'::DATE, p.min_date) as min_max_weeks
+                   FROM last_run lr
+                 INNER JOIN profile_results p
+                    ON (lr.table_groups_id = p.table_groups_id
+                    AND lr.last_run_date = p.run_date) ),
+     locked AS (SELECT schema_name, table_name, column_name
+                  FROM test_definitions
+				     WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+                   AND test_suite = '{TEST_SUITE}'
+				       AND test_type = '{TEST_TYPE}'
+                   AND lock_refresh = 'Y'),
+     newtests AS (SELECT *
+                  FROM curprof
+                  WHERE schema_name = '{DATA_SCHEMA}'
+                    AND {SELECTION_CRITERIA} )
+SELECT '{PROJECT_CODE}' as project_code,
+       '{TABLE_GROUPS_ID}'::UUID as table_groups_id,
+       n.profile_run_id,
+       '{TEST_TYPE}' AS test_type,
+       '{TEST_SUITE}' AS test_suite,
+       '{TEST_SUITE_ID}' AS test_suite_id,
+       n.schema_name, n.table_name, n.column_name,
+       0 as skip_errors, 'Y' as test_active, '{RUN_DATE}'::TIMESTAMP as last_auto_gen_date,
+       '{AS_OF_DATE}'::TIMESTAMP as profiling_as_of_date,
+       {DEFAULT_PARM_VALUES}
+FROM newtests n
+LEFT JOIN locked l
+  ON (n.schema_name = l.schema_name
+ AND  n.table_name = l.table_name
+ AND  n.column_name = l.column_name)
+WHERE l.schema_name IS NULL;

testgen/template/get_entities/get_connection.sql

@@ -0,0 +1,21 @@
+SELECT
+	id,
+	project_code as project_key,
+	connection_id,
+	connection_name,
+	sql_flavor,
+	project_host,
+	project_port,
+	project_user,
+	project_db,
+    project_pw_encrypted,
+	max_threads,
+	max_query_chars,
+    project_qc_schema,
+    url,
+    connect_by_url,
+    connect_by_key,
+    private_key,
+    private_key_passphrase
+FROM connections
+WHERE connection_id = {CONNECTION_ID};

testgen/template/get_entities/get_connections_list.sql

@@ -0,0 +1,9 @@
+SELECT
+	id,
+	project_code as project_key,
+	connection_id,
+	sql_flavor,
+	project_host,
+	max_threads,
+	max_query_chars
+FROM connections;

testgen/template/get_entities/get_latest.sql

@@ -0,0 +1,4 @@
+SELECT id
+FROM {ENTITY}
+ORDER BY {TIME_COLUMN} DESC
+LIMIT 1;

testgen/template/get_entities/get_profile.sql

@@ -0,0 +1,12 @@
+SELECT p.profile_run_id,
+       p.run_date,
+       p.schema_name,
+       p.table_name,
+--        p."position",
+       p.column_name,
+       p.general_type,
+       p.column_type,
+       p.datatype_suggestion
+FROM profile_results p
+WHERE profile_run_id = '{PROFILE_RUN_ID}'::UUID
+ORDER BY p.schema_name, p.table_name, p.position;

testgen/template/get_entities/get_profile_info.sql

@@ -0,0 +1,17 @@
+/*profile-info: project-code, connection-id
+Output: latest profiling details
+Optional: run_date (==profiling-run-id==), table-name, column-name, date-from, date-thru*/
+
+SELECT profile_run_id,
+       run_date,
+       schema_name,
+       table_name,
+--        position,
+       column_name,
+       general_type,
+       column_type,
+       functional_data_type
+  FROM profile_results
+ WHERE table_name ILIKE '{TABLE_NAME}'
+   AND profile_run_id = '{PROFILE_RUN}'::UUID
+ ORDER BY table_name, position;

testgen/template/get_entities/get_profile_list.sql

@@ -0,0 +1,17 @@
+/*profile-list: project-code, connection-id
+Output: list of all profiling runs conducted from profiling_runs
+Optional: table-name*/
+
+SELECT p.id as profile_run_id,
+       p.project_code as project_key,
+       schema_name, p.table_groups_id,
+       profiling_starttime as start_time, status,
+       COUNT(DISTINCT table_name) as tables,
+       COUNT(DISTINCT table_name || '.' || column_name) as columns
+  FROM profiling_runs p
+INNER JOIN profile_results r
+   ON (p.id = r.profile_run_id)
+ WHERE p.table_groups_id = '{TABLE_GROUPS_ID}'::UUID
+GROUP BY p.id, p.project_code, p.connection_id, schema_name, p.table_groups_id,
+       profiling_starttime, status
+ORDER BY profiling_starttime DESC;

testgen/template/get_entities/get_profile_screen.sql

@@ -0,0 +1,275 @@
+WITH
+   profiling as ( SELECT *
+                    FROM profile_results
+                   WHERE profile_run_id = '{PROFILE_RUN}'::UUID
+                     AND table_name ILIKE '{TABLE_NAME}' ),
+   profile_date as (SELECT MAX(run_date) as run_date
+                      FROM profiling),
+   mults AS ( SELECT p.project_code,
+                      p.table_groups_id,
+                      p.run_date,
+                      p.schema_name,
+                      p.column_name,
+                      COUNT(*)                      AS column_ct,
+                      COUNT(DISTINCT p.column_type) AS type_ct,
+                      COUNT(DISTINCT p.general_type) AS general_type_ct,
+                      MIN(p.column_type::TEXT)      AS min_type,
+                      MAX(p.column_type::TEXT)      AS max_type,
+                      MIN(p.distinct_pattern_ct)    AS min_pattern_ct,
+                      MAX(p.distinct_pattern_ct)    AS max_pattern_ct,
+                      SUM(p.distinct_pattern_ct)    AS sum_pattern_ct
+                 FROM profile_results p
+               INNER JOIN profile_date d
+                  ON (p.run_date <= d.run_date)
+                GROUP BY p.project_code, p.table_groups_id, p.run_date, p.schema_name, p.column_name
+               HAVING COUNT(*) > 1 ),
+   results as (SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Suggested Data Type'               AS qualification_test,
+                      p.datatype_suggestion::VARCHAR(200) AS detail
+                 FROM profiling p
+                WHERE LOWER(p.column_type) <> LOWER(p.datatype_suggestion)
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Non-Standard Blank Values'   AS qualification_test,
+                      (((('Filled Values: ' || p.filled_value_ct::VARCHAR(10)) || ', Null: ') ||
+                        p.null_value_ct::VARCHAR(10)) || ', Empty String: ') ||
+                      p.zero_length_ct::VARCHAR(10) AS detail
+                 FROM profiling p
+                WHERE p.filled_value_ct > 0
+                   OR p.zero_length_ct > 0
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Invalid Zip Code Format'      AS qualification_test,
+                      (((('Min Length: ' || p.min_length::VARCHAR(10)) || ', Max Length: ') ||
+                        p.max_length::VARCHAR(10)) || ', Filled Values: ') ||
+                      p.filled_value_ct::VARCHAR(10) AS detail
+                 FROM profiling p
+                WHERE p.column_name ILIKE '%zip%'
+                  AND (p.general_type <> 'A' OR p.filled_value_ct > 0 OR p.min_length >= 1 AND p.min_length <= 4 OR
+                       p.max_length > 10)
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Multiple Data Types per Column Name: Strict' AS qualification_test,
+                      (((((('Found ' || m.column_ct::VARCHAR(10)) || ' columns, ') ||
+                          m.type_ct::VARCHAR(10)) || ' types, ') || m.min_type) || ' to ') ||
+                      m.max_type                            AS detail
+                 FROM profiling p
+                 INNER JOIN mults m
+                           ON p.project_code = m.project_code
+                                 AND p.table_groups_id = m.table_groups_id
+                                 AND p.schema_name = m.schema_name
+                                 AND p.column_name = m.column_name
+                                 AND 1 < m.type_ct
+                                 AND 1 = m.general_type_ct
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Multiple Data Types per Column Name: Loose' AS qualification_test,
+                      (((((('Found ' || m.column_ct::VARCHAR(10)) || ' columns, ') ||
+                          m.type_ct::VARCHAR(10)) || ' types, ') || m.min_type) || ' to ') ||
+                      m.max_type                            AS detail
+                 FROM profiling p
+                 INNER JOIN mults m
+                           ON p.project_code = m.project_code
+                                 AND p.table_groups_id = m.table_groups_id
+                                 AND p.schema_name = m.schema_name
+                                 AND p.column_name = m.column_name
+                  AND 1 < m.general_type_ct
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'No column values present'    AS qualification_test,
+                      (((('Null: ' || p.null_value_ct::VARCHAR(10)) || ', Filled: ') ||
+                        p.filled_value_ct::VARCHAR(10)) || ', Zero Len: ') ||
+                      p.zero_length_ct::VARCHAR(10) AS detail
+                 FROM profiling p
+                WHERE (p.null_value_ct + p.filled_value_ct + p.zero_length_ct) = p.record_ct
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Pattern Inconsistency'       AS qualification_test,
+                      'Pattern: ' || p.top_patterns AS detail
+                 FROM profiling p
+                      LEFT JOIN mults m
+                                ON p.project_code = m.project_code
+                                      AND p.table_groups_id = m.table_groups_id
+                                      AND p.schema_name = m.schema_name
+                                      AND p.column_name = m.column_name
+                WHERE p.general_type = 'A'
+                  AND p.max_length > 3
+                  AND p.value_ct > (p.numeric_ct + p.filled_value_ct)
+                  AND (p.distinct_pattern_ct < 3 OR m.min_pattern_ct < 3)
+                  AND (p.distinct_pattern_ct > 1 OR m.sum_pattern_ct > m.column_ct::NUMERIC)
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Leading Spaces'                          AS qualification_test,
+                      'Found: ' || p.lead_space_ct::VARCHAR(10) AS detail
+                 FROM profiling p
+                WHERE p.lead_space_ct > 0
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Quoted Values'                          AS qualification_test,
+                      'Found: ' || p.quoted_value_ct::VARCHAR(10) AS detail
+                 FROM profiling p
+                WHERE quoted_value_ct > 0
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Mostly Numeric in String'                   AS qualification_test,
+                      'Numeric Percent: ' || ROUND(100.0 * p.numeric_ct::NUMERIC(18, 5) / p.value_ct::NUMERIC(18, 5),
+                                                   2)::VARCHAR(40) AS detail
+                 FROM profiling p
+                WHERE p.general_type = 'A'
+                  AND p.column_name NOT ILIKE '%zip'
+                  AND p.column_name NOT ILIKE '%id'
+                  AND p.column_name NOT ILIKE '%num'
+                  AND p.column_name NOT ILIKE '%sk'
+                  AND p.value_ct > p.numeric_ct
+                  AND p.numeric_ct::NUMERIC > (0.95 * p.value_ct::NUMERIC)
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Mostly Dates in String'                                                              AS qualification_test,
+                      'Date Percent: ' ||
+                      ROUND(100.0 * p.date_ct::NUMERIC(18, 5) / p.value_ct::NUMERIC(18, 5), 2)::VARCHAR(40) AS detail
+                 FROM profiling p
+                WHERE p.general_type = 'A'
+                  AND p.value_ct > p.date_ct
+                  AND p.date_ct::NUMERIC > (0.95 * p.value_ct::NUMERIC)
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Mostly not null, empty or filled values'             AS qualification_test,
+                      (p.record_ct - (p.value_ct - p.zero_length_ct - p.filled_value_ct))::VARCHAR(20) ||
+                      ' of ' || p.record_ct::VARCHAR(20) || ' blank values' AS detail
+                 FROM profiling p
+                WHERE (p.value_ct - p.zero_length_ct - p.filled_value_ct)::FLOAT / p.record_ct::FLOAT > 0.97
+                  AND (p.value_ct - p.zero_length_ct - p.filled_value_ct) < p.record_ct
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Mostly one value'                  AS qualification_test,
+                      'Freq | Value: ' || top_freq_values AS detail
+                 FROM profiling p
+                WHERE (100.0 * fn_parsefreq(p.top_freq_values, 1, 2)::FLOAT /
+                       p.value_ct::FLOAT) > 97::FLOAT
+                  AND (100.0 * fn_parsefreq(p.top_freq_values, 1, 2)::FLOAT /
+                       p.value_ct::FLOAT) < 100::FLOAT
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Too many boolean values'         AS qualification_test,
+                      'Top Freq: ' || p.top_freq_values AS detail
+                 FROM profiling p
+                WHERE p.general_type = 'A'
+                  AND p.distinct_value_ct >= 3
+                  AND p.distinct_value_ct <= 6
+                  AND (LOWER(p.top_freq_values) ILIKE '%| true |%' AND
+                       LOWER(p.top_freq_values) ILIKE '%| false |%' OR
+                       LOWER(p.top_freq_values) ILIKE '%| yes |&' AND LOWER(p.top_freq_values) ILIKE '%| no |&')
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Potential Duplicates'            AS qualification_test,
+                      'Top Freq: ' || p.top_freq_values AS detail
+                 FROM profiling p
+                WHERE p.distinct_value_ct > 1000
+                  AND fn_parsefreq(p.top_freq_values, 1, 2)::BIGINT BETWEEN 2 AND 4
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Non-Standardized Values in Categories/Codes'   AS qualification_test,
+                      'Distinct Values: ' || p.distinct_value_ct::VARCHAR
+                         || ', Standardized: ' || p.distinct_std_value_ct::VARCHAR AS detail
+                 FROM profiling p
+                WHERE p.general_type = 'A'
+                  AND p.distinct_std_value_ct <> p.distinct_value_ct
+                  AND p.functional_data_type IN ('Category','Code')
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Unlikely Date Values out of normal range'   AS qualification_test,
+                      'Date Range: ' || p.min_date::VARCHAR || ' thru ' || p.max_date AS detail
+                 FROM profiling p
+                WHERE p.general_type = 'D'
+                  AND (p.min_date BETWEEN '0001-01-02'::DATE AND '1900-01-01'::DATE
+                   OR p.max_date > CURRENT_DATE + INTERVAL '30 year')
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      '(Table-Wide)' as column_name,
+                      '' as column_type,
+                      'Recency - No Table Dates within 1 Year'   AS qualification_test,
+                      'Most Recent Date: ' || MAX(p.max_date)::VARCHAR AS detail
+                 FROM profiling p
+                WHERE p.general_type = 'D'
+               GROUP BY p.schema_name, p.table_name
+               HAVING MAX(p.max_date) < CURRENT_DATE - INTERVAL '1 year'
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Unexpected column contains US States' AS qualification_test,
+                      'Value Range: ' || p.min_text || ' thru ' || max_text AS detail
+                 FROM profiling p
+                WHERE p.std_pattern_match = 'STATE_USA'
+                      AND p.distinct_value_ct > 5
+                      AND NOT (p.column_name ILIKE '%state%' OR p.column_name ILIKE '%_st')
+                UNION ALL
+               SELECT p.schema_name,
+                      p.table_name,
+                      p.column_name,
+                      p.column_type,
+                      'Unexpected column contains emails' AS qualification_test,
+                      'Value Range: ' || p.min_text || ' thru ' || max_text AS detail
+                 FROM v_latest_profile_results p
+                WHERE p.table_groups_id = 'a6b876b5-750b-49d3-b4e2-a599eceefe84'::UUID
+                  AND p.std_pattern_match = 'EMAIL'
+                      AND NOT (column_name ILIKE '%email%' OR column_name ILIKE '%addr%')
+                )
+SELECT table_name, column_name, column_type,
+       qualification_test as screening_test, detail
+  FROM results
+ WHERE qualification_test <> 'Suggested Data Type'
+ ORDER BY qualification_test, schema_name, table_name, column_name;

testgen/template/get_entities/get_project_list.sql

@@ -0,0 +1,6 @@
+SELECT id,
+       project_code as project_key,
+       project_name,
+       observability_api_key
+FROM projects
+ORDER BY effective_from_date desc;

testgen/template/get_entities/get_table_group_list.sql

@@ -0,0 +1,10 @@
+SELECT
+    id as table_group_id,
+    project_code as project_key,
+    connection_id,
+    table_group_schema,
+    profiling_table_set,
+    profiling_include_mask as include_mask,
+    profiling_exclude_mask as exclude_mask
+FROM table_groups
+where project_code = '{PROJECT_CODE}';

testgen/template/get_entities/get_test_generation_list.sql

@@ -0,0 +1,18 @@
+/*test-generation-list: project-code, test-suite
+Output: list all test generation runs based on last_auto_run_date
+Optional: n/a*/
+
+Select test_suite as test_suite_key,
+       table_groups_id,
+       last_auto_gen_date,
+       d.profiling_as_of_date,
+       lock_refresh,
+       COUNT(DISTINCT schema_name || '.' || table_name) as tables,
+       COUNT(DISTINCT schema_name || '.' || table_name || '.' || column_name) as columns,
+       COUNT(*) as tests
+from test_definitions d
+where d.project_code = '{PROJECT_CODE}'
+  and test_suite = '{TEST_SUITE}'
+  and last_auto_gen_date IS NOT NULL
+GROUP BY table_groups_id, project_code, test_suite, last_auto_gen_date, d.profiling_as_of_date, lock_refresh
+order by last_auto_gen_date desc;