dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/profiling/pii_flag.sql

@@ -0,0 +1,133 @@
+-- Primary Screen:  Alpha
+WITH screen
+   AS ( SELECT id  AS profile_results_id,
+               table_name, column_name,
+               CASE
+                  WHEN functional_data_type IN ('Person Full Name', 'Person Given Name', 'Person Last Name') THEN 'B/NAME/Individual'
+                  
+                  WHEN LOWER(column_name) SIMILAR TO '%(maiden|surname)%'                      THEN 'B/NAME/Individual'
+                  
+                  WHEN functional_data_type = 'Historical Date'
+                     AND LOWER(column_name) SIMILAR TO '%(dob|birth)%'                         THEN 'B/DEMO/Birthdate'
+                  
+                  WHEN LOWER(column_name)
+                     SIMILAR TO '%(nationality|race|ethnicity|gender|sex|marital)%'            THEN 'B/DEMO/Demographic'
+                  
+                  WHEN LOWER(column_name) ILIKE '%med%record%'                                 THEN 'A/DEMO/Medical'
+                  
+                  WHEN LOWER(column_name) SIMILAR TO '%(password|pwd|auth)%'                   THEN 'A/ID/Security'
+                  
+                  WHEN max_length < 10
+                   AND avg_embedded_spaces < 0.1
+                   AND (column_name ILIKE 'pin%' OR column_name ILIKE '%pin')                  THEN 'A/ID/Security'
+                  
+                  WHEN std_pattern_match = 'SSN'
+                     AND LOWER(column_name) SIMILAR TO '%(ss|soc|sec)%'                        THEN 'A/ID/SSN'
+                  
+                  WHEN TRIM(fn_parsefreq(top_patterns, 1, 2))
+                          IN ('NNNNNNNNN', 'NNN-NN-NNNN', 'NNN NN NNNN')
+                     AND LEFT(min_text, 1) = '9'
+                     AND avg_length BETWEEN 8.8 AND 11.2
+                     AND LOWER(column_name) SIMILAR TO '%(tax|tin|fed)%'                       THEN 'A/ID/Tax'
+                  
+                  WHEN TRIM(fn_parsefreq(top_patterns, 1, 2))
+                          IN ('NNNNNNNNN', 'ANNNNNNNN')
+                     AND avg_length BETWEEN 8.8 AND 9.2
+                     AND LOWER(column_name) SIMILAR TO '%(passp|pp)%'                          THEN 'A/ID/Passport'
+                  
+                  WHEN std_pattern_match = 'CREDIT_CARD'
+                     AND LOWER(column_name) SIMILAR TO '%(credit|card|cc|acct|account)%'       THEN 'A/ID/Credit'
+                  
+                  WHEN TRIM(fn_parsefreq(top_patterns, 1, 2))
+                          ILIKE '[Aa]{6}[A-Za-z0-9]{2}N{0,3}'
+                     AND TRIM(fn_parsefreq(top_patterns, 2, 2))
+                          ILIKE '[Aa]{6}[A-Za-z0-9]{2}N{0,3}'
+                     AND avg_length BETWEEN 7.8 AND 11.2
+                     AND LOWER(column_name) SIMILAR TO '%(swift|bic)%'                         THEN 'A/ID/Bank'
+                  
+                  WHEN max_length <= 34
+                     AND UPPER(LEFT(TRIM(fn_parsefreq(top_patterns, 1, 2)), 2))
+                          = 'AA'
+                     AND (column_name ILIKE 'iban%' OR column_name ILIKE '%iban')              THEN 'A/ID/Bank'
+                  
+                  WHEN avg_length BETWEEN 5 AND 20
+                     AND LOWER(column_name) SIMILAR TO '%(bank|checking|saving|debit)%'        THEN 'A/ID/Bank'
+                  
+                  WHEN avg_embedded_spaces < 0.5
+                     AND avg_length < 20
+                     AND (LOWER(column_name) SIMILAR TO '%(dr|op)%lic%'
+                        OR LOWER(column_name) SIMILAR TO '%(driver|license|operator)%')        THEN 'A/ID/License'
+                  
+                  WHEN LOWER(column_name) IN ('patient_id', 'pat_id')                          THEN 'A/ID/Medical'
+
+                  WHEN LOWER(column_name) IN ('member_id')                                     THEN 'B/ID/Commercial'
+
+               END AS pii_flag
+          
+          FROM profile_results p
+         WHERE profile_run_id = '{PROFILE_RUN_ID}'
+           AND general_type = 'A' )
+UPDATE profile_results
+   SET pii_flag = screen.pii_flag
+  FROM screen
+ WHERE screen.pii_flag > ''
+   AND profile_results.id = screen.profile_results_id;
+
+-- Secondary Screen - Alpha
+  WITH table_pii_counts
+          AS ( SELECT table_name, COUNT(pii_flag) AS pii_ct
+                 FROM profile_results
+                WHERE profile_run_id = '{PROFILE_RUN_ID}'
+                GROUP BY table_name ),
+       screen
+          AS ( SELECT id  AS profile_results_id,
+                      p.table_name, p.column_name,
+                      CASE
+                         WHEN functional_data_type = 'Email'                                           THEN 'B/CONTACT/Email'
+                         WHEN functional_data_type IN ('Address', 'City', 'State', 'Zip')
+                                                                                                       THEN 'B/CONTACT/Address'
+                         WHEN functional_data_type = 'Phone'
+                                                                                                       THEN 'B/CONTACT/Phone'
+                         
+                         WHEN LOWER(column_name) SIMILAR TO '%(insur|health|med|patient)%'
+                                                                                                       THEN 'A/DEMO/Medical'
+                         
+                         WHEN LOWER(column_name) SIMILAR TO '%(vehicle|vin|auto|car)%'
+                            AND avg_length BETWEEN 16 AND 18
+                            AND max_length < 20
+                            AND TRIM(fn_parsefreq(top_patterns, 1, 2))
+                                   = 'AAANAAAAANNNNNNNN'                                               THEN 'B/ID/Auto'
+                         
+                         WHEN LOWER(column_name) SIMILAR TO
+                              '%(voice|fingerprint|retina|auth|biometric|iris|face_recog)%'
+                                                                                                       THEN 'A/ID/Security'
+                         
+                         WHEN LOWER(column_name) = 'dna'
+                            OR LOWER(column_name) ILIKE '%\_dna'
+                            OR LOWER(column_name) ILIKE 'dna\_%'
+                                                                                                       THEN 'A/DEMO/Demographic'
+                         
+                         WHEN column_name ILIKE '%rout%'
+                            AND avg_length BETWEEN 8.8 AND 11.2
+                            AND TRIM(fn_parsefreq(top_patterns, 1, 2))
+                                   IN ('NNNNNNNNN', 'NNNN-NNNN-N')                                     THEN 'C/ID/Bank'
+                         
+                         WHEN LOWER(column_name) SIMILAR TO '%(salary|income|wage)%'
+                                                                                                       THEN 'B/DEMO/Financial'
+                         
+                         WHEN LOWER(column_name) SIMILAR TO '%(user_id|userid)%'
+                                                                                                       THEN 'C/ID/Security'
+                      
+                      END AS pii_flag
+                 FROM profile_results p
+                      INNER JOIN table_pii_counts t
+                                 ON (p.table_name = t.table_name)
+                WHERE p.profile_run_id = '{PROFILE_RUN_ID}'
+                  AND p.general_type = 'A'
+                  AND p.pii_flag IS NULL
+                  AND t.pii_ct > 1 )
+UPDATE profile_results
+   SET pii_flag = screen.pii_flag
+  FROM screen
+ WHERE screen.pii_flag > ''
+   AND profile_results.id = screen.profile_results_id;

testgen/template/profiling/profile_anomalies_screen_column.sql

@@ -0,0 +1,22 @@
+INSERT INTO profile_anomaly_results
+   (project_code, table_groups_id, profile_run_id, anomaly_id,
+    schema_name, table_name, column_name, column_type, detail)
+SELECT p.project_code,
+       p.table_groups_id,
+       p.profile_run_id,
+       '{ANOMALY_ID}' as anomaly_id,
+       p.schema_name,
+       p.table_name,
+       p.column_name,
+       p.column_type,
+       {DETAIL_EXPRESSION} AS detail
+  FROM profile_results p
+LEFT JOIN v_inactive_anomalies i
+  ON (p.table_groups_id = i.table_groups_id
+ AND  p.schema_name = i.schema_name
+ AND  p.table_name = i.table_name
+ AND  p.column_name = i.column_name
+ AND  '{ANOMALY_ID}' = i.anomaly_id)
+ WHERE p.profile_run_id = '{PROFILE_RUN_ID}'::UUID
+   AND i.anomaly_id IS NULL
+   AND {ANOMALY_CRITERIA};

testgen/template/profiling/profile_anomalies_screen_multi_column.sql

@@ -0,0 +1,58 @@
+WITH mults AS (   SELECT p.project_code,
+                         p.table_groups_id,
+                         p.schema_name,
+                         p.column_name,
+                         COUNT(*)                       AS column_ct,
+                         COUNT(DISTINCT p.column_type)  AS type_ct,
+                         COUNT(DISTINCT p.general_type) AS general_type_ct,
+                         MIN(p.column_type::TEXT)       AS min_type,
+                         MAX(p.column_type::TEXT)       AS max_type,
+                         MIN(p.distinct_pattern_ct)     AS min_pattern_ct,
+                         MAX(p.distinct_pattern_ct)     AS max_pattern_ct,
+                         SUM(p.distinct_pattern_ct)     AS sum_pattern_ct,
+                         STRING_AGG(table_name, ', ' order by table_name) as table_list,
+                         MAX(RIGHT(REPEAT('0', 20) || SPLIT_PART(p.top_patterns, '|', 1), 20) || '|' || SPLIT_PART(p.top_patterns, '|', 2) )as very_top_pattern
+                    FROM profile_results p
+                    WHERE p.profile_run_id = '{PROFILE_RUN_ID}'::UUID
+                   GROUP BY p.project_code, p.table_groups_id, schema_name, p.column_name
+                  HAVING COUNT(*) > 1 ),
+    subset AS
+     (
+            SELECT p.project_code,
+                   p.table_groups_id,
+                   p.profile_run_id,
+                   '{ANOMALY_ID}' as anomaly_id,
+                   p.schema_name,
+                   p.table_name,
+                   p.column_name,
+                   p.column_type,
+                   p.top_patterns,
+                   ltrim(m.very_top_pattern, '0') as very_top_pattern,
+                   m.table_list,
+                   {DETAIL_EXPRESSION} AS detail
+              FROM profile_results p
+            INNER JOIN mults m
+               ON p.project_code = m.project_code
+              AND p.table_groups_id = m.table_groups_id
+              AND p.schema_name = m.schema_name
+              AND p.column_name = m.column_name
+            LEFT JOIN v_inactive_anomalies i
+              ON (p.table_groups_id = i.table_groups_id
+             AND  p.schema_name = i.schema_name
+             AND  p.table_name = i.table_name
+             AND  p.column_name = i.column_name
+             AND  '{ANOMALY_ID}' = i.anomaly_id)
+             WHERE p.profile_run_id = '{PROFILE_RUN_ID}'::UUID
+               AND i.anomaly_id IS NULL
+               AND {ANOMALY_CRITERIA}
+    )
+INSERT INTO profile_anomaly_results
+   (project_code, table_groups_id, profile_run_id, anomaly_id,
+    schema_name, table_name, column_name, column_type, detail)
+SELECT project_code, table_groups_id, profile_run_id, anomaly_id,
+       schema_name, '(multi-table)' as table_name,
+       column_name, '(multiple)' as column_type,
+       detail  || ' , Tables: ' || table_list AS detail
+FROM subset
+GROUP BY project_code, table_groups_id, profile_run_id, anomaly_id,
+         schema_name, column_name, table_list, detail;

testgen/template/profiling/profile_anomalies_screen_table.sql

@@ -0,0 +1,22 @@
+INSERT INTO profile_anomaly_results
+   (project_code, table_groups_id, profile_run_id, anomaly_id,
+    schema_name, table_name, column_name, detail, disposition)
+SELECT p.project_code,
+       p.table_groups_id,
+       p.profile_run_id,
+       '{ANOMALY_ID}' as anomaly_id,
+       p.schema_name,
+       p.table_name,
+       '(Table)' as column_name,
+       {DETAIL_EXPRESSION} AS detail,
+       CASE WHEN i.anomaly_id IS NULL THEN NULL ELSE 'Inactive' END as disposition
+  FROM profile_results p
+LEFT JOIN v_inactive_anomalies i
+  ON (p.table_groups_id = i.table_groups_id
+ AND  p.schema_name = i.schema_name
+ AND  p.table_name = i.table_name
+ AND  '{ANOMALY_ID}' = i.anomaly_id)
+ WHERE p.profile_run_id = '{PROFILE_RUN_ID}'::UUID
+GROUP BY p.project_code, p.table_groups_id, p.profile_run_id,
+         p.schema_name, p.table_name
+HAVING {ANOMALY_CRITERIA};

testgen/template/profiling/profile_anomalies_screen_table_dates.sql

@@ -0,0 +1,30 @@
+INSERT INTO profile_anomaly_results
+   (project_code, table_groups_id, profile_run_id, anomaly_id,
+    schema_name, table_name, column_name, detail)
+SELECT p.project_code,
+       p.table_groups_id,
+       p.profile_run_id,
+       '{ANOMALY_ID}' as anomaly_id,
+       p.schema_name,
+       p.table_name,
+       CASE
+         WHEN COUNT(p.column_name) > 2 THEN '(multi-column)'
+         ELSE STRING_AGG(p.column_name, ', ' ORDER BY p.position)
+       END as column_name,
+       {DETAIL_EXPRESSION}
+         || CASE
+              WHEN COUNT(p.column_name) > 2 THEN ', Columns: ' || STRING_AGG(p.column_name, ', ' ORDER BY p.position)
+              ELSE ''
+            END as detail
+  FROM profile_results p
+LEFT JOIN v_inactive_anomalies i
+  ON (p.table_groups_id = i.table_groups_id
+ AND  p.schema_name = i.schema_name
+ AND  p.table_name = i.table_name
+ AND  '{ANOMALY_ID}' = i.anomaly_id)
+ WHERE p.profile_run_id = '{PROFILE_RUN_ID}'::UUID
+   AND i.anomaly_id IS NULL
+   AND p.general_type = 'D'
+GROUP BY p.project_code, p.table_groups_id, p.profile_run_id,
+         p.schema_name, p.table_name, i.anomaly_id
+HAVING {ANOMALY_CRITERIA};

testgen/template/profiling/profile_anomalies_screen_variants.sql

@@ -0,0 +1,40 @@
+INSERT INTO profile_anomaly_results
+   (project_code, table_groups_id, profile_run_id, anomaly_id,
+    schema_name, table_name, column_name, column_type, detail)
+WITH all_matches
+   AS ( SELECT p.project_code,
+               p.table_groups_id,
+               p.profile_run_id,
+               p.schema_name,
+               p.table_name,
+               p.column_name,
+               p.column_type,
+               fn_extract_distinct_items(STRING_AGG(fn_extract_intersecting_items(LOWER(fn_extract_top_values(p.top_freq_values)),
+                                                                                  v.check_values, '|'),
+                                                    '|'),
+                                         '|') AS intersect_list
+          FROM profile_results p
+               CROSS JOIN variant_codings v
+               LEFT JOIN v_inactive_anomalies i
+                         ON (p.table_groups_id = i.table_groups_id
+                            AND p.schema_name = i.schema_name
+                            AND p.table_name = i.table_name
+                            AND p.column_name = i.column_name
+                            AND '{ANOMALY_ID}' = i.anomaly_id)
+         WHERE p.profile_run_id = '{PROFILE_RUN_ID}'::UUID
+           AND {ANOMALY_CRITERIA}
+           AND p.top_freq_values > ''
+           AND i.anomaly_id IS NULL
+           AND fn_count_intersecting_items(LOWER(fn_extract_top_values(p.top_freq_values)), v.check_values, '|') > 1
+         GROUP BY p.project_code,
+            p.table_groups_id,
+            p.profile_run_id,
+            p.schema_name,
+            p.table_name,
+            p.column_name,
+            p.column_type )
+SELECT project_code, table_groups_id, profile_run_id,
+       '{ANOMALY_ID}'  AS anomaly_id,
+       schema_name, table_name, column_name, column_type,
+       {DETAIL_EXPRESSION} AS detail
+  FROM all_matches;

testgen/template/profiling/profile_anomaly_types_get.sql

@@ -0,0 +1,3 @@
+SELECT id, anomaly_type, data_object, anomaly_criteria, detail_expression
+  FROM profile_anomaly_types t
+ORDER BY id;

testgen/template/profiling/project_get_table_sample_count.sql

@@ -0,0 +1,22 @@
+SELECT '{SAMPLING_TABLE}' as schema_table,
+       CASE
+         WHEN count(*) <= {PROFILE_SAMPLE_MIN_COUNT}
+           THEN -1
+           ELSE
+                CASE
+                  WHEN ROUND(CAST({PROFILE_SAMPLE_PERCENT} as FLOAT) * CAST(COUNT(*) as FLOAT) / 100.0, 0) > {PROFILE_SAMPLE_MIN_COUNT}
+                    THEN LEAST(999000, ROUND(CAST({PROFILE_SAMPLE_PERCENT} as FLOAT) * CAST(COUNT(*) as FLOAT) / 100.0, 0))
+                    ELSE {PROFILE_SAMPLE_MIN_COUNT}
+                END
+       END as sample_count,
+       CASE
+         WHEN count(*) <= {PROFILE_SAMPLE_MIN_COUNT}
+           THEN 1
+           ELSE (CAST(COUNT(*) as FLOAT)
+                  / CASE
+                      WHEN ROUND(CAST({PROFILE_SAMPLE_PERCENT} as FLOAT) * CAST(COUNT(*) as FLOAT) / 100.0, 0) > {PROFILE_SAMPLE_MIN_COUNT}
+                        THEN LEAST(999000, ROUND(CAST({PROFILE_SAMPLE_PERCENT} as FLOAT) * CAST(COUNT(*) as FLOAT) / 100.0, 0))
+                        ELSE {PROFILE_SAMPLE_MIN_COUNT}
+                    END )
+       END as sample_ratio
+from {SAMPLING_TABLE};

testgen/template/profiling/project_profile_run_record_insert.sql

@@ -0,0 +1,8 @@
+INSERT INTO profiling_runs (id, project_code, connection_id, table_groups_id, profiling_starttime, process_id)
+(SELECT '{PROFILE_RUN_ID}' :: UUID  as id,
+        '{PROJECT_CODE}' as project_code,
+         {CONNECTION_ID} as connection_id,
+        '{TABLE_GROUPS_ID}' :: UUID as table_groups_id,
+        '{RUN_DATE}' as profiling_starttime,
+        '{PROCESS_ID}' as process_id
+     );

testgen/template/profiling/project_profile_run_record_update.sql

@@ -0,0 +1,5 @@
+UPDATE profiling_runs
+SET status = CASE WHEN length('{EXCEPTION_MESSAGE}') = 0 then 'Complete' else 'Error' end,
+    profiling_endtime = '{NOW}',
+    log_message = '{EXCEPTION_MESSAGE}'
+where id = '{PROFILE_RUN_ID}' :: UUID;

testgen/template/profiling/project_profile_run_record_update_status.sql

@@ -0,0 +1,5 @@
+UPDATE profiling_runs
+SET status = '{STATUS}',
+    profiling_endtime = '{NOW}',
+    log_message = '{EXCEPTION_MESSAGE}'
+where id = '{PROFILE_RUN_ID}' :: UUID;

testgen/template/profiling/project_update_profile_results_to_estimates.sql

@@ -0,0 +1,32 @@
+
+-- Update sampled profile results for given profile_run to estimated values
+-- We don't update distinct counts, because these should already be representative
+-- in a random sample.
+
+update profile_results
+set sample_ratio = {PROFILE_SAMPLE_RATIO},
+    record_ct = ROUND(record_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    value_ct = ROUND(value_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    -- distinct_value_ct = ROUND(record_ct * {PROFILE_SAMPLE_RATIO} *(distinct_value_ct::numeric/record_ct::numeric), 0),
+    null_value_ct = ROUND(null_value_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    zero_value_ct = ROUND(zero_value_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    lead_space_ct = ROUND(lead_space_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    embedded_space_ct = ROUND(embedded_space_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    includes_digit_ct = ROUND(includes_digit_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    filled_value_ct = ROUND(filled_value_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    numeric_ct = ROUND(numeric_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    date_ct = ROUND(date_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    before_1yr_date_ct = ROUND(before_1yr_date_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    before_5yr_date_ct = ROUND(before_5yr_date_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    before_20yr_date_ct = ROUND(before_20yr_date_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    within_1yr_date_ct = ROUND(within_1yr_date_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    within_1mo_date_ct = ROUND(within_1mo_date_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    future_date_ct = ROUND(future_date_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    boolean_true_ct = ROUND(boolean_true_ct * {PROFILE_SAMPLE_RATIO}, 0),
+    date_days_present = ROUND(date_days_present * {PROFILE_SAMPLE_RATIO}, 0)
+where profile_run_id = '{PROFILE_RUN_ID}'
+and schema_name = split_part('{SAMPLING_TABLE}', '.', 1)
+and table_name = split_part('{SAMPLING_TABLE}', '.', 2)
+and sample_ratio IS NULL;
+
+

testgen/template/profiling/refresh_anomalies.sql

@@ -0,0 +1,33 @@
+
+
+WITH anomalies
+   AS ( SELECT profile_run_id,
+               COUNT(*) as anomaly_ct,
+               COUNT(DISTINCT schema_name || '.' || table_name) as anomaly_table_ct,
+               COUNT(DISTINCT schema_name || '.' || table_name || '.' || column_name) as anomaly_column_ct
+          FROM profile_anomaly_results
+         WHERE profile_run_id = '{PROFILE_RUN_ID}'::UUID
+        GROUP BY profile_run_id ),
+profiles
+   AS ( SELECT r.id as profile_run_id,
+               COUNT(DISTINCT p.schema_name || '.' || p.table_name) as table_ct,
+               COUNT(*) as column_ct
+          FROM profiling_runs r
+         INNER JOIN profile_results p
+            ON r.id = p.profile_run_id
+         WHERE r.id = '{PROFILE_RUN_ID}'::UUID
+         GROUP BY r.id ),
+stats
+   AS ( SELECT p.profile_run_id, table_ct, column_ct,
+               a.anomaly_ct, a.anomaly_table_ct, a.anomaly_column_ct
+          FROM profiles p
+               LEFT JOIN anomalies a
+                          ON (p.profile_run_id = a.profile_run_id) )
+UPDATE profiling_runs
+   SET table_ct = stats.table_ct,
+       column_ct = stats.column_ct,
+       anomaly_ct = COALESCE(stats.anomaly_ct, 0),
+       anomaly_table_ct = COALESCE(stats.anomaly_table_ct, 0),
+       anomaly_column_ct = COALESCE(stats.anomaly_column_ct, 0)
+  FROM stats
+ WHERE profiling_runs.id = stats.profile_run_id ;

testgen/template/profiling/refresh_data_chars_from_profiling.sql

@@ -0,0 +1,156 @@
+-- ==============================================================================
+-- |   Table Characteristics
+-- ==============================================================================
+
+-- Update existing records
+WITH new_chars
+   AS ( SELECT p.table_groups_id,
+               p.schema_name, p.table_name, p.functional_table_type,
+               run_date                  AS add_date,
+               MAX(record_ct)            AS record_ct,
+               COUNT(*)                  AS column_ct,
+               MAX(record_ct) * COUNT(*) AS data_point_ct
+          FROM v_latest_profile_results p
+         WHERE p.table_groups_id = '{TABLE_GROUPS_ID}'
+         GROUP BY p.table_groups_id,
+                  p.schema_name, p.table_name, p.functional_table_type, run_date )
+UPDATE data_table_chars
+   SET functional_table_type = n.functional_table_type,
+       record_ct = n.record_ct,
+       column_ct = n.column_ct,
+       data_point_ct = n.data_point_ct,
+       drop_date = NULL
+  FROM new_chars n
+INNER JOIN data_table_chars d
+   ON (n.table_groups_id = d.table_groups_id
+  AND  n.schema_name = d.schema_name
+  AND  n.table_name = d.table_name)
+ WHERE data_table_chars.table_id = d.table_id;
+
+-- Add new records
+WITH new_chars
+   AS ( SELECT p.table_groups_id,
+               p.schema_name, p.table_name, p.functional_table_type,
+               run_date                  AS add_date,
+               NULL::TIMESTAMP           AS drop_date,
+               MAX(record_ct)            AS record_ct,
+               COUNT(*)                  AS column_ct,
+               MAX(record_ct) * COUNT(*) AS data_point_ct
+          FROM v_latest_profile_results p
+         WHERE p.table_groups_id = '{TABLE_GROUPS_ID}'
+         GROUP BY p.table_groups_id,
+                  p.schema_name, p.table_name, p.functional_table_type, run_date )
+INSERT INTO data_table_chars
+  (table_groups_id, schema_name, table_name, functional_table_type, add_date,
+   record_ct, column_ct, data_point_ct)
+SELECT n.table_groups_id, n.schema_name, n.table_name, n.functional_table_type, n.add_date,
+       n.record_ct, n.column_ct, n.data_point_ct
+  FROM new_chars n
+LEFT JOIN data_table_chars d
+   ON (n.table_groups_id = d.table_groups_id
+  AND  n.schema_name = d.schema_name
+  AND  n.table_name = d.table_name)
+ WHERE d.table_id IS NULL;
+
+-- Mark dropped records
+WITH new_chars
+   AS ( SELECT p.table_groups_id,
+               p.schema_name, p.table_name
+          FROM v_latest_profile_results p
+         WHERE p.table_groups_id = '{TABLE_GROUPS_ID}'
+         GROUP BY p.table_groups_id,
+                  p.schema_name, p.table_name ),
+ last_run
+   AS ( SELECT table_groups_id, MAX(run_date) as last_run_date
+          FROM v_latest_profile_results
+         WHERE table_groups_id = '{TABLE_GROUPS_ID}'
+        GROUP BY table_groups_id)
+UPDATE data_table_chars
+   SET drop_date = l.last_run_date
+  FROM last_run l
+INNER JOIN data_table_chars d
+   ON (l.table_groups_id = d.table_groups_id)
+LEFT JOIN new_chars n
+  ON (d.table_groups_id = n.table_groups_id
+ AND  d.schema_name = n.schema_name
+ AND  d.table_name = n.table_name)
+ WHERE data_table_chars.table_id = d.table_id
+   AND n.table_name IS NULL;
+
+-- ==============================================================================
+-- |   Column Characteristics
+-- ==============================================================================
+
+-- Update existing records
+WITH new_chars
+   AS ( SELECT p.table_groups_id,
+               p.schema_name, p.table_name, p.column_name,
+               p.general_type, p.column_type, p.functional_data_type,
+               run_date
+          FROM v_latest_profile_results p
+         WHERE p.table_groups_id = '{TABLE_GROUPS_ID}')
+UPDATE data_column_chars
+   SET last_mod_date = CASE WHEN n.column_type <> d.column_type THEN n.run_date ELSE d.last_mod_date END,
+       general_type = n.general_type,
+       column_type = n.column_type,
+       functional_data_type = n.functional_data_type,
+       drop_date = NULL
+  FROM new_chars n
+INNER JOIN data_column_chars d
+   ON (n.table_groups_id = d.table_groups_id
+  AND  n.schema_name = d.schema_name
+  AND  n.table_name = d.table_name
+  AND  n.column_name = d.column_name)
+ WHERE data_column_chars.table_id = d.table_id
+   AND data_column_chars.column_name = d.column_name;
+
+-- Add new records
+WITH new_chars
+   AS ( SELECT p.table_groups_id,
+               p.schema_name, p.table_name, p.column_name,
+               p.general_type, p.column_type, p.functional_data_type,
+               run_date                  AS add_date
+          FROM v_latest_profile_results p
+         WHERE p.table_groups_id = '{TABLE_GROUPS_ID}')
+INSERT INTO data_column_chars
+  (table_groups_id, schema_name, table_name, table_id, column_name,
+   general_type, column_type, functional_data_type, add_date, last_mod_date)
+SELECT n.table_groups_id, n.schema_name, n.table_name,  dtc.table_id, n.column_name,
+       n.general_type, n.column_type, n.functional_data_type,
+       n.add_date, n.add_date as last_mod_date
+  FROM new_chars n
+INNER JOIN data_table_chars dtc
+   ON (n.table_groups_id = dtc.table_groups_id
+  AND  n.schema_name = dtc.schema_name
+  AND  n.table_name = dtc.table_name)
+ LEFT JOIN data_column_chars d
+   ON (n.table_groups_id = d.table_groups_id
+  AND  n.schema_name = d.schema_name
+  AND  n.table_name = d.table_name
+  AND  n.column_name = d.column_name)
+ WHERE d.table_id IS NULL;
+
+-- Mark dropped records
+WITH new_chars
+   AS ( SELECT p.table_groups_id,
+               p.schema_name, p.table_name, p.column_name
+          FROM v_latest_profile_results p
+         WHERE p.table_groups_id = '{TABLE_GROUPS_ID}'),
+ last_run
+   AS ( SELECT table_groups_id, MAX(run_date) as last_run_date
+          FROM v_latest_profile_results
+         WHERE table_groups_id = '{TABLE_GROUPS_ID}'
+        GROUP BY table_groups_id)
+UPDATE data_column_chars
+   SET drop_date = l.last_run_date
+  FROM last_run l
+INNER JOIN data_column_chars d
+   ON (l.table_groups_id = d.table_groups_id)
+LEFT JOIN new_chars n
+  ON (d.table_groups_id = n.table_groups_id
+ AND  d.schema_name = n.schema_name
+ AND  d.table_name = n.table_name
+ AND  d.column_name = n.column_name)
+ WHERE data_column_chars.table_id = d.table_id
+   AND data_column_chars.column_name = d.column_name
+   AND n.column_name IS NULL;

testgen/template/profiling/secondary_profiling_columns.sql

@@ -0,0 +1,12 @@
+-- Looking for columns not already freq'd,
+-- but with max_length * distinct_value_ct that fit in result
+SELECT schema_name,
+       table_name,
+       column_name
+  FROM profile_results p
+ WHERE p.profile_run_id = '{PROFILE_RUN_ID}'
+   AND p.top_freq_values IS NULL
+   AND p.general_type = 'A'
+   AND p.distinct_value_ct BETWEEN 2 and 70
+   AND p.max_length <= 70
+;

testgen/template/profiling/secondary_profiling_delete.sql

@@ -0,0 +1,4 @@
+DELETE FROM stg_secondary_profile_updates s
+             WHERE s.project_code = '{PROJECT_CODE}'
+               AND s.schema_name = '{DATA_SCHEMA}'
+               AND s.run_date = '{RUN_DATE}';

testgen/template/profiling/secondary_profiling_update.sql

@@ -0,0 +1,18 @@
+UPDATE profile_results
+   SET top_freq_values = u.top_freq_values,
+       distinct_value_hash = u.distinct_value_hash
+  FROM profile_results p
+INNER JOIN stg_secondary_profile_updates u
+   ON p.project_code = u.project_code
+  AND p.schema_name = u.schema_name
+  AND p.run_date = u.run_date
+  AND p.table_name = u.table_name
+  AND p.column_name = u.column_name
+WHERE p.project_code = profile_results.project_code
+  AND p.schema_name = profile_results.schema_name
+  AND p.run_date = profile_results.run_date
+  AND p.table_name = profile_results.table_name
+  AND p.column_name = profile_results.column_name
+  AND p.project_code = '{PROJECT_CODE}'
+   AND p.schema_name = '{DATA_SCHEMA}'
+   AND p.run_date = '{RUN_DATE}';