dataops-testgen 2.2.0__py3-none-any.whl

testgen/template/get_entities/get_test_info.sql

@@ -0,0 +1,41 @@
+/*test-info: project-code, test-suite
+Output: current detail of tests to perform for all columns within the test-suite
+Alternative: project-code, connection-id
+Optional: last_auto_run_date (==test-gen-run-id==), schema-name, table-name, column-name*/
+
+SELECT s.project_code as project_key,
+       s.cat_test_id,
+       s.test_suite as test_suite_key,
+       s.test_type,
+       COALESCE(s.test_description, tt.test_description) as test_description,
+       CASE
+           WHEN COALESCE(s.lock_refresh, 'N') = 'N' THEN 'Allowed'
+           ELSE 'Locked'
+           END                                           as test_refresh,
+       CASE
+           WHEN COALESCE(s.test_active, 'Y') = 'N' THEN 'Disabled'
+           ELSE 'Enabled'
+           END                                           as disabled,
+       COALESCE(s.watch_level, 'Warn')                   as watch_level,
+       s.schema_name,
+       s.table_name,
+       s.column_name,
+       tt.measure_uom,
+       s.threshold_value,
+       s.baseline_ct,
+       s.baseline_unique_ct,
+       s.baseline_value,
+       s.baseline_value_ct,
+       s.baseline_sum,
+       s.baseline_avg,
+       s.baseline_sd,
+       s.subset_condition,
+       s.check_result,
+       s.last_auto_gen_date,
+       s.profiling_as_of_date
+FROM test_definitions s
+         INNER JOIN test_types tt ON s.test_type = tt.test_type
+WHERE s.project_code = '{PROJECT_CODE}'
+  AND s.test_suite = '{TEST_SUITE}'
+ORDER BY s.schema_name, s.table_name,
+         s.column_name, s.test_type;

testgen/template/get_entities/get_test_results_for_run_cli.sql

@@ -0,0 +1,16 @@
+SELECT test_suite as test_suite_key,
+       table_name,
+       column_names as column_name,
+       r.test_type,
+       CASE
+         WHEN result_code = 1 THEN 'Pass'
+         WHEN result_code = 0 THEN 'Fail'
+       END as result,
+       COALESCE(r.result_message, '') as result_message,
+       result_measure,
+       tt.measure_uom
+  FROM test_results r
+INNER JOIN test_types tt ON r.test_type = tt.test_type
+ WHERE test_run_id = '{TEST_RUN_ID}'::UUID
+       {ERRORS_ONLY}
+ORDER BY r.schema_name, r.table_name, r.column_names, r.test_type;

testgen/template/get_entities/get_test_run_list.sql

@@ -0,0 +1,24 @@
+/*test-run-list: project-code, test-suite
+Output: list of test runs performed for a test_suite
+Alternative: project-code, table-name
+Optional: table-name, column-name, from-date, thru-date*/
+
+Select tr.test_suite as test_suite_key,
+       tr.test_starttime as test_time,
+       tr.status,
+       tr.id::VARCHAR(50) as test_run_id,
+       COUNT(DISTINCT lower(r.schema_name || '.' || table_name)) as table_ct,
+       COUNT(*) as result_ct,
+       SUM(CASE WHEN r.result_code = 0 THEN 1 END) as fail_ct,
+       SUM(CASE WHEN r.observability_status = 'Sent' THEN 1 END) as sent_to_obs,
+       process_id
+from test_runs tr
+INNER JOIN test_results r
+   ON (tr.id = r.test_run_id)
+where tr.project_code = '{PROJECT_CODE}'
+and tr.test_suite = '{TEST_SUITE}'
+GROUP BY tr.project_code,
+       tr.test_suite,
+       tr.test_starttime,
+       tr.status,
+       tr.id;

testgen/template/get_entities/get_test_suite.sql

@@ -0,0 +1,13 @@
+SELECT
+	id as test_suite_id,
+	project_code as project_key,
+	test_suite as test_suite_key,
+	connection_id,
+	test_suite_description,
+	test_action as default_test_action,
+	test_suite_schema,
+	component_key,
+	component_type
+ FROM test_suites
+WHERE project_code = '{PROJECT_CODE}'
+AND test_suite = '{TEST_SUITE}';

testgen/template/get_entities/get_test_suite_list.sql

@@ -0,0 +1,18 @@
+SELECT
+	ts.id as test_suite_id,
+	ts.project_code as project_key,
+	ts.test_suite as test_suite_key,
+	ts.connection_id,
+	ts.test_suite_description,
+	MAX(tr.test_starttime) as last_run
+FROM test_suites ts
+LEFT JOIN test_runs tr
+   ON ts.project_code = tr.project_code
+  AND ts.test_suite = tr.test_suite
+WHERE ts.project_code = '{PROJECT_CODE}'
+GROUP BY ts.id,
+	ts.project_code,
+	ts.test_suite,
+	ts.connection_id,
+	ts.test_suite_description
+ORDER BY ts.test_suite;

testgen/template/get_entities/list_test_types.sql

@@ -0,0 +1,4 @@
+SELECT t.test_type, t.test_description, dq_dimension
+  FROM test_types t
+ WHERE test_description > ''
+ORDER BY test_type;

testgen/template/observability/get_event_data.sql

@@ -0,0 +1,23 @@
+Select
+	   c.sql_flavor as sql_flavor,
+	   COALESCE(ts.component_key, tg.id::VARCHAR) as dataset_key,
+	   COALESCE(ts.component_name, tg.table_groups_name) as dataset_name,
+	   tg.table_group_schema as schema,
+	   c.connection_name as connection_name,
+	   c.project_db as project_db,
+	   tg.profile_sample_min_count as profile_sample_minimum_count,
+	   ts.table_groups_id as table_groups_id,
+	   tg.profile_use_sampling as profile_use_sampling,
+	   tg.profile_sample_percent as profile_sample_percent,
+	   pr.project_code as project_code,
+	   tg.profiling_table_set as profiling_table_set,
+	   tg.profiling_include_mask as profiling_include_mask,
+	   tg.profiling_exclude_mask as profiling_exclude_mask,
+       pr.observability_api_key as observability_api_key,
+       pr.observability_api_url as observability_api_url
+from test_suites ts
+join connections c on c.connection_id = ts.connection_id
+join projects pr on pr.project_code = ts.project_code
+join table_groups tg on tg.id = ts.table_groups_id
+where pr.project_code = '{PROJECT_CODE}'
+and ts.test_suite = '{TEST_SUITE}'

testgen/template/observability/get_test_results.sql

@@ -0,0 +1,41 @@
+SELECT
+	project_name,
+	component_tool,
+	"schema",
+	connection_name,
+	project_db,
+	sample_min_count,
+	group_id,
+	uses_sampling,
+	project_code,
+	sample_percentage,
+	profiling_table_set,
+	profiling_include_mask,
+	profiling_exclude_mask,
+	component_type,
+	component_key,
+	component_name,
+	column_names,
+	table_name,
+	test_suite,
+	input_parameters,
+	test_definition_id,
+	"type",
+	min_threshold,
+	max_threshold,
+	"name",
+	description,
+	start_time,
+	end_time,
+	dq_dimension,
+	result_status,
+    result_message,
+    result_id,
+	metric_value,
+	measure_uom,
+	measure_uom_description
+FROM v_queued_observability_results
+where project_code = '{PROJECT_CODE}'
+and test_suite = '{TEST_SUITE}'
+order by start_time asc
+limit {MAX_QTY_EVENTS}

testgen/template/observability/update_test_results_exported_to_observability.sql

@@ -0,0 +1,12 @@
+/*test-results: customer-code, test-group, result-ids
+Output: updates exported results */
+
+with selects
+  as ( SELECT UNNEST(ARRAY[{RESULT_IDS}]) AS selected_id )
+	update test_results set observability_status = 'Sent'
+	from test_results r
+	INNER JOIN selects s ON (r.result_id = s.selected_id)
+	where r.id = test_results.id
+	and r.observability_status = 'Queued'
+	and r.project_code = '{PROJECT_CODE}'
+	and r.test_suite = '{TEST_SUITE}'

testgen/template/parms/parms_profiling.sql

@@ -0,0 +1,34 @@
+SELECT cc.project_code,
+       cc.connection_id::VARCHAR(50) as connection_id,
+       cc.sql_flavor,
+       cc.url,
+       cc.connect_by_url,
+       cc.connect_by_key,
+       cc.private_key,
+       cc.private_key_passphrase,
+       cc.project_host,
+       cc.project_port,
+       cc.project_user,
+       cc.project_db,
+       tg.id::VARCHAR(50) as table_groups_id,
+       tg.table_group_schema,
+       CASE
+         WHEN tg.profiling_table_set ILIKE '''%''' THEN tg.profiling_table_set
+         ELSE fn_format_csv_quotes(tg.profiling_table_set)
+       END as profiling_table_set,
+       tg.profiling_include_mask,
+       tg.profiling_exclude_mask,
+       tg.profile_id_column_mask,
+       tg.profile_sk_column_mask,
+       tg.profile_use_sampling,
+       tg.profile_sample_percent,
+       tg.profile_sample_min_count,
+       cc.project_qc_schema,
+       tg.profile_do_pair_rules,
+       tg.profile_pair_rule_pct,
+       cc.max_threads
+  FROM table_groups tg
+  INNER JOIN connections cc
+         on cc.project_code = tg.project_code
+        and cc.connection_id = tg.connection_id
+ WHERE tg.id = '{TABLE_GROUPS_ID}'::UUID;

testgen/template/parms/parms_test_execution.sql

@@ -0,0 +1,13 @@
+SELECT g.project_code, g.connection_id::varchar(50),
+       cc.sql_flavor,
+       cc.project_host, cc.project_port,
+       cc.project_user, cc.project_db, tg.table_group_schema, cc.project_qc_schema,
+       cc.connect_by_key,
+       cc.private_key,
+       cc.private_key_passphrase,
+       cc.max_threads, cc.max_query_chars, cc.url, cc.connect_by_url
+  FROM test_suites g
+INNER JOIN connections cc    ON (g.connection_id = cc.connection_id)
+INNER join table_groups tg   ON (g.table_groups_id = tg.id)
+ WHERE g.project_code = '{PROJECT_CODE}'
+   AND g.test_suite = '{TEST_SUITE}';

testgen/template/parms/parms_test_gen.sql

@@ -0,0 +1,23 @@
+SELECT tg.project_code, tg.connection_id,
+       cc.sql_flavor,
+       cc.project_host,
+       cc.project_port,
+       cc.project_user,
+       cc.connect_by_key,
+       cc.private_key,
+       cc.private_key_passphrase,
+       cc.project_db,
+       tg.table_group_schema,
+       s.export_to_observability,
+       s.test_suite,
+       s.id::VARCHAR as test_suite_id,
+       cc.url,
+       cc.connect_by_url,
+       CURRENT_TIMESTAMP AT TIME ZONE 'UTC' - CAST(tg.profiling_delay_days AS integer) * INTERVAL '1 day' as profiling_as_of_date
+  FROM table_groups tg
+INNER JOIN connections cc
+   ON (tg.connection_id = cc.connection_id)
+LEFT JOIN test_suites s
+  ON (tg.connection_id = s.connection_id
+ AND '{TEST_SUITE}' = s.test_suite)
+WHERE tg.id = '{TABLE_GROUPS_ID}';

testgen/template/profiling/contingency_columns.sql

@@ -0,0 +1,7 @@
+-- All codes / categories with few distinct values
+SELECT schema_name, table_name, STRING_AGG(column_name, ',' ORDER BY column_name) as contingency_columns
+  FROM profile_results p
+ WHERE profile_run_id = '{PROFILE_RUN_ID}'::UUID
+   AND functional_data_type IN ('Code', 'Category')
+   AND distinct_value_ct BETWEEN 2 AND {CONTINGENCY_MAX_VALUES}
+GROUP BY schema_name, table_name;

testgen/template/profiling/datatype_suggestions.sql

@@ -0,0 +1,56 @@
+UPDATE profile_results
+               SET datatype_suggestion =
+                   CASE
+                     WHEN record_ct > 500 AND column_name not ILIKE '%id' THEN
+                            CASE
+                              WHEN general_type = 'A' AND column_name ILIKE '%zip%'
+                                 AND max_length <= 10 THEN 'VARCHAR(10)'
+                              WHEN general_type = 'A'
+                                AND numeric_ct > 0
+                                AND value_ct = numeric_ct + zero_length_ct
+                                AND POSITION('.' in top_freq_values) > 0  THEN 'DECIMAL(18,4)'
+                              WHEN general_type = 'A'
+                                AND numeric_ct > 0
+                                AND value_ct = numeric_ct + zero_length_ct
+                                AND max_length <= 6
+                                AND POSITION('.' in top_freq_values) = 0  THEN 'INTEGER'
+                              WHEN general_type = 'A'
+                                AND numeric_ct > 0
+                                AND value_ct = numeric_ct + zero_length_ct
+                                AND max_length > 6
+                                AND POSITION('.' in top_freq_values) = 0  THEN 'BIGINT'
+                              WHEN general_type = 'A'
+                                AND date_ct > 0
+                                AND value_ct = date_ct + zero_length_ct THEN 'DATE'
+                              WHEN general_type = 'A'
+                                AND max_length <= 5 THEN 'VARCHAR(10)'
+                              WHEN general_type = 'A'
+                               AND max_length IS NOT NULL
+                                   THEN 'VARCHAR('
+                                    || ( (1 + TRUNC( (max_length + 10) /20.0, 0)) * 20)::VARCHAR(10)
+                                    || ')'
+                              WHEN general_type = 'N'
+                               AND RTRIM(SPLIT_PART(column_type, ',', 2),')') > '0'
+                               AND fractional_sum = 0
+                               AND min_value >= -100
+                               AND max_value <= 100
+                                   THEN 'SMALLINT'
+                              WHEN general_type = 'N'
+                               AND RTRIM(SPLIT_PART(column_type, ',', 2),')') > '0'
+                               AND fractional_sum = 0
+                               AND min_value >= -100000000
+                               AND max_value <= 100000000
+                                   THEN 'INTEGER'
+                              WHEN general_type = 'N'
+                               AND RTRIM(SPLIT_PART(column_type, ',', 2),')') > '0'
+                               AND fractional_sum = 0
+                               AND (min_value < -100000000
+                                OR  max_value > 100000000)
+                                   THEN 'BIGINT'
+                              ELSE LOWER(column_type)
+                            END
+                     ELSE LOWER(column_type)
+                   END
+             WHERE project_code = '{PROJECT_CODE}'
+               AND schema_name = '{DATA_SCHEMA}'
+               AND run_date = '{RUN_DATE}';