dataops-testgen 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataops_testgen-2.2.0.dist-info/LICENSE +203 -0
- dataops_testgen-2.2.0.dist-info/METADATA +287 -0
- dataops_testgen-2.2.0.dist-info/NOTICE +5 -0
- dataops_testgen-2.2.0.dist-info/RECORD +270 -0
- dataops_testgen-2.2.0.dist-info/WHEEL +5 -0
- dataops_testgen-2.2.0.dist-info/entry_points.txt +2 -0
- dataops_testgen-2.2.0.dist-info/top_level.txt +1 -0
- testgen/__init__.py +0 -0
- testgen/__main__.py +770 -0
- testgen/commands/__init__.py +0 -0
- testgen/commands/queries/__init__.py +0 -0
- testgen/commands/queries/execute_cat_tests_query.py +95 -0
- testgen/commands/queries/execute_tests_query.py +160 -0
- testgen/commands/queries/generate_tests_query.py +94 -0
- testgen/commands/queries/profiling_query.py +366 -0
- testgen/commands/queries/test_parameter_validation_query.py +88 -0
- testgen/commands/run_execute_cat_tests.py +162 -0
- testgen/commands/run_execute_tests.py +168 -0
- testgen/commands/run_generate_tests.py +107 -0
- testgen/commands/run_get_entities.py +122 -0
- testgen/commands/run_launch_db_config.py +84 -0
- testgen/commands/run_observability_exporter.py +330 -0
- testgen/commands/run_profiling_bridge.py +495 -0
- testgen/commands/run_quick_start.py +168 -0
- testgen/commands/run_setup_profiling_tools.py +96 -0
- testgen/commands/run_test_definition.py +146 -0
- testgen/commands/run_test_parameter_validation.py +135 -0
- testgen/commands/run_upgrade_db_config.py +156 -0
- testgen/common/__init__.py +8 -0
- testgen/common/clean_sql.py +53 -0
- testgen/common/credentials.py +25 -0
- testgen/common/database/__init__.py +0 -0
- testgen/common/database/database_service.py +629 -0
- testgen/common/database/flavor/__init__.py +0 -0
- testgen/common/database/flavor/flavor_service.py +75 -0
- testgen/common/database/flavor/mssql_flavor_service.py +34 -0
- testgen/common/database/flavor/postgresql_flavor_service.py +5 -0
- testgen/common/database/flavor/redshift_flavor_service.py +22 -0
- testgen/common/database/flavor/snowflake_flavor_service.py +69 -0
- testgen/common/database/flavor/trino_flavor_service.py +21 -0
- testgen/common/date_service.py +68 -0
- testgen/common/display_service.py +85 -0
- testgen/common/docker_service.py +76 -0
- testgen/common/encrypt.py +55 -0
- testgen/common/get_pipeline_parms.py +57 -0
- testgen/common/logs.py +79 -0
- testgen/common/process_service.py +62 -0
- testgen/common/read_file.py +69 -0
- testgen/settings.py +440 -0
- testgen/template/dbsetup/010_create_base_schema.sql +2 -0
- testgen/template/dbsetup/020_create_standard_functions_sprocs.sql +179 -0
- testgen/template/dbsetup/030_initialize_new_schema_structure.sql +735 -0
- testgen/template/dbsetup/040_populate_new_schema_project.sql +59 -0
- testgen/template/dbsetup/050_populate_new_schema_metadata.sql +1517 -0
- testgen/template/dbsetup/060_create_standard_views.sql +248 -0
- testgen/template/dbsetup/070_create_default_users.sql +17 -0
- testgen/template/dbsetup/075_grant_role_rights.sql +43 -0
- testgen/template/dbsetup/080_set_current_revision.sql +5 -0
- testgen/template/dbupgrade/0100_incremental_upgrade.sql +5 -0
- testgen/template/dbupgrade/0101_incremental_upgrade.sql +15 -0
- testgen/template/dbupgrade/0102_incremental_upgrade.sql +4 -0
- testgen/template/dbupgrade/0103_incremental_upgrade.sql +22 -0
- testgen/template/dbupgrade/0104_incremental_upgrade.sql +44 -0
- testgen/template/dbupgrade/0105_incremental_upgrade.sql +1 -0
- testgen/template/dbupgrade/0106_incremental_upgrade.sql +5 -0
- testgen/template/dbupgrade/0107_incremental_upgrade.sql +3 -0
- testgen/template/dbupgrade_helpers/get_tg_revision.sql +2 -0
- testgen/template/exec_cat_tests/ex_cat_build_agg_table_tests.sql +116 -0
- testgen/template/exec_cat_tests/ex_cat_get_distinct_tables.sql +11 -0
- testgen/template/exec_cat_tests/ex_cat_results_parse.sql +69 -0
- testgen/template/exec_cat_tests/ex_cat_retrieve_agg_test_parms.sql +6 -0
- testgen/template/exec_cat_tests/ex_cat_test_query.sql +8 -0
- testgen/template/execution/ex_finalize_test_run_results.sql +37 -0
- testgen/template/execution/ex_get_tests_non_cat.sql +47 -0
- testgen/template/execution/ex_update_test_record_in_testrun_table.sql +27 -0
- testgen/template/execution/ex_write_test_record_to_testrun_table.sql +6 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_no_drops_generic.sql +48 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_num_incr_generic.sql +34 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_percent_above_generic.sql +49 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_percent_within_generic.sql +49 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_same_generic.sql +49 -0
- testgen/template/flavors/generic/exec_query_tests/ex_custom_query_generic.sql +39 -0
- testgen/template/flavors/generic/exec_query_tests/ex_data_match_2way_generic.sql +58 -0
- testgen/template/flavors/generic/exec_query_tests/ex_data_match_generic.sql +44 -0
- testgen/template/flavors/generic/exec_query_tests/ex_prior_match_generic.sql +37 -0
- testgen/template/flavors/generic/exec_query_tests/ex_relative_entropy_generic.sql +53 -0
- testgen/template/flavors/generic/exec_query_tests/ex_window_match_no_drops_generic.sql +46 -0
- testgen/template/flavors/generic/exec_query_tests/ex_window_match_same_generic.sql +59 -0
- testgen/template/flavors/generic/profiling/contingency_counts.sql +3 -0
- testgen/template/flavors/generic/validate_tests/ex_get_project_column_list_generic.sql +3 -0
- testgen/template/flavors/mssql/exec_query_tests/ex_relative_entropy_mssql.sql +53 -0
- testgen/template/flavors/mssql/profiling/project_ddf_query_mssql.sql +35 -0
- testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml +246 -0
- testgen/template/flavors/mssql/profiling/project_secondary_profiling_query_mssql.sql +36 -0
- testgen/template/flavors/mssql/setup_profiling_tools/00_drop_existing_functions_mssql.sql +8 -0
- testgen/template/flavors/mssql/setup_profiling_tools/01_create_functions_mssql.sql +12 -0
- testgen/template/flavors/mssql/setup_profiling_tools/02_create_functions_mssql.sql +54 -0
- testgen/template/flavors/mssql/setup_profiling_tools/create_qc_schema_mssql.sql +4 -0
- testgen/template/flavors/mssql/setup_profiling_tools/grant_execute_privileges_mssql.sql +1 -0
- testgen/template/flavors/postgresql/exec_query_tests/ex_window_match_no_drops_postgresql.sql +46 -0
- testgen/template/flavors/postgresql/exec_query_tests/ex_window_match_same_postgresql.sql +59 -0
- testgen/template/flavors/postgresql/profiling/project_ddf_query_postgresql.sql +42 -0
- testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml +225 -0
- testgen/template/flavors/postgresql/profiling/project_secondary_profiling_query_postgresql.sql +28 -0
- testgen/template/flavors/postgresql/setup_profiling_tools/create_functions_postgresql.sql +157 -0
- testgen/template/flavors/postgresql/setup_profiling_tools/create_qc_schema_postgresql.sql +1 -0
- testgen/template/flavors/postgresql/setup_profiling_tools/grant_execute_privileges_postgresql.sql +2 -0
- testgen/template/flavors/redshift/profiling/project_ddf_query_redshift.sql +38 -0
- testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml +221 -0
- testgen/template/flavors/redshift/profiling/project_secondary_profiling_query_redshift.sql +29 -0
- testgen/template/flavors/redshift/setup_profiling_tools/create_functions_redshift.sql +115 -0
- testgen/template/flavors/redshift/setup_profiling_tools/create_qc_schema_redshift.sql +1 -0
- testgen/template/flavors/redshift/setup_profiling_tools/grant_execute_privileges_redshift.sql +2 -0
- testgen/template/flavors/snowflake/profiling/project_ddf_query_snowflake.sql +38 -0
- testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml +220 -0
- testgen/template/flavors/snowflake/profiling/project_secondary_profiling_query_snowflake.sql +29 -0
- testgen/template/flavors/snowflake/setup_profiling_tools/create_functions_snowflake.sql +69 -0
- testgen/template/flavors/snowflake/setup_profiling_tools/create_qc_schema_snowflake.sql +1 -0
- testgen/template/flavors/snowflake/setup_profiling_tools/grant_execute_privileges_snowflake.sql +6 -0
- testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml +219 -0
- testgen/template/flavors/trino/setup_profiling_tools/create_functions_trino.sql +92 -0
- testgen/template/flavors/trino/setup_profiling_tools/create_qc_schema_trino.sql +1 -0
- testgen/template/gen_funny_cat_tests/gen_test_constant.sql +104 -0
- testgen/template/gen_funny_cat_tests/gen_test_distinct_value_ct.sql +98 -0
- testgen/template/gen_funny_cat_tests/gen_test_row_ct.sql +57 -0
- testgen/template/gen_funny_cat_tests/gen_test_row_ct_pct.sql +59 -0
- testgen/template/generation/gen_delete_old_tests.sql +5 -0
- testgen/template/generation/gen_insert_test_suite.sql +5 -0
- testgen/template/generation/gen_retrieve_or_insert_test_suite.sql +58 -0
- testgen/template/generation/gen_standard_test_type_list.sql +13 -0
- testgen/template/generation/gen_standard_tests.sql +48 -0
- testgen/template/get_entities/get_connection.sql +21 -0
- testgen/template/get_entities/get_connections_list.sql +9 -0
- testgen/template/get_entities/get_latest.sql +4 -0
- testgen/template/get_entities/get_profile.sql +12 -0
- testgen/template/get_entities/get_profile_info.sql +17 -0
- testgen/template/get_entities/get_profile_list.sql +17 -0
- testgen/template/get_entities/get_profile_screen.sql +275 -0
- testgen/template/get_entities/get_project_list.sql +6 -0
- testgen/template/get_entities/get_table_group_list.sql +10 -0
- testgen/template/get_entities/get_test_generation_list.sql +18 -0
- testgen/template/get_entities/get_test_info.sql +41 -0
- testgen/template/get_entities/get_test_results_for_run_cli.sql +16 -0
- testgen/template/get_entities/get_test_run_list.sql +24 -0
- testgen/template/get_entities/get_test_suite.sql +13 -0
- testgen/template/get_entities/get_test_suite_list.sql +18 -0
- testgen/template/get_entities/list_test_types.sql +4 -0
- testgen/template/observability/get_event_data.sql +23 -0
- testgen/template/observability/get_test_results.sql +41 -0
- testgen/template/observability/update_test_results_exported_to_observability.sql +12 -0
- testgen/template/parms/parms_profiling.sql +34 -0
- testgen/template/parms/parms_test_execution.sql +13 -0
- testgen/template/parms/parms_test_gen.sql +23 -0
- testgen/template/profiling/contingency_columns.sql +7 -0
- testgen/template/profiling/datatype_suggestions.sql +56 -0
- testgen/template/profiling/functional_datatype.sql +523 -0
- testgen/template/profiling/functional_tabletype_stage.sql +48 -0
- testgen/template/profiling/functional_tabletype_update.sql +8 -0
- testgen/template/profiling/pii_flag.sql +133 -0
- testgen/template/profiling/profile_anomalies_screen_column.sql +22 -0
- testgen/template/profiling/profile_anomalies_screen_multi_column.sql +58 -0
- testgen/template/profiling/profile_anomalies_screen_table.sql +22 -0
- testgen/template/profiling/profile_anomalies_screen_table_dates.sql +30 -0
- testgen/template/profiling/profile_anomalies_screen_variants.sql +40 -0
- testgen/template/profiling/profile_anomaly_types_get.sql +3 -0
- testgen/template/profiling/project_get_table_sample_count.sql +22 -0
- testgen/template/profiling/project_profile_run_record_insert.sql +8 -0
- testgen/template/profiling/project_profile_run_record_update.sql +5 -0
- testgen/template/profiling/project_profile_run_record_update_status.sql +5 -0
- testgen/template/profiling/project_update_profile_results_to_estimates.sql +32 -0
- testgen/template/profiling/refresh_anomalies.sql +33 -0
- testgen/template/profiling/refresh_data_chars_from_profiling.sql +156 -0
- testgen/template/profiling/secondary_profiling_columns.sql +12 -0
- testgen/template/profiling/secondary_profiling_delete.sql +4 -0
- testgen/template/profiling/secondary_profiling_update.sql +18 -0
- testgen/template/quick_start/populate_target_data.sql +1077 -0
- testgen/template/quick_start/recreate_target_data_schema.sql +167 -0
- testgen/template/quick_start/update_target_data.sql +100 -0
- testgen/template/updates/create_tmp_test_definition.sql +19 -0
- testgen/template/updates/get_test_def_parms.sql +38 -0
- testgen/template/updates/populate_stg_test_definitions.sql +184 -0
- testgen/template/validate_tests/ex_disable_tests_test_definitions.sql +5 -0
- testgen/template/validate_tests/ex_flag_tests_test_definitions.sql +64 -0
- testgen/template/validate_tests/ex_get_project_column_list_generic.sql +3 -0
- testgen/template/validate_tests/ex_get_test_column_list_tg.sql +65 -0
- testgen/template/validate_tests/ex_write_test_val_errors.sql +22 -0
- testgen/ui/__init__.py +0 -0
- testgen/ui/app.py +98 -0
- testgen/ui/assets/dk_logo.svg +46 -0
- testgen/ui/assets/question_mark.png +0 -0
- testgen/ui/assets/scripts.js +68 -0
- testgen/ui/assets/style.css +140 -0
- testgen/ui/bootstrap.py +109 -0
- testgen/ui/components/__init__.py +0 -0
- testgen/ui/components/frontend/css/KFOlCnqEu92Fr1MmEU9fBBc4.woff2 +0 -0
- testgen/ui/components/frontend/css/KFOlCnqEu92Fr1MmEU9fChc4EsA.woff2 +0 -0
- testgen/ui/components/frontend/css/KFOmCnqEu92Fr1Mu4mxK.woff2 +0 -0
- testgen/ui/components/frontend/css/KFOmCnqEu92Fr1Mu7GxKOzY.woff2 +0 -0
- testgen/ui/components/frontend/css/material-symbols-rounded.css +24 -0
- testgen/ui/components/frontend/css/material-symbols-rounded.woff2 +0 -0
- testgen/ui/components/frontend/css/roboto-font-faces.css +35 -0
- testgen/ui/components/frontend/css/shared.css +36 -0
- testgen/ui/components/frontend/img/dk_logo.svg +46 -0
- testgen/ui/components/frontend/index.html +17 -0
- testgen/ui/components/frontend/js/components/breadcrumbs.js +86 -0
- testgen/ui/components/frontend/js/components/button.js +66 -0
- testgen/ui/components/frontend/js/components/location.js +62 -0
- testgen/ui/components/frontend/js/components/select.js +75 -0
- testgen/ui/components/frontend/js/components/sidebar.js +358 -0
- testgen/ui/components/frontend/js/main.js +99 -0
- testgen/ui/components/frontend/js/streamlit.js +19 -0
- testgen/ui/components/frontend/js/van.min.js +1 -0
- testgen/ui/components/utils/__init__.py +0 -0
- testgen/ui/components/utils/callbacks.py +51 -0
- testgen/ui/components/utils/component.py +13 -0
- testgen/ui/components/widgets/__init__.py +6 -0
- testgen/ui/components/widgets/breadcrumbs.py +32 -0
- testgen/ui/components/widgets/location.py +65 -0
- testgen/ui/components/widgets/modal.py +97 -0
- testgen/ui/components/widgets/sidebar.py +69 -0
- testgen/ui/navigation/__init__.py +0 -0
- testgen/ui/navigation/menu.py +42 -0
- testgen/ui/navigation/page.py +20 -0
- testgen/ui/navigation/router.py +63 -0
- testgen/ui/queries/__init__.py +0 -0
- testgen/ui/queries/authentication_queries.py +47 -0
- testgen/ui/queries/connection_queries.py +121 -0
- testgen/ui/queries/profiling_queries.py +148 -0
- testgen/ui/queries/project_queries.py +9 -0
- testgen/ui/queries/table_group_queries.py +186 -0
- testgen/ui/queries/test_definition_queries.py +270 -0
- testgen/ui/queries/test_run_queries.py +32 -0
- testgen/ui/queries/test_suite_queries.py +145 -0
- testgen/ui/scripts/__init__.py +0 -0
- testgen/ui/scripts/patch_streamlit.py +111 -0
- testgen/ui/services/__init__.py +0 -0
- testgen/ui/services/authentication_service.py +119 -0
- testgen/ui/services/connection_service.py +220 -0
- testgen/ui/services/database_service.py +282 -0
- testgen/ui/services/form_service.py +1008 -0
- testgen/ui/services/javascript_service.py +44 -0
- testgen/ui/services/query_service.py +316 -0
- testgen/ui/services/string_service.py +12 -0
- testgen/ui/services/table_group_service.py +130 -0
- testgen/ui/services/test_definition_service.py +117 -0
- testgen/ui/services/test_run_service.py +13 -0
- testgen/ui/services/test_suite_service.py +76 -0
- testgen/ui/services/toolbar_service.py +77 -0
- testgen/ui/session.py +46 -0
- testgen/ui/views/__init__.py +0 -0
- testgen/ui/views/app_log_modal.py +92 -0
- testgen/ui/views/connections.py +72 -0
- testgen/ui/views/connections_base.py +367 -0
- testgen/ui/views/login.py +40 -0
- testgen/ui/views/not_found.py +16 -0
- testgen/ui/views/overview.py +34 -0
- testgen/ui/views/profiling_anomalies.py +501 -0
- testgen/ui/views/profiling_details.py +335 -0
- testgen/ui/views/profiling_modal.py +40 -0
- testgen/ui/views/profiling_results.py +206 -0
- testgen/ui/views/profiling_summary.py +177 -0
- testgen/ui/views/project_settings.py +74 -0
- testgen/ui/views/table_groups.py +530 -0
- testgen/ui/views/test_definitions.py +1020 -0
- testgen/ui/views/test_results.py +908 -0
- testgen/ui/views/test_runs.py +195 -0
- testgen/ui/views/test_suites.py +545 -0
- testgen/utils/__init__.py +0 -0
- testgen/utils/plugins.py +17 -0
- testgen/utils/singleton.py +14 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import streamlit as st
|
|
2
|
+
|
|
3
|
+
import testgen.ui.services.database_service as db
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def update_attribute(schema, test_definition_ids, attribute, value):
|
|
7
|
+
sql = f"""UPDATE {schema}.test_definitions
|
|
8
|
+
SET
|
|
9
|
+
{attribute}='{value}'
|
|
10
|
+
where
|
|
11
|
+
id in ({"'" + "','".join(test_definition_ids) + "'"})
|
|
12
|
+
;
|
|
13
|
+
"""
|
|
14
|
+
db.execute_sql(sql)
|
|
15
|
+
st.cache_data.clear()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@st.cache_data(show_spinner=False)
|
|
19
|
+
def get_test_definitions(schema, project_code, test_suite, table_name, column_name, test_definition_ids):
|
|
20
|
+
if table_name:
|
|
21
|
+
table_condition = f" AND d.table_name = '{table_name}'"
|
|
22
|
+
else:
|
|
23
|
+
table_condition = ""
|
|
24
|
+
if column_name:
|
|
25
|
+
column_condition = f" AND d.column_name = '{column_name}'"
|
|
26
|
+
else:
|
|
27
|
+
column_condition = ""
|
|
28
|
+
sql = f"""
|
|
29
|
+
SELECT
|
|
30
|
+
d.schema_name, d.table_name, d.column_name, t.test_name_short, t.test_name_long,
|
|
31
|
+
d.id::VARCHAR(50),
|
|
32
|
+
d.project_code, d.table_groups_id::VARCHAR(50), d.test_suite, d.test_suite_id::VARCHAR,
|
|
33
|
+
d.test_type, d.cat_test_id::VARCHAR(50),
|
|
34
|
+
d.test_active,
|
|
35
|
+
CASE WHEN d.test_active = 'Y' THEN 'Yes' ELSE 'No' END as test_active_display,
|
|
36
|
+
d.lock_refresh,
|
|
37
|
+
CASE WHEN d.lock_refresh = 'Y' THEN 'Yes' ELSE 'No' END as lock_refresh_display,
|
|
38
|
+
t.test_scope,
|
|
39
|
+
d.test_description,
|
|
40
|
+
d.profiling_as_of_date,
|
|
41
|
+
d.last_manual_update,
|
|
42
|
+
d.severity, COALESCE(d.severity, s.severity, t.default_severity) as urgency,
|
|
43
|
+
d.export_to_observability as export_to_observability_raw,
|
|
44
|
+
CASE
|
|
45
|
+
WHEN d.export_to_observability = 'Y' THEN 'Yes'
|
|
46
|
+
WHEN d.export_to_observability = 'N' THEN 'No'
|
|
47
|
+
WHEN d.export_to_observability IS NULL AND s.export_to_observability = 'Y' THEN 'Inherited (Yes)'
|
|
48
|
+
ELSE 'Inherited (No)'
|
|
49
|
+
END as export_to_observability,
|
|
50
|
+
-- test_action,
|
|
51
|
+
d.threshold_value, COALESCE(t.measure_uom_description, t.measure_uom) as export_uom,
|
|
52
|
+
d.baseline_ct, d.baseline_unique_ct, d.baseline_value,
|
|
53
|
+
d.baseline_value_ct, d.baseline_sum, d.baseline_avg, d.baseline_sd,
|
|
54
|
+
d.subset_condition,
|
|
55
|
+
d.groupby_names, d.having_condition, d.window_date_column, d.window_days,
|
|
56
|
+
d.match_schema_name, d.match_table_name, d.match_column_names,
|
|
57
|
+
d.match_subset_condition, d.match_groupby_names, d.match_having_condition,
|
|
58
|
+
d.skip_errors, d.custom_query,
|
|
59
|
+
COALESCE(d.test_description, t.test_description) as final_test_description,
|
|
60
|
+
t.default_parm_columns, t.selection_criteria,
|
|
61
|
+
d.profile_run_id::VARCHAR(50), d.test_action, d.test_definition_status,
|
|
62
|
+
d.watch_level, d.check_result, d.last_auto_gen_date,
|
|
63
|
+
d.test_mode
|
|
64
|
+
FROM {schema}.test_definitions d
|
|
65
|
+
INNER JOIN {schema}.test_types t ON (d.test_type = t.test_type)
|
|
66
|
+
INNER JOIN {schema}.test_suites s ON (d.test_suite_id = s.id)
|
|
67
|
+
WHERE True
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
if project_code:
|
|
71
|
+
sql += f""" AND d.project_code = '{project_code}'
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
if test_suite:
|
|
75
|
+
sql += f""" AND d.test_suite = '{test_suite}' {table_condition} {column_condition}
|
|
76
|
+
"""
|
|
77
|
+
if test_definition_ids:
|
|
78
|
+
sql += f""" AND d.id in ({"'" + "','".join(test_definition_ids) + "'"})
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
sql += """ORDER BY d.schema_name, d.table_name, d.column_name, d.test_type;
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
return db.retrieve_data(sql)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def update(schema, test_definition):
|
|
88
|
+
sql = f"""UPDATE {schema}.test_definitions
|
|
89
|
+
SET
|
|
90
|
+
cat_test_id = {test_definition["cat_test_id"]},
|
|
91
|
+
--last_auto_gen_date = NULLIF('test_definition["last_auto_gen_date"]', ''),
|
|
92
|
+
--profiling_as_of_date = NULLIF('test_definition["profiling_as_of_date"]', ''),
|
|
93
|
+
last_manual_update = CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
|
|
94
|
+
skip_errors = {test_definition["skip_errors"]},
|
|
95
|
+
custom_query = NULLIF($${test_definition["custom_query"]}$$, ''),
|
|
96
|
+
test_definition_status = NULLIF('{test_definition["test_definition_status"]}', ''),
|
|
97
|
+
export_to_observability = NULLIF('{test_definition["export_to_observability"]}', ''),
|
|
98
|
+
column_name = NULLIF($${test_definition["column_name"]}$$, ''),
|
|
99
|
+
watch_level = NULLIF('{test_definition["watch_level"]}', ''),
|
|
100
|
+
project_code = NULLIF('{test_definition["project_code"]}', ''),
|
|
101
|
+
table_groups_id = '{test_definition["table_groups_id"]}'::UUID,
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
if test_definition["profile_run_id"]:
|
|
105
|
+
sql += f""" profile_run_id = '{test_definition["profile_run_id"]}'::UUID,
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
sql += f""" test_type = NULLIF('{test_definition["test_type"]}', ''),
|
|
109
|
+
test_suite = NULLIF('{test_definition["test_suite"]}', ''),
|
|
110
|
+
test_description = NULLIF($${test_definition["test_description"]}$$, ''),
|
|
111
|
+
test_action = NULLIF('{test_definition["test_action"]}', ''),
|
|
112
|
+
test_mode = NULLIF('{test_definition["test_mode"]}', ''),
|
|
113
|
+
lock_refresh = NULLIF('{test_definition["lock_refresh"]}', ''),
|
|
114
|
+
schema_name = NULLIF('{test_definition["schema_name"]}', ''),
|
|
115
|
+
table_name = NULLIF('{test_definition["table_name"]}', ''),
|
|
116
|
+
test_active = NULLIF('{test_definition["test_active"]}', ''),
|
|
117
|
+
severity = NULLIF('{test_definition["severity"]}', ''),
|
|
118
|
+
check_result = NULLIF('{test_definition["check_result"]}', ''),
|
|
119
|
+
baseline_ct = NULLIF('{test_definition["baseline_ct"]}', ''),
|
|
120
|
+
baseline_unique_ct = NULLIF('{test_definition["baseline_unique_ct"]}', ''),
|
|
121
|
+
baseline_value = NULLIF($${test_definition["baseline_value"]}$$, ''),
|
|
122
|
+
baseline_value_ct = NULLIF('{test_definition["baseline_value_ct"]}', ''),
|
|
123
|
+
threshold_value = NULLIF($${test_definition["threshold_value"]}$$, ''),
|
|
124
|
+
baseline_sum = NULLIF('{test_definition["baseline_sum"]}', ''),
|
|
125
|
+
baseline_avg = NULLIF('{test_definition["baseline_avg"]}', ''),
|
|
126
|
+
baseline_sd = NULLIF('{test_definition["baseline_sd"]}', ''),
|
|
127
|
+
subset_condition = NULLIF($${test_definition["subset_condition"]}$$, ''),
|
|
128
|
+
groupby_names = NULLIF($${test_definition["groupby_names"]}$$, ''),
|
|
129
|
+
having_condition = NULLIF($${test_definition["having_condition"]}$$, ''),
|
|
130
|
+
window_date_column = NULLIF('{test_definition["window_date_column"]}', ''),
|
|
131
|
+
match_schema_name = NULLIF('{test_definition["match_schema_name"]}', ''),
|
|
132
|
+
match_table_name = NULLIF('{test_definition["match_table_name"]}', ''),
|
|
133
|
+
match_column_names = NULLIF($${test_definition["match_column_names"]}$$, ''),
|
|
134
|
+
match_subset_condition = NULLIF($${test_definition["match_subset_condition"]}$$, ''),
|
|
135
|
+
match_groupby_names = NULLIF($${test_definition["match_groupby_names"]}$$, ''),
|
|
136
|
+
match_having_condition = NULLIF($${test_definition["match_having_condition"]}$$, ''),
|
|
137
|
+
window_days = COALESCE({test_definition["window_days"]}, 0)
|
|
138
|
+
where
|
|
139
|
+
id = '{test_definition["id"]}'
|
|
140
|
+
;
|
|
141
|
+
"""
|
|
142
|
+
db.execute_sql(sql)
|
|
143
|
+
st.cache_data.clear()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def add(schema, test_definition):
|
|
147
|
+
sql = f"""INSERT INTO {schema}.test_definitions
|
|
148
|
+
(
|
|
149
|
+
--cat_test_id,
|
|
150
|
+
--last_auto_gen_date,
|
|
151
|
+
--profiling_as_of_date,
|
|
152
|
+
last_manual_update,
|
|
153
|
+
skip_errors,
|
|
154
|
+
custom_query,
|
|
155
|
+
test_definition_status,
|
|
156
|
+
export_to_observability,
|
|
157
|
+
column_name,
|
|
158
|
+
watch_level,
|
|
159
|
+
project_code,
|
|
160
|
+
table_groups_id,
|
|
161
|
+
profile_run_id,
|
|
162
|
+
test_type,
|
|
163
|
+
test_suite,
|
|
164
|
+
test_suite_id,
|
|
165
|
+
test_description,
|
|
166
|
+
test_action,
|
|
167
|
+
test_mode,
|
|
168
|
+
lock_refresh,
|
|
169
|
+
schema_name,
|
|
170
|
+
table_name,
|
|
171
|
+
test_active,
|
|
172
|
+
severity,
|
|
173
|
+
check_result,
|
|
174
|
+
baseline_ct,
|
|
175
|
+
baseline_unique_ct,
|
|
176
|
+
baseline_value,
|
|
177
|
+
baseline_value_ct,
|
|
178
|
+
threshold_value,
|
|
179
|
+
baseline_sum,
|
|
180
|
+
baseline_avg,
|
|
181
|
+
baseline_sd,
|
|
182
|
+
subset_condition,
|
|
183
|
+
groupby_names,
|
|
184
|
+
having_condition,
|
|
185
|
+
window_date_column,
|
|
186
|
+
match_schema_name,
|
|
187
|
+
match_table_name,
|
|
188
|
+
match_column_names,
|
|
189
|
+
match_subset_condition,
|
|
190
|
+
match_groupby_names,
|
|
191
|
+
match_having_condition,
|
|
192
|
+
window_days
|
|
193
|
+
)
|
|
194
|
+
SELECT
|
|
195
|
+
--{test_definition["cat_test_id"]} as cat_test_id,
|
|
196
|
+
--NULLIF('test_definition["last_auto_gen_date"]', '') as last_auto_gen_date,
|
|
197
|
+
--NULLIF('test_definition["profiling_as_of_date"]', '') as profiling_as_of_date,
|
|
198
|
+
CURRENT_TIMESTAMP AT TIME ZONE 'UTC' as last_manual_update,
|
|
199
|
+
{test_definition["skip_errors"]} as skip_errors,
|
|
200
|
+
NULLIF($${test_definition["custom_query"]}$$, '') as custom_query,
|
|
201
|
+
NULLIF('{test_definition["test_definition_status"]}', '') as test_definition_status,
|
|
202
|
+
NULLIF('{test_definition["export_to_observability"]}', '') as export_to_observability,
|
|
203
|
+
NULLIF('{test_definition["column_name"]}', '') as column_name,
|
|
204
|
+
NULLIF('{test_definition["watch_level"]}', '') as watch_level,
|
|
205
|
+
NULLIF('{test_definition["project_code"]}', '') as project_code,
|
|
206
|
+
'{test_definition["table_groups_id"]}'::UUID as table_groups_id,
|
|
207
|
+
NULL AS profile_run_id,
|
|
208
|
+
NULLIF('{test_definition["test_type"]}', '') as test_type,
|
|
209
|
+
NULLIF('{test_definition["test_suite"]}', '') as test_suite,
|
|
210
|
+
'{test_definition["test_suite_id"]}'::UUID as test_suite_id,
|
|
211
|
+
NULLIF('{test_definition["test_description"]}', '') as test_description,
|
|
212
|
+
NULLIF('{test_definition["test_action"]}', '') as test_action,
|
|
213
|
+
NULLIF('{test_definition["test_mode"]}', '') as test_mode,
|
|
214
|
+
NULLIF('{test_definition["lock_refresh"]}', '') as lock_refresh,
|
|
215
|
+
NULLIF('{test_definition["schema_name"]}', '') as schema_name,
|
|
216
|
+
NULLIF('{test_definition["table_name"]}', '') as table_name,
|
|
217
|
+
NULLIF('{test_definition["test_active"]}', '') as test_active,
|
|
218
|
+
NULLIF('{test_definition["severity"]}', '') as severity,
|
|
219
|
+
NULLIF('{test_definition["check_result"]}', '') as check_result,
|
|
220
|
+
NULLIF('{test_definition["baseline_ct"]}', '') as baseline_ct,
|
|
221
|
+
NULLIF('{test_definition["baseline_unique_ct"]}', '') as baseline_unique_ct,
|
|
222
|
+
NULLIF($${test_definition["baseline_value"]}$$, '') as baseline_value,
|
|
223
|
+
NULLIF($${test_definition["baseline_value_ct"]}$$, '') as baseline_value_ct,
|
|
224
|
+
NULLIF($${test_definition["threshold_value"]}$$, '') as threshold_value,
|
|
225
|
+
NULLIF($${test_definition["baseline_sum"]}$$, '') as baseline_sum,
|
|
226
|
+
NULLIF('{test_definition["baseline_avg"]}', '') as baseline_avg,
|
|
227
|
+
NULLIF('{test_definition["baseline_sd"]}', '') as baseline_sd,
|
|
228
|
+
NULLIF($${test_definition["subset_condition"]}$$, '') as subset_condition,
|
|
229
|
+
NULLIF($${test_definition["groupby_names"]}$$, '') as groupby_names,
|
|
230
|
+
NULLIF($${test_definition["having_condition"]}$$, '') as having_condition,
|
|
231
|
+
NULLIF('{test_definition["window_date_column"]}', '') as window_date_column,
|
|
232
|
+
NULLIF('{test_definition["match_schema_name"]}', '') as match_schema_name,
|
|
233
|
+
NULLIF('{test_definition["match_table_name"]}', '') as match_table_name,
|
|
234
|
+
NULLIF($${test_definition["match_column_names"]}$$, '') as match_column_names,
|
|
235
|
+
NULLIF($${test_definition["match_subset_condition"]}$$, '') as match_subset_condition,
|
|
236
|
+
NULLIF($${test_definition["match_groupby_names"]}$$, '') as match_groupby_names,
|
|
237
|
+
NULLIF($${test_definition["match_having_condition"]}$$, '') as match_having_condition,
|
|
238
|
+
COALESCE({test_definition["window_days"]}, 0) as window_days
|
|
239
|
+
;
|
|
240
|
+
"""
|
|
241
|
+
db.execute_sql(sql)
|
|
242
|
+
st.cache_data.clear()
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def get_test_definition_usage(schema, test_definition_ids):
|
|
246
|
+
test_definition_names_join = [f"'{item}'" for item in test_definition_ids]
|
|
247
|
+
sql = f"""
|
|
248
|
+
select distinct test_definition_id from {schema}.test_results where test_definition_id in ({",".join(test_definition_names_join)});
|
|
249
|
+
"""
|
|
250
|
+
return db.retrieve_data(sql)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def delete(schema, test_definition_ids):
|
|
254
|
+
if test_definition_ids is None or len(test_definition_ids) == 0:
|
|
255
|
+
raise ValueError("No Test Definition is specified.")
|
|
256
|
+
|
|
257
|
+
items = [f"'{item}'" for item in test_definition_ids]
|
|
258
|
+
sql = f"""DELETE FROM {schema}.test_definitions WHERE id in ({",".join(items)})"""
|
|
259
|
+
db.execute_sql(sql)
|
|
260
|
+
st.cache_data.clear()
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def cascade_delete(schema, test_suite_names):
|
|
264
|
+
if test_suite_names is None or len(test_suite_names) == 0:
|
|
265
|
+
raise ValueError("No Test Suite is specified.")
|
|
266
|
+
|
|
267
|
+
items = [f"'{item}'" for item in test_suite_names]
|
|
268
|
+
sql = f"""delete from {schema}.test_definitions where test_suite in ({",".join(items)})"""
|
|
269
|
+
db.execute_sql(sql)
|
|
270
|
+
st.cache_data.clear()
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import streamlit as st
|
|
2
|
+
|
|
3
|
+
import testgen.common.date_service as date_service
|
|
4
|
+
import testgen.ui.services.database_service as db
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def cascade_delete(schema: str, test_suite_names: list[str]) -> None:
|
|
8
|
+
if test_suite_names is None or len(test_suite_names) == 0:
|
|
9
|
+
raise ValueError("No Test Suite is specified.")
|
|
10
|
+
|
|
11
|
+
items = [f"'{item}'" for item in test_suite_names]
|
|
12
|
+
sql = f"""delete from {schema}.working_agg_cat_results where test_suite in ({",".join(items)});
|
|
13
|
+
delete from {schema}.working_agg_cat_tests where test_suite in ({",".join(items)});
|
|
14
|
+
delete from {schema}.test_runs where test_suite in ({",".join(items)});
|
|
15
|
+
delete from {schema}.test_results where test_suite in ({",".join(items)});
|
|
16
|
+
delete from {schema}.execution_queue where test_suite in ({",".join(items)});"""
|
|
17
|
+
db.execute_sql(sql)
|
|
18
|
+
st.cache_data.clear()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def update_status(schema: str, test_run_id: str, status: str) -> None:
|
|
22
|
+
if not all([test_run_id, status]):
|
|
23
|
+
raise ValueError("Missing query parameters.")
|
|
24
|
+
|
|
25
|
+
now = date_service.get_now_as_string()
|
|
26
|
+
|
|
27
|
+
sql = f"""UPDATE {schema}.test_runs
|
|
28
|
+
SET status = '{status}',
|
|
29
|
+
test_endtime = '{now}'
|
|
30
|
+
where id = '{test_run_id}' :: UUID;"""
|
|
31
|
+
db.execute_sql(sql)
|
|
32
|
+
st.cache_data.clear()
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import streamlit as st
|
|
3
|
+
|
|
4
|
+
import testgen.ui.services.database_service as db
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@st.cache_data(show_spinner=False)
|
|
8
|
+
def get_by_table_group(schema, project_code, table_group_id):
|
|
9
|
+
sql = f"""
|
|
10
|
+
SELECT
|
|
11
|
+
id::VARCHAR(50),
|
|
12
|
+
project_code, test_suite,
|
|
13
|
+
connection_id::VARCHAR(50),
|
|
14
|
+
table_groups_id::VARCHAR(50),
|
|
15
|
+
test_suite_description, test_action,
|
|
16
|
+
case when severity is null then 'Inherit' else severity end,
|
|
17
|
+
export_to_observability, test_suite_schema, component_key, component_type, component_name
|
|
18
|
+
FROM {schema}.test_suites
|
|
19
|
+
WHERE project_code = '{project_code}'
|
|
20
|
+
AND table_groups_id = '{table_group_id}'
|
|
21
|
+
ORDER BY test_suite;
|
|
22
|
+
"""
|
|
23
|
+
return db.retrieve_data(sql)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def edit(schema, test_suite):
|
|
27
|
+
sql = f"""UPDATE {schema}.test_suites
|
|
28
|
+
SET
|
|
29
|
+
test_suite='{test_suite["test_suite"]}',
|
|
30
|
+
test_suite_description='{test_suite["test_suite_description"]}',
|
|
31
|
+
test_action=NULLIF('{test_suite["test_action"]}', ''),
|
|
32
|
+
severity=NULLIF('{test_suite["severity"]}', 'Inherit'),
|
|
33
|
+
export_to_observability='{'Y' if test_suite["export_to_observability"] else 'N'}',
|
|
34
|
+
test_suite_schema=NULLIF('{test_suite["test_suite_schema"]}', ''),
|
|
35
|
+
component_key=NULLIF('{test_suite["component_key"]}', ''),
|
|
36
|
+
component_type=NULLIF('{test_suite["component_type"]}', ''),
|
|
37
|
+
component_name=NULLIF('{test_suite["component_name"]}', '')
|
|
38
|
+
where
|
|
39
|
+
id = '{test_suite["id"]}';
|
|
40
|
+
"""
|
|
41
|
+
db.execute_sql(sql)
|
|
42
|
+
st.cache_data.clear()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def add(schema, test_suite):
|
|
46
|
+
sql = f"""INSERT INTO {schema}.test_suites
|
|
47
|
+
(id,
|
|
48
|
+
project_code, test_suite, connection_id, table_groups_id, test_suite_description, test_action,
|
|
49
|
+
severity, export_to_observability, test_suite_schema, component_key, component_type,
|
|
50
|
+
component_name)
|
|
51
|
+
SELECT
|
|
52
|
+
gen_random_uuid(),
|
|
53
|
+
'{test_suite["project_code"]}',
|
|
54
|
+
'{test_suite["test_suite"]}',
|
|
55
|
+
'{test_suite["connection_id"]}',
|
|
56
|
+
'{test_suite["table_groups_id"]}',
|
|
57
|
+
NULLIF('{test_suite["test_suite_description"]}', ''),
|
|
58
|
+
NULLIF('{test_suite["test_action"]}', ''),
|
|
59
|
+
NULLIF('{test_suite["severity"]}', 'Inherit'),
|
|
60
|
+
'{'Y' if test_suite["export_to_observability"] else 'N' }'::character varying,
|
|
61
|
+
NULLIF('{test_suite["test_suite_schema"]}', ''),
|
|
62
|
+
NULLIF('{test_suite["component_key"]}', ''),
|
|
63
|
+
NULLIF('{test_suite["component_type"]}', ''),
|
|
64
|
+
NULLIF('{test_suite["component_name"]}', '')
|
|
65
|
+
;"""
|
|
66
|
+
db.execute_sql(sql)
|
|
67
|
+
st.cache_data.clear()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def delete(schema, test_suite_ids):
|
|
71
|
+
if test_suite_ids is None or len(test_suite_ids) == 0:
|
|
72
|
+
raise ValueError("No table group is specified.")
|
|
73
|
+
|
|
74
|
+
items = [f"'{item}'" for item in test_suite_ids]
|
|
75
|
+
sql = f"""DELETE FROM {schema}.test_suites WHERE id in ({",".join(items)})"""
|
|
76
|
+
db.execute_sql(sql)
|
|
77
|
+
st.cache_data.clear()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def cascade_delete(schema: str, test_suite_names: list[str]) -> None:
|
|
81
|
+
if test_suite_names is None or len(test_suite_names) == 0:
|
|
82
|
+
raise ValueError("No Test Suite is specified.")
|
|
83
|
+
|
|
84
|
+
items = [f"'{item}'" for item in test_suite_names]
|
|
85
|
+
sql = f"""delete from {schema}.test_suites where test_suite in ({",".join(items)})"""
|
|
86
|
+
db.execute_sql(sql)
|
|
87
|
+
st.cache_data.clear()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def get_test_suite_dependencies(schema: str, test_suite_names: list[str]) -> pd.DataFrame:
|
|
91
|
+
test_suite_names_join = [f"'{item}'" for item in test_suite_names]
|
|
92
|
+
sql = f"""
|
|
93
|
+
select distinct test_suite from {schema}.test_definitions where test_suite in ({",".join(test_suite_names_join)})
|
|
94
|
+
union
|
|
95
|
+
select distinct test_suite from {schema}.execution_queue where test_suite in ({",".join(test_suite_names_join)})
|
|
96
|
+
union
|
|
97
|
+
select distinct test_suite from {schema}.test_results where test_suite in ({",".join(test_suite_names_join)});
|
|
98
|
+
"""
|
|
99
|
+
return db.retrieve_data(sql)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def get_test_suite_usage(schema: str, test_suite_names: list[str]) -> pd.DataFrame:
|
|
104
|
+
test_suite_names_join = [f"'{item}'" for item in test_suite_names]
|
|
105
|
+
sql = f"""
|
|
106
|
+
select distinct test_suite from {schema}.test_runs where test_suite in ({",".join(test_suite_names_join)}) and status = 'Running'
|
|
107
|
+
"""
|
|
108
|
+
return db.retrieve_data(sql)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def get_test_suite_refresh_check(schema, table_groups_id, test_suite_name):
|
|
112
|
+
sql = f"""
|
|
113
|
+
SELECT COUNT(*) as test_ct,
|
|
114
|
+
SUM(CASE WHEN COALESCE(d.lock_refresh, 'N') = 'N' THEN 1 ELSE 0 END) as unlocked_test_ct,
|
|
115
|
+
SUM(CASE WHEN COALESCE(d.lock_refresh, 'N') = 'N' AND d.last_manual_update IS NOT NULL THEN 1 ELSE 0 END) as unlocked_edits_ct
|
|
116
|
+
FROM {schema}.test_definitions d
|
|
117
|
+
INNER JOIN {schema}.test_types t
|
|
118
|
+
ON (d.test_type = t.test_type)
|
|
119
|
+
WHERE d.table_groups_id = '{table_groups_id}'::UUID
|
|
120
|
+
AND d.test_suite = '{test_suite_name}'
|
|
121
|
+
AND t.run_type = 'CAT'
|
|
122
|
+
AND t.selection_criteria IS NOT NULL;
|
|
123
|
+
"""
|
|
124
|
+
return db.retrieve_data_list(sql)[0]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def get_generation_sets(schema):
|
|
128
|
+
sql = f"""
|
|
129
|
+
SELECT DISTINCT generation_set
|
|
130
|
+
FROM {schema}.generation_sets
|
|
131
|
+
ORDER BY generation_set;
|
|
132
|
+
"""
|
|
133
|
+
return db.retrieve_data(sql)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def lock_edited_tests(schema, test_suite_name):
|
|
137
|
+
sql = f"""
|
|
138
|
+
UPDATE {schema}.test_definitions
|
|
139
|
+
SET lock_refresh = 'Y'
|
|
140
|
+
WHERE test_suite = '{test_suite_name}'
|
|
141
|
+
AND last_manual_update IS NOT NULL
|
|
142
|
+
AND lock_refresh = 'N';
|
|
143
|
+
"""
|
|
144
|
+
db.execute_sql(sql)
|
|
145
|
+
return True
|
|
File without changes
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# ruff: noqa: TRY002
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
import pathlib
|
|
5
|
+
import shutil
|
|
6
|
+
|
|
7
|
+
import streamlit
|
|
8
|
+
from bs4 import BeautifulSoup, Tag
|
|
9
|
+
|
|
10
|
+
INJECTED_CLASS = "testgen-mods"
|
|
11
|
+
STREAMLIT_ROOT = pathlib.Path(streamlit.__file__).parent
|
|
12
|
+
STREAMLIT_INDEX = STREAMLIT_ROOT / "static" / "index.html"
|
|
13
|
+
STREAMLIT_JS_FOLDER = STREAMLIT_ROOT / "static" / "static" / "js"
|
|
14
|
+
STREAMLIT_CSS_FOLDER = STREAMLIT_ROOT / "static" / "static" / "css"
|
|
15
|
+
TESTGEN_ROOT = pathlib.Path(__file__).parent.parent.parent
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def patch(force: bool = False) -> list[str]:
|
|
19
|
+
operations = [
|
|
20
|
+
"ui/assets/style.css:insert",
|
|
21
|
+
"ui/assets/scripts.js:insert",
|
|
22
|
+
"ui/components/frontend/css/KFOmCnqEu92Fr1Mu7GxKOzY.woff2:copy",
|
|
23
|
+
"ui/components/frontend/css/KFOmCnqEu92Fr1Mu4mxK.woff2:copy",
|
|
24
|
+
"ui/components/frontend/css/KFOlCnqEu92Fr1MmEU9fChc4EsA.woff2:copy",
|
|
25
|
+
"ui/components/frontend/css/KFOlCnqEu92Fr1MmEU9fBBc4.woff2:copy",
|
|
26
|
+
"ui/components/frontend/css/material-symbols-rounded.woff2:copy",
|
|
27
|
+
"ui/components/frontend/css/roboto-font-faces.css:inject",
|
|
28
|
+
"ui/components/frontend/css/material-symbols-rounded.css:inject",
|
|
29
|
+
"ui/components/frontend/js/van.min.js:copy",
|
|
30
|
+
"ui/components/frontend/js/components/sidebar.js:inject",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
_patch_streamlit_index(*operations, force=force)
|
|
34
|
+
|
|
35
|
+
return [op.split(":")[0] for op in operations]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _patch_streamlit_index(*operations: list[str], force: bool = False) -> None:
|
|
39
|
+
"""
|
|
40
|
+
Patches the index.html inside streamlit package to inject Testgen's
|
|
41
|
+
own styles and scripts before rendering time.
|
|
42
|
+
|
|
43
|
+
The new tags are injected with a distinctive class so that on the
|
|
44
|
+
next streamlit re-run it skips injecting (making it a tag faster
|
|
45
|
+
than st.markdown method).
|
|
46
|
+
|
|
47
|
+
NOTE: keeps a .bak of the original index.html file
|
|
48
|
+
|
|
49
|
+
:param filename: list of path to valid .css and .js files
|
|
50
|
+
:param force: to use in development while actively changing the
|
|
51
|
+
injected files to force re-injection
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
html = BeautifulSoup(STREAMLIT_INDEX.read_text(), features="html.parser")
|
|
55
|
+
if force or not html.find_all(attrs={"class": INJECTED_CLASS}):
|
|
56
|
+
streamlit_index_backup = STREAMLIT_INDEX.with_suffix(".bak")
|
|
57
|
+
|
|
58
|
+
if not streamlit_index_backup.exists():
|
|
59
|
+
shutil.copy(STREAMLIT_INDEX, streamlit_index_backup)
|
|
60
|
+
else:
|
|
61
|
+
shutil.copy(streamlit_index_backup, STREAMLIT_INDEX)
|
|
62
|
+
html = BeautifulSoup(STREAMLIT_INDEX.read_text(), features="html.parser")
|
|
63
|
+
|
|
64
|
+
head = html.find(name="head")
|
|
65
|
+
if head:
|
|
66
|
+
actions = {
|
|
67
|
+
"insert": _inline_tag,
|
|
68
|
+
"copy": _sourced_tag,
|
|
69
|
+
"inject": functools.partial(_sourced_tag, inject=True),
|
|
70
|
+
}
|
|
71
|
+
for operation in operations:
|
|
72
|
+
filename, action = operation.split(":")
|
|
73
|
+
if (filepath := (TESTGEN_ROOT / filename)).exists():
|
|
74
|
+
if tag := actions[action](filepath, html):
|
|
75
|
+
head.append(tag)
|
|
76
|
+
|
|
77
|
+
STREAMLIT_INDEX.write_text(str(html))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _inline_tag(filepath: pathlib.Path, html: BeautifulSoup, **_) -> Tag:
|
|
81
|
+
tag_for_ext = {
|
|
82
|
+
".css": lambda: html.new_tag("style", **{"class": INJECTED_CLASS}),
|
|
83
|
+
".js": lambda: html.new_tag("script", **{"type": "module", "class": INJECTED_CLASS}),
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
tag = tag_for_ext[filepath.suffix]()
|
|
88
|
+
except:
|
|
89
|
+
raise Exception(f"Unsupported insert operation for file with extension {filepath.suffix}") from None
|
|
90
|
+
|
|
91
|
+
tag.string = filepath.read_text()
|
|
92
|
+
return tag
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _sourced_tag(filepath: pathlib.Path, html: BeautifulSoup, inject: bool = False) -> Tag | None:
|
|
96
|
+
tag_for_ext = {
|
|
97
|
+
".css": lambda: html.new_tag(
|
|
98
|
+
"link", **{"href": f"./static/css/{filepath.name}", "rel": "stylesheet", "class": INJECTED_CLASS}
|
|
99
|
+
),
|
|
100
|
+
".js": lambda: html.new_tag(
|
|
101
|
+
"script", **{"type": "module", "src": f"./static/js/{filepath.name}", "class": INJECTED_CLASS}
|
|
102
|
+
),
|
|
103
|
+
}
|
|
104
|
+
copy_to = ({".js": STREAMLIT_JS_FOLDER}).get(filepath.suffix, STREAMLIT_CSS_FOLDER)
|
|
105
|
+
|
|
106
|
+
shutil.copy(filepath, copy_to)
|
|
107
|
+
|
|
108
|
+
if not inject or filepath.suffix not in tag_for_ext:
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
return tag_for_ext[filepath.suffix]()
|
|
File without changes
|