dataops-testgen 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataops_testgen-2.2.0.dist-info/LICENSE +203 -0
- dataops_testgen-2.2.0.dist-info/METADATA +287 -0
- dataops_testgen-2.2.0.dist-info/NOTICE +5 -0
- dataops_testgen-2.2.0.dist-info/RECORD +270 -0
- dataops_testgen-2.2.0.dist-info/WHEEL +5 -0
- dataops_testgen-2.2.0.dist-info/entry_points.txt +2 -0
- dataops_testgen-2.2.0.dist-info/top_level.txt +1 -0
- testgen/__init__.py +0 -0
- testgen/__main__.py +770 -0
- testgen/commands/__init__.py +0 -0
- testgen/commands/queries/__init__.py +0 -0
- testgen/commands/queries/execute_cat_tests_query.py +95 -0
- testgen/commands/queries/execute_tests_query.py +160 -0
- testgen/commands/queries/generate_tests_query.py +94 -0
- testgen/commands/queries/profiling_query.py +366 -0
- testgen/commands/queries/test_parameter_validation_query.py +88 -0
- testgen/commands/run_execute_cat_tests.py +162 -0
- testgen/commands/run_execute_tests.py +168 -0
- testgen/commands/run_generate_tests.py +107 -0
- testgen/commands/run_get_entities.py +122 -0
- testgen/commands/run_launch_db_config.py +84 -0
- testgen/commands/run_observability_exporter.py +330 -0
- testgen/commands/run_profiling_bridge.py +495 -0
- testgen/commands/run_quick_start.py +168 -0
- testgen/commands/run_setup_profiling_tools.py +96 -0
- testgen/commands/run_test_definition.py +146 -0
- testgen/commands/run_test_parameter_validation.py +135 -0
- testgen/commands/run_upgrade_db_config.py +156 -0
- testgen/common/__init__.py +8 -0
- testgen/common/clean_sql.py +53 -0
- testgen/common/credentials.py +25 -0
- testgen/common/database/__init__.py +0 -0
- testgen/common/database/database_service.py +629 -0
- testgen/common/database/flavor/__init__.py +0 -0
- testgen/common/database/flavor/flavor_service.py +75 -0
- testgen/common/database/flavor/mssql_flavor_service.py +34 -0
- testgen/common/database/flavor/postgresql_flavor_service.py +5 -0
- testgen/common/database/flavor/redshift_flavor_service.py +22 -0
- testgen/common/database/flavor/snowflake_flavor_service.py +69 -0
- testgen/common/database/flavor/trino_flavor_service.py +21 -0
- testgen/common/date_service.py +68 -0
- testgen/common/display_service.py +85 -0
- testgen/common/docker_service.py +76 -0
- testgen/common/encrypt.py +55 -0
- testgen/common/get_pipeline_parms.py +57 -0
- testgen/common/logs.py +79 -0
- testgen/common/process_service.py +62 -0
- testgen/common/read_file.py +69 -0
- testgen/settings.py +440 -0
- testgen/template/dbsetup/010_create_base_schema.sql +2 -0
- testgen/template/dbsetup/020_create_standard_functions_sprocs.sql +179 -0
- testgen/template/dbsetup/030_initialize_new_schema_structure.sql +735 -0
- testgen/template/dbsetup/040_populate_new_schema_project.sql +59 -0
- testgen/template/dbsetup/050_populate_new_schema_metadata.sql +1517 -0
- testgen/template/dbsetup/060_create_standard_views.sql +248 -0
- testgen/template/dbsetup/070_create_default_users.sql +17 -0
- testgen/template/dbsetup/075_grant_role_rights.sql +43 -0
- testgen/template/dbsetup/080_set_current_revision.sql +5 -0
- testgen/template/dbupgrade/0100_incremental_upgrade.sql +5 -0
- testgen/template/dbupgrade/0101_incremental_upgrade.sql +15 -0
- testgen/template/dbupgrade/0102_incremental_upgrade.sql +4 -0
- testgen/template/dbupgrade/0103_incremental_upgrade.sql +22 -0
- testgen/template/dbupgrade/0104_incremental_upgrade.sql +44 -0
- testgen/template/dbupgrade/0105_incremental_upgrade.sql +1 -0
- testgen/template/dbupgrade/0106_incremental_upgrade.sql +5 -0
- testgen/template/dbupgrade/0107_incremental_upgrade.sql +3 -0
- testgen/template/dbupgrade_helpers/get_tg_revision.sql +2 -0
- testgen/template/exec_cat_tests/ex_cat_build_agg_table_tests.sql +116 -0
- testgen/template/exec_cat_tests/ex_cat_get_distinct_tables.sql +11 -0
- testgen/template/exec_cat_tests/ex_cat_results_parse.sql +69 -0
- testgen/template/exec_cat_tests/ex_cat_retrieve_agg_test_parms.sql +6 -0
- testgen/template/exec_cat_tests/ex_cat_test_query.sql +8 -0
- testgen/template/execution/ex_finalize_test_run_results.sql +37 -0
- testgen/template/execution/ex_get_tests_non_cat.sql +47 -0
- testgen/template/execution/ex_update_test_record_in_testrun_table.sql +27 -0
- testgen/template/execution/ex_write_test_record_to_testrun_table.sql +6 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_no_drops_generic.sql +48 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_num_incr_generic.sql +34 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_percent_above_generic.sql +49 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_percent_within_generic.sql +49 -0
- testgen/template/flavors/generic/exec_query_tests/ex_aggregate_match_same_generic.sql +49 -0
- testgen/template/flavors/generic/exec_query_tests/ex_custom_query_generic.sql +39 -0
- testgen/template/flavors/generic/exec_query_tests/ex_data_match_2way_generic.sql +58 -0
- testgen/template/flavors/generic/exec_query_tests/ex_data_match_generic.sql +44 -0
- testgen/template/flavors/generic/exec_query_tests/ex_prior_match_generic.sql +37 -0
- testgen/template/flavors/generic/exec_query_tests/ex_relative_entropy_generic.sql +53 -0
- testgen/template/flavors/generic/exec_query_tests/ex_window_match_no_drops_generic.sql +46 -0
- testgen/template/flavors/generic/exec_query_tests/ex_window_match_same_generic.sql +59 -0
- testgen/template/flavors/generic/profiling/contingency_counts.sql +3 -0
- testgen/template/flavors/generic/validate_tests/ex_get_project_column_list_generic.sql +3 -0
- testgen/template/flavors/mssql/exec_query_tests/ex_relative_entropy_mssql.sql +53 -0
- testgen/template/flavors/mssql/profiling/project_ddf_query_mssql.sql +35 -0
- testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml +246 -0
- testgen/template/flavors/mssql/profiling/project_secondary_profiling_query_mssql.sql +36 -0
- testgen/template/flavors/mssql/setup_profiling_tools/00_drop_existing_functions_mssql.sql +8 -0
- testgen/template/flavors/mssql/setup_profiling_tools/01_create_functions_mssql.sql +12 -0
- testgen/template/flavors/mssql/setup_profiling_tools/02_create_functions_mssql.sql +54 -0
- testgen/template/flavors/mssql/setup_profiling_tools/create_qc_schema_mssql.sql +4 -0
- testgen/template/flavors/mssql/setup_profiling_tools/grant_execute_privileges_mssql.sql +1 -0
- testgen/template/flavors/postgresql/exec_query_tests/ex_window_match_no_drops_postgresql.sql +46 -0
- testgen/template/flavors/postgresql/exec_query_tests/ex_window_match_same_postgresql.sql +59 -0
- testgen/template/flavors/postgresql/profiling/project_ddf_query_postgresql.sql +42 -0
- testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml +225 -0
- testgen/template/flavors/postgresql/profiling/project_secondary_profiling_query_postgresql.sql +28 -0
- testgen/template/flavors/postgresql/setup_profiling_tools/create_functions_postgresql.sql +157 -0
- testgen/template/flavors/postgresql/setup_profiling_tools/create_qc_schema_postgresql.sql +1 -0
- testgen/template/flavors/postgresql/setup_profiling_tools/grant_execute_privileges_postgresql.sql +2 -0
- testgen/template/flavors/redshift/profiling/project_ddf_query_redshift.sql +38 -0
- testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml +221 -0
- testgen/template/flavors/redshift/profiling/project_secondary_profiling_query_redshift.sql +29 -0
- testgen/template/flavors/redshift/setup_profiling_tools/create_functions_redshift.sql +115 -0
- testgen/template/flavors/redshift/setup_profiling_tools/create_qc_schema_redshift.sql +1 -0
- testgen/template/flavors/redshift/setup_profiling_tools/grant_execute_privileges_redshift.sql +2 -0
- testgen/template/flavors/snowflake/profiling/project_ddf_query_snowflake.sql +38 -0
- testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml +220 -0
- testgen/template/flavors/snowflake/profiling/project_secondary_profiling_query_snowflake.sql +29 -0
- testgen/template/flavors/snowflake/setup_profiling_tools/create_functions_snowflake.sql +69 -0
- testgen/template/flavors/snowflake/setup_profiling_tools/create_qc_schema_snowflake.sql +1 -0
- testgen/template/flavors/snowflake/setup_profiling_tools/grant_execute_privileges_snowflake.sql +6 -0
- testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml +219 -0
- testgen/template/flavors/trino/setup_profiling_tools/create_functions_trino.sql +92 -0
- testgen/template/flavors/trino/setup_profiling_tools/create_qc_schema_trino.sql +1 -0
- testgen/template/gen_funny_cat_tests/gen_test_constant.sql +104 -0
- testgen/template/gen_funny_cat_tests/gen_test_distinct_value_ct.sql +98 -0
- testgen/template/gen_funny_cat_tests/gen_test_row_ct.sql +57 -0
- testgen/template/gen_funny_cat_tests/gen_test_row_ct_pct.sql +59 -0
- testgen/template/generation/gen_delete_old_tests.sql +5 -0
- testgen/template/generation/gen_insert_test_suite.sql +5 -0
- testgen/template/generation/gen_retrieve_or_insert_test_suite.sql +58 -0
- testgen/template/generation/gen_standard_test_type_list.sql +13 -0
- testgen/template/generation/gen_standard_tests.sql +48 -0
- testgen/template/get_entities/get_connection.sql +21 -0
- testgen/template/get_entities/get_connections_list.sql +9 -0
- testgen/template/get_entities/get_latest.sql +4 -0
- testgen/template/get_entities/get_profile.sql +12 -0
- testgen/template/get_entities/get_profile_info.sql +17 -0
- testgen/template/get_entities/get_profile_list.sql +17 -0
- testgen/template/get_entities/get_profile_screen.sql +275 -0
- testgen/template/get_entities/get_project_list.sql +6 -0
- testgen/template/get_entities/get_table_group_list.sql +10 -0
- testgen/template/get_entities/get_test_generation_list.sql +18 -0
- testgen/template/get_entities/get_test_info.sql +41 -0
- testgen/template/get_entities/get_test_results_for_run_cli.sql +16 -0
- testgen/template/get_entities/get_test_run_list.sql +24 -0
- testgen/template/get_entities/get_test_suite.sql +13 -0
- testgen/template/get_entities/get_test_suite_list.sql +18 -0
- testgen/template/get_entities/list_test_types.sql +4 -0
- testgen/template/observability/get_event_data.sql +23 -0
- testgen/template/observability/get_test_results.sql +41 -0
- testgen/template/observability/update_test_results_exported_to_observability.sql +12 -0
- testgen/template/parms/parms_profiling.sql +34 -0
- testgen/template/parms/parms_test_execution.sql +13 -0
- testgen/template/parms/parms_test_gen.sql +23 -0
- testgen/template/profiling/contingency_columns.sql +7 -0
- testgen/template/profiling/datatype_suggestions.sql +56 -0
- testgen/template/profiling/functional_datatype.sql +523 -0
- testgen/template/profiling/functional_tabletype_stage.sql +48 -0
- testgen/template/profiling/functional_tabletype_update.sql +8 -0
- testgen/template/profiling/pii_flag.sql +133 -0
- testgen/template/profiling/profile_anomalies_screen_column.sql +22 -0
- testgen/template/profiling/profile_anomalies_screen_multi_column.sql +58 -0
- testgen/template/profiling/profile_anomalies_screen_table.sql +22 -0
- testgen/template/profiling/profile_anomalies_screen_table_dates.sql +30 -0
- testgen/template/profiling/profile_anomalies_screen_variants.sql +40 -0
- testgen/template/profiling/profile_anomaly_types_get.sql +3 -0
- testgen/template/profiling/project_get_table_sample_count.sql +22 -0
- testgen/template/profiling/project_profile_run_record_insert.sql +8 -0
- testgen/template/profiling/project_profile_run_record_update.sql +5 -0
- testgen/template/profiling/project_profile_run_record_update_status.sql +5 -0
- testgen/template/profiling/project_update_profile_results_to_estimates.sql +32 -0
- testgen/template/profiling/refresh_anomalies.sql +33 -0
- testgen/template/profiling/refresh_data_chars_from_profiling.sql +156 -0
- testgen/template/profiling/secondary_profiling_columns.sql +12 -0
- testgen/template/profiling/secondary_profiling_delete.sql +4 -0
- testgen/template/profiling/secondary_profiling_update.sql +18 -0
- testgen/template/quick_start/populate_target_data.sql +1077 -0
- testgen/template/quick_start/recreate_target_data_schema.sql +167 -0
- testgen/template/quick_start/update_target_data.sql +100 -0
- testgen/template/updates/create_tmp_test_definition.sql +19 -0
- testgen/template/updates/get_test_def_parms.sql +38 -0
- testgen/template/updates/populate_stg_test_definitions.sql +184 -0
- testgen/template/validate_tests/ex_disable_tests_test_definitions.sql +5 -0
- testgen/template/validate_tests/ex_flag_tests_test_definitions.sql +64 -0
- testgen/template/validate_tests/ex_get_project_column_list_generic.sql +3 -0
- testgen/template/validate_tests/ex_get_test_column_list_tg.sql +65 -0
- testgen/template/validate_tests/ex_write_test_val_errors.sql +22 -0
- testgen/ui/__init__.py +0 -0
- testgen/ui/app.py +98 -0
- testgen/ui/assets/dk_logo.svg +46 -0
- testgen/ui/assets/question_mark.png +0 -0
- testgen/ui/assets/scripts.js +68 -0
- testgen/ui/assets/style.css +140 -0
- testgen/ui/bootstrap.py +109 -0
- testgen/ui/components/__init__.py +0 -0
- testgen/ui/components/frontend/css/KFOlCnqEu92Fr1MmEU9fBBc4.woff2 +0 -0
- testgen/ui/components/frontend/css/KFOlCnqEu92Fr1MmEU9fChc4EsA.woff2 +0 -0
- testgen/ui/components/frontend/css/KFOmCnqEu92Fr1Mu4mxK.woff2 +0 -0
- testgen/ui/components/frontend/css/KFOmCnqEu92Fr1Mu7GxKOzY.woff2 +0 -0
- testgen/ui/components/frontend/css/material-symbols-rounded.css +24 -0
- testgen/ui/components/frontend/css/material-symbols-rounded.woff2 +0 -0
- testgen/ui/components/frontend/css/roboto-font-faces.css +35 -0
- testgen/ui/components/frontend/css/shared.css +36 -0
- testgen/ui/components/frontend/img/dk_logo.svg +46 -0
- testgen/ui/components/frontend/index.html +17 -0
- testgen/ui/components/frontend/js/components/breadcrumbs.js +86 -0
- testgen/ui/components/frontend/js/components/button.js +66 -0
- testgen/ui/components/frontend/js/components/location.js +62 -0
- testgen/ui/components/frontend/js/components/select.js +75 -0
- testgen/ui/components/frontend/js/components/sidebar.js +358 -0
- testgen/ui/components/frontend/js/main.js +99 -0
- testgen/ui/components/frontend/js/streamlit.js +19 -0
- testgen/ui/components/frontend/js/van.min.js +1 -0
- testgen/ui/components/utils/__init__.py +0 -0
- testgen/ui/components/utils/callbacks.py +51 -0
- testgen/ui/components/utils/component.py +13 -0
- testgen/ui/components/widgets/__init__.py +6 -0
- testgen/ui/components/widgets/breadcrumbs.py +32 -0
- testgen/ui/components/widgets/location.py +65 -0
- testgen/ui/components/widgets/modal.py +97 -0
- testgen/ui/components/widgets/sidebar.py +69 -0
- testgen/ui/navigation/__init__.py +0 -0
- testgen/ui/navigation/menu.py +42 -0
- testgen/ui/navigation/page.py +20 -0
- testgen/ui/navigation/router.py +63 -0
- testgen/ui/queries/__init__.py +0 -0
- testgen/ui/queries/authentication_queries.py +47 -0
- testgen/ui/queries/connection_queries.py +121 -0
- testgen/ui/queries/profiling_queries.py +148 -0
- testgen/ui/queries/project_queries.py +9 -0
- testgen/ui/queries/table_group_queries.py +186 -0
- testgen/ui/queries/test_definition_queries.py +270 -0
- testgen/ui/queries/test_run_queries.py +32 -0
- testgen/ui/queries/test_suite_queries.py +145 -0
- testgen/ui/scripts/__init__.py +0 -0
- testgen/ui/scripts/patch_streamlit.py +111 -0
- testgen/ui/services/__init__.py +0 -0
- testgen/ui/services/authentication_service.py +119 -0
- testgen/ui/services/connection_service.py +220 -0
- testgen/ui/services/database_service.py +282 -0
- testgen/ui/services/form_service.py +1008 -0
- testgen/ui/services/javascript_service.py +44 -0
- testgen/ui/services/query_service.py +316 -0
- testgen/ui/services/string_service.py +12 -0
- testgen/ui/services/table_group_service.py +130 -0
- testgen/ui/services/test_definition_service.py +117 -0
- testgen/ui/services/test_run_service.py +13 -0
- testgen/ui/services/test_suite_service.py +76 -0
- testgen/ui/services/toolbar_service.py +77 -0
- testgen/ui/session.py +46 -0
- testgen/ui/views/__init__.py +0 -0
- testgen/ui/views/app_log_modal.py +92 -0
- testgen/ui/views/connections.py +72 -0
- testgen/ui/views/connections_base.py +367 -0
- testgen/ui/views/login.py +40 -0
- testgen/ui/views/not_found.py +16 -0
- testgen/ui/views/overview.py +34 -0
- testgen/ui/views/profiling_anomalies.py +501 -0
- testgen/ui/views/profiling_details.py +335 -0
- testgen/ui/views/profiling_modal.py +40 -0
- testgen/ui/views/profiling_results.py +206 -0
- testgen/ui/views/profiling_summary.py +177 -0
- testgen/ui/views/project_settings.py +74 -0
- testgen/ui/views/table_groups.py +530 -0
- testgen/ui/views/test_definitions.py +1020 -0
- testgen/ui/views/test_results.py +908 -0
- testgen/ui/views/test_runs.py +195 -0
- testgen/ui/views/test_suites.py +545 -0
- testgen/utils/__init__.py +0 -0
- testgen/utils/plugins.py +17 -0
- testgen/utils/singleton.py +14 -0
testgen/__main__.py
ADDED
|
@@ -0,0 +1,770 @@
|
|
|
1
|
+
import getpass
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
import typing
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
|
|
9
|
+
import click
|
|
10
|
+
from click.core import Context
|
|
11
|
+
from progress.spinner import MoonSpinner
|
|
12
|
+
from trogon import tui
|
|
13
|
+
|
|
14
|
+
from testgen import settings
|
|
15
|
+
from testgen.commands.run_execute_tests import run_execution_steps
|
|
16
|
+
from testgen.commands.run_generate_tests import run_test_gen_queries
|
|
17
|
+
from testgen.commands.run_get_entities import (
|
|
18
|
+
run_get_results,
|
|
19
|
+
run_get_test_suite,
|
|
20
|
+
run_list_connections,
|
|
21
|
+
run_list_profiles,
|
|
22
|
+
run_list_projects,
|
|
23
|
+
run_list_test_generation,
|
|
24
|
+
run_list_test_runs,
|
|
25
|
+
run_list_test_suites,
|
|
26
|
+
run_list_test_types,
|
|
27
|
+
run_profile_info,
|
|
28
|
+
run_profile_screen,
|
|
29
|
+
run_table_group_list,
|
|
30
|
+
run_test_info,
|
|
31
|
+
)
|
|
32
|
+
from testgen.commands.run_launch_db_config import run_launch_db_config
|
|
33
|
+
from testgen.commands.run_observability_exporter import run_observability_exporter
|
|
34
|
+
from testgen.commands.run_profiling_bridge import run_profiling_queries
|
|
35
|
+
from testgen.commands.run_quick_start import run_quick_start, run_quick_start_increment
|
|
36
|
+
from testgen.commands.run_setup_profiling_tools import run_setup_profiling_tools
|
|
37
|
+
from testgen.commands.run_test_definition import get_test_def_parms, update_test_def_parms_dict
|
|
38
|
+
from testgen.commands.run_upgrade_db_config import get_schema_revision, is_db_revision_up_to_date, run_upgrade_db_config
|
|
39
|
+
from testgen.common import (
|
|
40
|
+
configure_logging,
|
|
41
|
+
display_service,
|
|
42
|
+
docker_service,
|
|
43
|
+
get_tg_db,
|
|
44
|
+
get_tg_host,
|
|
45
|
+
get_tg_schema,
|
|
46
|
+
logs,
|
|
47
|
+
)
|
|
48
|
+
from testgen.utils import plugins
|
|
49
|
+
|
|
50
|
+
LOG = logging.getLogger("testgen")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class Configuration:
|
|
55
|
+
verbose: bool = field(default=False)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# This is just sugar - @pass_obj or @pass_context would work too.
|
|
59
|
+
pass_configuration = click.make_pass_decorator(Configuration)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@tui()
|
|
63
|
+
@click.group(
|
|
64
|
+
help=f"This version: {settings.VERSION} \n\nLatest version: {docker_service.check_for_new_docker_release()} \n\nSchema revision: {get_schema_revision()}"
|
|
65
|
+
)
|
|
66
|
+
@click.option(
|
|
67
|
+
"-v",
|
|
68
|
+
"--verbose",
|
|
69
|
+
help="Enables more detailed logging. If used, must be entered before the command.",
|
|
70
|
+
is_flag=True,
|
|
71
|
+
default=False,
|
|
72
|
+
)
|
|
73
|
+
@click.pass_context
|
|
74
|
+
def cli(ctx: Context, verbose: bool):
|
|
75
|
+
if verbose:
|
|
76
|
+
configure_logging(level=logging.DEBUG)
|
|
77
|
+
else:
|
|
78
|
+
configure_logging(level=logging.INFO)
|
|
79
|
+
|
|
80
|
+
ctx.obj = Configuration(verbose=verbose)
|
|
81
|
+
status_ok, message = docker_service.check_basic_configuration()
|
|
82
|
+
if not status_ok:
|
|
83
|
+
click.secho(message, fg="red")
|
|
84
|
+
sys.exit(1)
|
|
85
|
+
|
|
86
|
+
if (
|
|
87
|
+
ctx.invoked_subcommand not in ["ui", "tui", "setup-system-db", "upgrade-system-version", "quick-start"]
|
|
88
|
+
and not is_db_revision_up_to_date()
|
|
89
|
+
):
|
|
90
|
+
click.secho("The system database schema is outdated. Automatically running the following command:", fg="red")
|
|
91
|
+
click.secho("testgen upgrade-system-version", fg="red")
|
|
92
|
+
do_upgrade_system_version()
|
|
93
|
+
click.secho("\nNow running the requested command...", fg="red")
|
|
94
|
+
LOG.debug("Current Step: Main Program")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@cli.command("run-profile", help="Generates a new profile of the table group.")
|
|
98
|
+
@pass_configuration
|
|
99
|
+
@click.option(
|
|
100
|
+
"-tg",
|
|
101
|
+
"--table-group-id",
|
|
102
|
+
required=False,
|
|
103
|
+
type=click.STRING,
|
|
104
|
+
help="The identifier for the table group used during a profile run. Use a table_group_id shown in list-table-groups.",
|
|
105
|
+
default=None,
|
|
106
|
+
)
|
|
107
|
+
def run_profile(configuration: Configuration, table_group_id: str):
|
|
108
|
+
click.echo(f"run-profile with table_group_id: {table_group_id}")
|
|
109
|
+
spinner = None
|
|
110
|
+
if not configuration.verbose:
|
|
111
|
+
spinner = MoonSpinner("Processing ... ")
|
|
112
|
+
message = run_profiling_queries(table_group_id, spinner=spinner)
|
|
113
|
+
click.echo("\n" + message)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@cli.command("run-test-generation", help="Generates or refreshes the tests for a table group.")
|
|
117
|
+
@click.option(
|
|
118
|
+
"-tg",
|
|
119
|
+
"--table-group-id",
|
|
120
|
+
help="The identifier for the table group used during a profile run. Use a table_group_id shown in list-table-groups.",
|
|
121
|
+
required=False,
|
|
122
|
+
type=click.STRING,
|
|
123
|
+
default=None,
|
|
124
|
+
)
|
|
125
|
+
@click.option(
|
|
126
|
+
"-ts",
|
|
127
|
+
"--test-suite-key",
|
|
128
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
129
|
+
required=False,
|
|
130
|
+
default=settings.DEFAULT_TEST_SUITE_KEY,
|
|
131
|
+
)
|
|
132
|
+
@click.option(
|
|
133
|
+
"-gs",
|
|
134
|
+
"--generation-set",
|
|
135
|
+
help="A defined subset of tests to generate for your purpose. Use a generation_set defined for your project.",
|
|
136
|
+
required=False,
|
|
137
|
+
default=None,
|
|
138
|
+
)
|
|
139
|
+
@pass_configuration
|
|
140
|
+
def run_test_generation(configuration: Configuration, table_group_id: str, test_suite_key: str, generation_set: str):
|
|
141
|
+
LOG.info("CurrentStep: Generate Tests - Main Procedure")
|
|
142
|
+
message = run_test_gen_queries(table_group_id, test_suite_key, generation_set)
|
|
143
|
+
LOG.info("Current Step: Generate Tests - Main Procedure Complete")
|
|
144
|
+
click.echo("\n" + message)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@cli.command("run-tests", help="Performs tests defined for a test suite.")
|
|
148
|
+
@click.option(
|
|
149
|
+
"-pk",
|
|
150
|
+
"--project-key",
|
|
151
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
152
|
+
required=False,
|
|
153
|
+
type=click.STRING,
|
|
154
|
+
default=settings.PROJECT_KEY,
|
|
155
|
+
)
|
|
156
|
+
@click.option(
|
|
157
|
+
"-ts",
|
|
158
|
+
"--test-suite-key",
|
|
159
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
160
|
+
required=False,
|
|
161
|
+
default=settings.DEFAULT_TEST_SUITE_KEY,
|
|
162
|
+
)
|
|
163
|
+
@pass_configuration
|
|
164
|
+
def run_tests(configuration: Configuration, project_key: str, test_suite_key: str):
|
|
165
|
+
click.echo(f"run-tests for suite: {test_suite_key}")
|
|
166
|
+
spinner = None
|
|
167
|
+
if not configuration.verbose:
|
|
168
|
+
spinner = MoonSpinner("Processing ... ")
|
|
169
|
+
message = run_execution_steps(project_key, test_suite_key, spinner=spinner)
|
|
170
|
+
click.echo("\n" + message)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@cli.command("list-profiles", help="Lists all profile runs for a table group.")
|
|
174
|
+
@click.option(
|
|
175
|
+
"-tg",
|
|
176
|
+
"--table-group-id",
|
|
177
|
+
help="The identifier for the table group used during a profile run. Use a table_group_id shown in list-table-groups.",
|
|
178
|
+
required=True,
|
|
179
|
+
type=click.STRING,
|
|
180
|
+
)
|
|
181
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
182
|
+
@pass_configuration
|
|
183
|
+
def list_profiles(configuration: Configuration, table_group_id: str, display: bool):
|
|
184
|
+
LOG.info("list_profiles:")
|
|
185
|
+
rows, header = run_list_profiles(table_group_id)
|
|
186
|
+
if display:
|
|
187
|
+
display_service.print_table(rows, header)
|
|
188
|
+
display_service.to_csv("list_profiles.csv", rows, header)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@cli.command("get-profile", help="Fetches details for a profile run.")
|
|
192
|
+
@click.option(
|
|
193
|
+
"-pr",
|
|
194
|
+
"--profile-run-id",
|
|
195
|
+
required=True,
|
|
196
|
+
type=click.STRING,
|
|
197
|
+
help="The identifier for a profile run. Use an ID shown in list-profiles.",
|
|
198
|
+
)
|
|
199
|
+
@click.option(
|
|
200
|
+
"-tn",
|
|
201
|
+
"--table-name",
|
|
202
|
+
help="Filter the profile run to view details for a single table. Enter the name of a table present in the profile run.",
|
|
203
|
+
type=click.STRING,
|
|
204
|
+
required=False,
|
|
205
|
+
default=None,
|
|
206
|
+
)
|
|
207
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
208
|
+
@pass_configuration
|
|
209
|
+
def get_profile(configuration: Configuration, profile_run_id: str, table_name: str, display: bool):
|
|
210
|
+
rows, header = run_profile_info(profile_run_id, table_name)
|
|
211
|
+
if display:
|
|
212
|
+
display_service.print_table(rows, header)
|
|
213
|
+
display_service.to_csv("get_profile.csv", rows, header)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@cli.command("get-profile-anomalies", help="Fetches details of anomalies that may have occurred for a given profile.")
|
|
217
|
+
@click.option(
|
|
218
|
+
"-pr",
|
|
219
|
+
"--profile-run-id",
|
|
220
|
+
required=True,
|
|
221
|
+
type=click.STRING,
|
|
222
|
+
help="The identifier for a profile run. Use an ID shown in list-profiles.",
|
|
223
|
+
)
|
|
224
|
+
@click.option(
|
|
225
|
+
"-tn",
|
|
226
|
+
"--table-name",
|
|
227
|
+
help="Filter the profile run to view details for a single table. Enter the name of a table present in the profile run.",
|
|
228
|
+
type=click.STRING,
|
|
229
|
+
required=False,
|
|
230
|
+
default=None,
|
|
231
|
+
)
|
|
232
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
233
|
+
@pass_configuration
|
|
234
|
+
def get_profile_screen(configuration: Configuration, profile_run_id: str, table_name: str, display: bool):
|
|
235
|
+
rows, header = run_profile_screen(profile_run_id, table_name)
|
|
236
|
+
if display:
|
|
237
|
+
display_service.print_table(rows, header)
|
|
238
|
+
display_service.to_csv("get_profile_anomalies.csv", rows, header)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
@cli.command(
|
|
242
|
+
"list-test-generation",
|
|
243
|
+
help="Lists a summary of test suite generation dates with details for table, column, and test counts.",
|
|
244
|
+
)
|
|
245
|
+
@click.option(
|
|
246
|
+
"-pk",
|
|
247
|
+
"--project-key",
|
|
248
|
+
required=False,
|
|
249
|
+
type=click.STRING,
|
|
250
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
251
|
+
default=settings.PROJECT_KEY,
|
|
252
|
+
)
|
|
253
|
+
@click.option(
|
|
254
|
+
"-ts",
|
|
255
|
+
"--test-suite-key",
|
|
256
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
257
|
+
type=click.STRING,
|
|
258
|
+
required=False,
|
|
259
|
+
default=settings.DEFAULT_TEST_SUITE_KEY,
|
|
260
|
+
)
|
|
261
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
262
|
+
@pass_configuration
|
|
263
|
+
def list_test_generation(configuration: Configuration, project_key: str, test_suite_key: str, display: bool):
|
|
264
|
+
rows, header = run_list_test_generation(project_key, test_suite_key)
|
|
265
|
+
if display:
|
|
266
|
+
display_service.print_table(rows, header)
|
|
267
|
+
display_service.to_csv("list_test_generation.csv", rows, header)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
@cli.command("get-test-properties", help="Fetches test details for a test suite (in an editable file).")
|
|
271
|
+
@click.option(
|
|
272
|
+
"-pk",
|
|
273
|
+
"--project-key",
|
|
274
|
+
required=False,
|
|
275
|
+
type=click.STRING,
|
|
276
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
277
|
+
default=settings.PROJECT_KEY,
|
|
278
|
+
)
|
|
279
|
+
@click.option(
|
|
280
|
+
"-ts",
|
|
281
|
+
"--test-suite-key",
|
|
282
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
283
|
+
type=click.STRING,
|
|
284
|
+
required=True,
|
|
285
|
+
default=settings.DEFAULT_TEST_SUITE_KEY,
|
|
286
|
+
)
|
|
287
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
288
|
+
@pass_configuration
|
|
289
|
+
def get_test_properties(configuration: Configuration, project_key: str, test_suite_key: str, display: bool):
|
|
290
|
+
LOG.info("CurrentStep: Main Program - List Test Properties")
|
|
291
|
+
yaml_dict = get_test_def_parms(project_key, test_suite_key)
|
|
292
|
+
display_service.to_yaml("get_test_parms.yaml", yaml_dict, display)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
@cli.command("update-test-properties", help="Updates test properties from changes made in the YAML file.")
|
|
296
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
297
|
+
@pass_configuration
|
|
298
|
+
def update_test_parms(configuration: Configuration, display: bool):
|
|
299
|
+
click.echo("update-test-properties is starting.")
|
|
300
|
+
LOG.info("CurrentStep: Update Test Def Parms")
|
|
301
|
+
yaml_dict = display_service.from_yaml("get_test_parms.yaml", display)
|
|
302
|
+
update_test_def_parms_dict(yaml_dict)
|
|
303
|
+
click.echo("update-test-properties has successfully finished.")
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@cli.command("list-tests", help="Lists the tests generated for a test suite.")
|
|
307
|
+
@click.option(
|
|
308
|
+
"-pk",
|
|
309
|
+
"--project-key",
|
|
310
|
+
required=False,
|
|
311
|
+
type=click.STRING,
|
|
312
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
313
|
+
default=settings.PROJECT_KEY,
|
|
314
|
+
)
|
|
315
|
+
@click.option(
|
|
316
|
+
"-ts",
|
|
317
|
+
"--test-suite-key",
|
|
318
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
319
|
+
type=click.STRING,
|
|
320
|
+
required=False,
|
|
321
|
+
default=settings.DEFAULT_TEST_SUITE_KEY,
|
|
322
|
+
)
|
|
323
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
324
|
+
@pass_configuration
|
|
325
|
+
def get_tests(configuration: Configuration, project_key: str, test_suite_key: str, display: bool):
|
|
326
|
+
rows, header = run_test_info(project_key, test_suite_key)
|
|
327
|
+
if display:
|
|
328
|
+
display_service.print_table(rows, header)
|
|
329
|
+
display_service.to_csv("list_tests.csv", rows, header)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
@cli.command("list-test-runs", help="Lists the test runs performed for a test suite.")
|
|
333
|
+
@click.option(
|
|
334
|
+
"-pk",
|
|
335
|
+
"--project-key",
|
|
336
|
+
required=False,
|
|
337
|
+
type=click.STRING,
|
|
338
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
339
|
+
default=settings.PROJECT_KEY,
|
|
340
|
+
)
|
|
341
|
+
@click.option(
|
|
342
|
+
"-ts",
|
|
343
|
+
"--test-suite-key",
|
|
344
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
345
|
+
type=click.STRING,
|
|
346
|
+
required=False,
|
|
347
|
+
default=settings.DEFAULT_TEST_SUITE_KEY,
|
|
348
|
+
)
|
|
349
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
350
|
+
@pass_configuration
|
|
351
|
+
def list_test_runs(configuration: Configuration, project_key: str, test_suite_key: str, display: bool):
|
|
352
|
+
rows, header = run_list_test_runs(project_key, test_suite_key)
|
|
353
|
+
if display:
|
|
354
|
+
display_service.print_table(rows, header)
|
|
355
|
+
display_service.to_csv("list_test_runs.csv", rows, header)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
@cli.command("quick-start", help="Use to generate sample target database, for demo purposes.")
|
|
359
|
+
@click.option(
|
|
360
|
+
"--delete-target-db",
|
|
361
|
+
help="Will delete the current target database, if it exists",
|
|
362
|
+
is_flag=True,
|
|
363
|
+
default=False,
|
|
364
|
+
)
|
|
365
|
+
@click.option(
|
|
366
|
+
"--iteration",
|
|
367
|
+
"-i",
|
|
368
|
+
default=0,
|
|
369
|
+
required=False,
|
|
370
|
+
help="The monthly data increment snapshot. Can be 0, 1, 2 or 3. 0 is the initial data.",
|
|
371
|
+
)
|
|
372
|
+
@click.option(
|
|
373
|
+
"--simulate-fast-forward",
|
|
374
|
+
"-s",
|
|
375
|
+
default=False,
|
|
376
|
+
is_flag=True,
|
|
377
|
+
required=False,
|
|
378
|
+
help="For demo purposes, simulates that some time pass by and the target data is changing. This will call the iterations in order.",
|
|
379
|
+
)
|
|
380
|
+
@click.option(
|
|
381
|
+
"--observability-api-url",
|
|
382
|
+
help="Observability API url to be able to export TestGen data to Observability using the command 'export-observability'",
|
|
383
|
+
type=click.STRING,
|
|
384
|
+
required=False,
|
|
385
|
+
default="",
|
|
386
|
+
)
|
|
387
|
+
@click.option(
|
|
388
|
+
"--observability-api-key",
|
|
389
|
+
help="Observability API key to be able to export TestGen data to Observability using the command 'export-observability'",
|
|
390
|
+
type=click.STRING,
|
|
391
|
+
required=False,
|
|
392
|
+
default="",
|
|
393
|
+
)
|
|
394
|
+
@pass_configuration
|
|
395
|
+
def quick_start(
|
|
396
|
+
configuration: Configuration,
|
|
397
|
+
delete_target_db: bool,
|
|
398
|
+
iteration: int,
|
|
399
|
+
simulate_fast_forward: bool,
|
|
400
|
+
observability_api_url: str,
|
|
401
|
+
observability_api_key: str,
|
|
402
|
+
):
|
|
403
|
+
if observability_api_url:
|
|
404
|
+
settings.OBSERVABILITY_API_URL = observability_api_url
|
|
405
|
+
if observability_api_key:
|
|
406
|
+
settings.OBSERVABILITY_API_KEY = observability_api_key
|
|
407
|
+
|
|
408
|
+
# Check if this is an increment or the initial state
|
|
409
|
+
if iteration == 0 and not simulate_fast_forward:
|
|
410
|
+
click.echo("quick-start command")
|
|
411
|
+
run_quick_start(delete_target_db)
|
|
412
|
+
|
|
413
|
+
if not simulate_fast_forward:
|
|
414
|
+
run_quick_start_increment(iteration)
|
|
415
|
+
else:
|
|
416
|
+
for iteration in range(1, 4):
|
|
417
|
+
click.echo(f"Running iteration: {iteration} / 3")
|
|
418
|
+
minutes_offset = 2 * iteration
|
|
419
|
+
run_quick_start_increment(iteration)
|
|
420
|
+
run_execution_steps(settings.PROJECT_KEY, settings.DEFAULT_TEST_SUITE_KEY, minutes_offset=minutes_offset)
|
|
421
|
+
|
|
422
|
+
click.echo("Quick start has successfully finished.")
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
@cli.command("setup-system-db", help="Use to initialize the TestGen system database.")
|
|
426
|
+
@click.option(
|
|
427
|
+
"--delete-db",
|
|
428
|
+
help="Will delete the current system database, if it exists",
|
|
429
|
+
is_flag=True,
|
|
430
|
+
default=False,
|
|
431
|
+
)
|
|
432
|
+
@click.option("--yes", "-y", default=False, is_flag=True, required=False, help="Force yes")
|
|
433
|
+
@pass_configuration
|
|
434
|
+
def setup_app_db(configuration: Configuration, delete_db: bool, yes: bool):
|
|
435
|
+
click.echo("setup-system-db command")
|
|
436
|
+
|
|
437
|
+
db = get_tg_db()
|
|
438
|
+
host = get_tg_host()
|
|
439
|
+
schema = get_tg_schema()
|
|
440
|
+
|
|
441
|
+
if not yes:
|
|
442
|
+
operation_message = "create"
|
|
443
|
+
|
|
444
|
+
if delete_db:
|
|
445
|
+
warning_text = "WARNING: This command will delete any existing TestGen data, drop the current TestGen app database, and create a new copy from scratch. "
|
|
446
|
+
click.secho(warning_text, fg="red")
|
|
447
|
+
|
|
448
|
+
operation_message = "DELETE and recreate"
|
|
449
|
+
|
|
450
|
+
message = f"Are you SURE you want to {operation_message} the app database '{db}' in the host '{host}'?"
|
|
451
|
+
if not click.confirm(click.style(message, fg="red")):
|
|
452
|
+
click.echo("Exiting without any operation performed.")
|
|
453
|
+
return
|
|
454
|
+
|
|
455
|
+
run_launch_db_config(delete_db)
|
|
456
|
+
|
|
457
|
+
projectDetails = {
|
|
458
|
+
"project_key": settings.PROJECT_KEY,
|
|
459
|
+
"sql_flavor": settings.PROJECT_SQL_FLAVOR,
|
|
460
|
+
"project_name": settings.PROJECT_NAME,
|
|
461
|
+
"project_db_name": settings.PROJECT_DATABASE_NAME,
|
|
462
|
+
"project_db_user": settings.PROJECT_DATABASE_USER,
|
|
463
|
+
"project_db_port": settings.PROJECT_DATABASE_PORT,
|
|
464
|
+
"project_db_host": settings.PROJECT_DATABASE_HOST,
|
|
465
|
+
"project_db_schema": settings.PROJECT_DATABASE_SCHEMA,
|
|
466
|
+
}
|
|
467
|
+
click.echo(f"App DB created: Host={host}, DB Name={db}, Schema={schema}")
|
|
468
|
+
click.echo(f"Project Details:{projectDetails}")
|
|
469
|
+
click.echo("setup-system-db has successfully finished.")
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
@cli.command(
|
|
473
|
+
"upgrade-system-version", help="Upgrades your system and services to match the current DataOps TestGen version."
|
|
474
|
+
)
|
|
475
|
+
@pass_configuration
|
|
476
|
+
def upgrade_system_version(configuration: Configuration):
|
|
477
|
+
do_upgrade_system_version()
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def do_upgrade_system_version():
|
|
481
|
+
LOG.info("setup_app_db command")
|
|
482
|
+
|
|
483
|
+
if run_upgrade_db_config():
|
|
484
|
+
click.echo("System and services were upgraded to match current TestGen version.")
|
|
485
|
+
else:
|
|
486
|
+
click.echo("System and services upgrade is not required.")
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
@cli.command(
|
|
490
|
+
"setup-target-db-functions", help="Use to set up the utility functions in the target database for running profiles."
|
|
491
|
+
)
|
|
492
|
+
@click.option(
|
|
493
|
+
"-c",
|
|
494
|
+
"--connection-id",
|
|
495
|
+
help="The identifier for the connection. Use a connection_id shown in list-connections.",
|
|
496
|
+
required=True,
|
|
497
|
+
type=click.STRING,
|
|
498
|
+
)
|
|
499
|
+
@click.option(
|
|
500
|
+
"-dr",
|
|
501
|
+
"--dry-run",
|
|
502
|
+
default=False,
|
|
503
|
+
is_flag=True,
|
|
504
|
+
required=False,
|
|
505
|
+
help="Dry run to show which schema will be modified",
|
|
506
|
+
)
|
|
507
|
+
@click.option(
|
|
508
|
+
"-cs",
|
|
509
|
+
"--create-qc-schema",
|
|
510
|
+
default=False,
|
|
511
|
+
is_flag=True,
|
|
512
|
+
required=False,
|
|
513
|
+
help="Create the QC utility schema required in the target database",
|
|
514
|
+
)
|
|
515
|
+
@click.option("--yes", "-y", default=False, is_flag=True, required=False, help="Force yes")
|
|
516
|
+
@click.option(
|
|
517
|
+
"--skip-asking-credentials",
|
|
518
|
+
"-s",
|
|
519
|
+
default=False,
|
|
520
|
+
is_flag=True,
|
|
521
|
+
required=False,
|
|
522
|
+
help="Skip request for special write credentials for target database, uses standard credentials instead",
|
|
523
|
+
)
|
|
524
|
+
@click.option(
|
|
525
|
+
"--skip-granting-privileges",
|
|
526
|
+
"-sgp",
|
|
527
|
+
default=False,
|
|
528
|
+
is_flag=True,
|
|
529
|
+
required=False,
|
|
530
|
+
help="Skip granting execute privileges to the user for the QC utility schema in the target database",
|
|
531
|
+
)
|
|
532
|
+
@pass_configuration
|
|
533
|
+
def setup_profiling_tools(
|
|
534
|
+
configuration: Configuration,
|
|
535
|
+
connection_id: str,
|
|
536
|
+
dry_run: bool,
|
|
537
|
+
create_qc_schema: bool,
|
|
538
|
+
yes: bool,
|
|
539
|
+
skip_asking_credentials: bool,
|
|
540
|
+
skip_granting_privileges: bool,
|
|
541
|
+
):
|
|
542
|
+
db_user = None
|
|
543
|
+
db_password = None
|
|
544
|
+
if not skip_asking_credentials:
|
|
545
|
+
db_user = input("Admin DB User?")
|
|
546
|
+
db_password = getpass.getpass("Admin DB Password?")
|
|
547
|
+
|
|
548
|
+
if not yes and not dry_run:
|
|
549
|
+
confirm = input(
|
|
550
|
+
f"Are you sure you want to setup the utility functions to be able to run the profile for connection {connection_id}? [yes/No]"
|
|
551
|
+
)
|
|
552
|
+
if confirm.lower() != "yes":
|
|
553
|
+
click.echo("Exiting without any operation performed.")
|
|
554
|
+
return
|
|
555
|
+
project_qc_schema = run_setup_profiling_tools(
|
|
556
|
+
connection_id, dry_run, create_qc_schema, db_user, db_password, skip_granting_privileges
|
|
557
|
+
)
|
|
558
|
+
if not dry_run:
|
|
559
|
+
message = f"Project DB has been set up. Modified schema: {project_qc_schema}"
|
|
560
|
+
else:
|
|
561
|
+
message = (
|
|
562
|
+
f"Project DB dry run completed, no changes applied. Modified schema would have been: {project_qc_schema}"
|
|
563
|
+
)
|
|
564
|
+
click.echo(message)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
@cli.command("get-test-results", help="Fetches results for a test run.")
|
|
568
|
+
@click.option(
|
|
569
|
+
"-tr",
|
|
570
|
+
"--test-run-id",
|
|
571
|
+
required=True,
|
|
572
|
+
type=click.STRING,
|
|
573
|
+
help="The identifier for a test run. Use a test_run_id shown in list-test-runs.",
|
|
574
|
+
)
|
|
575
|
+
@click.option(
|
|
576
|
+
"-f", "--fails-only", help="Filter test results to view only failed outcomes.", is_flag=True, default=False
|
|
577
|
+
)
|
|
578
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
579
|
+
@pass_configuration
|
|
580
|
+
def get_results(configuration: Configuration, test_run_id: str, fails_only: bool, display: bool):
|
|
581
|
+
rows, header = run_get_results(test_run_id, fails_only)
|
|
582
|
+
if display:
|
|
583
|
+
display_service.print_table(rows, header)
|
|
584
|
+
display_service.to_csv("get_test_results.csv", rows, header)
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
@cli.command("export-observability", help="Sends test results from TestGen to DataOps Observability.")
|
|
588
|
+
@click.option(
|
|
589
|
+
"-pk",
|
|
590
|
+
"--project-key",
|
|
591
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
592
|
+
required=False,
|
|
593
|
+
type=click.STRING,
|
|
594
|
+
default=settings.PROJECT_KEY,
|
|
595
|
+
)
|
|
596
|
+
@click.option(
|
|
597
|
+
"-ts",
|
|
598
|
+
"--test-suite-key",
|
|
599
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
600
|
+
required=False,
|
|
601
|
+
default=settings.DEFAULT_TEST_SUITE_KEY,
|
|
602
|
+
)
|
|
603
|
+
@pass_configuration
|
|
604
|
+
def export_data(configuration: Configuration, project_key: str, test_suite_key: str):
|
|
605
|
+
click.echo(f"export-observability for test suite: {test_suite_key}")
|
|
606
|
+
LOG.info("CurrentStep: Main Program - Observability Export")
|
|
607
|
+
run_observability_exporter(project_key, test_suite_key)
|
|
608
|
+
LOG.info("CurrentStep: Main Program - Observability Export - DONE")
|
|
609
|
+
click.echo("\nexport-observability completed successfully.\n")
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
@cli.command("list-test-types", help="Lists all available TestGen test types.")
|
|
613
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
614
|
+
@pass_configuration
|
|
615
|
+
def list_test_types(configuration: Configuration, display: bool):
|
|
616
|
+
rows, header = run_list_test_types()
|
|
617
|
+
if display:
|
|
618
|
+
display_service.print_table(rows, header)
|
|
619
|
+
display_service.to_csv("list_test_types.csv", rows, header)
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
@cli.command("list-projects", help="Lists all projects in the tenant.")
|
|
623
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
624
|
+
@pass_configuration
|
|
625
|
+
def list_projects(configuration: Configuration, display: bool):
|
|
626
|
+
rows, header = run_list_projects()
|
|
627
|
+
if display:
|
|
628
|
+
display_service.print_table(rows, header)
|
|
629
|
+
display_service.to_csv("list_projects.csv", rows, header)
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
@cli.command("list-connections", help="Lists all database connections in the tenant.")
|
|
633
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
634
|
+
@pass_configuration
|
|
635
|
+
def list_connections(configuration: Configuration, display: bool):
|
|
636
|
+
rows, header = run_list_connections()
|
|
637
|
+
if display:
|
|
638
|
+
display_service.print_table(rows, header)
|
|
639
|
+
display_service.to_csv("list_connections.csv", rows, header)
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
@cli.command("get-test-suite", help="Fetches details for a test suite.")
|
|
643
|
+
@click.option(
|
|
644
|
+
"-pk",
|
|
645
|
+
"--project-key",
|
|
646
|
+
required=False,
|
|
647
|
+
type=click.STRING,
|
|
648
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
649
|
+
default=settings.PROJECT_KEY,
|
|
650
|
+
)
|
|
651
|
+
@click.option(
|
|
652
|
+
"-ts",
|
|
653
|
+
"--test-suite-key",
|
|
654
|
+
help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
|
|
655
|
+
type=click.STRING,
|
|
656
|
+
required=False,
|
|
657
|
+
default=None,
|
|
658
|
+
)
|
|
659
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
660
|
+
@pass_configuration
|
|
661
|
+
def get_test_suite(configuration: Configuration, project_key: str, test_suite_key: str, display: bool):
|
|
662
|
+
rows, header = run_get_test_suite(project_key, test_suite_key)
|
|
663
|
+
if display:
|
|
664
|
+
display_service.print_table(rows, header)
|
|
665
|
+
display_service.to_csv("get_test_suite.csv", rows, header)
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
@cli.command("list-test-suites", help="Lists the test suites in a project.")
|
|
669
|
+
@click.option(
|
|
670
|
+
"-pk",
|
|
671
|
+
"--project-key",
|
|
672
|
+
required=False,
|
|
673
|
+
type=click.STRING,
|
|
674
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
675
|
+
default=settings.PROJECT_KEY,
|
|
676
|
+
)
|
|
677
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
678
|
+
@pass_configuration
|
|
679
|
+
def list_test_suites(configuration: Configuration, project_key: str, display: bool):
|
|
680
|
+
rows, header = run_list_test_suites(project_key)
|
|
681
|
+
if display:
|
|
682
|
+
display_service.print_table(rows, header)
|
|
683
|
+
display_service.to_csv("list_test_suites.csv", rows, header)
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
@cli.command("list-table-groups", help="Lists the tables groups in a project.")
|
|
687
|
+
@click.option(
|
|
688
|
+
"-pk",
|
|
689
|
+
"--project-key",
|
|
690
|
+
required=False,
|
|
691
|
+
type=click.STRING,
|
|
692
|
+
help="The identifier for a TestGen project. Use a project_key shown in list-projects.",
|
|
693
|
+
default=settings.PROJECT_KEY,
|
|
694
|
+
)
|
|
695
|
+
@click.option("-d", "--display", help="Show command output in the terminal.", is_flag=True, default=False)
|
|
696
|
+
@pass_configuration
|
|
697
|
+
def list_table_groups(configuration: Configuration, project_key: str, display: bool):
|
|
698
|
+
rows, header = run_table_group_list(project_key)
|
|
699
|
+
if display:
|
|
700
|
+
display_service.print_table(rows, header)
|
|
701
|
+
display_service.to_csv("list_table_groups.csv", rows, header)
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
@cli.group("ui", help="Manage the browser application")
|
|
705
|
+
def ui(): ...
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
@ui.command("run", help="Run the browser application with default settings")
|
|
709
|
+
@click.option("-d", "--debug", is_flag=True, default=False)
|
|
710
|
+
def run(debug: bool):
|
|
711
|
+
configure_logging(
|
|
712
|
+
level=logging.INFO,
|
|
713
|
+
log_format="%(message)s",
|
|
714
|
+
)
|
|
715
|
+
|
|
716
|
+
status_code: int = -1
|
|
717
|
+
logger = logging.getLogger("testgen")
|
|
718
|
+
stderr: typing.TextIO = typing.cast(typing.TextIO, logs.LogPipe(logger, logging.INFO))
|
|
719
|
+
stdout: typing.TextIO = typing.cast(typing.TextIO, logs.LogPipe(logger, logging.INFO))
|
|
720
|
+
|
|
721
|
+
use_ssl = os.path.isfile(settings.SSL_CERT_FILE) and os.path.isfile(settings.SSL_KEY_FILE)
|
|
722
|
+
|
|
723
|
+
try:
|
|
724
|
+
app_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ui/app.py")
|
|
725
|
+
status_code = subprocess.check_call(
|
|
726
|
+
[ # noqa: S607
|
|
727
|
+
"streamlit",
|
|
728
|
+
"run",
|
|
729
|
+
app_file,
|
|
730
|
+
"--ui.hideSidebarNav=true",
|
|
731
|
+
"--browser.gatherUsageStats=false",
|
|
732
|
+
f"--server.sslCertFile={settings.SSL_CERT_FILE}" if use_ssl else "",
|
|
733
|
+
f"--server.sslKeyFile={settings.SSL_KEY_FILE}" if use_ssl else "",
|
|
734
|
+
"--",
|
|
735
|
+
f"{'--debug' if debug else ''}",
|
|
736
|
+
],
|
|
737
|
+
stdout=stdout,
|
|
738
|
+
stderr=stderr,
|
|
739
|
+
)
|
|
740
|
+
except Exception:
|
|
741
|
+
LOG.exception(f"Testgen UI exited with status code {status_code}")
|
|
742
|
+
raise
|
|
743
|
+
finally:
|
|
744
|
+
if stderr:
|
|
745
|
+
stderr.close()
|
|
746
|
+
if stdout:
|
|
747
|
+
stdout.close()
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
@ui.command("plugins", help="List installed application plugins")
|
|
751
|
+
def list_ui_plugins():
|
|
752
|
+
installed_plugins = list(plugins.discover())
|
|
753
|
+
|
|
754
|
+
click.echo(click.style(len(installed_plugins), fg="bright_magenta") + click.style(" plugins installed", bold=True))
|
|
755
|
+
for plugin in installed_plugins:
|
|
756
|
+
click.echo(click.style(" + ", fg="bright_green") + f"{plugin.package: <30}" + f"\tversion: {plugin.version}")
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
@ui.command("patch-streamlit", help="Modifies Streamlit's internals with custom static files")
|
|
760
|
+
@click.option("-f", "--force", is_flag=True, default=False)
|
|
761
|
+
def patch_streamlit(force: bool) -> None:
|
|
762
|
+
from testgen.ui.scripts import patch_streamlit
|
|
763
|
+
|
|
764
|
+
patched_files = patch_streamlit.patch(force=force)
|
|
765
|
+
click.echo(click.style("Patched ", bold=True) + click.style(patch_streamlit.STREAMLIT_INDEX, fg="bright_magenta"))
|
|
766
|
+
click.echo(click.style(" + ", fg="bright_green") + click.style(f"patched {len(patched_files)} files", italic=True))
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
if __name__ == "__main__":
|
|
770
|
+
cli()
|