dataops-testgen 2.2.0__py3-none-any.whl

testgen/commands/run_setup_profiling_tools.py

@@ -0,0 +1,96 @@
+import logging
+
+from testgen.commands.run_get_entities import run_get_connection
+from testgen.common import AssignConnectParms, RunActionQueryList
+from testgen.common.database.database_service import get_queries_for_command
+
+LOG = logging.getLogger("testgen")
+
+
+def _get_params_mapping(project_qc_schema: str, user: str, user_role: str | None) -> dict:
+    return {
+        "DATA_QC_SCHEMA": project_qc_schema,
+        "DB_USER": user,
+        "DB_USER_ROLE": user_role,
+    }
+
+
+def get_setup_profiling_tools_queries(sql_flavor, create_qc_schema, skip_granting_privileges, project_qc_schema, user, user_role=None):
+    queries = []
+
+    params_mapping = _get_params_mapping(project_qc_schema, user, user_role)
+
+    if create_qc_schema:
+        queries.extend(
+            get_queries_for_command(
+                f"flavors/{sql_flavor}/setup_profiling_tools",
+                params_mapping,
+                mask=rf"^.*create_qc_schema_{sql_flavor}.sql$",
+            )
+        )
+
+    queries.extend(
+        get_queries_for_command(
+            f"flavors/{sql_flavor}/setup_profiling_tools", params_mapping, mask=rf"^.*functions_{sql_flavor}.sql$"
+        )
+    )
+
+    if not skip_granting_privileges:
+        queries.extend(
+            get_queries_for_command(
+                f"flavors/{sql_flavor}/setup_profiling_tools",
+                params_mapping,
+                mask=rf"^.*grant_execute_privileges_{sql_flavor}.sql$",
+            )
+        )
+
+    return queries
+
+
+def run_setup_profiling_tools(
+    connection_id: str | int,
+    dry_run: bool,
+    create_qc_schema: bool = True,
+    db_user: str | None = None,
+    db_password: str | None = None,
+    skip_granting_privileges: bool = False,
+    admin_private_key_passphrase: str | None = None,
+    admin_private_key: str | None = None,
+    user_role: str | None = None,
+) -> str:
+    connection = run_get_connection(str(connection_id))
+
+    # Set Project Connection Parms in common.db_bridgers from retrieved parms
+    LOG.info("CurrentStep: Assigning Connection Parms")
+    user = db_user or connection["project_user"]
+    connect_by_key = admin_private_key is not None or connection["connect_by_key"]
+    private_key_passphrase = admin_private_key_passphrase if admin_private_key is not None else connection["private_key_passphrase"]
+    private_key = admin_private_key if admin_private_key is not None else connection["private_key"]
+
+    AssignConnectParms(
+        connection["project_key"],
+        connection["connection_id"],
+        connection["project_host"],
+        connection["project_port"],
+        connection["project_db"],
+        connection["project_qc_schema"],
+        user,
+        connection["sql_flavor"],
+        connection["url"],
+        connection["connect_by_url"],
+        connect_by_key,
+        private_key,
+        private_key_passphrase,
+        "PROJECT",
+    )
+
+    project_qc_schema = connection["project_qc_schema"]
+    sql_flavor = connection["sql_flavor"]
+    user = connection["project_user"]
+
+    queries = get_setup_profiling_tools_queries(sql_flavor, create_qc_schema, skip_granting_privileges, project_qc_schema, user, user_role)
+
+    if not dry_run:
+        RunActionQueryList("PROJECT", queries, user_override=db_user, pwd_override=db_password)
+
+    return project_qc_schema

testgen/commands/run_test_definition.py

@@ -0,0 +1,146 @@
+from testgen.common import RetrieveDBResultsToDictList, RunActionQueryList, WriteListToDB, read_template_sql_file
+
+
+def get_test_def_parms(project_code, test_suite):
+    lstResults = run_test_def_parms(project_code, test_suite)
+
+    if lstResults is None:
+        raise ValueError("Test Definition Parameters not found")
+
+    yaml_dict = {}
+
+    for row in lstResults:
+        project_code = row["project_code"]
+        test_suite = row["test_suite"]
+        schema = row["schema_name"]
+        table_name = row["table_name"]
+        column_name = row["column_name"]
+        row_id = (row["id"],)
+        test_type = (row["test_type"],)
+        test_description = (row["test_description"],)
+        test_action = (row["test_action"],)
+        test_active = (row["test_active"],)
+        lock_refresh = (row["lock_refresh"],)
+        severity = (row["severity"],)
+        test_parameters = (row["test_parameters"],)
+
+        if project_code not in yaml_dict:
+            yaml_dict[project_code] = {}
+        if test_suite not in yaml_dict[project_code]:
+            yaml_dict[project_code][test_suite] = {}
+        if schema not in yaml_dict[project_code][test_suite]:
+            yaml_dict[project_code][test_suite][schema] = {}
+        if table_name not in yaml_dict[project_code][test_suite][schema]:
+            yaml_dict[project_code][test_suite][schema][table_name] = {}
+        if column_name not in yaml_dict[project_code][test_suite][schema][table_name]:
+            yaml_dict[project_code][test_suite][schema][table_name][column_name] = []
+
+        parm_columns = test_parameters[0].split(",")
+        parm_dict = {}
+
+        for column in parm_columns:
+            parm_dict[column] = row[column]
+
+        yaml_dict[project_code][test_suite][schema][table_name][column_name].append(
+            {
+                "id": str(row_id[0]),
+                "test_type": str(test_type[0]),
+                "test_description": str(test_description[0]),
+                "test_action": str(test_action[0]),
+                "test_active": str(test_active[0]),
+                "lock_refresh": str(lock_refresh[0]),
+                "severity": str(severity[0]),
+                "test_parameters": parm_dict,
+            }
+        )
+
+    return yaml_dict
+
+
+def run_test_def_parms(project_code, test_suite):
+    sql_template = read_template_sql_file("get_test_def_parms.sql", "updates")
+
+    sql_template = sql_template.replace("{PROJECT_CODE}", project_code)
+    sql_template = sql_template.replace("{TEST_SUITE}", test_suite)
+
+    return RetrieveDBResultsToDictList("DKTG", sql_template)
+
+
+def update_test_def_parms_dict(yaml_dict):
+    if yaml_dict is None:
+        raise ValueError("Test Definition Parameters not found")
+
+    updResults = update_test_definitions(yaml_dict)
+    RunActionQueryList("DKTG", updResults)
+
+
+def update_test_definitions(data):
+    list_columns = []
+    list_update_insert_queries = []
+
+    for project_code, test_suite_dict in data.items():
+        for test_suite, schema_dict in test_suite_dict.items():
+            for schema, table_dict in schema_dict.items():
+                for table, column_dict in table_dict.items():
+                    for column, attributes_list in column_dict.items():
+                        for attribute in attributes_list:
+                            id_col = attribute["id"]
+                            test_type = attribute["test_type"]
+                            test_description = attribute["test_description"]
+                            test_action = attribute["test_action"]
+                            test_active = attribute["test_active"]
+                            lock_refresh = attribute["lock_refresh"]
+                            severity = attribute["severity"]
+                            test_parameters = attribute["test_parameters"]
+
+                            column_keys = test_parameters.keys()
+                            column_values = test_parameters.values()
+
+                            for col, value in zip(column_keys, column_values, strict=False):
+                                list_columns.append(
+                                    [
+                                        project_code,
+                                        test_suite,
+                                        schema,
+                                        table,
+                                        column,
+                                        id_col,
+                                        test_type,
+                                        test_description,
+                                        test_action,
+                                        test_active,
+                                        lock_refresh,
+                                        severity,
+                                        col,
+                                        value,
+                                    ]
+                                )
+
+    col_list = [
+        "project_code",
+        "test_suite",
+        "schema_name",
+        "table_name",
+        "column_name",
+        "id",
+        "test_type",
+        "test_description",
+        "test_action",
+        "test_active",
+        "lock_refresh",
+        "severity",
+        "test_parameter",
+        "test_parameter_value",
+    ]
+
+    list_create_queries = []
+    create_table = read_template_sql_file("create_tmp_test_definition.sql", "updates")
+    list_create_queries.append(create_table)
+    RunActionQueryList("DKTG", list_create_queries)
+
+    # Write to tmp_test_definition
+    WriteListToDB("DKTG", list_columns, col_list, "tmp_test_definition")
+
+    sql_template = read_template_sql_file("populate_stg_test_definitions.sql", "updates")
+    list_update_insert_queries.append(sql_template)
+    return list_update_insert_queries

testgen/commands/run_test_parameter_validation.py

@@ -0,0 +1,135 @@
+import logging
+
+from testgen.commands.queries.test_parameter_validation_query import CTestParamValidationSQL
+from testgen.common import AssignConnectParms, RetrieveDBResultsToDictList, RetrieveTestExecParms, RunActionQueryList
+
+LOG = logging.getLogger("testgen")
+
+
+def run_parameter_validation_queries(
+    test_run_id="", test_time="", strProjectCode="", strTestSuite="", booRunFromTestExec=True
+):
+    LOG.info("CurrentStep: Retrieving TestzVal Parameters")
+    dctParms = RetrieveTestExecParms(strProjectCode, strTestSuite)
+    LOG.debug("Test Parameter Validation - Parameters retrieved")
+
+    # Set Project Connection Parms in db_bridgers from retrieved parms
+    LOG.info("CurrentStep: Assigning Connection Parms")
+    AssignConnectParms(
+        dctParms["project_code"],
+        dctParms["connection_id"],
+        dctParms["project_host"],
+        dctParms["project_port"],
+        dctParms["project_db"],
+        dctParms["table_group_schema"],
+        dctParms["project_user"],
+        dctParms["sql_flavor"],
+        dctParms["url"],
+        dctParms["connect_by_url"],
+        dctParms["connect_by_key"],
+        dctParms["private_key"],
+        dctParms["private_key_passphrase"],
+        "PROJECT",
+    )
+
+    LOG.debug("Validating parameters for Test Suite %s")
+
+    LOG.info("CurrentStep: Initializing Test Parameter Validation")
+    clsExecute = CTestParamValidationSQL(strProjectCode, dctParms["sql_flavor"], strTestSuite)
+    clsExecute.run_date = test_time
+    clsExecute.test_run_id = test_run_id
+    LOG.info("CurrentStep: Validation Class successfully initialized")
+
+    booClean = False
+
+    # Retrieve Test Column list
+    LOG.info("CurrentStep: Retrieve Test Columns for Validation")
+    strColumnList = clsExecute.GetTestValidationColumns(booClean)
+    lstTestColumns = RetrieveDBResultsToDictList("DKTG", strColumnList)
+
+    if len(lstTestColumns) == 0:
+        LOG.warning(f"No test columns are present to validate in Test Suite {strTestSuite}")
+        missing_columns = []
+    else:
+        # Derive test schema list -- make CSV string from list of columns
+        #  to be used as criteria for retrieving data dictionary
+        setSchemas = {s["columns"].split(".")[0] for s in lstTestColumns}
+        strSchemas = ", ".join([f"'{value}'" for value in setSchemas])
+        LOG.debug("Test column list successfully retrieved")
+
+        # Retrieve Project Column list
+        LOG.info("CurrentStep: Retrieve Test Columns for Validation")
+        clsExecute.test_schemas = strSchemas
+        strProjectColumnList = clsExecute.GetProjectTestValidationColumns()
+        if "where table_schema in ()" in strProjectColumnList:
+            raise ValueError("No schema specified in Validation Columns check")
+        lstProjectTestColumns = RetrieveDBResultsToDictList("PROJECT", strProjectColumnList)
+
+        if len(lstProjectTestColumns) == 0:
+            LOG.info("Project Test Column list is empty")
+
+        LOG.debug("Project column list successfully received")
+        LOG.info("CurrentStep: Compare column sets")
+        # load results into sets
+        result_set1 = {item["columns"].lower() for item in set(lstTestColumns)}
+        result_set2 = {item["columns"].lower() for item in set(lstProjectTestColumns)}
+
+        # Check if all columns exist in the table
+        missing_columns = result_set1.difference(result_set2)
+
+        if len(missing_columns) == 0:
+            LOG.info("No missing column in Project Column list.")
+
+        strMissingColumns = ", ".join(f"'{x}'" for x in missing_columns)
+        srtNoQuoteMissingCols = strMissingColumns.replace("'", "")
+
+    if missing_columns:
+        LOG.debug("Test Columns are missing in target database: %s", srtNoQuoteMissingCols)
+
+        # Extracting schema.tables that are missing from the result sets
+        tables_set1 = {elem.rsplit(".", 1)[0] for elem in result_set1}
+        tables_set2 = {elem.rsplit(".", 1)[0] for elem in result_set2}
+
+        # Check if all the tables exist in the schema
+        missing_tables = tables_set1.difference(tables_set2)
+
+        if missing_tables:
+            strMissingtables = ", ".join(f"'{x}'" for x in missing_tables)
+        else:
+            LOG.info("No missing tables in Project Column list.")
+            strMissingtables = "''"
+
+        # Flag test_definitions tests with missing columns:
+        LOG.info("CurrentStep: Flagging Tests That Failed Validation")
+        clsExecute.missing_columns = strMissingColumns
+        clsExecute.missing_tables = strMissingtables
+        # Flag Value is D if called from execute_tests_qry.py, otherwise N to disable now
+        if booRunFromTestExec:
+            clsExecute.flag_val = "D"
+            strTempMessage = "Tests that failed parameter validation have been flagged."
+        else:
+            clsExecute.flag_val = "N"
+            strTempMessage = "Tests that failed parameter validation have been set to inactive."
+        strFlagTests = clsExecute.FlagTestsWithFailedValidation()
+        RunActionQueryList("DKTG", [strFlagTests])
+        LOG.debug(strTempMessage)
+
+        # when run_parameter_validation_queries() is called from execute_tests_query.py:
+        # we disable tests and write validation errors to test_results table.
+        if booRunFromTestExec:
+            # Copy test results to DK DB, using temporary flagged -1 value to identify
+            LOG.info("CurrentStep: Saving error results for invalid tests")
+            strReportValErrors = clsExecute.ReportTestValidationErrors()
+            RunActionQueryList("DKTG", [strReportValErrors])
+            LOG.debug("Results inserted for invalid tests")
+
+            # Set to Inactive those test_definitions tests that are flagged D:  set to N
+
+            LOG.info("CurrentStep: Disabling Tests That Failed Validation")
+            strDisableTests = clsExecute.DisableTestsWithFailedValidation()
+            RunActionQueryList("DKTG", [strDisableTests])
+            LOG.debug("Tests that failed parameter validation have been disabled.")
+
+        LOG.info("Validation Complete: tests referencing missing columns have been disabled.")
+    else:
+        LOG.info("Validation Successful: No columns missing from target database.")

testgen/commands/run_upgrade_db_config.py

@@ -0,0 +1,156 @@
+import logging
+
+from testgen import settings
+from testgen.common import RetrieveSingleResultValue, RunActionQueryList, read_template_sql_file
+from testgen.common.credentials import get_tg_schema
+from testgen.common.database.database_service import replace_params
+from testgen.common.read_file import get_template_files
+
+LOG = logging.getLogger("testgen")
+
+
+def _get_params_mapping() -> dict:
+    return {
+        "SCHEMA_NAME": get_tg_schema(),
+        "TESTGEN_ADMIN_USER": settings.DATABASE_ADMIN_USER,
+        "TESTGEN_ADMIN_PASSWORD": settings.DATABASE_ADMIN_PASSWORD,
+        "OBSERVABILITY_URL": settings.OBSERVABILITY_API_URL,
+    }
+
+
+def _get_revision_prefix(params_mapping):
+    strQuery = read_template_sql_file("get_tg_revision.sql", "dbupgrade_helpers")
+    strQuery = replace_params(strQuery, params_mapping)
+
+    intNextRevision = RetrieveSingleResultValue("DKTG", strQuery)
+
+    return intNextRevision
+
+
+def _get_next_revision_prefix(params_mapping):
+    return _get_revision_prefix(params_mapping) + 1
+
+
+def get_schema_revision():
+    params_mapping = {"SCHEMA_NAME": get_tg_schema()}
+    try:
+        schema_revision = str(_get_revision_prefix(params_mapping))
+    except Exception:
+        schema_revision = "UNKNOWN"
+    return schema_revision
+
+
+def _format_revision_prefix(intNextRevision: int):
+    if intNextRevision < 1 or intNextRevision > 9999:
+        raise ValueError(f"The prefix {intNextRevision} for the upgrade script is out of range.")
+
+    return str(intNextRevision).zfill(4)
+
+
+def _get_upgrade_template_directory():
+    return "dbupgrade"
+
+
+def _get_upgrade_scripts(sub_directory: str, params_mapping: dict, mask: str = r"^.*sql$", min_val: str = ""):
+    files = sorted(get_template_files(mask=mask, sub_directory=sub_directory), key=lambda key: str(key))
+
+    max_prefix = ""
+    queries = []
+    for file in files:
+        if file.name > min_val:
+            template = file.read_text("utf-8")
+            query = replace_params(template, params_mapping)
+            queries.append(query)
+            max_prefix = file.name[0:4]
+
+    if len(queries) == 0:
+        LOG.debug(f"No sql files were found for the mask {mask} in subdirectory {sub_directory}")
+
+    return queries, max_prefix
+
+
+def _execute_upgrade_scripts(params_mapping, lstScripts):
+    # Run scripts using admin credentials
+    RunActionQueryList(
+        "DKTG",
+        lstScripts,
+        "S",
+        user_override=params_mapping["TESTGEN_ADMIN_USER"],
+        pwd_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
+    )
+    return True
+
+
+def _refresh_static_metadata(params_mapping):
+    # Refresh static metadata -- shouldn't affect user data
+    strQueryMetadata = read_template_sql_file("050_populate_new_schema_metadata.sql", "dbsetup")
+    strQueryMetadata = replace_params(strQueryMetadata, params_mapping)
+    # Recreate standard views
+    strQueryViews = read_template_sql_file("060_create_standard_views.sql", "dbsetup")
+    strQueryViews = replace_params(strQueryViews, params_mapping)
+    # Reassign rights to standard roles
+    strQueryRights = read_template_sql_file("075_grant_role_rights.sql", "dbsetup")
+    strQueryRights = replace_params(strQueryRights, params_mapping)
+
+    RunActionQueryList(
+        "DKTG",
+        [strQueryMetadata, strQueryViews, strQueryRights],
+        "S",
+        user_override=params_mapping["TESTGEN_ADMIN_USER"],
+        pwd_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
+    )
+
+
+def _update_revision_number(params_mapping, latest_prefix_applied):
+    # Update extant revision number to highest script prefix applied
+    strQuery = read_template_sql_file("080_set_current_revision.sql", "dbsetup")
+    strQuery = strQuery.replace("{DB_REVISION}", str(int(latest_prefix_applied)))
+    strQuery = replace_params(strQuery, params_mapping)
+
+    RunActionQueryList(
+        "DKTG",
+        [strQuery],
+        "S",
+        user_override=params_mapping["TESTGEN_ADMIN_USER"],
+        pwd_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
+    )
+
+
+def run_upgrade_db_config() -> bool:
+    LOG.info("Running run_upgrade_db_config")
+
+    params_mapping = _get_params_mapping()
+
+    # Look for prefix one higher than last revision extant in db
+    strNextPrefix = _format_revision_prefix(_get_next_revision_prefix(params_mapping))
+    # Retrieve template upgrade directory name
+    upgrade_dir = _get_upgrade_template_directory()
+
+    # Retrieve and execute upgrade scripts, if any
+    lstQueries, max_prefix = _get_upgrade_scripts(upgrade_dir, params_mapping, min_val=strNextPrefix)
+    LOG.info(f"Updating db config qty of queries: {len(lstQueries)}. New prefix: {max_prefix}. Queries: {lstQueries}")
+    if len(lstQueries) > 0:
+        has_been_upgraded = _execute_upgrade_scripts(params_mapping, lstQueries)
+    else:
+        has_been_upgraded = False
+
+    _refresh_static_metadata(params_mapping)
+
+    if has_been_upgraded:
+        # Update revision number to max prefix found in update scripts
+        _update_revision_number(params_mapping, max_prefix)
+        LOG.info("Application data was successfully upgraded, and static metadata was refreshed.")
+    else:
+        LOG.info("Database upgrade was not required. Static metadata was refreshed.")
+
+    return has_been_upgraded
+
+
+def is_db_revision_up_to_date():
+    params_mapping = {"SCHEMA_NAME": get_tg_schema()}
+    strNextPrefix = _format_revision_prefix(_get_next_revision_prefix(params_mapping))
+    upgrade_dir = _get_upgrade_template_directory()
+
+    # Retrieve and execute upgrade scripts, if any
+    lstQueries, max_prefix = _get_upgrade_scripts(upgrade_dir, params_mapping, min_val=strNextPrefix)
+    return len(lstQueries) == 0

testgen/common/__init__.py

@@ -0,0 +1,8 @@
+from testgen.common.database.database_service import *
+
+from .clean_sql import *
+from .credentials import *
+from .encrypt import *
+from .get_pipeline_parms import *
+from .logs import *
+from .read_file import *

testgen/common/clean_sql.py

@@ -0,0 +1,53 @@
+__all__ = ["AddQuotesToIdentifierCSV", "CleanSQL", "ConcatColumnList"]
+
+import re
+
+
+def CleanSQL(strInput: str) -> str:
+    # Use regular expression to remove comment text fenced by /*...*/
+    strInput = re.sub(r"/\*.*?\*/", "", strInput, flags=re.DOTALL)
+    # Use regular expression to remove comment text starting with --
+    strInput = re.sub(r"--.*$", "", strInput, flags=re.MULTILINE)
+    # Use regular expression to replace any tab with one space
+    strInput = re.sub(r"\t", " ", strInput)
+    # Use regular expression to remove spaces outside quotes
+    parts = re.split(r"""("[^"]*"|'[^']*')""", strInput)
+    parts[::2] = (" ".join(s.split()) for s in parts[::2])  # outside quotes
+    return " ".join(parts)
+
+
+def AddQuotesToIdentifierCSV(strInput: str) -> str:
+    # Keywords -- identifiers to quote
+    keywords = [
+        "select",
+        "from",
+        "where",
+        "order",
+        "by",
+        "having",
+    ]
+
+    quoted_values = []
+    for value in strInput.split(","):
+        value = value.strip()
+        if value.startswith('"') and value.endswith('"'):
+            quoted_values.append(value)
+        elif any(c.isupper() or c.isspace() or value.lower() in keywords for c in value):
+            quoted_values.append(f'"{value}"')
+        else:
+            quoted_values.append(value)
+    return ", ".join(quoted_values)
+
+
+def ConcatColumnList(str_column_list, str_null_value):
+    # Prepares SQL expression to concatenate comma-separated column list into single SQL expression
+    str_expression = ""
+    if str_column_list:
+        if "," in str_column_list:
+            # Split each comma separated column name into individual list items
+            cols = [s.strip() for s in str_column_list.split(",")]
+            str_each = [f"COALESCE({i}, '{str_null_value}')" for i in cols]
+            str_expression = "CONCAT(" + ", ".join(str_each) + ")"
+        else:
+            str_expression = str_column_list
+    return str_expression

testgen/common/credentials.py

@@ -0,0 +1,25 @@
+from testgen import settings
+
+
+def get_tg_host() -> str:
+    return settings.DATABASE_HOST
+
+
+def get_tg_port() -> str:
+    return settings.DATABASE_PORT
+
+
+def get_tg_db() -> str:
+    return settings.DATABASE_NAME
+
+
+def get_tg_schema() -> str:
+    return settings.DATABASE_SCHEMA
+
+
+def get_tg_username() -> str:
+    return settings.DATABASE_USER
+
+
+def get_tg_password() -> str:
+    return settings.DATABASE_PASSWORD