dataops-testgen 2.2.0__py3-none-any.whl

testgen/commands/run_execute_tests.py

@@ -0,0 +1,168 @@
+import logging
+import subprocess
+import threading
+import uuid
+
+import testgen.common.process_service as process_service
+from testgen import settings
+from testgen.commands.queries.execute_tests_query import CTestExecutionSQL
+from testgen.common import (
+    AssignConnectParms,
+    RetrieveDBResultsToDictList,
+    RetrieveTestExecParms,
+    RunActionQueryList,
+    RunThreadedRetrievalQueryList,
+    WriteListToDB,
+    date_service,
+)
+from testgen.common.database.database_service import empty_cache
+
+from .run_execute_cat_tests import run_cat_test_queries
+from .run_test_parameter_validation import run_parameter_validation_queries
+
+LOG = logging.getLogger("testgen")
+
+
+def run_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset=0, spinner=None):
+    booErrors = False
+    error_msg = ""
+
+    LOG.info("CurrentStep: Retrieving TestExec Parameters")
+    dctParms = RetrieveTestExecParms(strProjectCode, strTestSuite)
+
+    # Set Project Connection Parms in common.db_bridgers from retrieved parms
+    LOG.info("CurrentStep: Assigning Connection Parms")
+
+    AssignConnectParms(
+        dctParms["project_code"],
+        dctParms["connection_id"],
+        dctParms["project_host"],
+        dctParms["project_port"],
+        dctParms["project_db"],
+        dctParms["table_group_schema"],
+        dctParms["project_user"],
+        dctParms["sql_flavor"],
+        dctParms["url"],
+        dctParms["connect_by_url"],
+        dctParms["connect_by_key"],
+        dctParms["private_key"],
+        dctParms["private_key_passphrase"],
+        "PROJECT",
+    )
+
+    LOG.info("CurrentStep: Initializing Query Generator")
+
+    clsExecute = CTestExecutionSQL(strProjectCode, dctParms["sql_flavor"], strTestSuite, minutes_offset)
+    clsExecute.run_date = strTestTime
+    clsExecute.test_run_id = strTestRunID
+    clsExecute.process_id = process_service.get_current_process_id()
+    booClean = False
+
+    # Add a record in Test Run table for the new Test Run
+    strTestRunQuery = clsExecute.AddTestRecordtoTestRunTable()
+    lstTestRunQuery = [strTestRunQuery]
+    RunActionQueryList("DKTG", lstTestRunQuery)
+
+    try:
+        # Retrieve non-CAT Queries
+        LOG.info("CurrentStep: Retrieve Non-CAT Queries")
+        strQuery = clsExecute.GetTestsNonCAT(booClean)
+        lstTestSet = RetrieveDBResultsToDictList("DKTG", strQuery)
+
+        if len(lstTestSet) == 0:
+            LOG.debug("0 non-CAT Queries retrieved.")
+
+        if lstTestSet:
+            LOG.info("CurrentStep: Preparing Non-CAT Tests")
+            lstTestQueries = []
+            for dctTest in lstTestSet:
+                # Set Test Parms
+                clsExecute.ClearTestParms()
+                clsExecute.dctTestParms = dctTest
+                lstTestQueries.append(clsExecute.GetTestQuery(booClean))
+                if spinner:
+                    spinner.next()
+
+            # Execute list, returning test results
+            LOG.info("CurrentStep: Executing Non-CAT Test Queries")
+            lstTestResults, colResultNames, intErrors = RunThreadedRetrievalQueryList(
+                "PROJECT", lstTestQueries, dctParms["max_threads"], spinner
+            )
+
+            # Copy test results to DK DB
+            LOG.info("CurrentStep: Saving Non-CAT Test Results")
+            if lstTestResults:
+                WriteListToDB("DKTG", lstTestResults, colResultNames, "test_results")
+            if intErrors > 0:
+                booErrors = True
+                error_msg = (
+                    f"Errors were encountered executing Referential Tests. ({intErrors} errors occurred.) "
+                    "Please check log. "
+                )
+                LOG.warning(error_msg)
+        else:
+            LOG.info("No tests found")
+
+    except Exception as e:
+        sqlsplit = e.args[0].split("[SQL", 1)
+        errorline = sqlsplit[0].replace("'", "''") if len(sqlsplit) > 0 else "unknown error"
+        clsExecute.exception_message = f"{type(e).__name__}: {errorline}"
+        LOG.info("Updating the test run record with exception message")
+        lstTestRunQuery = [clsExecute.PushTestRunStatusUpdateSQL()]
+        RunActionQueryList("DKTG", lstTestRunQuery)
+        raise
+
+    else:
+        return booErrors, error_msg
+
+
+def run_execution_steps_in_background(project_code, test_suite):
+    msg = f"Starting run_execution_steps_in_background against test suite: {test_suite}"
+    if settings.IS_DEBUG:
+        LOG.info(msg + ". Running in debug mode (new thread instead of new process).")
+        empty_cache()
+        background_thread = threading.Thread(
+            target=run_execution_steps,
+            args=(
+                project_code,
+                test_suite
+            ),
+        )
+        background_thread.start()
+    else:
+        LOG.info(msg)
+        script = ["testgen", "run-tests", "--project-key", project_code, "--test-suite-key", test_suite]
+        subprocess.Popen(script)  # NOQA S603
+
+
+def run_execution_steps(strProjectCode, strTestSuite, minutes_offset=0, spinner=None):
+    # Initialize required parms for all three steps
+    booErrors = False
+    error_msg = ""
+
+    strTestRunID = str(uuid.uuid4())
+    strTestTime = date_service.get_now_as_string_with_offset(minutes_offset)
+
+    if spinner:
+        spinner.next()
+
+    LOG.info("CurrentStep: Execute Step - Test Validation")
+    run_parameter_validation_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, True)
+
+    LOG.info("CurrentStep: Execute Step - Test Execution")
+    booErrors, error_msg = run_test_queries(
+        strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset, spinner
+    )
+
+    LOG.info("CurrentStep: Execute Step - CAT Test Execution")
+    if run_cat_test_queries(
+        strTestRunID, strTestTime, strProjectCode, strTestSuite, error_msg, minutes_offset, spinner
+    ):
+        booErrors = True
+
+    if booErrors:
+        str_error_status = "with errors. Check log for details."
+    else:
+        str_error_status = "successfully."
+    message = f"Test Execution completed {str_error_status}"
+    return message

testgen/commands/run_generate_tests.py

@@ -0,0 +1,107 @@
+import logging
+
+from testgen.commands.queries.generate_tests_query import CDeriveTestsSQL
+from testgen.common import AssignConnectParms, RetrieveDBResultsToDictList, RetrieveTestGenParms, RunActionQueryList
+
+LOG = logging.getLogger("testgen")
+
+
+def run_test_gen_queries(strTableGroupsID, strTestSuite, strGenerationSet=None):
+    if strTableGroupsID is None:
+        raise ValueError("Table Group ID was not specified")
+
+    clsTests = CDeriveTestsSQL()
+
+    # Set General Parms
+    booClean = False
+
+    LOG.info("CurrentStep: Retrieving General Parameters for Test Suite " + strTestSuite)
+    dctParms = RetrieveTestGenParms(strTableGroupsID, strTestSuite)
+
+    # Set Project Connection Parms from retrieved parms
+    LOG.info("CurrentStep: Assigning Connection Parameters")
+    AssignConnectParms(
+        dctParms["project_code"],
+        dctParms["connection_id"],
+        dctParms["project_host"],
+        dctParms["project_port"],
+        dctParms["project_db"],
+        dctParms["table_group_schema"],
+        dctParms["project_user"],
+        dctParms["sql_flavor"],
+        dctParms["url"],
+        dctParms["connect_by_url"],
+        dctParms["connect_by_key"],
+        dctParms["private_key"],
+        dctParms["private_key_passphrase"],
+        "PROJECT",
+    )
+
+    # Set static parms
+    clsTests.project_code = dctParms["project_code"]
+    clsTests.test_suite = strTestSuite
+    clsTests.test_suite_id = dctParms["test_suite_id"]
+    clsTests.generation_set = strGenerationSet if strGenerationSet is not None else ""
+    clsTests.connection_id = str(dctParms["connection_id"])
+    clsTests.table_groups_id = strTableGroupsID
+    clsTests.sql_flavor = dctParms["sql_flavor"]
+
+    clsTests.data_schema = dctParms["table_group_schema"]
+    if dctParms["profiling_as_of_date"] is not None:
+        clsTests.as_of_date = dctParms["profiling_as_of_date"].strftime("%Y-%m-%d %H:%M:%S")
+
+    if dctParms["test_suite"] is None:
+        LOG.info("CurrentStep: Creating new Test Suite")
+        strQuery = clsTests.GetInsertTestSuiteSQL(booClean)
+        if strQuery:
+            RunActionQueryList("DKTG", [strQuery])
+        else:
+            raise ValueError("Test Suite not found and could not be created")
+
+    LOG.info("CurrentStep: Compiling Test Gen Queries")
+
+    lstFunnyTemplateQueries = clsTests.GetTestDerivationQueriesAsList(booClean)
+    lstGenericTemplateQueries = []
+
+    # Delete old Tests
+    strDeleteQuery = clsTests.GetDeleteOldTestsQuery(booClean)
+
+    # Retrieve test_types as parms from list of dictionaries:  test_type, selection_criteria, default_parm_columns,
+    # default_parm_values
+    strQuery = clsTests.GetTestTypesSQL(booClean)
+
+    # Execute Query
+    if strQuery:
+        lstTestTypes = RetrieveDBResultsToDictList("DKTG", strQuery)
+
+        if lstTestTypes is None:
+            raise ValueError("Test Type Parameters not found")
+        elif (
+            lstTestTypes[0]["test_type"] == ""
+            or lstTestTypes[0]["selection_criteria"] == ""
+            or lstTestTypes[0]["default_parm_columns"] == ""
+            or lstTestTypes[0]["default_parm_values"] == ""
+        ):
+            raise ValueError("Test Type parameters not correctly set")
+    else:
+        raise ValueError("Test Type Queries were not generated")
+
+    for dctTestParms in lstTestTypes:
+        clsTests.ClearTestParms()
+        clsTests.dctTestParms = dctTestParms
+        strQuery = clsTests.GetTestQueriesFromGenericFile(booClean)
+
+        if strQuery:
+            lstGenericTemplateQueries.append(strQuery)
+
+    LOG.info("TestGen CAT Queries were compiled")
+
+    # Make sure delete, then generic templates run before the funny templates
+    lstQueries = [strDeleteQuery, *lstGenericTemplateQueries, *lstFunnyTemplateQueries]
+
+    if lstQueries:
+        LOG.info("Running Test Generation Template Queries")
+        RunActionQueryList("DKTG", lstQueries)
+        return "Test generation completed successfully."
+    else:
+        return "No TestGen Queries were compiled."

testgen/commands/run_get_entities.py

@@ -0,0 +1,122 @@
+import logging
+
+from testgen.common import RetrieveDBResultsToList, read_template_sql_file
+from testgen.common.encrypt import DecryptText
+
+LOG = logging.getLogger("testgen")
+
+
+def run_list_profiles(table_groups_id):
+    sql_template = read_template_sql_file("get_profile_list.sql", "get_entities")
+
+    sql_template = sql_template.replace("{TABLE_GROUPS_ID}", table_groups_id)
+
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_list_test_types():
+    sql_template = read_template_sql_file("list_test_types.sql", "get_entities")
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_list_projects():
+    sql_template = read_template_sql_file("get_project_list.sql", "get_entities")
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_list_connections():
+    sql_template = read_template_sql_file("get_connections_list.sql", "get_entities")
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_get_connection(connection_id):
+    sql_template = read_template_sql_file("get_connection.sql", "get_entities")
+    sql_template = sql_template.replace("{CONNECTION_ID}", str(connection_id))
+    rows, _ = RetrieveDBResultsToList("DKTG", sql_template)
+    connection = rows.pop()._asdict()
+    connection["password"] = DecryptText(connection["project_pw_encrypted"]) if connection["project_pw_encrypted"] else None
+    connection["private_key"] = DecryptText(connection["private_key"]) if connection["private_key"] else None
+    connection["private_key_passphrase"] = DecryptText(connection["private_key_passphrase"]) if connection["private_key_passphrase"] else ""
+    return connection
+
+
+def run_table_group_list(project_code):
+    sql_template = read_template_sql_file("get_table_group_list.sql", "get_entities")
+    sql_template = sql_template.replace("{PROJECT_CODE}", project_code)
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_list_test_suites(project_code):
+    sql_template = read_template_sql_file("get_test_suite_list.sql", "get_entities")
+    sql_template = sql_template.replace("{PROJECT_CODE}", project_code)
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_get_test_suite(project_code, test_suite):
+    sql_template = read_template_sql_file("get_test_suite.sql", "get_entities")
+
+    sql_template = sql_template.replace("{PROJECT_CODE}", project_code)
+    sql_template = sql_template.replace("{TEST_SUITE}", test_suite)
+
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_profile_info(profile_run, table_name=None):
+    if not table_name:
+        table_name = "%"  # if no table_name, we select all the tables
+    sql_template = read_template_sql_file("get_profile_info.sql", "get_entities")
+
+    sql_template = sql_template.replace("{PROFILE_RUN}", str(profile_run))
+    sql_template = sql_template.replace("{TABLE_NAME}", table_name)
+
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_profile_screen(profile_run, table_name=None):
+    if not table_name:
+        table_name = "%"
+    sql_template = read_template_sql_file("get_profile_screen.sql", "get_entities")
+
+    sql_template = sql_template.replace("{PROFILE_RUN}", profile_run)
+    sql_template = sql_template.replace("{TABLE_NAME}", table_name)
+
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_list_test_generation(project_code, test_suite):
+    sql_template = read_template_sql_file("get_test_generation_list.sql", "get_entities")
+
+    sql_template = sql_template.replace("{PROJECT_CODE}", project_code)
+    sql_template = sql_template.replace("{TEST_SUITE}", test_suite)
+
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_test_info(project_code, test_suite):
+    sql_template = read_template_sql_file("get_test_info.sql", "get_entities")
+
+    sql_template = sql_template.replace("{PROJECT_CODE}", project_code)
+    sql_template = sql_template.replace("{TEST_SUITE}", test_suite)
+
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_list_test_runs(project_code, test_suite):
+    sql_template = read_template_sql_file("get_test_run_list.sql", "get_entities")
+
+    sql_template = sql_template.replace("{PROJECT_CODE}", project_code)
+    sql_template = sql_template.replace("{TEST_SUITE}", test_suite)
+
+    return RetrieveDBResultsToList("DKTG", sql_template)
+
+
+def run_get_results(test_run_id, booErrorsOnly):
+    sql_template = read_template_sql_file("get_test_results_for_run_cli.sql", "get_entities")
+
+    sql_template = sql_template.replace("{TEST_RUN_ID}", test_run_id)
+    if booErrorsOnly:
+        sql_template = sql_template.replace("{ERRORS_ONLY}", "AND result_code = 0")
+    else:
+        sql_template = sql_template.replace("{ERRORS_ONLY}", "")
+
+    return RetrieveDBResultsToList("DKTG", sql_template)

testgen/commands/run_launch_db_config.py

@@ -0,0 +1,84 @@
+import logging
+import os
+
+from testgen import settings
+from testgen.common import CreateDatabaseIfNotExists, RunActionQueryList, date_service
+from testgen.common.credentials import get_tg_db, get_tg_schema
+from testgen.common.database.database_service import get_queries_for_command
+from testgen.common.encrypt import EncryptText, encrypt_ui_password
+from testgen.common.read_file import get_template_files
+
+LOG = logging.getLogger("testgen")
+
+
+def _get_latest_revision_number():
+    files = sorted(get_template_files(mask=r"^.*sql$", sub_directory="dbupgrade"), key=lambda key: str(key))
+    last_file = os.path.basename(str(files[-1]))
+    return str(int(last_file.split("_")[0]))  # Drop leading zeroes via str-->int-->str
+
+
+def _get_params_mapping() -> dict:
+    ui_user_encrypted_password = encrypt_ui_password(settings.PASSWORD)
+
+    now = date_service.get_now_as_string()
+    return {
+        "UI_USER_NAME": settings.USERNAME,
+        "UI_USER_USERNAME": settings.USERNAME,
+        "UI_USER_EMAIL": "",
+        "UI_USER_ENCRYPTED_PASSWORD": ui_user_encrypted_password,
+        "SCHEMA_NAME": get_tg_schema(),
+        "START_DATE": now,
+        "PROJECT_CODE": settings.PROJECT_KEY,
+        "CONNECTION_ID": 1,
+        "SQL_FLAVOR": settings.PROJECT_SQL_FLAVOR,
+        "PROJECT_NAME": settings.PROJECT_NAME,
+        "PROJECT_DB": settings.PROJECT_DATABASE_NAME,
+        "PROJECT_USER": settings.PROJECT_DATABASE_USER,
+        "PROJECT_PORT": settings.PROJECT_DATABASE_PORT,
+        "PROJECT_HOST": settings.PROJECT_DATABASE_HOST,
+        "PROJECT_SCHEMA": settings.PROJECT_DATABASE_SCHEMA,
+        "PROFILING_TABLE_SET": settings.DEFAULT_PROFILING_TABLE_SET,
+        "PROFILING_INCLUDE_MASK": settings.DEFAULT_PROFILING_INCLUDE_MASK,
+        "PROFILING_EXCLUDE_MASK": settings.DEFAULT_PROFILING_EXCLUDE_MASK,
+        "PROFILING_ID_COLUMN_MASK": settings.DEFAULT_PROFILING_ID_COLUMN_MASK,
+        "PROFILING_SK_COLUMN_MASK": settings.DEFAULT_PROFILING_SK_COLUMN_MASK,
+        "PROFILING_USE_SAMPLING": settings.DEFAULT_PROFILING_USE_SAMPLING,
+        "PROFILING_SAMPLE_PERCENT": "",
+        "PROFILING_SAMPLE_MIN_COUNT": "",
+        "PROFILING_DELAY_DAYS": "",
+        "CONNECTION_NAME": settings.PROJECT_CONNECTION_NAME,
+        "PROJECT_QC_SCHEMA": settings.PROJECT_QC_SCHEMA,
+        "TABLE_GROUPS_NAME": settings.DEFAULT_TABLE_GROUPS_NAME,
+        "TEST_SUITE": settings.DEFAULT_TEST_SUITE_KEY,
+        "TEST_SUITE_DESCRIPTION": settings.DEFAULT_TEST_SUITE_DESCRIPTION,
+        "MAX_THREADS": settings.PROJECT_CONNECTION_MAX_THREADS,
+        "MAX_QUERY_CHARS": settings.PROJECT_CONNECTION_MAX_QUERY_CHAR,
+        "PROJECT_PW_ENCRYPTED": EncryptText(settings.PROJECT_DATABASE_PASSWORD),
+        "OBSERVABILITY_API_URL": settings.OBSERVABILITY_API_URL,
+        "OBSERVABILITY_API_KEY": settings.OBSERVABILITY_API_KEY,
+        "OBSERVABILITY_COMPONENT_KEY": settings.OBSERVABILITY_DEFAULT_COMPONENT_KEY,
+        "OBSERVABILITY_COMPONENT_TYPE": settings.OBSERVABILITY_DEFAULT_COMPONENT_TYPE,
+        "TESTGEN_ADMIN_USER": settings.DATABASE_ADMIN_USER,
+        "TESTGEN_ADMIN_PASSWORD": settings.DATABASE_ADMIN_PASSWORD,
+        "TESTGEN_USER": settings.DATABASE_EXECUTE_USER,
+        "TESTGEN_PASSWORD": settings.DATABASE_PASSWORD,
+        "TESTGEN_REPORT_USER": settings.DATABASE_REPORT_USER,
+        "TESTGEN_REPORT_PASSWORD": settings.DATABASE_PASSWORD,
+        "DB_REVISION": _get_latest_revision_number(),
+    }
+
+
+def run_launch_db_config(delete_db: bool) -> None:
+    params_mapping = _get_params_mapping()
+
+    CreateDatabaseIfNotExists(get_tg_db(), params_mapping, delete_db)
+
+    queries = get_queries_for_command("dbsetup", params_mapping)
+
+    RunActionQueryList(
+        "DKTG",
+        queries,
+        "S",
+        user_override=params_mapping["TESTGEN_ADMIN_USER"],
+        pwd_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
+    )