dataops-testgen 2.2.0__py3-none-any.whl

testgen/commands/queries/execute_cat_tests_query.py

@@ -0,0 +1,95 @@
+import typing
+
+from testgen.common import date_service, read_template_sql_file
+from testgen.common.database import database_service
+
+
+class CCATExecutionSQL:
+    project_code = ""
+    flavor = ""
+    concat_operator = ""
+    test_suite = ""
+    run_date = ""
+    test_run_id = ""
+    max_query_chars = ""
+    exception_message = ""
+
+    # Test Set Parameters
+    target_schema = ""
+    target_table = ""
+    replace_qc_schema = ""
+    dctTestParms: typing.ClassVar = {}
+
+    def __init__(self, strProjectCode, strTestSuite, strSQLFlavor, max_query_chars, minutes_offset=0):
+        # Defaults
+        self.test_suite = strTestSuite
+        self.project_code = strProjectCode
+        flavor_service = database_service.get_flavor_service(strSQLFlavor)
+        self.concat_operator = flavor_service.get_concat_operator()
+        self.flavor = strSQLFlavor
+        self.max_query_chars = max_query_chars
+        self.today = date_service.get_now_as_string_with_offset(minutes_offset)
+        self.minutes_offset = minutes_offset
+
+    def _ReplaceParms(self, strInputString):
+        strInputString = strInputString.replace("{MAX_QUERY_CHARS}", str(self.max_query_chars))
+        strInputString = strInputString.replace("{TEST_RUN_ID}", self.test_run_id)
+        strInputString = strInputString.replace("{PROJECT_CODE}", self.project_code)
+        strInputString = strInputString.replace("{TEST_SUITE}", self.test_suite)
+        # NOTE:  REPLACE_QC_SCHEMA is parm replaced to run build query: sets the actual value to replace.
+        #        DATA_QC_SCHEMA is parm in cat_test_conditions that build query replaces via SQL.
+        strInputString = strInputString.replace("{REPLACE_QC_SCHEMA}", self.replace_qc_schema)
+        strInputString = strInputString.replace("{SQL_FLAVOR}", self.flavor)
+        strInputString = strInputString.replace("{CONCAT_OPERATOR}", self.concat_operator)
+
+        strInputString = strInputString.replace("{SCHEMA_NAME}", self.target_schema)
+        strInputString = strInputString.replace("{TABLE_NAME}", self.target_table)
+
+        strInputString = strInputString.replace("{RUN_DATE}", self.run_date)
+        strInputString = strInputString.replace("{NOW_DATE}", "GETDATE()")
+        strInputString = strInputString.replace("{START_TIME}", self.today)
+        strInputString = strInputString.replace(
+            "{NOW}", date_service.get_now_as_string_with_offset(self.minutes_offset)
+        )
+        strInputString = strInputString.replace("{EXCEPTION_MESSAGE}", self.exception_message.strip())
+
+        for parm, value in self.dctTestParms.items():
+            strInputString = strInputString.replace("{" + parm.upper() + "}", str(value))
+
+        strInputString = strInputString.replace("{RUN_DATE}", self.run_date)
+
+        # Adding escape character where ':' is referenced
+        strInputString = strInputString.replace(":", "\\:")
+
+        return strInputString
+
+    def GetDistinctTablesSQL(self):
+        # Runs on DK DB
+        strQ = self._ReplaceParms(read_template_sql_file("ex_cat_get_distinct_tables.sql", "exec_cat_tests"))
+        return strQ
+
+    def GetAggregateTableTestSQL(self):
+        # Runs on DK DB
+        strQ = self._ReplaceParms(read_template_sql_file("ex_cat_build_agg_table_tests.sql", "exec_cat_tests"))
+        return strQ
+
+    def GetAggregateTestParmsSQL(self):
+        # Runs on DK DB
+        strQ = self._ReplaceParms(read_template_sql_file("ex_cat_retrieve_agg_test_parms.sql", "exec_cat_tests"))
+        return strQ
+
+    def PrepCATQuerySQL(self):
+        strQ = self._ReplaceParms(read_template_sql_file("ex_cat_test_query.sql", "exec_cat_tests"))
+        return strQ
+
+    def GetCATResultsParseSQL(self):
+        strQ = self._ReplaceParms(read_template_sql_file("ex_cat_results_parse.sql", "exec_cat_tests"))
+        return strQ
+
+    def FinalizeTestResultsSQL(self):
+        strQ = self._ReplaceParms(read_template_sql_file("ex_finalize_test_run_results.sql", "execution"))
+        return strQ
+
+    def PushTestRunStatusUpdateSQL(self):
+        strQ = self._ReplaceParms(read_template_sql_file("ex_update_test_record_in_testrun_table.sql", "execution"))
+        return strQ

testgen/commands/queries/execute_tests_query.py

@@ -0,0 +1,160 @@
+import typing
+
+from testgen.common import AddQuotesToIdentifierCSV, CleanSQL, ConcatColumnList, date_service, read_template_sql_file
+
+
+class CTestExecutionSQL:
+    flavor = ""
+    run_date = ""
+    project_code = ""
+    test_suite = ""
+    test_run_id = ""
+    exception_message = ""
+    process_id = ""
+
+    # Test Group Parameters
+    dctTestParms: typing.ClassVar = {}
+    sum_columns = ""
+    match_sum_columns = ""
+    multi_column_error_condition = ""
+
+    def __init__(self, strProjectCode, strFlavor, strTestSuite, minutes_offset=0):
+        self.project_code = strProjectCode
+        self.flavor = strFlavor
+        self.test_suite = strTestSuite
+        self.today = date_service.get_now_as_string_with_offset(minutes_offset)
+        self.minutes_offset = minutes_offset
+
+    def _AssembleDisplayParameters(self):
+
+        lst_parms = ["column_name", "skip_errors", "baseline_ct", "baseline_unique_ct", "baseline_value",
+                     "baseline_value_ct", "baseline_sum", "baseline_avg", "baseline_sd", "subset_condition",
+                     "groupby_names", "having_condition", "window_date_column", "window_days",
+                     "match_column_names", "match_subset_condition", "match_schema_name", "match_table_name",
+                     "match_groupby_names", "match_having_condition",
+                     ]
+        str_parms = "; ".join(f"{key}={self.dctTestParms[key]}"
+                             for key in lst_parms
+                             if key.lower() in self.dctTestParms and self.dctTestParms[key] not in [None, ""])
+        str_parms = str_parms.replace("'", "`")
+        return str_parms
+
+    def _ReplaceParms(self, strInputString: str):
+        strInputString = strInputString.replace("{PROJECT_CODE}", self.project_code)
+        strInputString = strInputString.replace("{TEST_SUITE}", self.test_suite)
+        strInputString = strInputString.replace("{SQL_FLAVOR}", self.flavor)
+        strInputString = strInputString.replace("{TEST_RUN_ID}", self.test_run_id)
+        strInputString = strInputString.replace("{INPUT_PARAMETERS}", self._AssembleDisplayParameters())
+
+        strInputString = strInputString.replace("{RUN_DATE}", self.run_date)
+        strInputString = strInputString.replace("{EXCEPTION_MESSAGE}", self.exception_message)
+        strInputString = strInputString.replace("{START_TIME}", self.today)
+        strInputString = strInputString.replace("{PROCESS_ID}", str(self.process_id))
+        strInputString = strInputString.replace(
+            "{NOW}", date_service.get_now_as_string_with_offset(self.minutes_offset)
+        )
+
+        column_designators = [
+            "COLUMN_NAME",
+            # "COLUMN_NAMES",
+            # "COL_NAME",
+            # "COL_NAMES",
+            # "MATCH_COLUMN_NAMES",
+            # "MATCH_GROUPBY_NAMES",
+            # "MATCH_SUM_COLUMNS",
+        ]
+
+        for parm, value in self.dctTestParms.items():
+            if value:
+                if parm.upper() in column_designators:
+                    strInputString = strInputString.replace("{" + parm.upper() + "}", AddQuotesToIdentifierCSV(value))
+                else:
+                    strInputString = strInputString.replace("{" + parm.upper() + "}", value)
+            else:
+                strInputString = strInputString.replace("{" + parm.upper() + "}", "")
+            if parm == "column_name":
+                # Shows contents without double-quotes for display and aggregate expressions
+                strInputString = strInputString.replace("{COLUMN_NAME_NO_QUOTES}", value if value else "")
+                # Concatenates column list into single expression for relative entropy
+                str_value = ConcatColumnList(value, "<NULL>")
+                strInputString = strInputString.replace("{CONCAT_COLUMNS}", str_value if str_value else "")
+            if parm == "match_groupby_names":
+                # Concatenates column list into single expression for relative entropy
+                str_value = ConcatColumnList(value, "<NULL>")
+                strInputString = strInputString.replace("{CONCAT_MATCH_GROUPBY}", str_value if str_value else "")
+            if parm == "subset_condition":
+                strInputString = strInputString.replace("{SUBSET_DISPLAY}", value.replace("'", "''") if value else "")
+
+
+        # Adding escape character where ':' is referenced
+        strInputString = strInputString.replace(":", "\\:")
+
+        return strInputString
+
+    def ClearTestParms(self):
+        # Test Set Parameters
+        pass
+
+    def GetTestsNonCAT(self, booClean):
+        # Runs on DK DB
+        strQ = self._ReplaceParms(read_template_sql_file("ex_get_tests_non_cat.sql", "execution"))
+        if booClean:
+            strQ = CleanSQL(strQ)
+
+        return strQ
+
+    def AddTestRecordtoTestRunTable(self):
+        strQ = self._ReplaceParms(read_template_sql_file("ex_write_test_record_to_testrun_table.sql", "execution"))
+
+        return strQ
+
+    def PushTestRunStatusUpdateSQL(self):
+        # Runs on DK DB
+        strQ = self._ReplaceParms(read_template_sql_file("ex_update_test_record_in_testrun_table.sql", "execution"))
+
+        return strQ
+
+    def _GetTestQueryFromTemplate(self, strTemplateFile: str):
+        # Runs on Project DB
+        if strTemplateFile.endswith("_generic.sql"):
+            template_flavor = "generic"
+        else:
+            template_flavor = self.flavor
+        strQ = self._ReplaceParms(
+            read_template_sql_file(strTemplateFile, f"flavors/{template_flavor}/exec_query_tests")
+        )
+        return strQ
+
+    def _ConstructAggregateMatchParms(self):
+        # Prepares column list for SQL to compare sums of each column
+
+        # Split each comma separated column name into individual list items
+        cols = [s.strip() for s in self.dctTestParms["column_name"].split(",")]
+        _ = [s.strip() for s in self.dctTestParms["match_column_names"].split(",")]
+
+        # Surround all column names with SUM() to generate proper SQL syntax
+        self.list_sum_columns = ["SUM(" + i + ") as " + i for i in cols]
+        self.sum_columns = ", ".join(self.list_sum_columns)
+
+        self.list_match_sum_columns = ["SUM(" + i + ") as " + i for i in cols]
+        self.match_sum_columns = ", ".join(self.list_match_sum_columns)
+
+        # Suffix all column names with '< 0' to generate proper SQL WHERE/HAVING clause syntax
+        self.list_multi_column_error_condition = [i + " < 0" for i in cols]
+        self.multi_column_error_condition = " or ".join(self.list_multi_column_error_condition)
+
+
+    def GetTestQuery(self, booClean: bool):
+        strTestType = self.dctTestParms["test_type"]
+        strTemplate = self.dctTestParms["template_name"]
+
+        if strTemplate == "":
+            raise ValueError(f"No query template assigned to test_type {strTestType}")
+
+        strQ = self._GetTestQueryFromTemplate(strTemplate)
+        # Final replace to cover parm within CUSTOM_QUERY parm
+        strQ = strQ.replace("{DATA_SCHEMA}", self.dctTestParms["schema_name"])
+
+        if booClean:
+            strQ = CleanSQL(strQ)
+        return strQ

testgen/commands/queries/generate_tests_query.py

@@ -0,0 +1,94 @@
+import logging
+import typing
+
+from testgen.common import CleanSQL, date_service, get_template_files, read_template_sql_file
+
+LOG = logging.getLogger("testgen")
+
+
+class CDeriveTestsSQL:
+    run_date = ""
+    project_code = ""
+    connection_id = ""
+    table_groups_id = ""
+    data_schema = ""
+    test_suite = ""
+    test_suite_id = ""
+    generation_set = ""
+    as_of_date = ""
+    sql_flavor = ""
+    dctTestParms: typing.ClassVar = {}
+
+    def __init__(self):
+        today = date_service.get_now_as_string()
+        self.run_date = today
+        self.as_of_date = today
+        self.dctTestParms = {}
+
+    def ClearTestParms(self):
+        # Test Set Parameters
+        self.dctTestParms = {}
+
+    def ReplaceParms(self, strInputString):
+        for parm, value in self.dctTestParms.items():
+            strInputString = strInputString.replace("{" + parm.upper() + "}", value)
+
+        strInputString = strInputString.replace("{PROJECT_CODE}", self.project_code)
+        strInputString = strInputString.replace("{SQL_FLAVOR}", self.sql_flavor)
+        strInputString = strInputString.replace("{CONNECTION_ID}", self.connection_id)
+        strInputString = strInputString.replace("{TABLE_GROUPS_ID}", self.table_groups_id)
+        strInputString = strInputString.replace("{RUN_DATE}", self.run_date)
+        strInputString = strInputString.replace("{TEST_SUITE}", self.test_suite)
+        strInputString = strInputString.replace("{TEST_SUITE_ID}", self.test_suite_id)
+        strInputString = strInputString.replace("{GENERATION_SET}", self.generation_set)
+        strInputString = strInputString.replace("{AS_OF_DATE}", self.as_of_date)
+        strInputString = strInputString.replace("{DATA_SCHEMA}", self.data_schema)
+
+        return strInputString
+
+    def GetInsertTestSuiteSQL(self, booClean):
+        strQuery = self.ReplaceParms(read_template_sql_file("gen_insert_test_suite.sql", "generation"))
+        if booClean:
+            strQuery = CleanSQL(strQuery)
+
+        return strQuery
+
+    def GetTestTypesSQL(self, booClean):
+        strQuery = self.ReplaceParms(read_template_sql_file("gen_standard_test_type_list.sql", "generation"))
+        if booClean:
+            strQuery = CleanSQL(strQuery)
+
+        return strQuery
+
+    def GetTestDerivationQueriesAsList(self, booClean):
+        # This assumes the queries run in no particular order,
+        # and will order them alphabetically by file name
+        lstQueries = sorted(
+            get_template_files(mask=r"^.*sql$", sub_directory="gen_funny_cat_tests"), key=lambda key: str(key)
+        )
+        lstTemplate = []
+
+        for script in lstQueries:
+            query = script.read_text("utf-8")
+            template = self.ReplaceParms(query)
+            lstTemplate.append(template)
+
+        if booClean:
+            lstTemplate = [CleanSQL(q) for q in lstTemplate]
+
+        if len(lstQueries) == 0:
+            LOG.warning("No funny CAT test generation templates were found")
+
+        return lstTemplate
+
+    def GetTestQueriesFromGenericFile(self, booClean: bool):
+        strQuery = self.ReplaceParms(read_template_sql_file("gen_standard_tests.sql", "generation"))
+        if booClean:
+            strQuery = CleanSQL(strQuery)
+        return strQuery
+
+    def GetDeleteOldTestsQuery(self, booClean: bool):
+        strQuery = self.ReplaceParms(read_template_sql_file("gen_delete_old_tests.sql", "generation"))
+        if booClean:
+            strQuery = CleanSQL(strQuery)
+        return strQuery