dataops-testgen 2.2.0__py3-none-any.whl

testgen/ui/services/javascript_service.py

@@ -0,0 +1,44 @@
+import logging
+
+from streamlit_javascript import st_javascript
+
+from testgen.ui.services.authentication_service import AUTH_TOKEN_COOKIE_NAME
+
+LOG = logging.getLogger("testgen")
+
+
+def copy_to_clipboard(text):
+    script = f"""await (async function () {{
+        window.parent.postMessage({{ type: 'TestgenCopyToClipboard', text: '{text}' }}, '*');
+        return 0;
+    }})()
+    """
+    execute_javascript(script)
+
+
+def clear_component_states():
+    execute_javascript(
+        f"""await (async function () {{
+            window.parent.postMessage({{ type: 'TestgenLogout', cookie: '{AUTH_TOKEN_COOKIE_NAME}' }}, '*');
+            return 0;
+        }})()
+        """
+    )
+
+
+def execute_javascript(script):
+    return_value = st_javascript(script)
+    if return_value != 0:
+        LOG.warning(f"execute_javascript returned with non zero value: {return_value}, script: {script}")
+
+
+def get_browser_locale_timezone():
+    from streamlit_javascript import st_javascript
+
+    return st_javascript(
+        """await (async () => {
+                const userTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+                console.log(userTimezone)
+                return userTimezone
+    })().then(returnValue => returnValue)"""
+    )

testgen/ui/services/query_service.py

@@ -0,0 +1,316 @@
+import testgen.ui.services.database_service as db
+
+"""
+Shared queries for standard lookups
+  - should be called by cached functions within page
+"""
+
+
+def run_project_lookup_query(str_schema):
+    str_sql = f"""
+        SELECT
+            id::VARCHAR(50),
+            project_code,
+            project_name,
+            effective_from_date,
+            observability_api_url,
+            observability_api_key
+        FROM {str_schema}.projects
+        ORDER BY project_name
+    """
+    return db.retrieve_data(str_sql)
+
+
+def get_project_by_code(schema: str, project_code: str):
+    str_sql = f"""
+      SELECT
+        id::VARCHAR(50),
+        project_code,
+        project_name,
+        effective_from_date,
+        observability_api_url,
+        observability_api_key
+      FROM {schema}.projects
+      WHERE project_code = {db.make_value_db_friendly(project_code)};
+    """
+    results = db.retrieve_data(str_sql)
+    if results.size <= 0:
+        return None
+    return results.iloc[0]
+
+
+def run_test_type_lookup_query(str_schema, str_test_type=None, boo_show_referential=True, boo_show_table=True,
+                               boo_show_column=True, boo_show_custom=True):
+    if str_test_type:
+        str_criteria = f" AND tt.test_type = '{str_test_type}'"
+    else:
+        str_criteria = ""
+
+    if (boo_show_referential and boo_show_table and boo_show_column and boo_show_custom) == False:
+        str_scopes = ""
+        str_scopes += "'referential'," if boo_show_referential else ""
+        str_scopes += "'table'," if boo_show_table else ""
+        str_scopes += "'column'," if boo_show_column else ""
+        str_scopes += "'custom'," if boo_show_custom else ""
+        if str_scopes > "":
+            str_criteria += f"AND tt.test_scope in ({str_scopes[:-1]})"
+
+    str_sql = f"""
+            SELECT tt.id, tt.test_type, tt.id as cat_test_id,
+                   tt.test_name_short, tt.test_name_long, tt.test_description,
+                   tt.measure_uom, COALESCE(tt.measure_uom_description, '') as measure_uom_description,
+                   tt.default_parm_columns, tt.default_severity,
+                   tt.run_type, tt.test_scope, tt.dq_dimension, tt.threshold_description, 
+                   tt.column_name_prompt, tt.column_name_help, 
+                   tt.default_parm_prompts, tt.default_parm_help, tt.usage_notes,
+                   CASE tt.test_scope WHEN 'referential' THEN '⧉ ' WHEN 'custom' THEN '⛭ ' WHEN 'table' THEN '⊞ ' WHEN 'column' THEN '≣ ' ELSE '? ' END  
+                    || tt.test_name_short || ': ' || lower(tt.test_name_long)
+                    || CASE WHEN tt.selection_criteria > '' THEN ' [auto-generated]' ELSE '' END as select_name
+              FROM {str_schema}.test_types tt
+             WHERE tt.active = 'Y' {str_criteria}
+            ORDER BY CASE tt.test_scope WHEN 'referential' THEN 1 WHEN 'custom' THEN 2 WHEN 'table' THEN 3 WHEN 'column' THEN 4 ELSE 5 END, 
+                     tt.test_name_short;
+    """
+    return db.retrieve_data(str_sql)
+
+
+def run_connections_lookup_query(str_schema, str_project_code):
+    str_sql = f"""
+           SELECT c.id::VARCHAR(50), c.connection_id, c.connection_name
+             FROM {str_schema}.connections c
+             WHERE c.project_code = '{str_project_code}'
+             ORDER BY connection_name
+    """
+    return db.retrieve_data(str_sql)
+
+
+def run_table_groups_lookup_query(str_schema, str_project_code, connection_id=None, table_group_id=None):
+    str_sql = f"""
+           SELECT tg.id::VARCHAR(50), tg.table_groups_name, tg.connection_id, tg.table_group_schema
+             FROM {str_schema}.table_groups tg
+    """
+
+    if connection_id:
+        str_sql += f"""
+             inner join {str_schema}.connections c on c.connection_id = tg.connection_id
+        """
+
+    str_sql += f"""
+                       WHERE tg.project_code = '{str_project_code}'
+    """
+
+    if table_group_id:
+        str_sql += f"""
+                AND tg.id = '{table_group_id}'::UUID
+        """
+
+    if connection_id:
+        str_sql += f"""
+                AND c.id = '{connection_id}'::UUID
+        """
+
+    str_sql += """
+           ORDER BY table_groups_name
+    """
+    return db.retrieve_data(str_sql)
+
+
+def run_table_lookup_query(str_schema, str_table_groups_id):
+    str_sql = f"""
+           SELECT table_name
+             FROM {str_schema}.data_table_chars
+            WHERE table_groups_id = '{str_table_groups_id}'::UUID
+              AND drop_date IS NULL
+           ORDER BY table_name
+    """
+    return db.retrieve_data(str_sql)
+
+
+def run_column_lookup_query(str_schema, str_table_groups_id, str_table_name):
+    str_sql = f"""
+           SELECT column_name
+             FROM {str_schema}.data_column_chars
+            WHERE table_groups_id = '{str_table_groups_id}'::UUID
+              AND table_name = '{str_table_name}'
+              AND drop_date IS NULL
+           ORDER BY column_name
+    """
+    return db.retrieve_data(str_sql)
+
+
+def run_test_suite_lookup_by_tgroup_query(str_schema, str_table_groups_id, test_suite_name=None):
+    str_sql = f"""
+           SELECT id::VARCHAR(50), test_suite, test_suite_schema, severity, export_to_observability
+             FROM {str_schema}.test_suites
+            WHERE table_groups_id = '{str_table_groups_id}'
+    """
+
+    if test_suite_name:
+        str_sql += f"""
+               AND test_suite = '{test_suite_name}'
+        """
+
+    str_sql += """
+           ORDER BY test_suite
+    """
+
+    return db.retrieve_data(str_sql)
+
+
+def run_test_suite_lookup_by_project_query(str_schema, str_project):
+    str_sql = f"""
+           SELECT s.id::VARCHAR(50), s.test_suite, s.test_suite_schema,
+                  s.test_suite
+                    || CASE
+                         WHEN tg.table_groups_name IS NULL THEN ''
+                         ELSE '(' || tg.table_groups_name || ')'
+                       END as test_suite_with_tg,
+                  s.test_suite_description
+             FROM {str_schema}.test_suites s
+           LEFT JOIN {str_schema}.table_groups tg
+             ON (s.table_groups_id = tg.id)
+            WHERE s.project_code = '{str_project}'
+           ORDER BY s.test_suite
+    """
+    return db.retrieve_data(str_sql)
+
+
+def run_last_test_run(str_schema, str_project_code):
+    str_sql = f"""
+           SELECT MIN(EXTRACT(DAY FROM (CURRENT_DATE - r.test_starttime))) as days_back
+             FROM {str_schema}.test_runs r
+            WHERE r.project_code = '{str_project_code}'
+              AND r.status = 'Complete';
+    """
+    return db.retrieve_data(str_sql)
+
+
+def run_test_run_lookup_by_date(str_schema, str_project_code, str_run_date):
+    str_sql = f"""
+           SELECT r.id::VARCHAR(50),
+                  r.test_starttime::VARCHAR || ' - ' || s.test_suite as test_run_desc
+             FROM {str_schema}.test_runs r
+            LEFT JOIN {str_schema}.test_suites s
+              ON (r.project_code = s.project_code
+             AND  r.test_suite = s.test_suite)
+            WHERE r.project_code = '{str_project_code}'
+              AND r.test_starttime::DATE = '{str_run_date}'
+           ORDER BY r.test_starttime DESC
+    """
+    return db.retrieve_data(str_sql)
+
+
+def update_anomaly_disposition_old(selected, str_schema, str_new_status):
+    int_batch_size = 50
+
+    def finalize_query(status, ids):
+        return f"""UPDATE {str_schema}.profile_anomaly_results
+                      SET disposition = NULLIF('{status}', 'No Decision')
+                    WHERE id IN ({ids.rstrip(',')});"""
+
+    lst_ids = [row["id"] for row in selected if "id" in row]
+    lst_updates = []
+    i = 0
+    str_ids = ""
+
+    for my_id in lst_ids:
+        i += 1
+        str_ids += f" '{my_id}'::UUID,"
+        if i >= int_batch_size:
+            lst_updates.append(finalize_query(str_new_status, str_ids))
+            # reset for next batch
+            i = 0
+    if i > 0:
+        lst_updates.append(finalize_query(str_new_status, str_ids))
+    for upd in lst_updates:
+        db.execute_sql(upd)
+
+    return True
+
+
+def update_anomaly_disposition(selected, str_schema, str_new_status):
+    def finalize_small_update(status, ids):
+        return f"""UPDATE {str_schema}.profile_anomaly_results
+                      SET disposition = NULLIF('{status}', 'No Decision')
+                    WHERE id IN ({ids});"""
+
+    def finalize_big_update(status, ids):
+        return f"""WITH selects
+                    as ( SELECT UNNEST(ARRAY [{ids}]) AS selected_id )
+                   UPDATE {str_schema}.profile_anomaly_results
+                      SET disposition = NULLIF('{status}', 'No Decision')
+                     FROM {str_schema}.profile_anomaly_results r
+                   INNER JOIN selects s
+                      ON (r.id = s.selected_id)
+                    WHERE r.id = profile_anomaly_results.id;"""
+
+    lst_ids = [row["id"] for row in selected if "id" in row]
+    lst_updates = []
+    str_ids = ""
+
+    if len(lst_ids) > 0:
+        for my_id in lst_ids:
+            str_ids += f" '{my_id}'::UUID,"
+        str_ids = str_ids.rstrip(",")
+        if len(lst_ids) > 4:
+            lst_updates.append(finalize_big_update(str_new_status, str_ids))
+        else:
+            lst_updates.append(finalize_small_update(str_new_status, str_ids))
+
+        for q in lst_updates:
+            db.execute_sql_raw(q)
+
+    return True
+
+
+def update_result_disposition(selected, str_schema, str_new_status):
+    active_yn = "N" if str_new_status == "Inactive" else "Y"
+
+    def finalize_small_update(status, ids):
+        return f"""UPDATE {str_schema}.test_results
+                      SET disposition = NULLIF('{status}', 'No Decision')
+                    WHERE id IN ({ids});"""
+
+    def finalize_big_update(status, ids):
+        return f"""WITH selects
+                    as ( SELECT UNNEST(ARRAY [{ids}]) AS selected_id )
+                   UPDATE {str_schema}.test_results
+                      SET disposition = NULLIF('{status}', 'No Decision')
+                     FROM {str_schema}.test_results r
+                   INNER JOIN selects s
+                      ON (r.id = s.selected_id)
+                    WHERE r.id = test_results.id;"""
+
+    def finalize_test_update(ids):
+        str_lock_test = ", lock_refresh = 'N'" if active_yn == "Y" else ", lock_refresh = 'Y'"
+        return f"""WITH selects
+                    as ( SELECT UNNEST(ARRAY [{ids}]) AS selected_id )
+                   UPDATE {str_schema}.test_definitions
+                      SET test_active = '{active_yn}',
+                          last_manual_update = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' {str_lock_test}
+                     FROM {str_schema}.test_definitions d
+                   INNER JOIN {str_schema}.test_results r
+                      ON (d.id = r.test_definition_id)
+                   INNER JOIN selects s
+                      ON (r.id = s.selected_id)
+                    WHERE d.id = test_definitions.id"""
+
+    lst_ids = [row["test_result_id"] for row in selected if "test_result_id" in row]
+    lst_updates = []
+    str_ids = ""
+
+    for my_id in lst_ids:
+        str_ids += f" '{my_id}'::UUID,"
+    str_ids = str_ids.rstrip(",")
+
+    if len(lst_ids) > 0:
+        if len(lst_ids) > 4:
+            lst_updates.append(finalize_big_update(str_new_status, str_ids))
+        else:
+            lst_updates.append(finalize_small_update(str_new_status, str_ids))
+        lst_updates.append(finalize_test_update(str_ids))
+
+        for q in lst_updates:
+            db.execute_sql_raw(q)
+
+    return True

testgen/ui/services/string_service.py

@@ -0,0 +1,12 @@
+def empty_if_null(item):
+    if item is None:
+        return ""
+    else:
+        return item
+
+
+def snake_case_to_title_case(snake_case):
+    words = snake_case.split("_")
+    title_case_words = [word.capitalize() for word in words]
+    title_case = " ".join(title_case_words)
+    return title_case

testgen/ui/services/table_group_service.py

@@ -0,0 +1,130 @@
+import streamlit as st
+
+import testgen.ui.queries.table_group_queries as table_group_queries
+import testgen.ui.services.connection_service as connection_service
+import testgen.ui.services.test_suite_service as test_suite_service
+from testgen.common.database.database_service import RetrieveDBResultsToDictList
+
+
+def get_by_id(table_group_id: str):
+    schema = st.session_state["dbschema"]
+    return table_group_queries.get_by_id(schema, table_group_id)
+
+
+def get_by_connection(project_code, connection_id):
+    schema = st.session_state["dbschema"]
+    return table_group_queries.get_by_connection(schema, project_code, connection_id)
+
+
+def edit(table_group):
+    schema = st.session_state["dbschema"]
+    table_group_queries.edit(schema, table_group)
+
+
+def add(table_group):
+    schema = st.session_state["dbschema"]
+    table_group_queries.add(schema, table_group)
+
+
+def cascade_delete(table_group_names, dry_run=False):
+    schema = st.session_state["dbschema"]
+    test_suite_names = get_test_suite_names_by_table_group_names(table_group_names)
+    can_be_deleted = not table_group_has_dependencies(schema, table_group_names, test_suite_names)
+    if not dry_run:
+        test_suite_service.cascade_delete(test_suite_names)
+        table_group_queries.cascade_delete(schema, table_group_names)
+    return can_be_deleted
+
+
+def table_group_has_dependencies(schema, table_group_names, test_suite_names):
+    test_suite_usage_result = test_suite_service.has_test_suite_dependencies(schema, test_suite_names)
+    if not table_group_names:
+        table_group_usage_result = False
+    else:
+        table_group_usage_result = not table_group_queries.get_table_group_dependencies(schema, table_group_names).empty
+    return test_suite_usage_result or table_group_usage_result
+
+
+def are_table_groups_in_use(table_group_names):
+    if not table_group_names:
+        return False
+
+    schema = st.session_state["dbschema"]
+
+    test_suite_names = get_test_suite_names_by_table_group_names(table_group_names)
+    test_suites_in_use = test_suite_service.are_test_suites_in_use(test_suite_names)
+
+    table_groups_in_use_result = table_group_queries.get_table_group_usage(schema, table_group_names)
+    table_groups_in_use = not table_groups_in_use_result.empty
+
+    return test_suites_in_use or table_groups_in_use
+
+
+def get_test_suite_names_by_table_group_names(table_group_names):
+    if not table_group_names:
+        return []
+    schema = st.session_state["dbschema"]
+    test_suite_names = table_group_queries.get_test_suite_names_by_table_group_names(schema, table_group_names)
+    return test_suite_names.to_dict()["test_suite"].values()
+
+
+def test_table_group(table_group, connection_id, project_code):
+    # get connection data
+    connection = connection_service.get_by_id(connection_id, hide_passwords=False)
+    connection_id = str(connection["connection_id"])
+
+    # get table group data
+    table_group_schema = table_group["table_group_schema"]
+    table_group_id = table_group["id"]
+    project_qc_schema = connection["project_qc_schema"]
+    profiling_table_set = table_group["profiling_table_set"]
+    profiling_include_mask = table_group["profiling_include_mask"]
+    profiling_exclude_mask = table_group["profiling_exclude_mask"]
+    profile_id_column_mask = table_group["profile_id_column_mask"]
+    profile_sk_column_mask = table_group["profile_sk_column_mask"]
+    profile_use_sampling = "Y" if table_group["profile_use_sampling"] else "N"
+    profile_sample_percent = table_group["profile_sample_percent"]
+    profile_sample_min_count = table_group["profile_sample_min_count"]
+
+    clsProfiling = connection_service.init_profiling_sql(project_code, connection, table_group_schema)
+
+    # Set General Parms
+    clsProfiling.table_groups_id = table_group_id
+    clsProfiling.connection_id = connection_id
+    clsProfiling.parm_do_sample = "N"
+    clsProfiling.parm_sample_size = 0
+    clsProfiling.parm_vldb_flag = "N"
+    clsProfiling.parm_do_freqs = "Y"
+    clsProfiling.parm_max_freq_length = 25
+    clsProfiling.parm_do_patterns = "Y"
+    clsProfiling.parm_max_pattern_length = 25
+    clsProfiling.profile_run_id = ""
+    clsProfiling.data_qc_schema = project_qc_schema
+    clsProfiling.data_schema = table_group_schema
+    clsProfiling.parm_table_set = get_profiling_table_set_with_quotes(profiling_table_set)
+    clsProfiling.parm_table_include_mask = profiling_include_mask
+    clsProfiling.parm_table_exclude_mask = profiling_exclude_mask
+    clsProfiling.profile_id_column_mask = profile_id_column_mask
+    clsProfiling.profile_sk_column_mask = profile_sk_column_mask
+    clsProfiling.profile_use_sampling = profile_use_sampling
+    clsProfiling.profile_sample_percent = profile_sample_percent
+    clsProfiling.profile_sample_min_count = profile_sample_min_count
+
+    query = clsProfiling.GetDDFQuery()
+    table_group_results = RetrieveDBResultsToDictList("PROJECT", query)
+
+    qc_results = connection_service.test_qc_connection(project_code, connection, init_profiling=False)
+
+    return table_group_results, qc_results
+
+
+def get_profiling_table_set_with_quotes(profiling_table_set):
+    if not profiling_table_set:
+        return profiling_table_set
+
+    aux_list = []
+    split = profiling_table_set.split(",")
+    for item in split:
+        aux_list.append(f"'{item}'")
+    profiling_table_set = ",".join(aux_list)
+    return profiling_table_set

testgen/ui/services/test_definition_service.py

@@ -0,0 +1,117 @@
+import streamlit as st
+
+import testgen.ui.queries.test_definition_queries as test_definition_queries
+import testgen.ui.services.connection_service as connection_service
+import testgen.ui.services.database_service as database_service
+import testgen.ui.services.table_group_service as table_group_service
+import testgen.ui.services.test_run_service as test_run_service
+
+
+def update_attribute(test_definition_ids, attribute, value):
+    schema = st.session_state["dbschema"]
+    raw_value = "Y" if value else "N"
+    test_definition_queries.update_attribute(schema, test_definition_ids, attribute, raw_value)
+
+
+def get_test_definitions(
+    project_code=None, test_suite=None, table_name=None, column_name=None, test_definition_ids=None
+):
+    schema = st.session_state["dbschema"]
+    return test_definition_queries.get_test_definitions(
+        schema, project_code, test_suite, table_name, column_name, test_definition_ids
+    )
+
+
+def delete(test_definition_ids, dry_run=False):
+    schema = st.session_state["dbschema"]
+    usage_result = test_definition_queries.get_test_definition_usage(schema, test_definition_ids)
+    can_be_deleted = usage_result.empty
+    if not dry_run and can_be_deleted:
+        test_definition_queries.delete(schema, test_definition_ids)
+    return can_be_deleted
+
+
+def cascade_delete(test_suite_names):
+    schema = st.session_state["dbschema"]
+    test_run_service.cascade_delete(test_suite_names)
+    test_definition_queries.cascade_delete(schema, test_suite_names)
+
+
+def add(test_definition):
+    schema = st.session_state["dbschema"]
+    prepare_to_persist(test_definition)
+    test_definition_queries.add(schema, test_definition)
+
+
+def update(test_definition):
+    schema = st.session_state["dbschema"]
+    prepare_to_persist(test_definition)
+    return test_definition_queries.update(schema, test_definition)
+
+
+def prepare_to_persist(test_definition):
+    # severity
+    if test_definition["severity"] and test_definition["severity"].startswith("Inherited"):
+        test_definition["severity"] = None
+
+    test_definition["export_to_observability"] = prepare_boolean_for_update(
+        test_definition["export_to_observability_raw"]
+    )
+    test_definition["lock_refresh"] = prepare_boolean_for_update(test_definition["lock_refresh"])
+    test_definition["test_active"] = prepare_boolean_for_update(test_definition["test_active"])
+
+    if test_definition["custom_query"] is not None:
+        test_definition["custom_query"] = test_definition["custom_query"].strip()
+        if test_definition["custom_query"].endswith(";"):
+            test_definition["custom_query"] = test_definition["custom_query"][:-1]
+
+    empty_if_null(test_definition)
+
+
+def empty_if_null(test_definition):
+    for k, v in test_definition.items():
+        if v is None:
+            test_definition[k] = ""
+
+
+def prepare_boolean_for_update(value):
+    if "Yes" == value or "Y" == value or value is True:
+        return "Y"
+    elif "No" == value or "N" == value or value is False:
+        return "N"
+    else:
+        return None
+
+
+def validate_test(test_definition):
+    schema = test_definition["schema_name"]
+    table_name = test_definition["table_name"]
+
+    if test_definition["test_type"] == "Condition_Flag":
+        condition = test_definition["custom_query"]
+        sql_query = f"""SELECT COALESCE(CAST(SUM(CASE WHEN {condition} THEN 1 ELSE 0 END) AS VARCHAR(1000) ) || '|' ,'<NULL>|') FROM {schema}.{table_name}"""
+    else:
+        sql_query = test_definition["custom_query"]
+        sql_query = sql_query.replace("{DATA_SCHEMA}", schema)
+
+    table_group_id = test_definition["table_groups_id"]
+    table_group_df = table_group_service.get_by_id(table_group_id)
+
+    connection_id = table_group_df.iloc[0]["connection_id"]
+
+    connection = connection_service.get_by_id(connection_id, hide_passwords=False)
+
+    database_service.retrieve_target_db_data(
+        connection["sql_flavor"],
+        connection["project_host"],
+        connection["project_port"],
+        connection["project_db"],
+        connection["project_user"],
+        connection["password"],
+        connection["url"],
+        connection["connect_by_url"],
+        connection["connect_by_key"],
+        connection["private_key"],
+        connection["private_key_passphrase"],
+        sql_query,
+    )

testgen/ui/services/test_run_service.py

@@ -0,0 +1,13 @@
+import streamlit as st
+
+import testgen.ui.queries.test_run_queries as test_run_queries
+
+
+def cascade_delete(test_suite_names):
+    schema = st.session_state["dbschema"]
+    test_run_queries.cascade_delete(schema, test_suite_names)
+
+
+def update_status(test_run_id, status):
+    schema = st.session_state["dbschema"]
+    test_run_queries.update_status(schema, test_run_id, status)