dataops-testgen 2.2.0__py3-none-any.whl

testgen/ui/queries/connection_queries.py

@@ -0,0 +1,121 @@
+import pandas as pd
+import streamlit as st
+
+import testgen.ui.services.database_service as db
+
+
+def get_by_id(connection_id):
+    str_schema = st.session_state["dbschema"]
+    str_sql = f"""
+           SELECT id::VARCHAR(50), project_code, connection_id, connection_name,
+                  sql_flavor, project_host, project_port, project_user, project_qc_schema,
+                  project_db, project_pw_encrypted, NULL as password,
+                  max_threads, max_query_chars, url, connect_by_url, connect_by_key, private_key, private_key_passphrase
+             FROM {str_schema}.connections
+             WHERE connection_id = '{connection_id}'
+    """
+    return db.retrieve_data(str_sql)
+
+
+def get_connections(project_code):
+    str_schema = st.session_state["dbschema"]
+    str_sql = f"""
+           SELECT id::VARCHAR(50), project_code, connection_id, connection_name,
+                  sql_flavor, project_host, project_port, project_user, project_qc_schema,
+                  project_db, project_pw_encrypted, NULL as password,
+                  max_threads, max_query_chars, connect_by_url, url, connect_by_key, private_key,
+                  private_key_passphrase
+             FROM {str_schema}.connections
+             WHERE project_code = '{project_code}'
+           ORDER BY connection_id
+    """
+    return db.retrieve_data(str_sql)
+
+
+def get_table_group_names_by_connection(schema: str, connection_ids: list[str]) -> pd.DataFrame:
+    items = [f"'{item}'" for item in connection_ids]
+    str_sql = f"""select table_groups_name from {schema}.table_groups where connection_id in ({",".join(items)})"""
+    return db.retrieve_data(str_sql)
+
+
+def edit_connection(schema, connection, encrypted_password, encrypted_private_key, encrypted_private_key_passphrase):
+    sql = f"""UPDATE  {schema}.connections SET
+        project_code = '{connection["project_code"]}',
+        sql_flavor = '{connection["sql_flavor"]}',
+        project_host = '{connection["project_host"]}',
+        project_port = '{connection["project_port"]}',
+        project_user = '{connection["project_user"]}',
+        project_db = '{connection["project_db"]}',
+        project_qc_schema = '{connection["project_qc_schema"]}',
+        connection_name = '{connection["connection_name"]}',
+        max_threads = '{connection["max_threads"]}',
+        max_query_chars = '{connection["max_query_chars"]}',
+        url = '{connection["url"]}',
+        connect_by_key = '{connection["connect_by_key"]}',
+        connect_by_url = '{connection["connect_by_url"]}'"""
+
+    if encrypted_password:
+        sql += f""", project_pw_encrypted = '{encrypted_password}' """
+
+    if encrypted_private_key:
+        sql += f""", private_key = '{encrypted_private_key}' """
+
+    if encrypted_private_key_passphrase:
+        sql += f""", private_key_passphrase = '{encrypted_private_key_passphrase}' """
+
+    sql += f""" WHERE connection_id = '{connection["connection_id"]}';"""
+    db.execute_sql(sql)
+    st.cache_data.clear()
+
+
+def add_connection(schema, connection, encrypted_password, encrypted_private_key, encrypted_private_key_passphrase):
+
+    sql_header = f"""INSERT INTO {schema}.connections
+        (project_code, sql_flavor, url, connect_by_url, connect_by_key,  
+        project_host, project_port, project_user, project_db, project_qc_schema,
+        connection_name,"""
+
+    sql_footer = f""" SELECT
+        '{connection["project_code"]}' as project_code,
+        '{connection["sql_flavor"]}' as sql_flavor,
+        '{connection["url"]}' as url,
+        {connection["connect_by_url"]} as connect_by_url,
+        {connection["connect_by_key"]} as connect_by_key,
+        '{connection["project_host"]}' as project_host,
+        '{connection["project_port"]}' as project_port,
+        '{connection["project_user"]}' as project_user,
+        '{connection["project_db"]}' as project_db,
+        '{connection["project_qc_schema"]}' as project_qc_schema,
+        '{connection["connection_name"]}' as connection_name, """
+
+    if encrypted_password:
+        sql_header += "project_pw_encrypted, "
+        sql_footer += f""" '{encrypted_password}' as project_pw_encrypted, """
+
+    if encrypted_private_key:
+        sql_header += "private_key, "
+        sql_footer += f""" '{encrypted_private_key}' as private_key, """
+
+    if encrypted_private_key_passphrase:
+        sql_header += "private_key_passphrase, "
+        sql_footer += f""" '{encrypted_private_key_passphrase}' as private_key_passphrase, """
+
+    sql_header += """max_threads, max_query_chars) """
+
+    sql_footer += f""" '{connection["max_threads"]}' as max_threads,
+        '{connection["max_query_chars"]}' as max_query_chars;"""
+
+    sql = sql_header + sql_footer
+
+    db.execute_sql(sql)
+    st.cache_data.clear()
+
+
+def delete_connections(schema, connection_ids):
+    if connection_ids is None or len(connection_ids) == 0:
+        raise ValueError("No connection is specified.")
+
+    items = [f"'{item}'" for item in connection_ids]
+    sql = f"""DELETE FROM {schema}.connections WHERE connection_id in ({",".join(items)})"""
+    db.execute_sql(sql)
+    st.cache_data.clear()

testgen/ui/queries/profiling_queries.py

@@ -0,0 +1,148 @@
+import streamlit as st
+
+import testgen.ui.services.database_service as db
+import testgen.ui.services.query_service as dq
+
+
+@st.cache_data(show_spinner=False)
+def run_table_groups_lookup_query(str_project_code):
+    str_schema = st.session_state["dbschema"]
+    return dq.run_table_groups_lookup_query(str_schema, str_project_code)
+
+
+@st.cache_data(show_spinner=False)
+def get_latest_profile_run(str_table_group):
+    str_schema = st.session_state["dbschema"]
+    str_sql = f"""
+            WITH last_profile_run
+               AS (SELECT table_groups_id, MAX(profiling_starttime) as last_profile_run_date
+                     FROM {str_schema}.profiling_runs
+                   GROUP BY table_groups_id)
+            SELECT id as profile_run_id
+              FROM {str_schema}.profiling_runs r
+            INNER JOIN last_profile_run l
+               ON (r.table_groups_id = l.table_groups_id
+              AND  r.profiling_starttime = l.last_profile_run_date)
+             WHERE r.table_groups_id = '{str_table_group}';
+"""
+    str_profile_run_id = db.retrieve_single_result(str_sql)
+
+    return str_profile_run_id
+
+
+@st.cache_data(show_spinner=False)
+def get_db_profile_run_choices(str_table_groups_id):
+    str_schema = st.session_state["dbschema"]
+    # Define the query
+    str_sql = f"""
+            SELECT DISTINCT profiling_starttime as profile_run_date, id
+              FROM {str_schema}.profiling_runs pr
+             WHERE pr.table_groups_id = '{str_table_groups_id}'
+            ORDER BY profiling_starttime DESC;
+    """
+    # Retrieve and return data as df
+    return db.retrieve_data(str_sql)
+
+
+@st.cache_data(show_spinner=False)
+def run_table_lookup_query(str_table_groups_id):
+    str_schema = st.session_state["dbschema"]
+    str_sql = f"""
+           SELECT DISTINCT table_name
+             FROM {str_schema}.profile_results
+            WHERE table_groups_id = '{str_table_groups_id}'::UUID
+           ORDER BY table_name
+    """
+    return db.retrieve_data(str_sql)
+
+
+@st.cache_data(show_spinner=False)
+def run_column_lookup_query(str_table_groups_id, str_table_name):
+    str_schema = st.session_state["dbschema"]
+    return dq.run_column_lookup_query(str_schema, str_table_groups_id, str_table_name)
+
+
+@st.cache_data(show_spinner=False)
+def lookup_db_parentage_from_run(str_profile_run_id):
+    str_schema = st.session_state["dbschema"]
+    # Define the query
+    str_sql = f"""
+            SELECT profiling_starttime as profile_run_date, g.table_groups_name
+              FROM {str_schema}.profiling_runs pr
+             INNER JOIN {str_schema}.table_groups g
+                ON pr.table_groups_id = g.id
+             WHERE pr.id = '{str_profile_run_id}'
+    """
+    df = db.retrieve_data(str_sql)
+    if not df.empty:
+        return df.at[0, "profile_run_date"], df.at[0, "table_groups_name"]
+
+
+@st.cache_data(show_spinner="Retrieving Data")
+def get_profiling_detail(str_profile_run_id, str_table_name, str_column_name):
+    str_schema = st.session_state["dbschema"]
+    str_sql = f"""
+          SELECT   -- Identifiers
+                   id::VARCHAR, dk_id,
+                   p.project_code, connection_id, p.table_groups_id::VARCHAR,
+                   p.profile_run_id::VARCHAR,
+                   run_date, sample_ratio,
+                   -- Column basics
+                   p.schema_name, p.table_name, position, p.column_name,
+                   p.column_type, general_type as general_type_abbr,
+                   CASE general_type
+                     WHEN 'A' THEN 'Alpha'
+                     WHEN 'N' THEN 'Numeric'
+                     WHEN 'D' THEN 'Date'
+                     WHEN 'T' THEN 'Time'
+                     WHEN 'B' THEN 'Boolean'
+                              ELSE 'N/A'
+                   END as general_type,
+                   functional_table_type as semantic_table_type, 
+                   functional_data_type as semantic_data_type,
+                   datatype_suggestion,
+                   CASE WHEN s.column_name IS NOT NULL THEN 'Yes' END as anomalies,
+                   -- Shared counts
+                   record_ct, value_ct, distinct_value_ct, null_value_ct,
+                   -- Shared except for B and X
+                   min_length, max_length, avg_length,
+                   -- Alpha counts
+                   distinct_std_value_ct,
+                   numeric_ct, date_ct,
+                   filled_value_ct as dummy_value_ct,
+                   CASE WHEN general_type = 'A' THEN COALESCE(zero_length_ct, 0) END as zero_length_ct,
+                   CASE WHEN general_type = 'A' THEN COALESCE(lead_space_ct, 0) END as lead_space_ct,
+                   CASE WHEN general_type = 'A' THEN COALESCE(quoted_value_ct, 0) END as quoted_value_ct,
+                   CASE WHEN general_type = 'A' THEN COALESCE(includes_digit_ct, 0) END as includes_digit_ct,
+                   CASE WHEN general_type = 'A' THEN COALESCE(embedded_space_ct, 0) END as embedded_space_ct,
+                   avg_embedded_spaces,
+                   min_text, max_text,
+                   std_pattern_match,
+                   top_patterns,
+                   top_freq_values, distinct_value_hash,
+                   distinct_pattern_ct,
+                   -- A and N
+                   zero_value_ct,
+                   -- Numeric
+                   min_value, min_value_over_0, max_value,
+                   avg_value, stdev_value, percentile_25, percentile_50, percentile_75,
+                   fractional_sum,
+                   -- Dates
+                   min_date, max_date,
+                   before_1yr_date_ct, before_5yr_date_ct, within_1yr_date_ct, within_1mo_date_ct, future_date_ct,
+                   date_days_present, date_weeks_present, date_months_present,
+                   -- Boolean
+                   boolean_true_ct
+           FROM {str_schema}.profile_results p
+          LEFT JOIN (SELECT DISTINCT profile_run_id, table_name, column_name
+                       FROM {str_schema}.profile_anomaly_results) s
+          ON (p.profile_run_id = s.profile_run_id
+          AND p.table_name = s.table_name
+          AND p.column_name = s.column_name)
+          WHERE p.profile_run_id = '{str_profile_run_id}'::UUID
+            AND p.table_name ILIKE '{str_table_name}'
+            AND p.column_name ILIKE '{str_column_name}'
+          ORDER BY p.schema_name, p.table_name, position;
+    """
+
+    return db.retrieve_data(str_sql)

testgen/ui/queries/project_queries.py

@@ -0,0 +1,9 @@
+import streamlit as st
+
+import testgen.ui.services.query_service as query_service
+
+
+@st.cache_data(show_spinner=False)
+def get_projects():
+    str_schema = st.session_state["dbschema"]
+    return query_service.run_project_lookup_query(str_schema)

testgen/ui/queries/table_group_queries.py

@@ -0,0 +1,186 @@
+import streamlit as st
+
+import testgen.ui.services.database_service as db
+
+
+def _get_select_statement(schema):
+    return f"""
+               SELECT id::VARCHAR(50), project_code, connection_id, table_groups_name,
+                      table_group_schema,
+                      profiling_include_mask, profiling_exclude_mask,
+                      profiling_table_set,
+                      profile_id_column_mask, profile_sk_column_mask,
+                      data_source, source_system, data_location, business_domain, 
+                      transform_level, source_process, stakeholder_group,
+                      profile_use_sampling, profile_sample_percent, profile_sample_min_count,
+                      profiling_delay_days
+               FROM {schema}.table_groups
+               """
+
+
+@st.cache_data(show_spinner=False)
+def get_by_id(schema, table_group_id):
+    sql = _get_select_statement(schema)
+    sql += f"""WHERE id = '{table_group_id}'
+           ORDER BY table_groups_name
+    """
+    return db.retrieve_data(sql)
+
+
+def get_test_suite_names_by_table_group_names(schema, table_group_names):
+    items = [f"'{item}'" for item in table_group_names]
+    sql = f"""select test_suite
+from {schema}.test_suites ts
+inner join {schema}.table_groups tg on tg.id = ts.table_groups_id
+where tg.table_groups_name in ({",".join(items)})
+    """
+    return db.retrieve_data(sql)
+
+
+def get_table_group_dependencies(schema, table_group_names):
+    if table_group_names is None or len(table_group_names) == 0:
+        raise ValueError("No Table Group is specified.")
+
+    table_group_items = [f"'{item}'" for item in table_group_names]
+    sql = f"""select ppr.profile_run_id from {schema}.profile_pair_rules ppr
+    INNER JOIN {schema}.profiling_runs pr ON pr.id = ppr.profile_run_id
+    INNER JOIN {schema}.table_groups tg ON tg.id = pr.table_groups_id
+    where tg.table_groups_name in ({",".join(table_group_items)})
+    union
+    select par.table_groups_id from {schema}.profile_anomaly_results par INNER JOIN {schema}.table_groups tg ON tg.id = par.table_groups_id where tg.table_groups_name in ({",".join(table_group_items)})
+    union
+    select pr.table_groups_id from {schema}.profile_results pr INNER JOIN {schema}.table_groups tg ON tg.id = pr.table_groups_id where tg.table_groups_name in ({",".join(table_group_items)})
+    union
+    select pr.table_groups_id from {schema}.profiling_runs pr INNER JOIN {schema}.table_groups tg ON tg.id = pr.table_groups_id where tg.table_groups_name in ({",".join(table_group_items)})
+    union
+    select dtc.table_groups_id from {schema}.data_table_chars dtc INNER JOIN {schema}.table_groups tg ON tg.id = dtc.table_groups_id where tg.table_groups_name in ({",".join(table_group_items)})
+    union
+    select dcs.table_groups_id from {schema}.data_column_chars dcs INNER JOIN {schema}.table_groups tg ON tg.id = dcs.table_groups_id where tg.table_groups_name in ({",".join(table_group_items)});"""
+    return db.retrieve_data(sql)
+
+
+def get_table_group_usage(schema, table_group_names):
+    items = [f"'{item}'" for item in table_group_names]
+    sql = f"""select distinct pr.id from {schema}.profiling_runs pr
+INNER JOIN {schema}.table_groups tg ON tg.id = pr.table_groups_id
+where tg.table_groups_name in ({",".join(items)}) and pr.status = 'Running'"""
+    return db.retrieve_data(sql)
+
+
+@st.cache_data(show_spinner=False)
+def get_by_connection(schema, project_code, connection_id):
+    sql = _get_select_statement(schema)
+    sql += f"""WHERE project_code = '{project_code}'
+             AND connection_id = '{connection_id}'
+            ORDER BY table_groups_name
+     """
+    return db.retrieve_data(sql)
+
+
+def edit(schema, table_group):
+    sql = f"""UPDATE {schema}.table_groups
+                SET
+                    table_groups_name='{table_group["table_groups_name"]}',
+                    table_group_schema='{table_group["table_group_schema"]}',
+                    profiling_table_set=NULLIF('{table_group["profiling_table_set"]}', ''),
+                    profiling_include_mask='{table_group["profiling_include_mask"]}',
+                    profiling_exclude_mask='{table_group["profiling_exclude_mask"]}',
+                    profile_id_column_mask='{table_group["profile_id_column_mask"]}',
+                    profile_sk_column_mask='{table_group["profile_sk_column_mask"]}',
+                    profile_use_sampling='{'Y' if table_group["profile_use_sampling"] else 'N'}',
+                    profile_sample_percent='{table_group["profile_sample_percent"]}',
+                    profile_sample_min_count={int(table_group["profile_sample_min_count"])},
+                    profiling_delay_days='{table_group["profiling_delay_days"]}',
+                    data_source='{table_group["data_source"]}',
+                    source_system='{table_group["source_system"]}',
+                    data_location='{table_group["data_location"]}',
+                    business_domain='{table_group["business_domain"]}',
+                    transform_level='{table_group["transform_level"]}',
+                    source_process='{table_group["source_process"]}',
+                    stakeholder_group='{table_group["stakeholder_group"]}'
+                where
+                    id = '{table_group["id"]}'
+                ;
+                    """
+    db.execute_sql(sql)
+    st.cache_data.clear()
+
+
+def add(schema, table_group):
+    sql = f"""INSERT INTO {schema}.table_groups
+        (id,
+        project_code,
+        connection_id,
+        table_groups_name,
+        table_group_schema,
+        profiling_table_set,
+        profiling_include_mask,
+        profiling_exclude_mask,
+        profile_id_column_mask,
+        profile_sk_column_mask,
+        profile_use_sampling,
+        profile_sample_percent,
+        profile_sample_min_count,
+        profiling_delay_days,
+        data_source,
+        source_system,
+        data_location,
+        business_domain,
+        transform_level,
+        source_process,
+        stakeholder_group)
+    SELECT
+        gen_random_uuid(),
+        '{table_group["project_code"]}',
+        '{table_group["connection_id"]}',
+        '{table_group["table_groups_name"]}',
+        '{table_group["table_group_schema"]}',
+        NULLIF('{table_group["profiling_table_set"]}', ''),
+        '{table_group["profiling_include_mask"]}',
+        '{table_group["profiling_exclude_mask"]}',
+        '{table_group["profile_id_column_mask"]}'::character varying(2000),
+        '{table_group["profile_sk_column_mask"]}'::character varying,
+        '{'Y' if table_group["profile_use_sampling"]=='True' else 'N' }'::character varying,
+        '{table_group["profile_sample_percent"]}'::character varying,
+        {table_group["profile_sample_min_count"]}, '{table_group["profiling_delay_days"]}'::character varying,
+        '{table_group["data_source"]}',
+        '{table_group["source_system"]}',
+        '{table_group["data_location"]}',
+        '{table_group["business_domain"]}',
+        '{table_group["transform_level"]}',
+        '{table_group["source_process"]}',
+        '{table_group["stakeholder_group"]}'
+        ;"""
+    db.execute_sql(sql)
+    st.cache_data.clear()
+
+
+def delete(schema, table_group_ids):
+    if table_group_ids is None or len(table_group_ids) == 0:
+        raise ValueError("No table group is specified.")
+
+    items = [f"'{item}'" for item in table_group_ids]
+    sql = f"""DELETE FROM {schema}.table_groups WHERE id in ({",".join(items)})"""
+    db.execute_sql(sql)
+    st.cache_data.clear()
+
+
+def cascade_delete(schema, table_group_names):
+    if table_group_names is None or len(table_group_names) == 0:
+        raise ValueError("No Table Group is specified.")
+
+    table_group_items = [f"'{item}'" for item in table_group_names]
+    sql = f"""delete from {schema}.profile_pair_rules ppr
+USING {schema}.profiling_runs pr, {schema}.table_groups tg
+WHERE
+pr.id = ppr.profile_run_id
+AND tg.id = pr.table_groups_id
+AND tg.table_groups_name in ({",".join(table_group_items)});
+delete from {schema}.profile_anomaly_results par USING {schema}.table_groups tg where tg.id = par.table_groups_id and tg.table_groups_name in ({",".join(table_group_items)});
+delete from {schema}.profile_results pr USING {schema}.table_groups tg where tg.id = pr.table_groups_id and tg.table_groups_name in ({",".join(table_group_items)});
+delete from {schema}.profiling_runs pr USING {schema}.table_groups tg where tg.id = pr.table_groups_id and tg.table_groups_name in ({",".join(table_group_items)});
+delete from {schema}.data_table_chars dtc USING {schema}.table_groups tg where tg.id = dtc.table_groups_id and tg.table_groups_name in ({",".join(table_group_items)});
+delete from {schema}.data_column_chars dcs USING {schema}.table_groups tg where tg.id = dcs.table_groups_id and tg.table_groups_name in ({",".join(table_group_items)});
+delete from {schema}.table_groups where table_groups_name in ({",".join(table_group_items)});"""
+    db.execute_sql(sql)
+    st.cache_data.clear()