dataops-testgen 2.2.0__py3-none-any.whl

testgen/common/read_file.py

@@ -0,0 +1,69 @@
+__all__ = ["get_template_files", "read_template_sql_file", "read_template_yaml_file"]
+
+import logging
+import re
+from collections.abc import Generator
+from functools import cache
+from importlib.abc import Traversable
+from importlib.resources import as_file, files
+
+import yaml
+
+LOG = logging.getLogger("testgen")
+
+
+def _get_template_package_resource(
+    template_file_name: str | None = None,
+    sub_directory: str | None = None,
+    path: str | None = None,
+) -> Traversable:
+    if path is None:
+        path = "testgen.template"
+    if sub_directory:
+        path = f"{path}.{sub_directory.replace('/', '.')}"
+    if template_file_name:
+        return files(path).joinpath(template_file_name)
+    else:
+        return files(path)
+
+
+@cache
+def read_template_sql_file(template_file_name: str, sub_directory: str | None = None) -> str:
+    file = _get_template_package_resource(template_file_name, sub_directory)
+    LOG.debug("Reading SQL resource: %s", str(file))
+    try:
+        contents = file.read_text(encoding="utf-8").strip()
+    except FileNotFoundError as e:
+        raise ValueError(f"template.{sub_directory}.{template_file_name}: File not found") from e
+
+    if not contents.strip():
+        raise ValueError(f"template.{sub_directory}.{template_file_name}: file is empty")
+
+    return contents
+
+
+def get_template_files(mask: str, sub_directory: str | None = None, path: str | None = None) -> Generator[Traversable, None, None]:
+    folder = _get_template_package_resource(template_file_name=None, sub_directory=sub_directory, path=path)
+    LOG.debug("Reading SQL folder resource: %s", str(folder))
+    for entry in folder.iterdir():
+        if entry.is_file() and re.search(mask, str(entry)):
+            yield entry
+
+
+@cache
+def read_template_yaml_file(template_file_name: str, sub_directory: str | None = None) -> dict:
+    if not template_file_name.endswith(("yaml", "yml")):
+        raise ValueError(f"{template_file_name}: does not have a yaml/yml suffix; is it yaml?")
+    resource_file = _get_template_package_resource(template_file_name=template_file_name, sub_directory=sub_directory)
+    LOG.debug("Reading Yaml resource: %s", str(resource_file))
+
+    try:
+        with as_file(resource_file) as f:
+            with f.open("r") as file:
+                template = yaml.safe_load(file)
+    except FileNotFoundError as e:
+        raise ValueError(f"{template_file_name}: File not found") from e
+    if not template:
+        raise ValueError(f"{template_file_name}: File is empty")
+
+    return template

testgen/settings.py

@@ -0,0 +1,440 @@
+import os
+
+IS_DEBUG_LOG_LEVEL: bool = os.getenv("TESTGEN_DEBUG_LOG_LEVEL", "no").lower() == "yes"
+"""
+When set, logs will be at debug level.
+defaults to: `no`
+"""
+
+IS_DEBUG: bool = os.getenv("TESTGEN_DEBUG", "no").lower() == "yes"
+"""
+When True invalidates the cache with the bootstrapped application
+causing the changes to the routing and plugins to take effect on every
+render.
+
+from env variable: `TESTGEN_DEBUG`
+defaults to: `True`
+"""
+
+LOG_TO_FILE: bool = os.getenv("TESTGEN_LOG_TO_FILE", "yes").lower() == "yes"
+"""
+When set, rotating file logs will be generated.
+defaults to: `True`
+"""
+
+LOG_FILE_PATH: str = os.getenv("TESTGEN_LOG_FILE_PATH", "/var/lib/testgen/log")
+"""
+When set, rotating file logs will be generated under this path.
+
+"""
+
+LOG_FILE_MAX_QTY: str = os.getenv("TESTGEN_LOG_FILE_MAX_QTY", "90")
+"""
+Maximum log files to keep, defaults to 90 days (one file per day).
+"""
+
+APP_ENCRYPTION_SALT: str = os.getenv("TG_DECRYPT_SALT")
+"""
+Salt used to encrypt and decrypt user secrets. Only allows ascii
+characters.
+
+A minimun length of 16 characters is recommended.
+
+from env variable: `TG_DECRYPT_SALT`
+"""
+
+APP_ENCRYPTION_SECRET: str = os.getenv("TG_DECRYPT_PASSWORD")
+"""
+Secret passcode used in combination with `APP_ENCRYPTION_SALT` to
+encrypt and decrypt user secrets. Only allows ascii characters.
+
+from env variable: `TG_DECRYPT_PASSWORD`
+"""
+
+USERNAME: str = os.getenv("TESTGEN_USERNAME")
+"""
+Username to log into the web application
+
+from env variable: `TESTGEN_USERNAME`
+"""
+
+PASSWORD: str = os.getenv("TESTGEN_PASSWORD")
+"""
+Password to log into the web application
+
+from env variable: `TESTGEN_PASSWORD`
+"""
+
+DATABASE_USER: str = os.getenv("TG_METADATA_DB_USER", USERNAME)
+"""
+User to connect to the testgen application postgres database.
+
+from env variable: `TG_METADATA_DB_USER`
+defaults to: `environ[USERNAME]`
+"""
+
+DATABASE_PASSWORD: str = os.getenv("TG_METADATA_DB_PASSWORD", PASSWORD)
+"""
+Password to connect to the testgen application postgres database.
+
+from env variable: `TG_METADATA_DB_PASSWORD`
+defaults to: `environ[PASSWORD]`
+"""
+
+DATABASE_ADMIN_USER: str = os.getenv("DATABASE_ADMIN_USER", DATABASE_USER)
+"""
+User with admin privileges in the testgen application postgres database
+used to create roles, users, database and schema. Required if the user
+in `TG_METADATA_DB_USER` does not have the required privileges.
+
+from env variable: `DATABASE_ADMIN_USER`
+defaults to: `environ[DATABASE_USER]`
+"""
+
+DATABASE_ADMIN_PASSWORD: str = os.getenv("DATABASE_ADMIN_PASSWORD", DATABASE_PASSWORD)
+"""
+Password for the admin user to connect to the testgen application
+postgres database.
+
+from env variable: `DATABASE_ADMIN_PASSWORD`
+defaults to: `environ[DATABASE_PASSWORD]`
+"""
+
+DATABASE_EXECUTE_USER: str = os.getenv("DATABASE_EXECUTE_USER", "testgen_execute")
+"""
+User to be created into the testgen application postgres database. Will
+be granted:
+
+_ read/write to tables `test_results`, `test_suites` and `test_definitions`
+_ read_only to all other tables
+
+from env variable: `DATABASE_EXECUTE_USER`
+defaults to: `testgen_execute`
+"""
+
+DATABASE_REPORT_USER: str = os.getenv("DATABASE_REPORT_USER", "testgen_report")
+"""
+User to be created into the testgen application postgres database. Will
+be granted read_only access to all tables.
+
+from env variable: `DATABASE_REPORT_USER`
+defaults to: `testgen_report`
+"""
+
+DATABASE_HOST: str = os.getenv("TG_METADATA_DB_HOST", "localhost")
+"""
+Hostname where the testgen application postgres database is running in.
+
+from env variable: `TG_METADATA_DB_HOST`
+defaults to: `localhost`
+"""
+
+DATABASE_PORT: str = os.getenv("TG_METADATA_DB_PORT", "5432")
+"""
+Port at which the testgen application postgres database is exposed by
+the host.
+
+from env variable: `TG_METADATA_DB_PORT`
+defaults to: `5432`
+"""
+
+DATABASE_NAME: str = os.getenv("TG_METADATA_DB_NAME", "datakitchen")
+"""
+Name of the database in postgres on which to store testgen metadata.
+
+from env variable: `TG_METADATA_DB_NAME`
+defaults to: `datakitchen`
+"""
+
+DATABASE_SCHEMA: str = os.getenv("TG_METADATA_DB_SCHEMA", "testgen")
+"""
+Name of the schema inside the postgres database on which to store
+testgen metadata.
+
+from env variable: `TG_METADATA_DB_SCHEMA`
+defaults to: `testgen`
+"""
+
+PROJECT_KEY: str = os.getenv("PROJECT_KEY", "DEFAULT")
+"""
+Code used to uniquely identify the auto generated project.
+
+from env variable: `PROJECT_KEY`
+defaults to: `DEFAULT`
+"""
+
+PROJECT_NAME: str = os.getenv("PROJECT_NAME", "Demo")
+"""
+Name to assign to the auto generated project.
+
+from env variable: `DEFAULT_PROJECT_NAME`
+defaults to: `Demo`
+"""
+
+PROJECT_SQL_FLAVOR: str = os.getenv("PROJECT_SQL_FLAVOR", "postgresql")
+"""
+SQL flavor of the database the auto generated project will run tests
+against.
+
+Supported flavors are:
+_ redshift
+_ snowflake
+_ mssql
+_ postgresql
+
+from env variable: `PROJECT_SQL_FLAVOR`
+defaults to: `postgresql`
+"""
+
+PROJECT_CONNECTION_NAME: str = os.getenv("PROJECT_CONNECTION_NAME", "default")
+"""
+Name assigned to identify the connection to the project database.
+
+from env variable: `PROJECT_CONNECTION_NAME`
+defaults to: `default`
+"""
+
+PROJECT_CONNECTION_MAX_THREADS: int = int(os.getenv("PROJECT_CONNECTION_MAX_THREADS", "4"))
+"""
+Maximum number of concurrent queries executed when fetching data from
+the project database.
+
+from env variable: `PROJECT_CONNECTION_MAX_THREADS`
+defaults to: `4`
+"""
+
+PROJECT_CONNECTION_MAX_QUERY_CHAR: int = int(os.getenv("PROJECT_CONNECTION_MAX_QUERY_CHAR", "5000"))
+"""
+Determine how many tests are grouped together in a single query.
+Increase for better performance or decrease to better isolate test
+failures. Accepted values are 500 to 14 000.
+
+from env variable: `PROJECT_CONNECTION_MAX_QUERY_CHAR`
+defaults to: `5000`
+"""
+
+PROJECT_QC_SCHEMA: str = os.getenv("PROJECT_QC_SCHEMA", "qc")
+"""
+Name of the schema to be created in the project database.
+
+from env variable: `PROJECT_QC_SCHEMA`
+defaults to: `qc`
+"""
+
+PROJECT_DATABASE_NAME: str = os.getenv("PROJECT_DATABASE_NAME", "demo_db")
+"""
+Name of the database the auto generated project will run test
+against.
+
+from env variable: `PROJECT_DATABASE_NAME`
+defaults to: `demo_db`
+"""
+
+PROJECT_DATABASE_SCHEMA: str = os.getenv("PROJECT_DATABASE_SCHEMA", "demo")
+"""
+Name of the schema inside the project database the tests will be run
+against.
+
+from env variable: `PROJECT_DATABASE_SCHEMA`
+defaults to: `demo`
+"""
+
+PROJECT_DATABASE_USER: str = os.getenv("PROJECT_DATABASE_USER", DATABASE_USER)
+"""
+User to be used by the auto generated project to connect to the database
+under testing.
+
+from env variable: `PROJECT_DATABASE_USER`
+defaults to: `environ[DATABASE_USER]`
+"""
+
+PROJECT_DATABASE_PASSWORD: str = os.getenv("PROJECT_DATABASE_PASSWORD", DATABASE_PASSWORD)
+"""
+Password to be used by the auto generated project to connect to the
+database under testing.
+
+from env variable: `PROJECT_DATABASE_USER`
+defaults to: `environ[DATABASE_PASSWORD]`
+"""
+
+PROJECT_DATABASE_HOST: str = os.getenv("PROJECT_DATABASE_HOST", DATABASE_HOST)
+"""
+Hostname where the database under testing is running in.
+
+from env variable: `PROJECT_DATABASE_HOST`
+defaults to: `environ[DATABASE_HOST]`
+"""
+
+PROJECT_DATABASE_PORT: str = os.getenv("PROJECT_DATABASE_PORT", DATABASE_PORT)
+"""
+Port at which the database under testing is exposed by the host.
+
+from env variable: `PROJECT_DATABASE_PORT`
+defaults to: `environ[DATABASE_PORT]`
+"""
+
+SKIP_DATABASE_CERTIFICATE_VERIFICATION: bool = os.getenv("TG_TARGET_DB_TRUST_SERVER_CERTIFICATE", "no").lower() == "yes"
+"""
+When True for supported SQL flavors, set up the SQLAlchemy connection to
+trust the database server certificate.
+
+from env variable: `TG_TARGET_DB_TRUST_SERVER_CERTIFICATE`
+defaults to: `True`
+"""
+
+DEFAULT_TABLE_GROUPS_NAME: str = os.getenv("DEFAULT_TABLE_GROUPS_NAME", "default")
+"""
+Name assigned to the auto generated table group.
+
+from env variable: `DEFAULT_TABLE_GROUPS_NAME`
+defaults to: `default`
+"""
+
+DEFAULT_TEST_SUITE_KEY: str = os.getenv("DEFAULT_TEST_SUITE_NAME", "default-suite-1")
+"""
+Key to be assgined to the auto generated test suite.
+
+from env variable: `DEFAULT_TEST_SUITE_NAME`
+defaults to: `default-suite-1`
+"""
+
+DEFAULT_TEST_SUITE_DESCRIPTION: str = os.getenv("DEFAULT_TEST_SUITE_DESCRIPTION", "default_suite_desc")
+"""
+Description for the auto generated test suite.
+
+from env variable: `DEFAULT_TEST_SUITE_DESCRIPTION`
+defaults to: `default_suite_desc`
+"""
+
+DEFAULT_PROFILING_TABLE_SET = os.getenv("DEFAULT_PROFILING_TABLE_SET", "")
+"""
+Comma separated list of specific table names to include when running
+profiling for the project database.
+
+from env variable: `DEFAULT_PROFILING_TABLE_SET`
+"""
+
+DEFAULT_PROFILING_INCLUDE_MASK = os.getenv("DEFAULT_PROFILING_INCLUDE_MASK", "%")
+"""
+A SQL filter supported by the project database's `LIKE` operator for
+table names to include.
+
+from env variable: `DEFAULT_PROFILING_INCLUDE_MASK`
+defaults to: `%`
+"""
+
+DEFAULT_PROFILING_EXCLUDE_MASK = os.getenv("DEFAULT_PROFILING_EXCLUDE_MASK", "tmp%")
+"""
+A SQL filter supported by the project database's `LIKE` operator for
+table names to exclude.
+
+from env variable: `DEFAULT_PROFILING_EXCLUDE_MASK`
+defaults to: `tmp%`
+"""
+
+DEFAULT_PROFILING_ID_COLUMN_MASK = os.getenv("DEFAULT_PROFILING_ID_COLUMN_MASK", "%id")
+"""
+A SQL filter supported by the project database's `LIKE` operator
+representing ID columns.
+
+from env variable: `DEFAULT_PROFILING_ID_COLUMN_MASK`
+defaults to: `%id`
+"""
+
+DEFAULT_PROFILING_SK_COLUMN_MASK = os.getenv("DEFAULT_PROFILING_SK_COLUMN_MASK", "%sk")
+"""
+A SQL filter supported by the project database's `LIKE` operator
+representing surrogate key columns.
+
+from env variable: `DEFAULT_PROFILING_SK_COLUMN_MASK`
+defaults to: `%sk`
+"""
+
+DEFAULT_PROFILING_USE_SAMPLING: str = os.getenv("DEFAULT_PROFILING_USE_SAMPLING", "N")
+"""
+Toggle on to base profiling on a sample of records instead of the full
+table. Accepts `Y` or `N`
+
+from env variable: `DEFAULT_PROFILING_USE_SAMPLING`
+defaults to: `N`
+"""
+
+OBSERVABILITY_API_URL: str = os.getenv("OBSERVABILITY_API_URL", "")
+"""
+API URL of your instance of Observability where to send events to for
+the project.
+
+You can configure this from the UI settings page.
+
+from env variable: `OBSERVABILITY_API_URL`
+"""
+
+OBSERVABILITY_API_KEY: str = os.getenv("OBSERVABILITY_API_KEY", "")
+"""
+Authentication key with permissions to send events created in your
+instance of Observability.
+
+You can configure this from the UI settings page.
+
+from env variable: `OBSERVABILITY_API_KEY`
+"""
+
+OBSERVABILITY_VERIFY_SSL: bool = os.getenv("TG_EXPORT_TO_OBSERVABILITY_VERIFY_SSL", "yes").lower() in ["yes", "true"]
+"""
+When False, exporting events to your instance of Observabilty will skip
+SSL verification.
+
+from env variable: `TG_EXPORT_TO_OBSERVABILITY_VERIFY_SSL`
+defaults to: `True`
+"""
+
+OBSERVABILITY_EXPORT_LIMIT: int = int(os.getenv("TG_OBSERVABILITY_EXPORT_MAX_QTY", "5000"))
+"""
+When exporting to your instance of Observabilty, the maximum number of
+events that will be sent to the events API on a single export.
+
+from env variable: `TG_OBSERVABILITY_EXPORT_MAX_QTY`
+defaults to: `5000`
+"""
+
+OBSERVABILITY_DEFAULT_COMPONENT_TYPE: str = os.getenv("OBSERVABILITY_DEFAULT_COMPONENT_TYPE", "dataset")
+"""
+When exporting to your instance of Observabilty, the type of event that
+will be sent to the events API.
+
+from env variable: `OBSERVABILITY_DEFAULT_COMPONENT_TYPE`
+defaults to: `dataset`
+"""
+
+OBSERVABILITY_DEFAULT_COMPONENT_KEY: str = os.getenv("OBSERVABILITY_DEFAULT_COMPONENT_KEY", "default")
+"""
+When exporting to your instance of Observabilty, the key sent to the
+events API to identify the components.
+
+from env variable: `OBSERVABILITY_DEFAULT_COMPONENT_KEY`
+defaults to: `default`
+"""
+
+CHECK_FOR_LATEST_VERSION: bool = os.getenv("TG_DOCKER_RELEASE_CHECK_ENABLED", "yes").lower() == "yes"
+"""
+When True, enables calling Docker Hub API to fetch the latest released
+image tag. The fetched tag is displayed in the UI menu.
+
+from env variable: `TG_DOCKER_RELEASE_CHECK_ENABLED`
+defaults to: `True`
+"""
+
+VERSION: str = os.getenv("TESTGEN_VERSION", "unknown")
+"""
+Current deployed version. The value is displayed in the UI menu.
+
+from env variable: `TESTGEN_VERSION`
+defaults to: `unknown`
+"""
+
+SSL_CERT_FILE: str = os.getenv("SSL_CERT_FILE", "")
+SSL_KEY_FILE: str = os.getenv("SSL_KEY_FILE", "")
+"""
+File paths for SSL certificate and private key to support HTTPS.
+Both files must be provided.
+"""

testgen/template/dbsetup/010_create_base_schema.sql

@@ -0,0 +1,2 @@
+-- Create base tg schema
+create schema if not exists {SCHEMA_NAME};

testgen/template/dbsetup/020_create_standard_functions_sprocs.sql

@@ -0,0 +1,179 @@
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.datediff(difftype character varying, firstdate timestamp without time zone, seconddate timestamp without time zone) returns bigint
+    language plpgsql
+as
+$$
+   BEGIN
+      RETURN
+      CASE
+        WHEN UPPER(difftype) IN ('DAY', 'DD')
+              THEN DATE_PART('day', seconddate - firstdate)
+        WHEN UPPER(difftype) IN ('WEEK','WK')
+              THEN TRUNC(DATE_PART('day', seconddate - firstdate)/7)
+        WHEN UPPER(difftype) IN ('MON', 'MM')
+              THEN 12 * (DATE_PART('year', seconddate) - DATE_PART('year', firstdate))
+                    + (DATE_PART('month', seconddate) - DATE_PART('month', firstdate))
+        WHEN UPPER(difftype) IN ('QUARTER', 'QTR')
+              THEN 4 * (DATE_PART('year', seconddate) - DATE_PART('year', firstdate))
+                    + (DATE_PART('qtr', seconddate) - DATE_PART('month', firstdate))
+        WHEN UPPER(difftype) IN ('YEAR', 'YY')
+              THEN DATE_PART('year', seconddate) - DATE_PART('year', firstdate)
+      END;
+   END;
+$$;
+
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_charcount(instring character varying, searchstring character varying) returns bigint
+    language plpgsql
+as
+$$
+   BEGIN
+      RETURN (CHAR_LENGTH(instring) - CHAR_LENGTH(REPLACE(instring, searchstring, ''))) / CHAR_LENGTH(searchstring);
+   END;
+$$;
+
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_parsefreq(top_freq_values VARCHAR(1000), rowno INTEGER, colno INTEGER) returns VARCHAR(1000)
+LANGUAGE SQL
+STABLE
+as
+$$
+    WITH first AS
+    (
+        SELECT SPLIT_PART(top_freq_values, CHR(10), rowno) AS first_row
+    )
+    SELECT
+        CASE
+            WHEN colno = 1 THEN CAST(TRIM(LEADING '|' FROM SUBSTRING(first_row, POSITION('|' IN first_row), LENGTH(first_row) - POSITION('|' IN REVERSE(first_row)))) AS VARCHAR)
+            WHEN colno = 2 THEN CAST(TRIM(SUBSTRING(first_row, LENGTH(first_row) - POSITION('|' IN REVERSE(first_row)) + 2)) AS VARCHAR)
+            ELSE NULL
+            END
+    FROM first
+$$;
+
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_PrepColumnName(value_to_check TEXT)
+    RETURNS TEXT AS
+$$
+DECLARE
+    keyword_arr TEXT[] := ARRAY ['ALL', 'ALTER', 'ANALYSE', 'ANALYZE', 'AND', 'ANY', 'ARRAY', 'AS', 'ASC', 'ASYMMETRIC',
+                                 'AUTHORIZATION', 'BINARY', 'BOTH', 'CASE', 'CAST', 'CHECK', 'COLLATE', 'COLLATION',
+                                 'COLUMN', 'CONCURRENTLY', 'CONSTRAINT', 'CREATE', 'CROSS', 'CURRENT_CATALOG',
+                                 'CURRENT_DATE', 'CURRENT_ROLE', 'CURRENT_SCHEMA', 'CURRENT_TIME', 'CURRENT_TIMESTAMP',
+                                 'CURRENT_USER', 'CREDENTIALS',
+                                 'DEFAULT', 'DEFERRABLE', 'DESC', 'DISTINCT', 'DO', 'DROP',
+                                 'ELSE', 'END', 'EXCEPT', 'FALSE', 'FETCH', 'FOR', 'FOREIGN', 'FREEZE', 'FROM', 'FULL',
+                                 'GRANT', 'GROUP', 'HAVING', 'ILIKE', 'IN', 'INITIALLY', 'INNER', 'INTERSECT', 'INTO',
+                                 'IS', 'ISNULL', 'JOIN', 'LATERAL', 'LEADING', 'LEFT', 'LIKE', 'LIMIT', 'LOCALTIME',
+                                 'LOCALTIMESTAMP', 'NATURAL', 'NOT', 'NOTNULL', 'NULL', 'OFFSET', 'ON', 'ONLY', 'OR',
+                                 'ORDER', 'OUTER', 'OVERLAPS', 'PLACING', 'PRIMARY', 'REFERENCES', 'RETURNING', 'RIGHT',
+                                 'SELECT', 'SESSION_USER', 'SIMILAR', 'SOME', 'SYMMETRIC', 'TABLE', 'TABLESAMPLE',
+                                 'THEN', 'TIMESTAMP', 'TIMEZONE', 'TO', 'TRAILING', 'TRUE', 'UNION', 'UNIQUE', 'USER', 'USING',
+                                 'VARIADIC', 'VERBOSE', 'WHEN', 'WHERE', 'WINDOW', 'WITH']; -- Add more keywords here
+BEGIN
+    -- Check if the value matches any of the keywords (case-insensitive)
+    IF value_to_check ILIKE ANY (keyword_arr) THEN
+        RETURN '"' || value_to_check || '"';
+    -- Check if the value contains a space or a comma or it starts with a number
+    ELSIF value_to_check !~ '^[a-zA-Z_][a-zA-Z0-9_]*$' THEN
+        RETURN '"' || value_to_check || '"';
+    ELSE
+        RETURN value_to_check;
+    END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_pct(numerator NUMERIC, denominator NUMERIC, decs INTEGER DEFAULT 0) returns NUMERIC
+    language plpgsql
+as
+$$
+   BEGIN
+      RETURN ROUND((100.0 * numerator/denominator), decs);
+   END;
+$$;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_quote_literal_escape(var_value varchar, sql_flavor varchar) RETURNS varchar
+    LANGUAGE plpgsql
+AS
+$$
+DECLARE
+    escaped_value         varchar;
+    lower_case_sql_flavor varchar;
+BEGIN
+    lower_case_sql_flavor := LOWER(sql_flavor);
+
+    IF lower_case_sql_flavor = 'postgres'  OR lower_case_sql_flavor = 'postgresql' THEN
+        escaped_value := QUOTE_LITERAL(var_value);
+    ELSIF lower_case_sql_flavor = 'redshift' OR lower_case_sql_flavor = 'snowflake' THEN
+        escaped_value := TRIM(LEADING 'E' FROM QUOTE_LITERAL(var_value));
+    ELSIF lower_case_sql_flavor = 'mssql' THEN
+        escaped_value := '''' || REPLACE(var_value, '''', '''''') || '''';
+    ELSE
+        RAISE EXCEPTION 'Invalid sql_flavor name: %', sql_flavor;
+    END IF;
+
+    RETURN escaped_value;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_format_csv_no_quotes(str_csv TEXT) returns TEXT
+LANGUAGE SQL
+IMMUTABLE
+as
+$$
+    SELECT
+        REGEXP_REPLACE(
+                       REGEXP_REPLACE(str_csv::VARCHAR, '''', '', 'g'),  -- Remove single quotes
+                       '\s*,\s*',  -- Match comma, with or without surrounding spaces
+                       ', ',       -- Replace with comma followed by a space
+                       'g'         -- Global replace
+                      ) AS formatted_value
+$$;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_format_csv_quotes(str_csv TEXT) returns TEXT
+LANGUAGE SQL
+IMMUTABLE
+as
+$$
+    SELECT
+       '''' || REGEXP_REPLACE(str_csv::VARCHAR, '\s*,\s*', ''', ''', 'g') || ''''
+   AS formatted_value
+$$;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_count_intersecting_items(list1 VARCHAR, list2 VARCHAR, separator VARCHAR)
+RETURNS BIGINT AS $$
+SELECT COUNT(*)
+FROM (
+    SELECT unnest(string_to_array(list1, separator)) AS element
+    INTERSECT
+    SELECT unnest(string_to_array(list2, separator))
+) AS intersection
+$$ LANGUAGE sql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_extract_intersecting_items(list1 VARCHAR, list2 VARCHAR, separator VARCHAR)
+RETURNS VARCHAR AS $$
+SELECT STRING_AGG(DISTINCT element, separator) as shared_vals
+FROM (
+    SELECT unnest(string_to_array(list1, separator)) AS element
+    INTERSECT
+    SELECT unnest(string_to_array(list2, separator))
+) AS intersection
+$$ LANGUAGE sql;
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_extract_distinct_items(list VARCHAR, separator VARCHAR)
+RETURNS VARCHAR AS $$
+SELECT STRING_AGG(DISTINCT element, separator) as distinct_items
+FROM (
+    SELECT unnest(string_to_array(list, separator)) AS element
+) AS all_items
+$$ LANGUAGE sql;
+
+
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.fn_extract_top_values(input_string TEXT)
+RETURNS TEXT AS $$
+SELECT string_agg(trim(split_part(value, '|', 2)), '|') AS values_only
+FROM (
+  SELECT unnest(regexp_split_to_array(input_string, E'\n')) AS value
+) AS t
+WHERE trim(value) <> ''
+$$ LANGUAGE sql;