databricks-labs-lakebridge 0.10.0__py3-none-any.whl

databricks/labs/lakebridge/deployment/recon.py

@@ -0,0 +1,145 @@
+import logging
+from importlib.resources import files
+
+from databricks.labs.blueprint.installation import Installation
+from databricks.labs.blueprint.installer import InstallState
+from databricks.labs.blueprint.wheels import ProductInfo
+from databricks.labs.blueprint.wheels import find_project_root
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.errors import InvalidParameterValue, NotFound
+
+import databricks.labs.lakebridge.resources
+from databricks.labs.lakebridge.config import ReconcileConfig
+from databricks.labs.lakebridge.deployment.dashboard import DashboardDeployment
+from databricks.labs.lakebridge.deployment.job import JobDeployment
+from databricks.labs.lakebridge.deployment.table import TableDeployment
+
+logger = logging.getLogger(__name__)
+
+_RECON_PREFIX = "Reconciliation"
+RECON_JOB_NAME = f"{_RECON_PREFIX} Runner"
+
+
+class ReconDeployment:
+    def __init__(
+        self,
+        ws: WorkspaceClient,
+        installation: Installation,
+        install_state: InstallState,
+        product_info: ProductInfo,
+        table_deployer: TableDeployment,
+        job_deployer: JobDeployment,
+        dashboard_deployer: DashboardDeployment,
+    ):
+        self._ws = ws
+        self._installation = installation
+        self._install_state = install_state
+        self._product_info = product_info
+        self._table_deployer = table_deployer
+        self._job_deployer = job_deployer
+        self._dashboard_deployer = dashboard_deployer
+
+    def install(self, recon_config: ReconcileConfig | None, wheel_paths: list[str]):
+        if not recon_config:
+            logger.warning("Recon Config is empty.")
+            return
+        logger.info("Installing reconcile components.")
+        self._deploy_tables(recon_config)
+        self._deploy_dashboards(recon_config)
+        remorph_wheel_path = [whl for whl in wheel_paths if "remorph" in whl][0]
+        self._deploy_jobs(recon_config, remorph_wheel_path)
+        self._install_state.save()
+        logger.info("Installation of reconcile components completed successfully.")
+
+    def uninstall(self, recon_config: ReconcileConfig | None):
+        if not recon_config:
+            return
+        logger.info("Uninstalling reconcile components.")
+        self._remove_dashboards()
+        self._remove_jobs()
+        logging.info(
+            f"Won't remove reconcile metadata schema `{recon_config.metadata_config.schema}` "
+            f"from catalog `{recon_config.metadata_config.catalog}`. Please remove it and the tables inside manually."
+        )
+        logging.info(
+            f"Won't remove configured reconcile secret scope `{recon_config.secret_scope}`. "
+            f"Please remove it manually."
+        )
+
+    def _deploy_tables(self, recon_config: ReconcileConfig):
+        logger.info("Deploying reconciliation metadata tables.")
+        catalog = recon_config.metadata_config.catalog
+        schema = recon_config.metadata_config.schema
+        resources = files(databricks.labs.lakebridge.resources)
+        query_dir = resources.joinpath("reconcile/queries/installation")
+
+        sqls_to_deploy = [
+            "main.sql",
+            "metrics.sql",
+            "details.sql",
+            "aggregate_metrics.sql",
+            "aggregate_details.sql",
+            "aggregate_rules.sql",
+        ]
+
+        for sql_file in sqls_to_deploy:
+            table_sql_file = query_dir.joinpath(sql_file)
+            self._table_deployer.deploy_table_from_ddl_file(catalog, schema, sql_file.strip(".sql"), table_sql_file)
+
+    def _deploy_dashboards(self, recon_config: ReconcileConfig):
+        logger.info("Deploying reconciliation dashboards.")
+        dashboard_base_dir = (
+            find_project_root(__file__) / "src/databricks/labs/lakebridge/resources/reconcile/dashboards"
+        )
+        self._dashboard_deployer.deploy(dashboard_base_dir, recon_config)
+
+    def _get_dashboards(self) -> list[tuple[str, str]]:
+        return list(self._install_state.dashboards.items())
+
+    def _remove_dashboards(self):
+        logger.info("Removing reconciliation dashboards.")
+        for dashboard_ref, dashboard_id in self._get_dashboards():
+            try:
+                logger.info(f"Removing dashboard with id={dashboard_id}.")
+                del self._install_state.dashboards[dashboard_ref]
+                self._ws.lakeview.trash(dashboard_id)
+            except (InvalidParameterValue, NotFound):
+                logger.warning(f"Dashboard with id={dashboard_id} doesn't exist anymore for some reason.")
+                continue
+
+    def _deploy_jobs(self, recon_config: ReconcileConfig, remorph_wheel_path: str):
+        logger.info("Deploying reconciliation jobs.")
+        self._job_deployer.deploy_recon_job(RECON_JOB_NAME, recon_config, remorph_wheel_path)
+        for job_name, job_id in self._get_deprecated_jobs():
+            try:
+                logger.info(f"Removing job_id={job_id}, as it is no longer needed.")
+                del self._install_state.jobs[job_name]
+                self._ws.jobs.delete(job_id)
+            except (InvalidParameterValue, NotFound):
+                logger.warning(f"{job_name} doesn't exist anymore for some reason.")
+                continue
+
+    def _get_jobs(self) -> list[tuple[str, int]]:
+        return [
+            (job_name, int(job_id))
+            for job_name, job_id in self._install_state.jobs.items()
+            if job_name.startswith(_RECON_PREFIX)
+        ]
+
+    def _get_deprecated_jobs(self) -> list[tuple[str, int]]:
+        return [
+            (job_name, int(job_id))
+            for job_name, job_id in self._install_state.jobs.items()
+            if job_name.startswith(_RECON_PREFIX) and job_name != RECON_JOB_NAME
+        ]
+
+    def _remove_jobs(self):
+        logger.info("Removing Reconciliation Jobs.")
+        for job_name, job_id in self._get_jobs():
+            try:
+                logger.info(f"Removing job {job_name} with job_id={job_id}.")
+                del self._install_state.jobs[job_name]
+                self._ws.jobs.delete(int(job_id))
+            except (InvalidParameterValue, NotFound):
+                logger.warning(f"{job_name} doesn't exist anymore for some reason.")
+                continue

databricks/labs/lakebridge/deployment/table.py

@@ -0,0 +1,30 @@
+import logging
+from importlib.abc import Traversable
+
+from databricks.labs.lsql.backends import SqlBackend
+
+logger = logging.getLogger(__name__)
+
+
+class TableDeployment:
+    def __init__(self, sql_backend: SqlBackend):
+        self._sql_backend = sql_backend
+
+    def deploy_table_from_ddl_file(
+        self,
+        catalog: str,
+        schema: str,
+        table_name: str,
+        ddl_query_filepath: Traversable,
+    ):
+        """
+        Deploys a table to the given catalog and schema
+        :param catalog: The table catalog
+        :param schema: The table schema
+        :param table_name: The table to deploy
+        :param ddl_query_filepath: DDL file path
+        """
+        query = ddl_query_filepath.read_text()
+        logger.info(f"Deploying table {table_name} in {catalog}.{schema}")
+        logger.info(f"SQL Backend used for deploying table: {type(self._sql_backend).__name__}")
+        self._sql_backend.execute(query, catalog=catalog, schema=schema)

databricks/labs/lakebridge/deployment/upgrade_common.py

@@ -0,0 +1,124 @@
+import logging
+import re
+from importlib.resources import files
+
+import databricks.labs.lakebridge.resources
+
+from databricks.labs.blueprint.tui import Prompts
+from databricks.sdk import WorkspaceClient
+from databricks.labs.lakebridge.helpers import db_sql
+
+logger = logging.getLogger(__name__)
+
+
+def replace_patterns(sql_text: str) -> str:
+    """
+    Replace the STRUCT and MAP datatypes in the SQL text with empty string
+    """
+    # Pattern to match nested STRUCT and MAP datatypes
+    pattern = r'(STRUCT<[^<>]*?(?:<[^<>]*?>[^<>]*?)*>|MAP<[^<>]*?(?:<[^<>]*?>[^<>]*?)*>)'
+    parsed_sql_text = re.sub(pattern, "", sql_text, flags=re.DOTALL)
+    return parsed_sql_text
+
+
+def extract_columns_with_datatype(sql_text: str) -> list[str]:
+    """
+    Extract the columns with datatype from the SQL text
+    Example:
+        Input: CREATE TABLE main (
+            recon_table_id BIGINT NOT NULL,
+            report_type STRING NOT NULL
+            );
+       Output:  [recon_table_id BIGINT NOT NULL,
+                      report_type STRING NOT NULL]
+    """
+    return sql_text[sql_text.index("(") + 1 : sql_text.index(")")].strip().split(",")
+
+
+def extract_column_name(column_with_datatype: str) -> str:
+    """
+    Extract the column name from the column with datatype.
+    Example:
+        Input: \n    recon_table_id BIGINT NOT NULL,
+        Output: recon_table_id
+    """
+    return column_with_datatype.strip("\n").strip().split(" ")[0]
+
+
+def table_original_query(table_name: str, full_table_name: str) -> str:
+    """
+    Get the main table DDL from the main.sql file
+    :return: str
+    """
+    resources = files(databricks.labs.lakebridge.resources)
+    query_dir = resources.joinpath("reconcile/queries/installation")
+    return (
+        query_dir.joinpath(f"{table_name}.sql")
+        .read_text()
+        .replace(f"CREATE TABLE IF NOT EXISTS {table_name}", f"CREATE OR REPLACE TABLE {full_table_name}")
+    )
+
+
+def current_table_columns(table_name: str, full_table_name: str) -> list[str]:
+    """
+    Extract the column names from the main table DDL
+    :return: column_names: list[str]
+    """
+    main_sql = replace_patterns(table_original_query(table_name, full_table_name))
+    main_table_columns = [
+        extract_column_name(main_table_column) for main_table_column in extract_columns_with_datatype(main_sql)
+    ]
+    return main_table_columns
+
+
+def installed_table_columns(ws: WorkspaceClient, table_identifier: str) -> list[str]:
+    """
+    Fetch the column names from the installed table on Databricks Workspace using SQL Backend
+    :return: column_names: list[str]
+    """
+    main_table_columns = list(db_sql.get_sql_backend(ws).fetch(f"DESC {table_identifier}"))
+    return [row.col_name for row in main_table_columns]
+
+
+def check_table_mismatch(
+    installed_table,
+    current_table,
+) -> bool:
+    # Compare the current main table columns with the installed main table columns
+    if len(installed_table) != len(current_table) or sorted(installed_table) != sorted(current_table):
+        return True
+    return False
+
+
+def recreate_table_sql(
+    table_identifier: str,
+    installed_table: list[str],
+    current_table: list[str],
+    prompts: Prompts,
+) -> str | None:
+    """
+    * Verify all the current main table columns are present in the installed main table and then use CTAS to recreate the main table
+       * If any of the current main table columns are missing in the installed main table, prompt the user to recreate the main table:
+            - If the user confirms, recreate the main table using the main DDL file, else log an error message and exit
+    :param table_identifier:
+    :param installed_table:
+    :param current_table:
+    :param prompts:
+    :return:
+    """
+    table_name = table_identifier.split('.')[-1]
+    sql: str | None = (
+        f"CREATE OR REPLACE TABLE {table_identifier} AS SELECT {','.join(current_table)} FROM {table_identifier}"
+    )
+
+    if not set(current_table).issubset(installed_table):
+        if prompts.confirm(
+            f"The `{table_identifier}` table columns are not as expected. Do you want to recreate the `{table_identifier}` table?"
+        ):
+            sql = table_original_query(table_name, table_identifier)
+        else:
+            logger.error(
+                f"The `{table_identifier}` table columns are not as expected. Please check and recreate the `{table_identifier}` table."
+            )
+            sql = None
+    return sql

databricks/labs/lakebridge/discovery/table.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class TableFQN:
+    catalog: str | None
+    schema: str
+    name: str
+
+    @property
+    def fqn(self) -> str:
+        if self.catalog:
+            return f"{self.catalog}.{self.schema}.{self.name}"
+        return f"{self.schema}.{self.name}"
+
+
+@dataclass
+class FieldInfo:
+    name: str
+    data_type: str
+    nullable: bool | None = None
+    metadata: dict[str, Any] | None = None
+    comment: str | None = None
+
+
+@dataclass
+class TableDefinition:
+    fqn: TableFQN
+    location: str | None = None
+    table_format: str | None = None
+    view_text: str | None = None
+    columns: list[FieldInfo] = field(default_factory=list)
+    primary_keys: list[str] | None = None
+    size_gb: int | None = None
+    comment: str | None = None

databricks/labs/lakebridge/discovery/table_definition.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from typing import Any
+
+from databricks.labs.lakebridge.discovery.table import TableDefinition
+
+
+class TableDefinitionService(ABC):
+
+    def __init__(self, connection: Any):
+        self.connection = connection
+
+    @abstractmethod
+    def get_table_definition(self, catalog_name: str) -> Iterable[TableDefinition]:
+        pass
+
+    @abstractmethod
+    def _get_table_definition_query(self, catalog_name: str) -> str:
+        pass
+
+    @abstractmethod
+    def get_all_catalog(self) -> Iterable[str]:
+        pass

databricks/labs/lakebridge/discovery/tsql_table_definition.py

@@ -0,0 +1,185 @@
+from collections.abc import Iterable
+
+from databricks.labs.lakebridge.connections.database_manager import DatabaseManager
+from databricks.labs.lakebridge.discovery.table import TableDefinition, TableFQN, FieldInfo
+from databricks.labs.lakebridge.discovery.table_definition import TableDefinitionService
+
+
+class TsqlTableDefinitionService(TableDefinitionService):
+
+    # Hexadecimal value of § is U+00A7.Hexadecimal value of ‡ (double dagger) is U+2021
+    def _get_table_definition_query(self, catalog_name: str) -> str:
+        query = f"""
+        WITH column_info AS (
+            SELECT
+                TABLE_CATALOG,
+                TABLE_SCHEMA,
+                TABLE_NAME,
+                STRING_AGG(
+                    CONCAT(
+                        column_name,
+                        '§',
+                        CASE
+                            WHEN numeric_precision IS NOT NULL AND numeric_scale IS NOT NULL THEN CONCAT(data_type, '(', numeric_precision, ',', numeric_scale, ')')
+                            WHEN LOWER(data_type) = 'text' THEN CONCAT('varchar', '(', CHARACTER_MAXIMUM_LENGTH, ')')
+                            ELSE data_type
+                        END,
+                        '§',
+                        CASE
+                            WHEN cis.IS_NULLABLE = 'YES' THEN 'true'
+                            ELSE 'false'
+                        END,
+                        '§',
+                        ISNULL(CAST(ep_col.value AS NVARCHAR(MAX)), '')
+                    ),
+                    '‡'
+                ) WITHIN GROUP (ORDER BY ordinal_position) AS DERIVED_SCHEMA
+            FROM
+                {catalog_name}.sys.tables t
+                INNER JOIN {catalog_name}.sys.columns c ON t.object_id = c.object_id
+                INNER JOIN {catalog_name}.INFORMATION_SCHEMA.COLUMNS cis ON t.name = cis.TABLE_NAME AND c.name = cis.COLUMN_NAME
+                OUTER APPLY (
+                    SELECT TOP 1 value
+                    FROM {catalog_name}.sys.extended_properties
+                    WHERE major_id = t.object_id AND minor_id = 0
+                    ORDER BY name DESC
+                ) ep_tbl
+                OUTER APPLY (
+                    SELECT TOP 1 value
+                    FROM {catalog_name}.sys.extended_properties
+                    WHERE major_id = c.object_id AND minor_id = c.column_id
+                    ORDER BY name DESC
+                ) ep_col
+            GROUP BY
+                TABLE_CATALOG,
+                TABLE_SCHEMA,
+                TABLE_NAME
+        ),
+        table_file_info AS (
+            SELECT
+                s.name AS TABLE_SCHEMA,
+                t.name AS TABLE_NAME,
+                f.physical_name AS location,
+                f.type_desc AS TABLE_FORMAT,
+                CAST(ROUND(SUM(a.used_pages) * 8.0 / 1024, 2) AS DECIMAL(18, 2)) AS SIZE_GB
+            FROM
+                {catalog_name}.sys.tables t
+                INNER JOIN {catalog_name}.sys.indexes i ON t.object_id = i.object_id
+                INNER JOIN {catalog_name}.sys.partitions p ON i.object_id = p.object_id AND i.index_id = p.index_id
+                INNER JOIN {catalog_name}.sys.allocation_units a ON p.partition_id = a.container_id
+                INNER JOIN {catalog_name}.sys.schemas s ON t.schema_id = s.schema_id
+                INNER JOIN {catalog_name}.sys.database_files f ON a.data_space_id = f.data_space_id
+                LEFT JOIN {catalog_name}.sys.extended_properties ep ON ep.major_id = t.object_id AND ep.minor_id = 0
+            GROUP BY
+                s.name,
+                t.name,
+                f.name,
+                f.physical_name,
+                f.type_desc
+        ),
+        table_comment_info AS (
+            SELECT
+                s.name AS TABLE_SCHEMA,
+                t.name AS TABLE_NAME,
+                CAST(ep.value AS NVARCHAR(MAX)) AS TABLE_COMMENT
+            FROM
+                {catalog_name}.sys.tables t
+                INNER JOIN {catalog_name}.sys.schemas s ON t.schema_id = s.schema_id
+                OUTER APPLY (
+                    SELECT TOP 1 value
+                    FROM {catalog_name}.sys.extended_properties
+                    WHERE major_id = t.object_id AND minor_id = 0
+                    ORDER BY name DESC
+                ) ep
+        ),
+        table_pk_info AS (
+            SELECT
+                TC.TABLE_CATALOG,
+                TC.TABLE_SCHEMA,
+                TC.TABLE_NAME,
+                STRING_AGG(KU.COLUMN_NAME,':') as PK_COLUMN_NAME
+                FROM {catalog_name}.INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS TC
+                JOIN {catalog_name}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS KU
+                ON TC.CONSTRAINT_NAME = KU.CONSTRAINT_NAME
+                AND TC.TABLE_NAME = KU.TABLE_NAME
+                WHERE TC.CONSTRAINT_TYPE = 'PRIMARY KEY' group by TC.TABLE_CATALOG, TC.TABLE_SCHEMA, TC.TABLE_NAME)
+        SELECT
+            sft.TABLE_CATALOG,
+            sft.TABLE_SCHEMA,
+            sft.TABLE_NAME,
+            tfi.location,
+            tfi.TABLE_FORMAT,
+            '' as view_definition,
+            column_info.DERIVED_SCHEMA,
+            tfi.SIZE_GB,
+            tci.TABLE_COMMENT,
+            tpK.PK_COLUMN_NAME
+        FROM
+            column_info
+            JOIN {catalog_name}.INFORMATION_SCHEMA.TABLES sft ON column_info.TABLE_CATALOG = sft.TABLE_CATALOG AND column_info.TABLE_SCHEMA = sft.TABLE_SCHEMA AND column_info.TABLE_NAME = sft.TABLE_NAME
+            LEFT JOIN table_file_info tfi ON column_info.TABLE_SCHEMA = tfi.TABLE_SCHEMA AND column_info.TABLE_NAME = tfi.TABLE_NAME
+            LEFT JOIN table_comment_info tci ON column_info.TABLE_SCHEMA = tci.TABLE_SCHEMA AND column_info.TABLE_NAME = tci.TABLE_NAME
+            LEFT JOIN table_pk_info tpK ON column_info.TABLE_SCHEMA = tpK.TABLE_SCHEMA AND column_info.TABLE_NAME = tpK.TABLE_NAME
+
+        UNION ALL
+        SELECT
+            sfv.TABLE_CATALOG,
+            sfv.TABLE_SCHEMA,
+            sfv.TABLE_NAME,
+            '' location,
+            '' TABLE_FORMAT,
+            sfv.view_definition,
+            '' DERIVED_SCHEMA,
+            0 SIZE_GB,
+            '' TABLE_COMMENT,
+            '' PK_COLUMN_NAME
+        FROM {catalog_name}.INFORMATION_SCHEMA.VIEWS sfv
+        """
+        return query
+
+    def get_table_definition(self, catalog_name: str) -> Iterable[TableDefinition]:
+        sql = self._get_table_definition_query(catalog_name)
+        tsql_connection = self.connection
+        result = tsql_connection.execute_query(sql)
+
+        column_names = list(result.keys())
+        table_definitions = []
+
+        for row in result:
+            result = dict(zip(column_names, row))
+            table_fqn = TableFQN(
+                catalog=result["TABLE_CATALOG"], schema=result["TABLE_SCHEMA"], name=result["TABLE_NAME"]
+            )
+            columns = result["DERIVED_SCHEMA"].split("‡") if result["DERIVED_SCHEMA"] else None
+            field_info = []
+            if columns is not None:
+                for column in columns:
+                    column_info = column.split("§")
+                    field = FieldInfo(
+                        name=column_info[0],
+                        data_type=column_info[1],
+                        nullable=column_info[2],
+                        comment=column_info[3],
+                    )
+                    field_info.append(field)
+
+            pks = result["PK_COLUMN_NAME"].split(":") if result["PK_COLUMN_NAME"] else None
+            table_definition = TableDefinition(
+                fqn=table_fqn,
+                location=result["location"],
+                table_format=result["TABLE_FORMAT"],
+                view_text=result["view_definition"],
+                columns=field_info,
+                size_gb=result["SIZE_GB"],
+                comment=result["TABLE_COMMENT"],
+                primary_keys=pks,
+            )
+            table_definitions.append(table_definition)
+        return table_definitions
+
+    def get_all_catalog(self) -> Iterable[str]:
+        cursor: DatabaseManager = self.connection
+        result = cursor.connector.execute_query("""select name from sys.databases""")
+        catalogs = [row[0] for row in result]
+        print(catalogs)
+        return catalogs

databricks/labs/lakebridge/errors/exceptions.py

@@ -0,0 +1 @@
+class IllegalStateException(BaseException): ...

databricks/labs/lakebridge/helpers/db_sql.py

@@ -0,0 +1,24 @@
+import logging
+import os
+
+from databricks.labs.lsql.backends import (
+    DatabricksConnectBackend,
+    RuntimeBackend,
+    SqlBackend,
+    StatementExecutionBackend,
+)
+from databricks.sdk import WorkspaceClient
+
+logger = logging.getLogger(__name__)
+
+
+def get_sql_backend(ws: WorkspaceClient, warehouse_id: str | None = None) -> SqlBackend:
+    warehouse_id = warehouse_id or ws.config.warehouse_id
+    if warehouse_id:
+        logger.info(f"Using SQL backend with warehouse_id: {warehouse_id}")
+        return StatementExecutionBackend(ws, warehouse_id)
+    if "DATABRICKS_RUNTIME_VERSION" in os.environ:
+        logger.info("Using SQL backend with Databricks Runtime.")
+        return RuntimeBackend()
+    logger.info("Using SQL backend with Databricks Connect.")
+    return DatabricksConnectBackend(ws)

databricks/labs/lakebridge/helpers/execution_time.py

@@ -0,0 +1,20 @@
+import inspect
+import logging
+import time
+from functools import wraps
+
+logger = logging.getLogger(__name__)
+
+
+def timeit(func):
+    @wraps(func)
+    def timeit_wrapper(*args, **kwargs):
+        start_time = time.perf_counter()
+        result = func(*args, **kwargs)
+        end_time = time.perf_counter()
+        total_time = end_time - start_time
+        name = inspect.getmodule(func).__name__.split(".")[3].capitalize()
+        logger.info(f"{name} took {total_time:.4f} seconds")
+        return result
+
+    return timeit_wrapper

databricks/labs/lakebridge/helpers/file_utils.py

@@ -0,0 +1,64 @@
+from pathlib import Path
+from collections.abc import Generator
+
+
+def is_sql_file(file: str | Path) -> bool:
+    """
+    Checks if the given file is a SQL file.
+
+    :param file: The name of the file to check.
+    :return: True if the file is a SQL file (i.e., its extension is either .sql or .ddl), False otherwise.
+    """
+    file_extension = Path(file).suffix
+    return file_extension.lower() in {".sql", ".ddl"}
+
+
+def is_dbt_project_file(file: Path):
+    # it's ok to hardcode the file name here, see https://docs.getdbt.com/reference/dbt_project.yml
+    return file.name == "dbt_project.yml"
+
+
+def make_dir(path: str | Path) -> None:
+    """
+    Creates a directory at the specified path if it does not already exist.
+
+    :param path: The path where the directory should be created.
+    """
+    Path(path).mkdir(parents=True, exist_ok=True)
+
+
+def dir_walk(root: Path):
+    """
+    Walks the directory tree rooted at the given path, yielding a tuple containing the root directory, a list of
+    :param root: Path
+    :return: tuple of  root, subdirectory , files
+    """
+    sub_dirs = [d for d in root.iterdir() if d.is_dir()]
+    files = [f for f in root.iterdir() if f.is_file()]
+    yield root, sub_dirs, files
+
+    for each_dir in sub_dirs:
+        yield from dir_walk(each_dir)
+
+
+def get_sql_file(input_path: str | Path) -> Generator[Path, None, None]:
+    """
+    Returns Generator that yields the names of all SQL files in the given directory.
+    :param input_path: Path
+    :return: List of SQL files
+    """
+    for _, _, files in dir_walk(Path(input_path)):
+        for filename in files:
+            if is_sql_file(filename):
+                yield filename
+
+
+def read_file(filename: str | Path) -> str:
+    """
+    Reads the contents of the given file and returns it as a string.
+    :param filename: Input File Path
+    :return: File Contents as String
+    """
+    # pylint: disable=unspecified-encoding
+    with Path(filename).open() as file:
+        return file.read()