recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/pigeon/loader/base.py

@@ -0,0 +1,83 @@
+import os
+
+from recurvedata.pigeon.schema import Schema
+from recurvedata.pigeon.utils import LoggingMixin, fs, sql
+
+
+class BaseLoader(LoggingMixin):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def before_execute(self):
+        pass
+
+    def after_execute(self):
+        pass
+
+    def execute(self):
+        self.before_execute()
+        self.execute_impl()
+        self.after_execute()
+
+    def execute_impl(self):
+        raise NotImplementedError("execute_impl must be implemented by subclass")
+
+
+class CSVToDBAPIMixin(object):
+    @property
+    def schema_filename(self) -> str:
+        return fs.schema_filename(self.filename)
+
+    def _prepare_target_table(self):
+        # add schema for azure data warehouse
+        if self.connector.has_table(table=self.table, schema=getattr(self, "schema", None)):
+            return
+
+        self.logger.info("table not found, try to create it")
+        ddl = self._infer_create_table_ddl()
+        if not ddl:
+            raise ValueError("table not found, create_table_ddl is required")
+        ddl = ddl.strip().rstrip(";")
+        self.logger.info("create table ddl: %s\n", ddl)
+        with self.connector.cursor() as cursor:
+            cursor.execute(ddl)
+
+    def _infer_create_table_ddl(self):
+        if not self.create_table_ddl:
+            # infer by schema
+            schema_file = self.schema_filename
+            self.logger.info("infer ddl by schema file %s", schema_file)
+            return self._generate_ddl_from_schema(schema_file)
+
+        if "CREATE TABLE" in self.create_table_ddl.upper():
+            self.logger.info("self.create_table_ddl contains `CREATE TABLE`, use it")
+            create_table_ddl = self.create_table_ddl
+
+            # Safely get schema, defaulting to None if not present
+            schema = getattr(self, "schema", None)
+            if not schema:
+                return create_table_ddl
+
+            self.logger.info(f"add schema {schema} to create table ddl")
+            return sql.add_schema_to_create_table(create_table_ddl, schema, self.connector.quote_identifier)
+
+        if os.path.isfile(self.create_table_ddl):
+            self.logger.info("self.create_table_ddl is a filename, treat it as schema file")
+            return self._generate_ddl_from_schema(self.create_table_ddl)
+        return None
+
+    def _generate_ddl_from_schema(self, schema_file):
+        if not os.path.exists(schema_file):
+            self.logger.error("file not exists, not able to infer DDL")
+            return None
+
+        try:
+            schema = Schema.load(schema_file)
+        except Exception:
+            self.logger.exception("failed to load schema from %s", schema_file)
+            return None
+
+        ddl_options = getattr(self, "ddl_options", {})
+        table_name = getattr(self, "full_table_name", self.table)
+        ddl = self.connector.generate_create_table_ddl(table_name, schema, **ddl_options)
+        return ddl

recurvedata/pigeon/loader/csv_to_azure_synapse.py

@@ -0,0 +1,214 @@
+from typing import TYPE_CHECKING, List, Union
+
+from recurvedata.pigeon import const
+from recurvedata.pigeon.loader.base import BaseLoader, CSVToDBAPIMixin
+from recurvedata.pigeon.utils import ensure_query_list, ensure_str_list, fs
+
+if TYPE_CHECKING:
+    from recurvedata.pigeon.connector.azure_synapse import AzureSynapseConnector
+
+allowed_modes = (
+    const.LOAD_OVERWRITE,
+    const.LOAD_MERGE,
+    const.LOAD_APPEND
+)
+
+
+class CSVToAzureSynapseLoader(BaseLoader, CSVToDBAPIMixin):
+    def __init__(
+            self,
+            table: str,
+            filename: str,
+            azure_synapse_connector: 'AzureSynapseConnector' = None,
+            schema: str = None,
+            create_table_ddl: str = None,
+            mode: str = const.LOAD_MERGE,
+            primary_keys: Union[str, List[str]] = None,
+            columns: Union[str, List[str]] = None,
+            compress: bool = True,
+            delete_file: bool = True,
+            dedup: bool = False,
+            dedup_uniq_keys: Union[str, List[str]] = None,
+            dedup_orderby: Union[str, List[str]] = None,
+            pre_queries: Union[str, List[str]] = None,
+            post_queries: Union[str, List[str]] = None,
+            *args, **kwargs
+    ):
+        if '.' in table:
+            self.schema, self.table = table.split('.')
+        else:
+            self.schema = schema or 'dbo'
+            self.table = table
+
+        # determine table name of target table and staging table
+        self.compress = compress
+        self.filename = filename  # full file path
+
+        self.azure_synapse = azure_synapse_connector  # TODO: use default
+
+        # determine table ddl stuff
+        self.create_table_ddl = create_table_ddl
+
+        # merge stuff
+        if mode not in allowed_modes:
+            raise ValueError(f'mode should be one of ({allowed_modes})')
+
+        self.mode = mode
+        self.primary_keys = ensure_str_list(primary_keys)
+        self.columns = ensure_str_list(columns)
+
+        # dedup stuff
+        self.dedup = dedup
+        self.dedup_uniq_keys = ensure_str_list(dedup_uniq_keys)
+        self.dedup_orderby = dedup_orderby
+        if self.dedup and not self.dedup_uniq_keys:
+            raise ValueError('dedup_uniq_keys should not be empty if dedup is true')
+
+        self.pre_queries = ensure_query_list(pre_queries) or []
+        self.post_queries = ensure_query_list(post_queries) or []
+
+        self.delete_file = delete_file
+
+        super().__init__()
+
+    def execute_impl(self):
+        if fs.is_file_empty(self.filename):
+            self.logger.error('file not exists or has no content. %s', self.filename)
+            fs.remove_files_safely(fs.schema_filename(self.filename))
+            return
+        self._prepare_target_table()
+        self._prepare_staging_table()
+        self._merge_into_target_table()
+
+        # do cleaning things
+        if self.delete_file:
+            self.logger.info('delete local file %s', self.filename)
+            fs.remove_files_safely(self.filename)
+            fs.remove_files_safely(fs.schema_filename(self.filename))
+
+    @property
+    def connector(self):
+        return self.azure_synapse
+
+    @property
+    def staging_table(self):
+        return f'{self.table}_staging'
+
+    @property
+    def full_staging_table_name(self):
+        return f'{self.schema}.{self.staging_table}'
+
+    @property
+    def full_table_name(self):
+        return f'{self.schema}.{self.table}'
+
+    @property
+    def quoted_full_staging_table(self):
+        return self.connector.quote_identifier(self.full_staging_table_name)
+
+    @property
+    def quoted_full_table_name(self):
+        return self.connector.quote_identifier(self.full_table_name)
+
+    def _prepare_staging_table(self):
+        schema, table = self.full_staging_table_name.split('.')
+        drop = self._make_drop_table_query(schema, table)
+        ddl = f'SELECT TOP 0 * INTO {self.quoted_full_staging_table} FROM {self.quoted_full_table_name}'
+        self.connector.execute([drop, ddl])
+
+        self.logger.info(f'load {self.filename} into staging table {self.full_staging_table_name}')
+        self.connector.load_csv(table=self.full_staging_table_name,
+                                filename=self.filename,
+                                columns=self.columns)
+
+        if self.dedup:
+            dedup_query = self._construct_dedup_query()
+            self.connector.execute(dedup_query, autocommit=False, commit_on_close=True)
+
+    def _construct_dedup_query(self):
+        partitions_cols = []
+        for col in self.dedup_uniq_keys:
+            partitions_cols.append(self.connector.quote_identifier(col))
+        partition_by = ', '.join(partitions_cols)
+        columns = ' ,'.join(self.azure_synapse.get_columns(schema=self.schema, table=self.staging_table))
+        tmp_table = f'{self.full_staging_table_name}_tmp'
+        quoted_tmp_table = self.connector.quote_identifier(tmp_table)
+        quoted_bak_table = self.connector.quote_identifier(f'{self.staging_table}_bak')
+
+        queries = f"""
+            {self._make_drop_table_query(self.schema, tmp_table)};
+
+            CREATE TABLE {quoted_tmp_table} AS
+            SELECT {', '.join(self.connector.quote_identifier(x) for x in columns)}
+            FROM (
+              SELECT *, ROW_NUMBER() OVER (PARTITION BY {partition_by} ORDER BY {self.dedup_orderby}) rn
+              FROM {self.quoted_full_staging_table}
+            ) AS t
+            WHERE rn = 1;
+
+            RENAME OBJECT {self.quoted_full_staging_table} TO {quoted_bak_table};
+            RENAME OBJECT {quoted_tmp_table} TO {self.staging_table};
+            DROP TABLE {quoted_bak_table};
+        """
+        return queries
+
+    def _make_drop_table_query(self, schema: str, table: str) -> str:
+        if '.' in table:
+            schema, table = table.split('.')
+        if not schema:
+            schema = self.schema
+        full_table = f'{schema}.{table}'
+        query = f'''
+            IF EXISTS (
+              SELECT * FROM sys.tables
+              WHERE schema_name(schema_id) = '{schema}' AND name = '{table}'
+            )
+            DROP TABLE {self.connector.quote_identifier(full_table)}
+        '''
+        return query
+
+    def _merge_into_target_table(self):
+        target = self.quoted_full_table_name
+        staging = self.quoted_full_staging_table
+
+        append_sql = f'INSERT INTO {target} SELECT * FROM {staging}'
+        if self.mode == const.LOAD_OVERWRITE:
+            queries = [f'TRUNCATE TABLE {target}', append_sql]
+        elif self.mode == const.LOAD_MERGE:
+            queries = self._ingest_by_merging()
+        else:
+            # APPEND
+            queries = [append_sql]
+
+        queries.append(f'DROP TABLE {staging}')
+
+        queries = self.pre_queries + queries + self.post_queries
+        self.logger.info('running Azure Synapse queries...')
+        self.connector.execute(queries, autocommit=True, commit_on_close=True)
+        self.logger.info('done.')
+
+    def _ingest_by_merging(self):
+        merge_table = f'{self.full_table_name}_merge'
+        quote = self.connector.quote_identifier
+        join = ' AND '.join([f'a.{quote(x)} = b.{quote(x)}' for x in self.primary_keys])
+
+        drop_merge_table = self._make_drop_table_query(self.schema, merge_table)
+        # TODO: maybe use the builtin MERGE by T-SQL?
+        queries = f"""
+            {drop_merge_table};
+
+            CREATE TABLE {quote(merge_table)} WITH (DISTRIBUTION = ROUND_ROBIN)
+            AS
+            SELECT a.*
+            FROM {self.quoted_full_table_name} AS a
+            LEFT JOIN {self.quoted_full_staging_table} AS b ON {join}
+            WHERE b.{quote(self.primary_keys[0])} IS NULL
+            UNION ALL
+            SELECT * FROM {self.quoted_full_staging_table};
+
+            TRUNCATE TABLE {self.quoted_full_table_name};
+            INSERT INTO {self.quoted_full_table_name} SELECT * FROM {quote(merge_table)};
+
+            {drop_merge_table};
+        """
+        return queries.split(';')

recurvedata/pigeon/loader/csv_to_clickhouse.py

@@ -0,0 +1,152 @@
+from recurvedata.pigeon import const
+from recurvedata.pigeon.connector import new_clickhouse_connector
+from recurvedata.pigeon.csv import CSV
+from recurvedata.pigeon.loader.base import BaseLoader, CSVToDBAPIMixin
+from recurvedata.pigeon.utils import ensure_query_list, ensure_str_list, fs
+from recurvedata.pigeon.utils.sql import bak_table_of, reconcile_table_of, staging_table_of
+
+allowed_modes = (const.LOAD_OVERWRITE, const.LOAD_MERGE, const.LOAD_APPEND)
+
+
+class CSVToClickHouseLoader(BaseLoader, CSVToDBAPIMixin):
+    def __init__(
+        self,
+        database,
+        table,
+        filename,
+        connector=None,
+        create_table_ddl=None,
+        mode=const.LOAD_OVERWRITE,
+        primary_keys=None,
+        skiprows=0,
+        using_insert=False,
+        insert_batch_size=10000,
+        insert_concurrency=1,
+        delete_file=False,
+        table_engine="Log",
+        pre_queries=None,
+        post_queries=None,
+        native=False,
+    ):
+        self.database = database
+        self.table = table
+
+        if isinstance(filename, CSV):
+            filename = filename.path
+        self.filename = filename
+        self.csvfile = CSV(self.filename)
+
+        if connector is None:
+            connector = new_clickhouse_connector(database=self.database, native=native)
+        else:
+            connector.database = self.database
+        self.connector = connector
+
+        self.create_table_ddl = create_table_ddl
+        self.ddl_options = {"ENGINE": table_engine}
+
+        if mode not in allowed_modes:
+            raise ValueError("mode should be one of ({})".format(allowed_modes))
+
+        self.mode = mode
+        self.primary_keys = ensure_str_list(primary_keys)
+        if self.mode == const.LOAD_MERGE and not self.primary_keys:
+            raise ValueError("primary_keys should not be empty in mode {}".format(const.LOAD_MERGE))
+
+        # self.columns = columns or self.csvfile.header
+        # self.skiprows = int(skiprows or self.csvfile.has_header)
+        self.skiprows = int(skiprows)
+        self.using_insert = using_insert
+        self.insert_batch_size = insert_batch_size
+        self.insert_concurrency = insert_concurrency
+        self.delete_file = delete_file
+
+        self.pre_queries = ensure_query_list(pre_queries) or []
+        self.post_queries = ensure_query_list(post_queries) or []
+
+        super().__init__()
+
+    @property
+    def staging_table(self):
+        return staging_table_of(self.table)
+
+    def execute_impl(self):
+        if fs.is_file_empty(self.filename):
+            self.logger.error("file not exists or has no content. %s", self.filename)
+            fs.remove_files_safely(fs.schema_filename(self.filename))
+            return
+        self._prepare_target_table()
+        self._prepare_staging_table()
+        self._load_to_staging()
+        self._merge_into_target_table()
+
+        # do cleaning things
+        if self.delete_file:
+            self.logger.info("delete local file %s", self.filename)
+            fs.remove_files_safely(self.filename)
+            fs.remove_files_safely(fs.schema_filename(self.filename))
+
+    def _prepare_staging_table(self):
+        queries = """
+            DROP TABLE IF EXISTS {staging};
+            CREATE TABLE {staging} AS {table};
+        """.format(
+            staging=self.staging_table, table=self.table
+        )
+        self.connector.execute(queries, autocommit=True)
+
+    def _load_to_staging(self):
+        self.connector.load_csv(
+            table=self.staging_table,
+            filename=self.csvfile.path,
+            lineterminator=self.csvfile.dialect.lineterminator,
+            skiprows=self.skiprows,
+            null_values=("NULL", r"\N"),
+            using_insert=self.using_insert,
+            batch_size=self.insert_batch_size,
+            concurrency=self.insert_concurrency,
+        )
+
+    def _merge_into_target_table(self):
+        queries = []
+        if self.mode == const.LOAD_MERGE:
+            queries.extend(self._ingest_by_merging())
+        elif self.mode == const.LOAD_OVERWRITE:
+            bak_table = bak_table_of(self.table)
+            queries.append(f"DROP TABLE IF EXISTS {bak_table}")
+            queries.append(f"RENAME TABLE {self.table} TO {bak_table}")
+            queries.append(f"RENAME TABLE {self.staging_table} TO {self.table}")
+            queries.append(f"DROP TABLE IF EXISTS {bak_table}")
+        else:
+            queries.append(f"INSERT INTO {self.table} SELECT * FROM {self.staging_table}")
+            queries.append(f"DROP TABLE {self.staging_table}")
+
+        queries = self.pre_queries + queries + self.post_queries
+        self.connector.execute(queries)
+
+    def _ingest_by_merging(self):
+        reconcile = reconcile_table_of(self.table)
+        bak = bak_table_of(self.table)
+        using = ", ".join(self.primary_keys)
+        sql = """
+            DROP TABLE IF EXISTS {reconcile};
+            CREATE TABLE {reconcile} AS {table};
+
+            INSERT INTO {reconcile}
+            SELECT * FROM {table} WHERE NOT ({using}) IN (SELECT {using} FROM {staging})
+            UNION ALL
+            SELECT * FROM {staging};
+
+            RENAME TABLE {table} TO {bak};
+            RENAME TABLE {reconcile} TO {table};
+            DROP TABLE IF EXISTS {bak};
+            DROP TABLE {staging};
+        """.format(
+            reconcile=self.connector.quote_identifier(reconcile),
+            table=self.connector.quote_identifier(self.table),
+            staging=self.connector.quote_identifier(self.staging_table),
+            bak=self.connector.quote_identifier(bak),
+            using=using,
+        )
+        queries = sql.split(";")
+        return queries

recurvedata/pigeon/loader/csv_to_doris.py

@@ -0,0 +1,215 @@
+from typing import TYPE_CHECKING, Any, List, Optional
+
+from recurvedata.pigeon import const
+from recurvedata.pigeon.loader.csv_to_mysql import CSVToMySQLLoader
+from recurvedata.pigeon.utils import md5hash
+from recurvedata.pigeon.utils.sql import bak_table_of, reconcile_table_of, staging_table_of
+
+if TYPE_CHECKING:
+    from recurvedata.pigeon.connector.doris import DorisConnector
+
+allowed_modes = (const.LOAD_OVERWRITE, const.LOAD_MERGE, const.LOAD_APPEND)
+
+
+class CSVToDorisLoader(CSVToMySQLLoader):
+    def __init__(
+        self,
+        database: str,
+        table: str,
+        filename: str,
+        connector: Optional["DorisConnector"] = None,
+        create_table_ddl: Optional[str] = None,
+        mode: str = const.LOAD_OVERWRITE,
+        primary_keys: Optional[List[str]] = None,
+        skiprows: int = 0,
+        columns: Optional[List[str]] = None,
+        using_insert: bool = False,
+        insert_batch_size: int = 1000,
+        insert_concurrency: int = 1,
+        delete_file: bool = False,
+        pre_queries: Optional[List[str]] = None,
+        post_queries: Optional[List[str]] = None,
+        load_strict_mode: bool = False,
+        max_filter_ratio: float = 0,
+        *args: Any,
+        **kwargs: Any,
+    ):
+        if not connector:
+            raise ValueError(f"connector is required for {self.__class__.__name__}")
+        self.load_strict_mode: bool = load_strict_mode
+        connector.load_strict_mode = load_strict_mode
+        self.logger.info(f"load_strict_mode: {load_strict_mode}")
+        connector.max_filter_ratio = max_filter_ratio
+        self.logger.info(f"max_filter_ratio: {max_filter_ratio}")
+        table_suffix: str = md5hash(filename)[:6]
+        self.__staging_table: str = staging_table_of(table) + "_" + table_suffix
+        self.__reconcile_table: str = reconcile_table_of(table) + "_" + table_suffix
+        self.__bak_table: str = bak_table_of(table) + "_" + table_suffix
+
+        # Check table name length
+        if any(
+            [
+                len(self.__staging_table) > 64,
+                len(self.__reconcile_table) > 64,
+                len(self.__bak_table) > 64,
+            ]
+        ):
+            self.logger.error(
+                f"table name {self.__staging_table} 's length: {len(self.__staging_table)}\n"
+                f"table name {self.__reconcile_table}'s length: {len(self.__reconcile_table)}\n"
+                f"table name {self.__bak_table}'s length: {len(self.__bak_table)}\n"
+            )
+            raise ValueError("length of intermediate table name is greater than 64!")
+
+        super().__init__(
+            database=database,
+            table=table,
+            filename=filename,
+            connector=connector,
+            create_table_ddl=create_table_ddl,
+            mode=mode,
+            primary_keys=primary_keys,
+            skiprows=skiprows,
+            columns=columns,
+            using_insert=using_insert,
+            insert_batch_size=insert_batch_size,
+            insert_concurrency=insert_concurrency,
+            delete_file=delete_file,
+            pre_queries=pre_queries,
+            post_queries=post_queries,
+            *args,
+            **kwargs,
+        )
+
+    @property
+    def staging_table(self) -> str:
+        """
+        Override method to return table name with random suffix
+        """
+        return self.__staging_table
+
+    def _merge_into_target_table(self) -> None:
+        queries = []
+        if self.mode == const.LOAD_MERGE:
+            queries.extend(self._ingest_by_merging())
+        elif self.mode == const.LOAD_OVERWRITE:
+            bak_table = self.__bak_table
+            queries.extend(
+                [
+                    f"DROP TABLE IF EXISTS {bak_table}",
+                    f"ALTER TABLE {self.table} RENAME {bak_table}",
+                    f"ALTER TABLE {self.staging_table} RENAME {self.table}",
+                    f"DROP TABLE IF EXISTS {bak_table}",
+                ]
+            )
+        else:  # APPEND mode
+            queries.append(f"INSERT INTO {self.table} SELECT * FROM {self.staging_table}")
+            self.connector.execute(self.pre_queries + queries, autocommit=True, commit_on_close=False)
+
+            queries.clear()
+            queries.append(f"DROP TABLE {self.staging_table}")
+            self.connector.execute(queries + self.post_queries, autocommit=True, commit_on_close=False)
+            return
+
+        queries = self.pre_queries + queries + self.post_queries
+        self.logger.info("running MySQL queries within a transaction")
+        self.connector.execute(queries, autocommit=False, commit_on_close=True)
+
+    def _ingest_by_merging(self) -> List[str]:
+        """Merge with deduplication based on primary keys using Doris-compatible syntax"""
+        # First, deduplicate staging table based on primary keys using window function
+        pk_columns = ", ".join(self.primary_keys)
+
+        # Get all columns from staging table (excluding the rn column we'll add)
+        cols = self.connector.get_columns(self.staging_table)
+        cols_str = ", ".join(self.connector.quote_identifier(x) for x in cols)
+
+        # Create a temporary table with deduplicated data
+        tmp_table = f"{self.staging_table}_dedup"
+        dedup_sql = f"""
+            DROP TABLE IF EXISTS {tmp_table};
+            CREATE TABLE {tmp_table} LIKE {self.staging_table};
+            INSERT INTO {tmp_table}
+            SELECT {cols_str} FROM (
+                SELECT *, ROW_NUMBER() OVER(PARTITION BY {pk_columns} ORDER BY {pk_columns}) AS rn
+                FROM {self.staging_table}
+            ) t
+            WHERE rn = 1;
+        """
+
+        # Replace staging table with deduplicated data
+        replace_sql = f"""
+            DROP TABLE {self.staging_table};
+            ALTER TABLE {tmp_table} RENAME {self.staging_table};
+        """
+
+        # Simple merge: backup target table, then merge and deduplicate in one step
+        bak = self.__bak_table
+        table = self.connector.quote_identifier(self.table)
+        staging = self.connector.quote_identifier(self.staging_table)
+        bak = self.connector.quote_identifier(bak)
+
+        # Simple and efficient merge: backup + merge + deduplicate in one operation
+        merge_sql = f"""
+            -- Backup target table
+            DROP TABLE IF EXISTS {bak};
+            ALTER TABLE {table} RENAME {bak};
+
+            -- Create new target table and insert deduplicated merged data in one step
+            CREATE TABLE {table} AS
+            SELECT {cols_str} FROM (
+                SELECT *, ROW_NUMBER() OVER(PARTITION BY {pk_columns} ORDER BY {pk_columns}) AS rn
+                FROM (
+                    SELECT * FROM {bak}
+                    UNION ALL
+                    SELECT * FROM {staging}
+                ) combined
+            ) t WHERE rn = 1;
+
+            -- Clean up
+            DROP TABLE {bak};
+            DROP TABLE {staging};
+        """
+
+        return [dedup_sql, replace_sql, merge_sql]
+
+    def execute(self) -> None:
+        """
+        Override method to implement exception handling
+        """
+        self.before_execute()
+        try:
+            self.execute_impl()
+        except Exception as e:
+            self.handle_exception()
+            raise e
+        self.after_execute()
+
+    def handle_exception(self) -> None:
+        """
+        Ensure safe cleanup of all intermediate tables after an exception occurs
+        """
+        qry_exists_sql = """
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = '{database}' AND table_name = '{table}';
+        """
+        is_table_exists = self.connector.fetchall(qry_exists_sql.format(database=self.database, table=self.table))
+        is_bak_exists = self.connector.fetchall(qry_exists_sql.format(database=self.database, table=self.__bak_table))
+
+        if is_table_exists:
+            # Directly clean up intermediate tables
+            queries = [
+                f"DROP TABLE IF EXISTS {self.__bak_table}",
+                f"DROP TABLE IF EXISTS {self.__staging_table}",
+                f"DROP TABLE IF EXISTS {self.__reconcile_table}",
+            ]
+        elif is_bak_exists:
+            # Restore from backup table
+            queries = [
+                f"ALTER TABLE {self.__bak_table} RENAME {self.table}",
+                f"DROP TABLE IF EXISTS {self.__staging_table}",
+                f"DROP TABLE IF EXISTS {self.__reconcile_table}",
+            ]
+        else:
+            queries = [f"DROP TABLE IF EXISTS {self.__staging_table}", f"DROP TABLE IF EXISTS {self.__reconcile_table}"]
+        self.connector.execute(queries, autocommit=False, commit_on_close=True)