dbhydra 2.1.3__py3-none-any.whl → 2.2.1__py3-none-any.whl

dbhydra/dbhydra_core.py

@@ -13,7 +13,8 @@ from dbhydra.src.mongo_db import MongoDb
 from dbhydra.src.postgres_db import PostgresDb
 from dbhydra.src.xlsx_db import XlsxDb, XlsxDB
 from dbhydra.src.abstract_db import AbstractDb
-from dbhydra.src.tables import SqlServerTable, PostgresTable, MysqlTable, XlsxTable, AbstractTable, MongoTable, BigQueryTable, Table, AbstractSelectable, AbstractJoinable
+from dbhydra.src.tables import (SqlServerTable, PostgresTable, MysqlTable, XlsxTable, AbstractTable, MongoTable,
+                                BigQueryTable, Table, AbstractSelectable, AbstractJoinable, PYTHON_TO_MYSQL_DATA_MAPPING)
 ##### Do not remove imports - they are expored in the package
 
 

dbhydra/src/abstract_db.py

@@ -105,6 +105,7 @@ class AbstractDb(abc.ABC):
         
         self.active_transactions=[]
         self.last_table_inserted_into: Optional[str] = None
+        self.identifier_quote = ''
 
     @abc.abstractmethod
     def connect_locally(self):

dbhydra/src/abstract_table.py

@@ -132,12 +132,13 @@ class AbstractSelectable:
             
 
     def select_all(self):
+        quote = self.db1.identifier_quote
         all_cols_query = ""
         for col in self.columns:
-            all_cols_query = all_cols_query + col + ","
+            all_cols_query = all_cols_query + quote + col + quote + ","
         if all_cols_query[-1] == ",":
             all_cols_query = all_cols_query[:-1]
-        list1 = self.select(f"SELECT {all_cols_query} FROM " + self.name)
+        list1 = self.select(f"SELECT {all_cols_query} FROM {quote}{self.name}{quote};")
         return (list1)
 
     def select_to_df(self):
@@ -244,7 +245,6 @@ class AbstractTable(AbstractJoinable, abc.ABC):
             self.column_type_dict={self.columns[i]:self.types[i] for i,x in enumerate(self.columns)}
         else:
             self.column_type_dict={}
-        
 
     # Temporary disabled, please make sure this is implemented where needed, don't introduce breaking changes please
     # @abc.abstractmethod
@@ -421,11 +421,12 @@ class AbstractTable(AbstractJoinable, abc.ABC):
 
 
     def delete(self, where=None):
+        quote = self.db1.identifier_quote
 
         if where is None:
-            query = "DELETE FROM " + self.name
+            query = "DELETE FROM {quote}{self.name}{quote}"
         else:
-            query = "DELETE FROM " + self.name + " WHERE " + where
+            query = f"DELETE FROM {quote}{self.name}{quote} WHERE {where}"
         return self.execute(query)
 
 

dbhydra/src/migrator.py

@@ -1,15 +1,46 @@
-import pandas as pd
+import os
 import math
 import json
+import pandas as pd
+
+from typing import Optional
+from deepdiff import DeepDiff
+from dataclasses import dataclass, asdict
 
+PENDING_MIGRATION_DEFAULT_PATH = "./db/migrations/pending_migration.json"
+MIGRATION_HISTORY_DEFAULT_PATH = "./db/migrations/migration_history.json"
+
+# @dataclass
+# class Migration:
+#     forward: list[dict]
+#     backward: list[dict]
 
 class Migrator:
-    """It was somewhat tested only for MySQL and Postgres dialect"""
+    """
+    A class for managing database migrations.
+
+    This class provides functionality to create, manage, and execute database migrations
+    using a migration system compatible with MySQL and Postgres dialects. It allows for
+    creating forward and backward migrations, reading and writing migrations to JSON files,
+    and executing migrations based on changes detected in database structures.
+
+    Note: This class is compatible with MySQL and Postgres dialects and has been somewhat tested
+    with those databases. It may require adjustments for other database systems.
+
+    Attributes:
+        db: The database connection object used for executing migrations.
+    """
     
-    def __init__(self, db=None):
+    def __init__(self, db):
         self.db = db
-        self.migration_number = 1
-        self.migration_list = []
+        
+        # Used in older implementations, TODO: decide whether to keep both approaches, unify them or pick one
+        self._migration_number = 1
+        self._migration_list = []
+        
+        # Used in newer approach
+        self._pending_forward_migration_list = []#Migration(forward=[], backward=[])
+        self._pending_forward_migration_list = []#Migration(forward=[], backward=[])
 
     def process_migration_dict(self, migration_dict):
         matching_table_class = self.db.matching_table_class #E.g. MysqlTable
@@ -42,9 +73,10 @@ class Migrator:
             table.initialize_types()
             table.drop_column(options["column_name"])
 
+    # Old approach methods START
     def next_migration(self):
-        self.migration_number += 1
-        self.migration_list = []
+        self._migration_number += 1
+        self._migration_list = []
 
     def migrate(self, migration_list):
         for i, migration_dict in enumerate(migration_list):
@@ -59,21 +91,30 @@ class Migrator:
         return (result)
 
     def migration_list_to_json(self, filename=None):
-        result = json.dumps(self.migration_list)
+        result = json.dumps(self._migration_list)
 
         if filename is None or filename == "" or filename.isspace():
-            with open("migrations/migration-" + str(self.migration_number) + ".json", "w+") as f:
+            with open("migrations/migration-" + str(self._migration_number) + ".json", "w+") as f:
                 f.write(result)
         else:
             with open(f"migrations/{filename}.json", "w+") as f:
                 f.write(result)
 
+
+
+
+
+
+
+
+
+    ##### Auxilliary? #####
     def create_migrations_from_df(self, name, dataframe):
 
         columns, return_types = self.extract_columns_and_types_from_df(dataframe)
 
         migration_dict = {"create": {"table_name": name, "columns": columns, "types": return_types}}
-        self.migration_list.append(migration_dict)
+        self._migration_list.append(migration_dict)
         self.migration_list_to_json()
         # return columns, return_types
 
@@ -111,4 +152,274 @@ class Migrator:
             return_types.insert(0, "int")
 
         return columns, return_types
+    # Old approach methods END
+  
+    
+  
+    
+  
+    
+  
+    
+  
+    
+  
+    
+    # def set_pending_migration(self, migration_dict: dict[str, list]):
+    #     self._pending_migration = Migration(**migration_dict)
+    
+    def migrate_forward(self):
+        """
+        Applies forward migrations from the pending migration object.
+
+        Iterates through each migration dictionary in the pending migration's forward list,
+        processes the migration, saves it to migration history, and clears the pending migration.
+
+        Returns:
+            None
+        """
+            
+        for migration_dict in self._pending_forward_migration_list:
+            self.process_migration_dict(migration_dict)
+            
+        #self._save_migration_to_history(migration=self._pending_migration)
+        self._clear_pending_migration()
+            
+    def migrate_backward(self):
+        """
+        Applies backward migrations from the pending migration object.
+
+        Iterates through each migration dictionary in the pending migration's backward list,
+        processes the migration, saves it to migration history, and clears the pending migration.
+
+        Returns:
+            None
+        """
+        
+        for migration_dict in self._pending_backward_migration_list:
+            self.process_migration_dict(migration_dict)
+            
+        #history_migration = Migration(forward=self._pending_migration.backward, backward=self._pending_migration.forward)
+        #self._save_migration_to_history(migration=history_migration)
+        self._clear_pending_migration()
+        
+    # def migrate_n_steps_back_in_history(self, n: int, migration_history_json: str = MIGRATION_HISTORY_DEFAULT_PATH):        
+    #     migration_history = self._read_migration_history_json(migration_history_json)
+        
+    #     if len(migration_history) < n:
+    #         raise ValueError(f"Provided n (= {n}) is larger than migration history length (= {len(migration_history)}).")
+
+    #     total_backward_migration = Migration(forward=[], backward=[])
+    #     migrations = migration_history[-n:] # Take last n elements of migration history for execution
+        
+    #     # Loop in reversed order as we execute backward migrations in reversed order compared to forward ones
+    #     for migration_dict in reversed(migrations):
+    #         total_backward_migration.forward.append(migration_dict["forward"])
+    #         total_backward_migration.backward.append(migration_dict["backward"])
+            
+    #     self.set_pending_migration(asdict(total_backward_migration))
+    #     self.migrate_backward()
+    
+    # def load_migration_from_json(self, json_file_path: str = PENDING_MIGRATION_DEFAULT_PATH):
+    #     with open(json_file_path, "r") as file:
+    #         migration_dict = json.load(file)
+            
+    #     self.set_pending_migration(migration_dict)
+    
+    # def save_pending_migration_to_json(self, file_path: str = PENDING_MIGRATION_DEFAULT_PATH):
+    #     if not file_path.endswith(".json"):
+    #         raise ValueError("pending migration file must be of '.json' type.")
+        
+    #     self._build_folder_structure_for_file_path(file_path)
+        
+    #     with open(file_path, "w+") as file:
+    #         json.dump(asdict(self._pending_migration), file, indent=2)
+    
+    def create_table_migration(self, table_name: str, old_column_type_dict: Optional[dict], new_column_type_dict: Optional[dict]):
+        """
+        Creates a migration for a database table based on its old and new column_type_dicts.
+
+        Args:
+            table_name (str): The name of the database table.
+            old_column_type_dict (Optional[dict]): The old column_type_dict of the table.
+            new_column_type_dict (Optional[dict]): The new column_type_dict of the table.
+
+            If old_column_type_dict is None and new_column_type_dict is not None: CREATE table
+            If old_column_type_dict is not None and new_column_type_dict is None: DROP table
+
+        Returns:
+            Migration: The generated migration object.
+
+        Raises:
+            ValueError: If the table_name argument is empty.
+        """
+        
+        def _extract_column_name_from_deepdiff_key(deepdiff_key: str) -> str:
+            """
+            Extracts the column name from a key generated by deepdiff.
+
+            Args:
+                deepdiff_key (str): The key generated by deepdiff.
+
+            Returns:
+                str: The extracted column name.
+
+            Example:
+                >>> migrator = Migrator()
+                >>> column_name = migrator._extract_column_name_from_deepdiff_key("root['table']['column']")
+                >>> print(column_name)
+                'column'
+            """
+            
+            # Split the item_key by '[' and ']' to isolate the column name
+            # The column name is expected to be the last element after splitting
+            column_name = deepdiff_key.split('[')[-1].strip("']")
+            return column_name
+        
+        def _convert_deepdiff_dict_into_migration_lists(table_name: str, deepdiff_dict: dict):
+            """
+            Converts deepdiff dictionary from the new and old table column_type_dicts comparison into a Migration object.
+
+            Args:
+                table_name (str): A name of the examined DB table.
+                deepdiff_dict (dict): A dictionary from DeepDiff comparison of the old and new table column_type_dict.
+
+            Returns:
+                Migration: A Migration object containing forward and backward migrations for the given table.
+                
+            Example:
+                >>> table_name = 'results'
+                >>> deepdiff_dict = {'dictionary_item_removed': {"root['hehexd']": 'double'}}
+                >>> migrator = Migrator()
+                >>> asdict(migrator._convert_deepdiff_dict_into_migration)
+                >>> {
+                    'forward': [
+                        {'drop_column': {'table_name': 'results', 'column_name': 'hehexd'}}
+                        ],
+                    'backward': [
+                        {'add_column': {'table_name': 'results', 'column_name': 'hehexd', 'column_type': 'double'}}
+                        ]
+                    }
+            """
+            forward_migration_list, backward_migration_list = [], []
+
+            forward_conversions = {
+                "dictionary_item_added": "add_column",
+                "dictionary_item_removed": "drop_column",
+                "values_changed": "modify_column"
+                }
+            backward_conversions = {
+                "dictionary_item_added": "drop_column",
+                "dictionary_item_removed": "add_column",
+                "values_changed": "modify_column"
+                }
+
+            for action_name, deepdiff_action in deepdiff_dict.items():
+                for deepdiff_key in deepdiff_action.keys():
+                    column_name = _extract_column_name_from_deepdiff_key(deepdiff_key)
+                    forward_action, backward_action = forward_conversions[action_name], backward_conversions[action_name]
+                    
+                    if action_name=="dictionary_item_added":
+                        column_type = deepdiff_action[deepdiff_key]
+                        forward_migration_list.append({forward_action: {"table_name": table_name, "column_name": column_name, "column_type": column_type}})
+                        backward_migration_list.append({backward_action: {"table_name": table_name, "column_name": column_name}})
+                    elif action_name=="dictionary_item_removed":
+                        column_type = deepdiff_action[deepdiff_key]
+                        forward_migration_list.append({forward_action: {"table_name": table_name, "column_name": column_name}})
+                        backward_migration_list.append({backward_action: {"table_name": table_name, "column_name": column_name, "column_type": column_type}})
+                    elif action_name=="values_changed":
+                        column_type = deepdiff_action[deepdiff_key]["old_value"]
+                        column_new_type = deepdiff_action[deepdiff_key]["new_value"]
+                        
+                        # HACK: Do not create migrations for cases such as varchar(2047) --> nvarchar(2047)
+                        is_varchar_in_types = "varchar" in column_type and "varchar" in column_new_type
+                        is_max_length_equal = (
+                            column_type[column_type.index("("): column_type.index(")")] 
+                            and column_new_type[column_new_type.index("("): column_new_type.index(")")]
+                            ) if is_varchar_in_types else False
+                        is_varchar_nvarchar_conversion = is_varchar_in_types and is_max_length_equal
+                        
+                        if not is_varchar_nvarchar_conversion:
+                            forward_migration_list.append({forward_action: {"table_name": table_name, "column_name": column_name,
+                                                                "column_type": column_new_type}})
+                            backward_migration_list.append({backward_action: {"table_name": table_name, "column_name": column_name,
+                                                                "column_type": column_type}})
+                
+            return forward_migration_list, backward_migration_list
+        
+        
+        
+        
+        if not table_name:
+            raise ValueError("The 'table_name' argument must be a non-empty string.")
+
+        if not old_column_type_dict and new_column_type_dict:
+            # non-empty initial column_type_dict --> empty new column_type_dict
+            columns, types = list(new_column_type_dict.keys()), list(new_column_type_dict.values())
+            forward_migration_list = [{"create": {"table_name": table_name, "columns": columns, "types": types}}]
+            backward_migration_list = [{"drop": {"table_name": table_name}}]
+            
+        elif not new_column_type_dict:
+            # new column_type_dict is empty ==> drop the table
+            forward_migration_list = [{"drop": {"table_name": table_name}}]
+            backward_migration_list = [{"create": {"table_name": table_name, "columns": columns, "types": types}}]
+            
+            
+        else:
+            diff = DeepDiff(old_column_type_dict, new_column_type_dict, verbose_level=2)
+            forward_migration_list, backward_migration_list = _convert_deepdiff_dict_into_migration_lists(table_name, diff)
+            
+        #migration = Migration(forward=forward_migration_list, backward=backward_migration_list)
+            
+        self._append_migration_to_pending_migration(forward_migration_list, backward_migration_list)
+
+        return forward_migration_list, backward_migration_list
+
+    
+    
+ 
+    
+    def _append_migration_to_pending_migration(self, forward_migration_list, backward_migration_list):
+        self._pending_forward_migration_list += forward_migration_list
+        self._pending_backward_migration_list += backward_migration_list
+        
+        
+    def _clear_pending_migration(self):
+        self._pending_forward_migration_list = []
+        self._pending_backward_migration_list = []
+        
+    # def _read_migration_history_json(self, file_path: str = MIGRATION_HISTORY_DEFAULT_PATH):
+    #     if not file_path.endswith(".json"):
+    #         raise ValueError("Migration history file must be of '.json' type.")
+        
+    #     if not os.path.exists(file_path):
+    #         raise FileNotFoundError(f"Migration history file '{file_path}' does not exist.")
+        
+    #     try:
+    #         with open(file_path, "r") as file:
+    #             migration_history = json.load(file)
+    #     except json.JSONDecodeError:
+    #         migration_history = []
+            
+    #     return migration_history
+        
+    # def _save_migration_to_history(self, migration: Migration, file_path: str = MIGRATION_HISTORY_DEFAULT_PATH):            
+    #     try:
+    #         migration_history = self._read_migration_history_json(file_path)
+    #     except FileNotFoundError:
+    #         self._build_folder_structure_for_file_path(file_path)
+    #         migration_history = []
+            
+    #     migration_history.append(asdict(migration))
+        
+    #     with open(file_path, "w") as file:
+    #         json.dump(migration_history, file, indent=2)
+            
+    # def _build_folder_structure_for_file_path(self, file_path: str):
+    #     folder_path = os.path.dirname(file_path)
+    #     if not os.path.exists(folder_path):
+    #         print(f"Folder path to the file '{file_path}' does not exist. Creating the file and the folder structure.")
+    #         os.makedirs(folder_path)
+        
+
 

dbhydra/src/mysql_db.py

@@ -22,6 +22,10 @@ class MysqlDb(AbstractDb):
     'Jsonable': "json"
     }
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.identifier_quote = '`'
+
     def connect_locally(self):
         self.connection = pymysql.connect(host=self.DB_SERVER, user=self.DB_USERNAME, password=self.DB_PASSWORD,
                                           database=self.DB_DATABASE)
@@ -39,6 +43,9 @@ class MysqlDb(AbstractDb):
         print("DB connection established")
 
     def create_new_db(self):
+        self.connection = pymysql.connect(host=self.DB_SERVER, port=self.DB_PORT, user=self.DB_USERNAME, 
+                                          charset="utf8mb4", password=self.DB_PASSWORD)
+        self.cursor = self.connection.cursor()
         create_db_command = "CREATE DATABASE " + self.DB_DATABASE
         self.execute(create_db_command)
 

dbhydra/src/tables.py

@@ -2,7 +2,7 @@ import pandas as pd
 import numpy as np
 from typing import Optional, Any
 import abc
-
+import time
 #xlsx imports
 import pathlib
 
@@ -655,7 +655,7 @@ class MysqlTable(AbstractTable):
     """
     def get_data_types_and_character_lengths(self):
         information_schema_table = MysqlTable(self.db1, 'INFORMATION_SCHEMA.COLUMNS', ['DATA_TYPE'], ['nvarchar(50)'])
-        query = f"SELECT DATA_TYPE,character_maximum_length FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{self.db1.DB_DATABASE}' AND TABLE_NAME  = '" + self.name + "'"
+        query = f"SELECT DATA_TYPE,character_maximum_length FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '{self.db1.DB_DATABASE}' AND TABLE_NAME  = '{self.name}'"
         types = information_schema_table.select(query)
         data_types = [x[0] for x in types]
         data_lengths = [x[1] for x in types]
@@ -725,12 +725,12 @@ class MysqlTable(AbstractTable):
         Returns the number of records in table
         """
 
-        num_of_records = self.select(f"SELECT COUNT(*) FROM {self.name};")
+        num_of_records = self.select(f"SELECT COUNT(*) FROM `{self.name}`;")
 
         return num_of_records[0][0]
 
     def drop(self):
-        query = "DROP TABLE " + self.name + ";"
+        query = "DROP TABLE `" + self.name + "`;"
         print(query)
         self.db1.execute(query)
 
@@ -742,9 +742,9 @@ class MysqlTable(AbstractTable):
 
         column_type_pairs = list(zip(self.columns, self.types))[1:]
         fields = ", ".join(
-            [f"{column} {type_.upper()}" for column, type_ in column_type_pairs]
+            [f"`{column}` {type_.upper()}" for column, type_ in column_type_pairs]
         )
-        query = f"CREATE TABLE {self.name} ({self.id_column_name} INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, {fields})"
+        query = f"CREATE TABLE `{self.name}` ({self.id_column_name} INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, {fields})"
 
         print(query)
         try:
@@ -759,14 +759,11 @@ class MysqlTable(AbstractTable):
         total_output=[]
         for k in range(len(rows)):
             if k % batch == 0:
-                query = "INSERT INTO " + self.name + " ("
+                query = "INSERT INTO `" + self.name + "` ("
                 for i in range(start_index, len(self.columns)):
                     if i < len(rows[k]) + 1:
-                        # column name containing space needs to be wrapped in `...`, otherwise causes syntax error
-                        if " " in self.columns[i]:
-                            column_name = '`' + self.columns[i] + '`'
-                        else:
-                            column_name = self.columns[i]
+                        # column name containing space/reserved keyword needs to be wrapped in `...`, otherwise causes syntax error
+                        column_name = '`' + self.columns[i] + '`'
                         query += column_name + ","
                 if len(rows) < len(self.columns):
                     print(len(self.columns) - len(rows), "columns were not specified")
@@ -846,17 +843,17 @@ class MysqlTable(AbstractTable):
     def add_foreign_key(self, foreign_key):
         parent_id = foreign_key['parent_id']
         parent = foreign_key['parent']
-        query = "ALTER TABLE " + self.name + " MODIFY " + parent_id + " INT UNSIGNED"
+        query = "ALTER TABLE `" + self.name + "` MODIFY " + parent_id + " INT UNSIGNED"
         print(query)
         self.db1.execute(query)
-        query = "ALTER TABLE " + self.name + " ADD FOREIGN KEY (" + parent_id + ") REFERENCES " + parent + "("+self.id_column_name+")"
+        query = "ALTER TABLE `" + self.name + "` ADD FOREIGN KEY (" + parent_id + ") REFERENCES " + parent + "("+self.id_column_name+")"
         print(query)
         self.db1.execute(query)
 
     @save_migration
     def add_column(self, column_name, column_type):
         assert len(column_name) > 1
-        command = "ALTER TABLE " + self.name + " ADD COLUMN " + column_name + " " + column_type
+        command = "ALTER TABLE `" + self.name + "` ADD COLUMN `" + column_name + "` " + column_type
         try:
             self.db1.execute(command)
             self.columns.append(column_name)
@@ -867,7 +864,7 @@ class MysqlTable(AbstractTable):
     @save_migration
     def drop_column(self, column_name):
         assert len(column_name) > 1
-        command = "ALTER TABLE " + self.name + " DROP COLUMN " + column_name
+        command = "ALTER TABLE `" + self.name + "` DROP COLUMN " + column_name
         try:
             print(command)
             self.db1.execute(command)
@@ -881,7 +878,7 @@ class MysqlTable(AbstractTable):
     @save_migration
     def modify_column(self, column_name, new_column_type):
         assert len(column_name) > 1
-        command = "ALTER TABLE " + self.name + " MODIFY COLUMN " + column_name + " " + new_column_type
+        command = "ALTER TABLE `" + self.name + "` MODIFY COLUMN `" + column_name + "` " + new_column_type
         print(command)
         try:
             self.db1.execute(command)
@@ -894,9 +891,10 @@ class MysqlTable(AbstractTable):
 ############### XLSX ##################
 
 class XlsxTable(AbstractTable):
-    def __init__(self, db1, name, columns=None, types=None, id_column_name = "id"):
+    def __init__(self, db1, name, columns=None, types=None, id_column_name = "id", number_of_retries=5):
         super().__init__(db1, name, columns, types)
         self.id_column_name = id_column_name
+        self.NUMBER_OF_RETRIES = number_of_retries
 
         table_filename = f"{self.name}.csv" if self.db1.is_csv else f"{self.name}.xlsx"
         self.table_directory_path: pathlib.Path = self.db1.db_directory_path / table_filename
@@ -960,23 +958,32 @@ class XlsxTable(AbstractTable):
             column for column, type_ in self.column_type_dict.items() if type_ == "datetime"
         ]
 
-        try:
-            if self.db1.is_csv:
-                df = pd.read_csv(
-                    self.table_directory_path, dtype=column_type_map,
-                    parse_dates=date_columns, encoding='utf-8'
-                )
-            else:
-                df = pd.read_excel(
-                    self.table_directory_path, dtype=column_type_map, 
-                    parse_dates=date_columns
-                )
-
-            df.replace({np.nan: None}, inplace=True)
-        except Exception as e:
-            print(f"Error while reading data into XlsxTable: {e}")
-            df = pd.DataFrame(columns=self.columns)
+        # BUG: If XlsxTable is being accessed by multiple threads, read operation
+        # might fail due to race conditions. Add retry mechanism to handle these cases.
+        for attempt in range(self.NUMBER_OF_RETRIES):
+            try:
+                df = self._select(column_type_map, date_columns)
+            except Exception:
+                # print(f"Error while reading data into XlsxTable: {e}")
+                # df = pd.DataFrame(columns=self.columns)
+                if attempt < self.NUMBER_OF_RETRIES - 1:
+                    time.sleep(0.1)
+                else:
+                    print(f"Failed to read data from {self.table_directory_path}, returning empty DataFrame")
+                    df = pd.DataFrame(columns=self.columns)
+        return df
 
+    def _select(self, column_type_map, date_columns):
+        if self.db1.is_csv:
+            df = pd.read_csv(
+                self.table_directory_path, dtype=column_type_map, parse_dates=date_columns,
+                encoding='utf-8'
+            )
+        else:
+            df = pd.read_excel(
+                self.table_directory_path, dtype=column_type_map, parse_dates=date_columns
+            )
+        df.replace({np.nan: None}, inplace=True)
         return df
 
     def insert_from_df(self, df, batch=1, try_mode=False, debug_mode=False, adjust_df=False, insert_id=False):

dbhydra/test_migrator.py

@@ -0,0 +1,27 @@
+import dbhydra.dbhydra_core as dh
+
+
+db1=dh.MysqlDb("config-mysql.ini")
+with db1.connect_to_db():
+    
+    nodes_table = dh.MysqlTable(db1, "nodes",columns=["id","name"],types=["int","int"])
+    #nodes_table.create()
+    
+    db1.initialize_migrator()
+    
+    print(nodes_table.column_type_dict)
+    
+    new_column_type_dict={"id":"int","name":"nvarchar","age":"int"}
+    
+    migration1=db1.migrator.create_table_migration("nodes", nodes_table.column_type_dict, new_column_type_dict)
+    db1.migrator.save_current_migration_to_json()
+    migration2=db1.migrator.create_table_migration("nodes", new_column_type_dict, nodes_table.column_type_dict)
+    db1.migrator.save_current_migration_to_json()    
+    print(migration1)
+    print(migration2)
+    
+    
+    
+    
+    
+    

{dbhydra-2.1.3.dist-info → dbhydra-2.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbhydra
-Version: 2.1.3
+Version: 2.2.1
 Summary: Data science friendly ORM combining Python
 Home-page: https://github.com/DovaX/dbhydra
 Author: DovaX

{dbhydra-2.1.3.dist-info → dbhydra-2.2.1.dist-info}/RECORD

@@ -1,15 +1,16 @@
 dbhydra/__init__.py,sha256=DCocEeXf4QxdVlBRlNiFvuP5IZJ5aa77_DbUR-_4C14,65
-dbhydra/dbhydra_core.py,sha256=QP4x9ObXCNHOJSolGVOUXZ3EbNM5kWhMNESi5kwKy-g,2405
+dbhydra/dbhydra_core.py,sha256=26xBOo3sl--xFa-IrnE3AmBjB3ut5CXUJ1add438ups,2470
+dbhydra/test_migrator.py,sha256=e3Nnb2mCd3CfjhjSexNg1tXVJMjkl5cCoYcuhbfZ4pM,803
 dbhydra/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dbhydra/src/abstract_db.py,sha256=sREaYBRnFWKxuoo60G_nhPjkkg8MKxTcgeVIX5FMUyc,5865
-dbhydra/src/abstract_table.py,sha256=7Eh-etJqsIxGbenNQ7v3NV6oHL1cNQyK7GAr4NNXxpE,16954
+dbhydra/src/abstract_db.py,sha256=lEP24vWz0HdGjQgSoHnawNK_NvRlSLuvzVSiiawssuw,5901
+dbhydra/src/abstract_table.py,sha256=c3pkBTgMOLGDgLH4YEfqM0x33puh-gkML9Rid8xzdFs,17081
 dbhydra/src/bigquery_db.py,sha256=77XsgvYbANlvYaJnuVve-kz-PNBx_CHoYCL-eYnA8e4,1834
-dbhydra/src/migrator.py,sha256=l2MBmiAsv2hv97YmA5IvfbPUK2aAPt46jFulLXBUFHc,4412
+dbhydra/src/migrator.py,sha256=QzaODEFfraD9_6HN_Osaidaj-nLYQryCYYWwJtUu3n8,18931
 dbhydra/src/mongo_db.py,sha256=mP48zRjI7mXKpm45R8prroZI-Eo7JKf0KJqGX-oTy3w,1922
-dbhydra/src/mysql_db.py,sha256=5qmJXvyEk4567RkDd8jeAgJ-7fZi_aIJoQ8wBn_NUB8,2784
+dbhydra/src/mysql_db.py,sha256=xFYy1Ty7iS3GXSncFoaKve4QN1SMJiuDjGyMbb-b1bw,3152
 dbhydra/src/postgres_db.py,sha256=L7MaBq_6ArwDSP_5LaEqK58oLxZ1X7FgIokcDOSB7wk,1805
 dbhydra/src/sqlserver_db.py,sha256=9Xi3NAliqM79MTV8fpNQb0nWMH8Bqjl1leJSEqgyT94,3611
-dbhydra/src/tables.py,sha256=gZRet-aiV5Qtya8ASf8HCyAzi3PEFTNOxrDJAqL8Kbo,46075
+dbhydra/src/tables.py,sha256=QZK76rv_d0MpXGNnuAezouXN8dO0nPrkVMmKVsHxj68,46656
 dbhydra/src/xlsx_db.py,sha256=z6d-IjMYMmXC591Mt5DcxIYWyluanjPRFd-sXtjjXww,3514
 dbhydra/src/errors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbhydra/src/errors/exceptions.py,sha256=LVpfbTd3NHfQIM-D5TFAU6hOZwGQ3b5DwFD4B6vtf2U,149
@@ -17,8 +18,8 @@ dbhydra/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbhydra/tests/test_cases.py,sha256=eAFGaHaIaab3md3HHm2_ryb_HHfObtcXDAEzLh4qWx8,508
 dbhydra/tests/test_mongo.py,sha256=M8TD72M0iQAk7ZcLTWwLmcmmF_zwALnYEGTWjhQlq0s,1979
 dbhydra/tests/test_sql.py,sha256=aPFXyA0jh8o9VG3B5f9fNz7qDbuVPZ9TcE2twn5dAeQ,3126
-dbhydra-2.1.3.dist-info/LICENSE,sha256=k49Yga8CP889JJaHlOpGFzr_be2nqMoep2chYeIDctk,1091
-dbhydra-2.1.3.dist-info/METADATA,sha256=pP00Quio2Fsk2Pv0SRTzTcZfGSniyvKXWvyUXpfvWBk,2298
-dbhydra-2.1.3.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-dbhydra-2.1.3.dist-info/top_level.txt,sha256=oO4Gf1T8_txIsIlp11GI0k7PtBIMb9GRwb5ObF4MLVg,8
-dbhydra-2.1.3.dist-info/RECORD,,
+dbhydra-2.2.1.dist-info/LICENSE,sha256=k49Yga8CP889JJaHlOpGFzr_be2nqMoep2chYeIDctk,1091
+dbhydra-2.2.1.dist-info/METADATA,sha256=fXT5IdyIT6MA0US_YolueauSs0KAtZJE5392uF7G03c,2298
+dbhydra-2.2.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+dbhydra-2.2.1.dist-info/top_level.txt,sha256=oO4Gf1T8_txIsIlp11GI0k7PtBIMb9GRwb5ObF4MLVg,8
+dbhydra-2.2.1.dist-info/RECORD,,