dbhydra 2.2.0__py3-none-any.whl → 2.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbhydra/dbhydra_core.py +2 -1
- dbhydra/src/migrator.py +217 -187
- dbhydra/src/mysql_db.py +3 -0
- dbhydra/src/tables.py +1 -2
- dbhydra/test_migrator.py +27 -0
- {dbhydra-2.2.0.dist-info → dbhydra-2.2.1.dist-info}/METADATA +1 -1
- {dbhydra-2.2.0.dist-info → dbhydra-2.2.1.dist-info}/RECORD +10 -9
- {dbhydra-2.2.0.dist-info → dbhydra-2.2.1.dist-info}/LICENSE +0 -0
- {dbhydra-2.2.0.dist-info → dbhydra-2.2.1.dist-info}/WHEEL +0 -0
- {dbhydra-2.2.0.dist-info → dbhydra-2.2.1.dist-info}/top_level.txt +0 -0
dbhydra/dbhydra_core.py
CHANGED
|
@@ -13,7 +13,8 @@ from dbhydra.src.mongo_db import MongoDb
|
|
|
13
13
|
from dbhydra.src.postgres_db import PostgresDb
|
|
14
14
|
from dbhydra.src.xlsx_db import XlsxDb, XlsxDB
|
|
15
15
|
from dbhydra.src.abstract_db import AbstractDb
|
|
16
|
-
from dbhydra.src.tables import SqlServerTable, PostgresTable, MysqlTable, XlsxTable, AbstractTable, MongoTable,
|
|
16
|
+
from dbhydra.src.tables import (SqlServerTable, PostgresTable, MysqlTable, XlsxTable, AbstractTable, MongoTable,
|
|
17
|
+
BigQueryTable, Table, AbstractSelectable, AbstractJoinable, PYTHON_TO_MYSQL_DATA_MAPPING)
|
|
17
18
|
##### Do not remove imports - they are expored in the package
|
|
18
19
|
|
|
19
20
|
|
dbhydra/src/migrator.py
CHANGED
|
@@ -7,13 +7,13 @@ from typing import Optional
|
|
|
7
7
|
from deepdiff import DeepDiff
|
|
8
8
|
from dataclasses import dataclass, asdict
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
PENDING_MIGRATION_DEFAULT_PATH = "./db/migrations/pending_migration.json"
|
|
11
11
|
MIGRATION_HISTORY_DEFAULT_PATH = "./db/migrations/migration_history.json"
|
|
12
12
|
|
|
13
|
-
@dataclass
|
|
14
|
-
class Migration:
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
# @dataclass
|
|
14
|
+
# class Migration:
|
|
15
|
+
# forward: list[dict]
|
|
16
|
+
# backward: list[dict]
|
|
17
17
|
|
|
18
18
|
class Migrator:
|
|
19
19
|
"""
|
|
@@ -39,7 +39,8 @@ class Migrator:
|
|
|
39
39
|
self._migration_list = []
|
|
40
40
|
|
|
41
41
|
# Used in newer approach
|
|
42
|
-
self.
|
|
42
|
+
self._pending_forward_migration_list = []#Migration(forward=[], backward=[])
|
|
43
|
+
self._pending_forward_migration_list = []#Migration(forward=[], backward=[])
|
|
43
44
|
|
|
44
45
|
def process_migration_dict(self, migration_dict):
|
|
45
46
|
matching_table_class = self.db.matching_table_class #E.g. MysqlTable
|
|
@@ -99,6 +100,15 @@ class Migrator:
|
|
|
99
100
|
with open(f"migrations/{filename}.json", "w+") as f:
|
|
100
101
|
f.write(result)
|
|
101
102
|
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
##### Auxilliary? #####
|
|
102
112
|
def create_migrations_from_df(self, name, dataframe):
|
|
103
113
|
|
|
104
114
|
columns, return_types = self.extract_columns_and_types_from_df(dataframe)
|
|
@@ -143,88 +153,99 @@ class Migrator:
|
|
|
143
153
|
|
|
144
154
|
return columns, return_types
|
|
145
155
|
# Old approach methods END
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
|
|
146
163
|
|
|
147
|
-
|
|
148
|
-
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# def set_pending_migration(self, migration_dict: dict[str, list]):
|
|
169
|
+
# self._pending_migration = Migration(**migration_dict)
|
|
149
170
|
|
|
150
171
|
def migrate_forward(self):
|
|
151
172
|
"""
|
|
152
|
-
Applies forward migrations from the
|
|
173
|
+
Applies forward migrations from the pending migration object.
|
|
153
174
|
|
|
154
|
-
Iterates through each migration dictionary in the
|
|
155
|
-
processes the migration, saves it to migration history, and clears the
|
|
175
|
+
Iterates through each migration dictionary in the pending migration's forward list,
|
|
176
|
+
processes the migration, saves it to migration history, and clears the pending migration.
|
|
156
177
|
|
|
157
178
|
Returns:
|
|
158
179
|
None
|
|
159
180
|
"""
|
|
160
181
|
|
|
161
|
-
for migration_dict in self.
|
|
182
|
+
for migration_dict in self._pending_forward_migration_list:
|
|
162
183
|
self.process_migration_dict(migration_dict)
|
|
163
184
|
|
|
164
|
-
self._save_migration_to_history(migration=self.
|
|
165
|
-
self.
|
|
185
|
+
#self._save_migration_to_history(migration=self._pending_migration)
|
|
186
|
+
self._clear_pending_migration()
|
|
166
187
|
|
|
167
188
|
def migrate_backward(self):
|
|
168
189
|
"""
|
|
169
|
-
Applies backward migrations from the
|
|
190
|
+
Applies backward migrations from the pending migration object.
|
|
170
191
|
|
|
171
|
-
Iterates through each migration dictionary in the
|
|
172
|
-
processes the migration, saves it to migration history, and clears the
|
|
192
|
+
Iterates through each migration dictionary in the pending migration's backward list,
|
|
193
|
+
processes the migration, saves it to migration history, and clears the pending migration.
|
|
173
194
|
|
|
174
195
|
Returns:
|
|
175
196
|
None
|
|
176
197
|
"""
|
|
177
198
|
|
|
178
|
-
for migration_dict in self.
|
|
199
|
+
for migration_dict in self._pending_backward_migration_list:
|
|
179
200
|
self.process_migration_dict(migration_dict)
|
|
180
201
|
|
|
181
|
-
history_migration = Migration(forward=self.
|
|
182
|
-
self._save_migration_to_history(migration=history_migration)
|
|
183
|
-
self.
|
|
202
|
+
#history_migration = Migration(forward=self._pending_migration.backward, backward=self._pending_migration.forward)
|
|
203
|
+
#self._save_migration_to_history(migration=history_migration)
|
|
204
|
+
self._clear_pending_migration()
|
|
184
205
|
|
|
185
|
-
def migrate_n_steps_back_in_history(self, n: int, migration_history_json: str = MIGRATION_HISTORY_DEFAULT_PATH):
|
|
186
|
-
|
|
206
|
+
# def migrate_n_steps_back_in_history(self, n: int, migration_history_json: str = MIGRATION_HISTORY_DEFAULT_PATH):
|
|
207
|
+
# migration_history = self._read_migration_history_json(migration_history_json)
|
|
187
208
|
|
|
188
|
-
|
|
189
|
-
|
|
209
|
+
# if len(migration_history) < n:
|
|
210
|
+
# raise ValueError(f"Provided n (= {n}) is larger than migration history length (= {len(migration_history)}).")
|
|
190
211
|
|
|
191
|
-
|
|
192
|
-
|
|
212
|
+
# total_backward_migration = Migration(forward=[], backward=[])
|
|
213
|
+
# migrations = migration_history[-n:] # Take last n elements of migration history for execution
|
|
193
214
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
215
|
+
# # Loop in reversed order as we execute backward migrations in reversed order compared to forward ones
|
|
216
|
+
# for migration_dict in reversed(migrations):
|
|
217
|
+
# total_backward_migration.forward.append(migration_dict["forward"])
|
|
218
|
+
# total_backward_migration.backward.append(migration_dict["backward"])
|
|
198
219
|
|
|
199
|
-
|
|
200
|
-
|
|
220
|
+
# self.set_pending_migration(asdict(total_backward_migration))
|
|
221
|
+
# self.migrate_backward()
|
|
201
222
|
|
|
202
|
-
def load_migration_from_json(self, json_file_path: str =
|
|
203
|
-
|
|
204
|
-
|
|
223
|
+
# def load_migration_from_json(self, json_file_path: str = PENDING_MIGRATION_DEFAULT_PATH):
|
|
224
|
+
# with open(json_file_path, "r") as file:
|
|
225
|
+
# migration_dict = json.load(file)
|
|
205
226
|
|
|
206
|
-
|
|
227
|
+
# self.set_pending_migration(migration_dict)
|
|
207
228
|
|
|
208
|
-
def
|
|
209
|
-
|
|
210
|
-
|
|
229
|
+
# def save_pending_migration_to_json(self, file_path: str = PENDING_MIGRATION_DEFAULT_PATH):
|
|
230
|
+
# if not file_path.endswith(".json"):
|
|
231
|
+
# raise ValueError("pending migration file must be of '.json' type.")
|
|
211
232
|
|
|
212
|
-
|
|
233
|
+
# self._build_folder_structure_for_file_path(file_path)
|
|
213
234
|
|
|
214
|
-
|
|
215
|
-
|
|
235
|
+
# with open(file_path, "w+") as file:
|
|
236
|
+
# json.dump(asdict(self._pending_migration), file, indent=2)
|
|
216
237
|
|
|
217
|
-
def create_table_migration(self, table_name: str,
|
|
238
|
+
def create_table_migration(self, table_name: str, old_column_type_dict: Optional[dict], new_column_type_dict: Optional[dict]):
|
|
218
239
|
"""
|
|
219
|
-
Creates a migration for a database table based on its old and new
|
|
240
|
+
Creates a migration for a database table based on its old and new column_type_dicts.
|
|
220
241
|
|
|
221
242
|
Args:
|
|
222
243
|
table_name (str): The name of the database table.
|
|
223
|
-
|
|
224
|
-
|
|
244
|
+
old_column_type_dict (Optional[dict]): The old column_type_dict of the table.
|
|
245
|
+
new_column_type_dict (Optional[dict]): The new column_type_dict of the table.
|
|
225
246
|
|
|
226
|
-
If
|
|
227
|
-
If
|
|
247
|
+
If old_column_type_dict is None and new_column_type_dict is not None: CREATE table
|
|
248
|
+
If old_column_type_dict is not None and new_column_type_dict is None: DROP table
|
|
228
249
|
|
|
229
250
|
Returns:
|
|
230
251
|
Migration: The generated migration object.
|
|
@@ -232,164 +253,173 @@ class Migrator:
|
|
|
232
253
|
Raises:
|
|
233
254
|
ValueError: If the table_name argument is empty.
|
|
234
255
|
"""
|
|
235
|
-
|
|
256
|
+
|
|
257
|
+
def _extract_column_name_from_deepdiff_key(deepdiff_key: str) -> str:
|
|
258
|
+
"""
|
|
259
|
+
Extracts the column name from a key generated by deepdiff.
|
|
260
|
+
|
|
261
|
+
Args:
|
|
262
|
+
deepdiff_key (str): The key generated by deepdiff.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
str: The extracted column name.
|
|
266
|
+
|
|
267
|
+
Example:
|
|
268
|
+
>>> migrator = Migrator()
|
|
269
|
+
>>> column_name = migrator._extract_column_name_from_deepdiff_key("root['table']['column']")
|
|
270
|
+
>>> print(column_name)
|
|
271
|
+
'column'
|
|
272
|
+
"""
|
|
273
|
+
|
|
274
|
+
# Split the item_key by '[' and ']' to isolate the column name
|
|
275
|
+
# The column name is expected to be the last element after splitting
|
|
276
|
+
column_name = deepdiff_key.split('[')[-1].strip("']")
|
|
277
|
+
return column_name
|
|
278
|
+
|
|
279
|
+
def _convert_deepdiff_dict_into_migration_lists(table_name: str, deepdiff_dict: dict):
|
|
280
|
+
"""
|
|
281
|
+
Converts deepdiff dictionary from the new and old table column_type_dicts comparison into a Migration object.
|
|
282
|
+
|
|
283
|
+
Args:
|
|
284
|
+
table_name (str): A name of the examined DB table.
|
|
285
|
+
deepdiff_dict (dict): A dictionary from DeepDiff comparison of the old and new table column_type_dict.
|
|
286
|
+
|
|
287
|
+
Returns:
|
|
288
|
+
Migration: A Migration object containing forward and backward migrations for the given table.
|
|
289
|
+
|
|
290
|
+
Example:
|
|
291
|
+
>>> table_name = 'results'
|
|
292
|
+
>>> deepdiff_dict = {'dictionary_item_removed': {"root['hehexd']": 'double'}}
|
|
293
|
+
>>> migrator = Migrator()
|
|
294
|
+
>>> asdict(migrator._convert_deepdiff_dict_into_migration)
|
|
295
|
+
>>> {
|
|
296
|
+
'forward': [
|
|
297
|
+
{'drop_column': {'table_name': 'results', 'column_name': 'hehexd'}}
|
|
298
|
+
],
|
|
299
|
+
'backward': [
|
|
300
|
+
{'add_column': {'table_name': 'results', 'column_name': 'hehexd', 'column_type': 'double'}}
|
|
301
|
+
]
|
|
302
|
+
}
|
|
303
|
+
"""
|
|
304
|
+
forward_migration_list, backward_migration_list = [], []
|
|
305
|
+
|
|
306
|
+
forward_conversions = {
|
|
307
|
+
"dictionary_item_added": "add_column",
|
|
308
|
+
"dictionary_item_removed": "drop_column",
|
|
309
|
+
"values_changed": "modify_column"
|
|
310
|
+
}
|
|
311
|
+
backward_conversions = {
|
|
312
|
+
"dictionary_item_added": "drop_column",
|
|
313
|
+
"dictionary_item_removed": "add_column",
|
|
314
|
+
"values_changed": "modify_column"
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
for action_name, deepdiff_action in deepdiff_dict.items():
|
|
318
|
+
for deepdiff_key in deepdiff_action.keys():
|
|
319
|
+
column_name = _extract_column_name_from_deepdiff_key(deepdiff_key)
|
|
320
|
+
forward_action, backward_action = forward_conversions[action_name], backward_conversions[action_name]
|
|
321
|
+
|
|
322
|
+
if action_name=="dictionary_item_added":
|
|
323
|
+
column_type = deepdiff_action[deepdiff_key]
|
|
324
|
+
forward_migration_list.append({forward_action: {"table_name": table_name, "column_name": column_name, "column_type": column_type}})
|
|
325
|
+
backward_migration_list.append({backward_action: {"table_name": table_name, "column_name": column_name}})
|
|
326
|
+
elif action_name=="dictionary_item_removed":
|
|
327
|
+
column_type = deepdiff_action[deepdiff_key]
|
|
328
|
+
forward_migration_list.append({forward_action: {"table_name": table_name, "column_name": column_name}})
|
|
329
|
+
backward_migration_list.append({backward_action: {"table_name": table_name, "column_name": column_name, "column_type": column_type}})
|
|
330
|
+
elif action_name=="values_changed":
|
|
331
|
+
column_type = deepdiff_action[deepdiff_key]["old_value"]
|
|
332
|
+
column_new_type = deepdiff_action[deepdiff_key]["new_value"]
|
|
333
|
+
|
|
334
|
+
# HACK: Do not create migrations for cases such as varchar(2047) --> nvarchar(2047)
|
|
335
|
+
is_varchar_in_types = "varchar" in column_type and "varchar" in column_new_type
|
|
336
|
+
is_max_length_equal = (
|
|
337
|
+
column_type[column_type.index("("): column_type.index(")")]
|
|
338
|
+
and column_new_type[column_new_type.index("("): column_new_type.index(")")]
|
|
339
|
+
) if is_varchar_in_types else False
|
|
340
|
+
is_varchar_nvarchar_conversion = is_varchar_in_types and is_max_length_equal
|
|
341
|
+
|
|
342
|
+
if not is_varchar_nvarchar_conversion:
|
|
343
|
+
forward_migration_list.append({forward_action: {"table_name": table_name, "column_name": column_name,
|
|
344
|
+
"column_type": column_new_type}})
|
|
345
|
+
backward_migration_list.append({backward_action: {"table_name": table_name, "column_name": column_name,
|
|
346
|
+
"column_type": column_type}})
|
|
347
|
+
|
|
348
|
+
return forward_migration_list, backward_migration_list
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
|
|
236
353
|
if not table_name:
|
|
237
354
|
raise ValueError("The 'table_name' argument must be a non-empty string.")
|
|
238
355
|
|
|
239
|
-
if not
|
|
240
|
-
# non-empty initial
|
|
241
|
-
columns, types = list(
|
|
242
|
-
|
|
243
|
-
|
|
356
|
+
if not old_column_type_dict and new_column_type_dict:
|
|
357
|
+
# non-empty initial column_type_dict --> empty new column_type_dict
|
|
358
|
+
columns, types = list(new_column_type_dict.keys()), list(new_column_type_dict.values())
|
|
359
|
+
forward_migration_list = [{"create": {"table_name": table_name, "columns": columns, "types": types}}]
|
|
360
|
+
backward_migration_list = [{"drop": {"table_name": table_name}}]
|
|
244
361
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
backward_migration = [{"create": {"table_name": table_name, "columns": columns, "types": types}}]
|
|
362
|
+
elif not new_column_type_dict:
|
|
363
|
+
# new column_type_dict is empty ==> drop the table
|
|
364
|
+
forward_migration_list = [{"drop": {"table_name": table_name}}]
|
|
365
|
+
backward_migration_list = [{"create": {"table_name": table_name, "columns": columns, "types": types}}]
|
|
250
366
|
|
|
251
|
-
migration = Migration(forward=forward_migration, backward=backward_migration)
|
|
252
|
-
else:
|
|
253
|
-
diff = DeepDiff(old_structure, new_structure, verbose_level=2)
|
|
254
|
-
migration = self._convert_deepdiff_dict_into_migration(table_name, diff)
|
|
255
367
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
def _convert_deepdiff_dict_into_migration(self, table_name: str, deepdiff_dict: dict) -> Migration:
|
|
261
|
-
"""
|
|
262
|
-
Converts deepdiff dictionary from the new and old table structures comparison into a Migration object.
|
|
263
|
-
|
|
264
|
-
Args:
|
|
265
|
-
table_name (str): A name of the examined DB table.
|
|
266
|
-
deepdiff_dict (dict): A dictionary from DeepDiff comparison of the old and new table structure.
|
|
267
|
-
|
|
268
|
-
Returns:
|
|
269
|
-
Migration: A Migration object containing forward and backward migrations for the given table.
|
|
368
|
+
else:
|
|
369
|
+
diff = DeepDiff(old_column_type_dict, new_column_type_dict, verbose_level=2)
|
|
370
|
+
forward_migration_list, backward_migration_list = _convert_deepdiff_dict_into_migration_lists(table_name, diff)
|
|
270
371
|
|
|
271
|
-
|
|
272
|
-
>>> table_name = 'results'
|
|
273
|
-
>>> deepdiff_dict = {'dictionary_item_removed': {"root['hehexd']": 'double'}}
|
|
274
|
-
>>> migrator = Migrator()
|
|
275
|
-
>>> asdict(migrator._convert_deepdiff_dict_into_migration)
|
|
276
|
-
>>> {
|
|
277
|
-
'forward': [
|
|
278
|
-
{'drop_column': {'table_name': 'results', 'column_name': 'hehexd'}}
|
|
279
|
-
],
|
|
280
|
-
'backward': [
|
|
281
|
-
{'add_column': {'table_name': 'results', 'column_name': 'hehexd', 'column_type': 'double'}}
|
|
282
|
-
]
|
|
283
|
-
}
|
|
284
|
-
"""
|
|
285
|
-
forward_migration, backward_migration = [], []
|
|
286
|
-
|
|
287
|
-
forward_conversions = {
|
|
288
|
-
"dictionary_item_added": "add_column",
|
|
289
|
-
"dictionary_item_removed": "drop_column",
|
|
290
|
-
"values_changed": "modify_column"
|
|
291
|
-
}
|
|
292
|
-
backward_conversions = {
|
|
293
|
-
"dictionary_item_added": "drop_column",
|
|
294
|
-
"dictionary_item_removed": "add_column",
|
|
295
|
-
"values_changed": "modify_column"
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
for action_name, deepdiff_action in deepdiff_dict.items():
|
|
299
|
-
for deepdiff_key in deepdiff_action.keys():
|
|
300
|
-
column_name = self._extract_column_name_from_deepdiff_key(deepdiff_key)
|
|
301
|
-
forward_action, backward_action = forward_conversions[action_name], backward_conversions[action_name]
|
|
302
|
-
|
|
303
|
-
if action_name=="dictionary_item_added":
|
|
304
|
-
column_type = deepdiff_action[deepdiff_key]
|
|
305
|
-
forward_migration.append({forward_action: {"table_name": table_name, "column_name": column_name, "column_type": column_type}})
|
|
306
|
-
backward_migration.append({backward_action: {"table_name": table_name, "column_name": column_name}})
|
|
307
|
-
elif action_name=="dictionary_item_removed":
|
|
308
|
-
column_type = deepdiff_action[deepdiff_key]
|
|
309
|
-
forward_migration.append({forward_action: {"table_name": table_name, "column_name": column_name}})
|
|
310
|
-
backward_migration.append({backward_action: {"table_name": table_name, "column_name": column_name, "column_type": column_type}})
|
|
311
|
-
elif action_name=="values_changed":
|
|
312
|
-
column_type = deepdiff_action[deepdiff_key]["old_value"]
|
|
313
|
-
column_new_type = deepdiff_action[deepdiff_key]["new_value"]
|
|
314
|
-
|
|
315
|
-
# HACK: Do not create migrations for cases such as varchar(2047) --> nvarchar(2047)
|
|
316
|
-
is_varchar_in_types = "varchar" in column_type and "varchar" in column_new_type
|
|
317
|
-
is_max_length_equal = (
|
|
318
|
-
column_type[column_type.index("("): column_type.index(")")]
|
|
319
|
-
and column_new_type[column_new_type.index("("): column_new_type.index(")")]
|
|
320
|
-
) if is_varchar_in_types else False
|
|
321
|
-
is_varchar_nvarchar_conversion = is_varchar_in_types and is_max_length_equal
|
|
322
|
-
|
|
323
|
-
if not is_varchar_nvarchar_conversion:
|
|
324
|
-
forward_migration.append({forward_action: {"table_name": table_name, "column_name": column_name,
|
|
325
|
-
"column_type": column_new_type}})
|
|
326
|
-
backward_migration.append({backward_action: {"table_name": table_name, "column_name": column_name,
|
|
327
|
-
"column_type": column_type}})
|
|
372
|
+
#migration = Migration(forward=forward_migration_list, backward=backward_migration_list)
|
|
328
373
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
def _extract_column_name_from_deepdiff_key(self, deepdiff_key: str) -> str:
|
|
332
|
-
"""
|
|
333
|
-
Extracts the column name from a key generated by deepdiff.
|
|
374
|
+
self._append_migration_to_pending_migration(forward_migration_list, backward_migration_list)
|
|
334
375
|
|
|
335
|
-
|
|
336
|
-
deepdiff_key (str): The key generated by deepdiff.
|
|
376
|
+
return forward_migration_list, backward_migration_list
|
|
337
377
|
|
|
338
|
-
Returns:
|
|
339
|
-
str: The extracted column name.
|
|
340
|
-
|
|
341
|
-
Example:
|
|
342
|
-
>>> migrator = Migrator()
|
|
343
|
-
>>> column_name = migrator._extract_column_name_from_deepdiff_key("root['table']['column']")
|
|
344
|
-
>>> print(column_name)
|
|
345
|
-
'column'
|
|
346
|
-
"""
|
|
347
|
-
|
|
348
|
-
# Split the item_key by '[' and ']' to isolate the column name
|
|
349
|
-
# The column name is expected to be the last element after splitting
|
|
350
|
-
column_name = deepdiff_key.split('[')[-1].strip("']")
|
|
351
|
-
return column_name
|
|
352
378
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _append_migration_to_pending_migration(self, forward_migration_list, backward_migration_list):
|
|
383
|
+
self._pending_forward_migration_list += forward_migration_list
|
|
384
|
+
self._pending_backward_migration_list += backward_migration_list
|
|
385
|
+
|
|
357
386
|
|
|
358
|
-
def
|
|
359
|
-
self.
|
|
387
|
+
def _clear_pending_migration(self):
|
|
388
|
+
self._pending_forward_migration_list = []
|
|
389
|
+
self._pending_backward_migration_list = []
|
|
360
390
|
|
|
361
|
-
def _read_migration_history_json(self, file_path: str = MIGRATION_HISTORY_DEFAULT_PATH):
|
|
362
|
-
|
|
363
|
-
|
|
391
|
+
# def _read_migration_history_json(self, file_path: str = MIGRATION_HISTORY_DEFAULT_PATH):
|
|
392
|
+
# if not file_path.endswith(".json"):
|
|
393
|
+
# raise ValueError("Migration history file must be of '.json' type.")
|
|
364
394
|
|
|
365
|
-
|
|
366
|
-
|
|
395
|
+
# if not os.path.exists(file_path):
|
|
396
|
+
# raise FileNotFoundError(f"Migration history file '{file_path}' does not exist.")
|
|
367
397
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
398
|
+
# try:
|
|
399
|
+
# with open(file_path, "r") as file:
|
|
400
|
+
# migration_history = json.load(file)
|
|
401
|
+
# except json.JSONDecodeError:
|
|
402
|
+
# migration_history = []
|
|
373
403
|
|
|
374
|
-
|
|
404
|
+
# return migration_history
|
|
375
405
|
|
|
376
|
-
def _save_migration_to_history(self, migration: Migration, file_path: str = MIGRATION_HISTORY_DEFAULT_PATH):
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
406
|
+
# def _save_migration_to_history(self, migration: Migration, file_path: str = MIGRATION_HISTORY_DEFAULT_PATH):
|
|
407
|
+
# try:
|
|
408
|
+
# migration_history = self._read_migration_history_json(file_path)
|
|
409
|
+
# except FileNotFoundError:
|
|
410
|
+
# self._build_folder_structure_for_file_path(file_path)
|
|
411
|
+
# migration_history = []
|
|
382
412
|
|
|
383
|
-
|
|
413
|
+
# migration_history.append(asdict(migration))
|
|
384
414
|
|
|
385
|
-
|
|
386
|
-
|
|
415
|
+
# with open(file_path, "w") as file:
|
|
416
|
+
# json.dump(migration_history, file, indent=2)
|
|
387
417
|
|
|
388
|
-
def _build_folder_structure_for_file_path(self, file_path: str):
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
418
|
+
# def _build_folder_structure_for_file_path(self, file_path: str):
|
|
419
|
+
# folder_path = os.path.dirname(file_path)
|
|
420
|
+
# if not os.path.exists(folder_path):
|
|
421
|
+
# print(f"Folder path to the file '{file_path}' does not exist. Creating the file and the folder structure.")
|
|
422
|
+
# os.makedirs(folder_path)
|
|
393
423
|
|
|
394
424
|
|
|
395
425
|
|
dbhydra/src/mysql_db.py
CHANGED
|
@@ -43,6 +43,9 @@ class MysqlDb(AbstractDb):
|
|
|
43
43
|
print("DB connection established")
|
|
44
44
|
|
|
45
45
|
def create_new_db(self):
|
|
46
|
+
self.connection = pymysql.connect(host=self.DB_SERVER, port=self.DB_PORT, user=self.DB_USERNAME,
|
|
47
|
+
charset="utf8mb4", password=self.DB_PASSWORD)
|
|
48
|
+
self.cursor = self.connection.cursor()
|
|
46
49
|
create_db_command = "CREATE DATABASE " + self.DB_DATABASE
|
|
47
50
|
self.execute(create_db_command)
|
|
48
51
|
|
dbhydra/src/tables.py
CHANGED
|
@@ -966,9 +966,8 @@ class XlsxTable(AbstractTable):
|
|
|
966
966
|
except Exception:
|
|
967
967
|
# print(f"Error while reading data into XlsxTable: {e}")
|
|
968
968
|
# df = pd.DataFrame(columns=self.columns)
|
|
969
|
-
if attempt < self.NUMBER_OF_RETRIES:
|
|
969
|
+
if attempt < self.NUMBER_OF_RETRIES - 1:
|
|
970
970
|
time.sleep(0.1)
|
|
971
|
-
continue
|
|
972
971
|
else:
|
|
973
972
|
print(f"Failed to read data from {self.table_directory_path}, returning empty DataFrame")
|
|
974
973
|
df = pd.DataFrame(columns=self.columns)
|
dbhydra/test_migrator.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import dbhydra.dbhydra_core as dh
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
db1=dh.MysqlDb("config-mysql.ini")
|
|
5
|
+
with db1.connect_to_db():
|
|
6
|
+
|
|
7
|
+
nodes_table = dh.MysqlTable(db1, "nodes",columns=["id","name"],types=["int","int"])
|
|
8
|
+
#nodes_table.create()
|
|
9
|
+
|
|
10
|
+
db1.initialize_migrator()
|
|
11
|
+
|
|
12
|
+
print(nodes_table.column_type_dict)
|
|
13
|
+
|
|
14
|
+
new_column_type_dict={"id":"int","name":"nvarchar","age":"int"}
|
|
15
|
+
|
|
16
|
+
migration1=db1.migrator.create_table_migration("nodes", nodes_table.column_type_dict, new_column_type_dict)
|
|
17
|
+
db1.migrator.save_current_migration_to_json()
|
|
18
|
+
migration2=db1.migrator.create_table_migration("nodes", new_column_type_dict, nodes_table.column_type_dict)
|
|
19
|
+
db1.migrator.save_current_migration_to_json()
|
|
20
|
+
print(migration1)
|
|
21
|
+
print(migration2)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
dbhydra/__init__.py,sha256=DCocEeXf4QxdVlBRlNiFvuP5IZJ5aa77_DbUR-_4C14,65
|
|
2
|
-
dbhydra/dbhydra_core.py,sha256=
|
|
2
|
+
dbhydra/dbhydra_core.py,sha256=26xBOo3sl--xFa-IrnE3AmBjB3ut5CXUJ1add438ups,2470
|
|
3
|
+
dbhydra/test_migrator.py,sha256=e3Nnb2mCd3CfjhjSexNg1tXVJMjkl5cCoYcuhbfZ4pM,803
|
|
3
4
|
dbhydra/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
5
|
dbhydra/src/abstract_db.py,sha256=lEP24vWz0HdGjQgSoHnawNK_NvRlSLuvzVSiiawssuw,5901
|
|
5
6
|
dbhydra/src/abstract_table.py,sha256=c3pkBTgMOLGDgLH4YEfqM0x33puh-gkML9Rid8xzdFs,17081
|
|
6
7
|
dbhydra/src/bigquery_db.py,sha256=77XsgvYbANlvYaJnuVve-kz-PNBx_CHoYCL-eYnA8e4,1834
|
|
7
|
-
dbhydra/src/migrator.py,sha256=
|
|
8
|
+
dbhydra/src/migrator.py,sha256=QzaODEFfraD9_6HN_Osaidaj-nLYQryCYYWwJtUu3n8,18931
|
|
8
9
|
dbhydra/src/mongo_db.py,sha256=mP48zRjI7mXKpm45R8prroZI-Eo7JKf0KJqGX-oTy3w,1922
|
|
9
|
-
dbhydra/src/mysql_db.py,sha256=
|
|
10
|
+
dbhydra/src/mysql_db.py,sha256=xFYy1Ty7iS3GXSncFoaKve4QN1SMJiuDjGyMbb-b1bw,3152
|
|
10
11
|
dbhydra/src/postgres_db.py,sha256=L7MaBq_6ArwDSP_5LaEqK58oLxZ1X7FgIokcDOSB7wk,1805
|
|
11
12
|
dbhydra/src/sqlserver_db.py,sha256=9Xi3NAliqM79MTV8fpNQb0nWMH8Bqjl1leJSEqgyT94,3611
|
|
12
|
-
dbhydra/src/tables.py,sha256=
|
|
13
|
+
dbhydra/src/tables.py,sha256=QZK76rv_d0MpXGNnuAezouXN8dO0nPrkVMmKVsHxj68,46656
|
|
13
14
|
dbhydra/src/xlsx_db.py,sha256=z6d-IjMYMmXC591Mt5DcxIYWyluanjPRFd-sXtjjXww,3514
|
|
14
15
|
dbhydra/src/errors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
16
|
dbhydra/src/errors/exceptions.py,sha256=LVpfbTd3NHfQIM-D5TFAU6hOZwGQ3b5DwFD4B6vtf2U,149
|
|
@@ -17,8 +18,8 @@ dbhydra/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
17
18
|
dbhydra/tests/test_cases.py,sha256=eAFGaHaIaab3md3HHm2_ryb_HHfObtcXDAEzLh4qWx8,508
|
|
18
19
|
dbhydra/tests/test_mongo.py,sha256=M8TD72M0iQAk7ZcLTWwLmcmmF_zwALnYEGTWjhQlq0s,1979
|
|
19
20
|
dbhydra/tests/test_sql.py,sha256=aPFXyA0jh8o9VG3B5f9fNz7qDbuVPZ9TcE2twn5dAeQ,3126
|
|
20
|
-
dbhydra-2.2.
|
|
21
|
-
dbhydra-2.2.
|
|
22
|
-
dbhydra-2.2.
|
|
23
|
-
dbhydra-2.2.
|
|
24
|
-
dbhydra-2.2.
|
|
21
|
+
dbhydra-2.2.1.dist-info/LICENSE,sha256=k49Yga8CP889JJaHlOpGFzr_be2nqMoep2chYeIDctk,1091
|
|
22
|
+
dbhydra-2.2.1.dist-info/METADATA,sha256=fXT5IdyIT6MA0US_YolueauSs0KAtZJE5392uF7G03c,2298
|
|
23
|
+
dbhydra-2.2.1.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
|
|
24
|
+
dbhydra-2.2.1.dist-info/top_level.txt,sha256=oO4Gf1T8_txIsIlp11GI0k7PtBIMb9GRwb5ObF4MLVg,8
|
|
25
|
+
dbhydra-2.2.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|