SQLPyHelper 0.1.6__tar.gz → 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/PKG-INFO +2 -2
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/README.md +1 -1
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/SQLPyHelper.egg-info/PKG-INFO +2 -2
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/SQLPyHelper.egg-info/SOURCES.txt +2 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/setup.py +1 -1
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/sqlpyhelper/__init__.py +2 -1
- sqlpyhelper-0.1.7/sqlpyhelper/migration.py +382 -0
- sqlpyhelper-0.1.7/test/test_migration.py +317 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/LICENSE +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/SQLPyHelper.egg-info/dependency_links.txt +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/SQLPyHelper.egg-info/entry_points.txt +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/SQLPyHelper.egg-info/requires.txt +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/SQLPyHelper.egg-info/top_level.txt +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/pyproject.toml +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/setup.cfg +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/sqlpyhelper/automation_utils.py +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/sqlpyhelper/cli.py +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/sqlpyhelper/db_helper.py +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/sqlpyhelper/py.typed +0 -0
- {sqlpyhelper-0.1.6 → sqlpyhelper-0.1.7}/test/test_sqlpyhelper.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SQLPyHelper
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.7
|
|
4
4
|
Summary: A simple SQL database helper package for Python.
|
|
5
5
|
Home-page: https://github.com/adebayopeter/sqlpyhelper
|
|
6
6
|
Author: Adebayo Olaonipekun
|
|
@@ -56,11 +56,11 @@ Dynamic: summary
|
|
|
56
56
|
# SQLPyHelper
|
|
57
57
|
|
|
58
58
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
59
|
+
[](https://sqlpyhelper.readthedocs.io/en/latest/)
|
|
59
60
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
60
61
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
61
62
|
[](https://github.com/adebayopeter/sqlpyhelper/blob/main/LICENSE)
|
|
62
63
|
[](https://github.com/adebayopeter/sqlpyhelper)
|
|
63
|
-
[](https://sqlpyhelper.readthedocs.io/en/latest/)
|
|
64
64
|
|
|
65
65
|
SQLPyHelper is a lightweight Python library that gives you a single, consistent API across **SQLite, PostgreSQL, MySQL, SQL Server, and Oracle** — without the overhead of an ORM.
|
|
66
66
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# SQLPyHelper
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
4
|
+
[](https://sqlpyhelper.readthedocs.io/en/latest/)
|
|
4
5
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
5
6
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
6
7
|
[](https://github.com/adebayopeter/sqlpyhelper/blob/main/LICENSE)
|
|
7
8
|
[](https://github.com/adebayopeter/sqlpyhelper)
|
|
8
|
-
[](https://sqlpyhelper.readthedocs.io/en/latest/)
|
|
9
9
|
|
|
10
10
|
SQLPyHelper is a lightweight Python library that gives you a single, consistent API across **SQLite, PostgreSQL, MySQL, SQL Server, and Oracle** — without the overhead of an ORM.
|
|
11
11
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SQLPyHelper
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.7
|
|
4
4
|
Summary: A simple SQL database helper package for Python.
|
|
5
5
|
Home-page: https://github.com/adebayopeter/sqlpyhelper
|
|
6
6
|
Author: Adebayo Olaonipekun
|
|
@@ -56,11 +56,11 @@ Dynamic: summary
|
|
|
56
56
|
# SQLPyHelper
|
|
57
57
|
|
|
58
58
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
59
|
+
[](https://sqlpyhelper.readthedocs.io/en/latest/)
|
|
59
60
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
60
61
|
[](https://pypi.org/project/sqlpyhelper/)
|
|
61
62
|
[](https://github.com/adebayopeter/sqlpyhelper/blob/main/LICENSE)
|
|
62
63
|
[](https://github.com/adebayopeter/sqlpyhelper)
|
|
63
|
-
[](https://sqlpyhelper.readthedocs.io/en/latest/)
|
|
64
64
|
|
|
65
65
|
SQLPyHelper is a lightweight Python library that gives you a single, consistent API across **SQLite, PostgreSQL, MySQL, SQL Server, and Oracle** — without the overhead of an ORM.
|
|
66
66
|
|
|
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as f:
|
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
7
|
name='SQLPyHelper',
|
|
8
|
-
version='0.1.
|
|
8
|
+
version='0.1.7',
|
|
9
9
|
description='A simple SQL database helper package for Python.',
|
|
10
10
|
long_description=long_description,
|
|
11
11
|
long_description_content_type="text/markdown",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Match the version in setup.py
|
|
2
|
-
__version__ = "0.1.
|
|
2
|
+
__version__ = "0.1.7"
|
|
3
3
|
|
|
4
4
|
from sqlpyhelper.db_helper import ( # noqa: F401
|
|
5
5
|
BackupError,
|
|
@@ -7,3 +7,4 @@ from sqlpyhelper.db_helper import ( # noqa: F401
|
|
|
7
7
|
QueryError,
|
|
8
8
|
SQLPyHelperError,
|
|
9
9
|
)
|
|
10
|
+
from sqlpyhelper.migration import MigrationError # noqa: F401
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""
|
|
2
|
+
sqlpyhelper.migration
|
|
3
|
+
~~~~~~~~~~~~~~~~~~~~~
|
|
4
|
+
Cross-database table migration utilities.
|
|
5
|
+
|
|
6
|
+
Copies data (and optionally schema) from one database to another.
|
|
7
|
+
Supports SQLite, PostgreSQL, MySQL, SQL Server, and Oracle.
|
|
8
|
+
|
|
9
|
+
Example usage::
|
|
10
|
+
|
|
11
|
+
from sqlpyhelper.db_helper import SQLPyHelper
|
|
12
|
+
from sqlpyhelper.migration import migrate_table
|
|
13
|
+
|
|
14
|
+
with SQLPyHelper(db_type="sqlite", database="local.db") as source:
|
|
15
|
+
with SQLPyHelper(db_type="postgres", host="localhost",
|
|
16
|
+
user="user", password="pass",
|
|
17
|
+
database="mydb") as target:
|
|
18
|
+
|
|
19
|
+
migrate_table(
|
|
20
|
+
source=source,
|
|
21
|
+
target=target,
|
|
22
|
+
table="users",
|
|
23
|
+
)
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import logging
|
|
27
|
+
from typing import Any, Optional
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger("sqlpyhelper.migration")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class MigrationError(Exception):
|
|
33
|
+
"""Raised when a migration operation fails."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------------------
|
|
37
|
+
# Type mapping
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
# Maps (source_db_type, generic_type) -> target SQL type string.
|
|
41
|
+
# Generic types are normalised from the source cursor description.
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
_TYPE_MAP: dict[str, dict[str, str]] = {
|
|
45
|
+
"sqlite": {
|
|
46
|
+
"integer": "INTEGER",
|
|
47
|
+
"real": "REAL",
|
|
48
|
+
"text": "TEXT",
|
|
49
|
+
"blob": "BLOB",
|
|
50
|
+
"numeric": "NUMERIC",
|
|
51
|
+
},
|
|
52
|
+
"postgres": {
|
|
53
|
+
"integer": "INTEGER",
|
|
54
|
+
"real": "DOUBLE PRECISION",
|
|
55
|
+
"text": "TEXT",
|
|
56
|
+
"blob": "BYTEA",
|
|
57
|
+
"numeric": "NUMERIC",
|
|
58
|
+
"varchar": "TEXT",
|
|
59
|
+
"bool": "BOOLEAN",
|
|
60
|
+
"date": "DATE",
|
|
61
|
+
"timestamp": "TIMESTAMP",
|
|
62
|
+
},
|
|
63
|
+
"mysql": {
|
|
64
|
+
"integer": "INT",
|
|
65
|
+
"real": "DOUBLE",
|
|
66
|
+
"text": "TEXT",
|
|
67
|
+
"blob": "BLOB",
|
|
68
|
+
"numeric": "DECIMAL",
|
|
69
|
+
"varchar": "VARCHAR(255)",
|
|
70
|
+
"bool": "TINYINT(1)",
|
|
71
|
+
"date": "DATE",
|
|
72
|
+
"timestamp": "DATETIME",
|
|
73
|
+
},
|
|
74
|
+
"sqlserver": {
|
|
75
|
+
"integer": "INT",
|
|
76
|
+
"real": "FLOAT",
|
|
77
|
+
"text": "NVARCHAR(MAX)",
|
|
78
|
+
"blob": "VARBINARY(MAX)",
|
|
79
|
+
"numeric": "DECIMAL",
|
|
80
|
+
"varchar": "NVARCHAR(255)",
|
|
81
|
+
"bool": "BIT",
|
|
82
|
+
"date": "DATE",
|
|
83
|
+
"timestamp": "DATETIME2",
|
|
84
|
+
},
|
|
85
|
+
"oracle": {
|
|
86
|
+
"integer": "NUMBER",
|
|
87
|
+
"real": "FLOAT",
|
|
88
|
+
"text": "CLOB",
|
|
89
|
+
"blob": "BLOB",
|
|
90
|
+
"numeric": "NUMBER",
|
|
91
|
+
"varchar": "VARCHAR2(255)",
|
|
92
|
+
"bool": "NUMBER(1)",
|
|
93
|
+
"date": "DATE",
|
|
94
|
+
"timestamp": "TIMESTAMP",
|
|
95
|
+
},
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _normalise_type(raw_type: Optional[str]) -> str:
|
|
100
|
+
"""
|
|
101
|
+
Normalise a raw column type string from a cursor description
|
|
102
|
+
into a generic type key used for cross-database mapping.
|
|
103
|
+
"""
|
|
104
|
+
if raw_type is None:
|
|
105
|
+
return "text"
|
|
106
|
+
t = raw_type.lower().split("(")[0].strip()
|
|
107
|
+
if t in (
|
|
108
|
+
"int",
|
|
109
|
+
"integer",
|
|
110
|
+
"int4",
|
|
111
|
+
"int8",
|
|
112
|
+
"bigint",
|
|
113
|
+
"smallint",
|
|
114
|
+
"tinyint",
|
|
115
|
+
"number",
|
|
116
|
+
):
|
|
117
|
+
return "integer"
|
|
118
|
+
if t in ("real", "float", "double", "double precision", "float4", "float8"):
|
|
119
|
+
return "real"
|
|
120
|
+
if t in (
|
|
121
|
+
"text",
|
|
122
|
+
"clob",
|
|
123
|
+
"nvarchar",
|
|
124
|
+
"nvarchar2",
|
|
125
|
+
"ntext",
|
|
126
|
+
"longtext",
|
|
127
|
+
"mediumtext",
|
|
128
|
+
):
|
|
129
|
+
return "text"
|
|
130
|
+
if t in ("varchar", "varchar2", "character varying", "char", "nchar"):
|
|
131
|
+
return "varchar"
|
|
132
|
+
if t in ("blob", "bytea", "varbinary", "binary", "longblob", "mediumblob"):
|
|
133
|
+
return "blob"
|
|
134
|
+
if t in ("numeric", "decimal"):
|
|
135
|
+
return "numeric"
|
|
136
|
+
if t in ("bool", "boolean"):
|
|
137
|
+
return "bool"
|
|
138
|
+
if t in ("date",):
|
|
139
|
+
return "date"
|
|
140
|
+
if t in ("timestamp", "datetime", "datetime2", "timestamp without time zone"):
|
|
141
|
+
return "timestamp"
|
|
142
|
+
return "text" # safe fallback
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _map_type(raw_type: Optional[str], target_db: str) -> str:
|
|
146
|
+
"""Map a source column type to the appropriate type for the target database."""
|
|
147
|
+
generic = _normalise_type(raw_type)
|
|
148
|
+
target_types = _TYPE_MAP.get(target_db, _TYPE_MAP["sqlite"])
|
|
149
|
+
return target_types.get(generic, "TEXT")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _get_column_info(source: Any, table: str) -> list[tuple[str, str]]:
|
|
153
|
+
"""
|
|
154
|
+
Return a list of (column_name, raw_type_string) tuples
|
|
155
|
+
by inspecting the source database schema.
|
|
156
|
+
"""
|
|
157
|
+
db_type = source.db_type
|
|
158
|
+
|
|
159
|
+
if db_type == "sqlite":
|
|
160
|
+
source.execute_query(f"PRAGMA table_info({table})")
|
|
161
|
+
rows = source.fetch_all()
|
|
162
|
+
# PRAGMA table_info returns: (cid, name, type, notnull, dflt_value, pk)
|
|
163
|
+
return [(row[1], row[2]) for row in rows]
|
|
164
|
+
|
|
165
|
+
elif db_type == "postgres":
|
|
166
|
+
source.execute_query(
|
|
167
|
+
"""
|
|
168
|
+
SELECT column_name, data_type
|
|
169
|
+
FROM information_schema.columns
|
|
170
|
+
WHERE table_name = %s
|
|
171
|
+
ORDER BY ordinal_position
|
|
172
|
+
""",
|
|
173
|
+
(table,),
|
|
174
|
+
)
|
|
175
|
+
return source.fetch_all()
|
|
176
|
+
|
|
177
|
+
elif db_type == "mysql":
|
|
178
|
+
source.execute_query(
|
|
179
|
+
"""
|
|
180
|
+
SELECT column_name, data_type
|
|
181
|
+
FROM information_schema.columns
|
|
182
|
+
WHERE table_name = %s AND table_schema = DATABASE()
|
|
183
|
+
ORDER BY ordinal_position
|
|
184
|
+
""",
|
|
185
|
+
(table,),
|
|
186
|
+
)
|
|
187
|
+
return source.fetch_all()
|
|
188
|
+
|
|
189
|
+
elif db_type == "sqlserver":
|
|
190
|
+
source.execute_query(
|
|
191
|
+
"""
|
|
192
|
+
SELECT column_name, data_type
|
|
193
|
+
FROM information_schema.columns
|
|
194
|
+
WHERE table_name = %s
|
|
195
|
+
ORDER BY ordinal_position
|
|
196
|
+
""",
|
|
197
|
+
(table,),
|
|
198
|
+
)
|
|
199
|
+
return source.fetch_all()
|
|
200
|
+
|
|
201
|
+
elif db_type == "oracle":
|
|
202
|
+
source.execute_query(
|
|
203
|
+
"""
|
|
204
|
+
SELECT column_name, data_type
|
|
205
|
+
FROM user_tab_columns
|
|
206
|
+
WHERE table_name = UPPER(:1)
|
|
207
|
+
ORDER BY column_id
|
|
208
|
+
""",
|
|
209
|
+
(table,),
|
|
210
|
+
)
|
|
211
|
+
return source.fetch_all()
|
|
212
|
+
|
|
213
|
+
else:
|
|
214
|
+
raise MigrationError(f"Cannot inspect schema for db_type={db_type!r}")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _build_create_table_sql(
|
|
218
|
+
table: str,
|
|
219
|
+
columns: list[tuple[str, str]],
|
|
220
|
+
target_db: str,
|
|
221
|
+
) -> str:
|
|
222
|
+
"""Build a CREATE TABLE IF NOT EXISTS statement for the target database."""
|
|
223
|
+
col_defs = ", ".join(
|
|
224
|
+
f"{col_name} {_map_type(col_type, target_db)}" for col_name, col_type in columns
|
|
225
|
+
)
|
|
226
|
+
return f"CREATE TABLE IF NOT EXISTS {table} ({col_defs})"
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _build_insert_sql(
|
|
230
|
+
table: str,
|
|
231
|
+
column_names: list[str],
|
|
232
|
+
target_db: str,
|
|
233
|
+
) -> str:
|
|
234
|
+
"""Build a parameterised INSERT statement for the target database."""
|
|
235
|
+
cols = ", ".join(column_names)
|
|
236
|
+
placeholder = "?" if target_db == "sqlite" else "%s"
|
|
237
|
+
placeholders = ", ".join([placeholder] * len(column_names))
|
|
238
|
+
return f"INSERT INTO {table} ({cols}) VALUES ({placeholders})"
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
# ---------------------------------------------------------------------------
|
|
242
|
+
# Public API
|
|
243
|
+
# ---------------------------------------------------------------------------
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def migrate_table(
|
|
247
|
+
source: Any,
|
|
248
|
+
target: Any,
|
|
249
|
+
table: str,
|
|
250
|
+
create_table: bool = True,
|
|
251
|
+
batch_size: int = 500,
|
|
252
|
+
truncate_target: bool = False,
|
|
253
|
+
) -> dict[str, Any]:
|
|
254
|
+
"""
|
|
255
|
+
Migrate a table from one database to another.
|
|
256
|
+
|
|
257
|
+
Copies all rows from ``source`` to ``target``. Optionally creates
|
|
258
|
+
the target table using best-effort type mapping from the source schema.
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
source: A connected SQLPyHelper instance (the data source).
|
|
262
|
+
target: A connected SQLPyHelper instance (the destination).
|
|
263
|
+
table: Name of the table to migrate.
|
|
264
|
+
create_table: If True, creates the table in the target database
|
|
265
|
+
using best-effort type mapping. If False, the table
|
|
266
|
+
must already exist in the target. Default: True.
|
|
267
|
+
batch_size: Number of rows to insert per batch. Default: 500.
|
|
268
|
+
truncate_target: If True, deletes all existing rows in the target
|
|
269
|
+
table before inserting. Default: False.
|
|
270
|
+
|
|
271
|
+
Returns:
|
|
272
|
+
A dict with migration statistics::
|
|
273
|
+
|
|
274
|
+
{
|
|
275
|
+
"table": "users",
|
|
276
|
+
"rows_migrated": 1234,
|
|
277
|
+
"batches": 3,
|
|
278
|
+
"source_db": "sqlite",
|
|
279
|
+
"target_db": "postgres",
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
Raises:
|
|
283
|
+
MigrationError: If the migration fails for any reason.
|
|
284
|
+
|
|
285
|
+
Example::
|
|
286
|
+
|
|
287
|
+
from sqlpyhelper.db_helper import SQLPyHelper
|
|
288
|
+
from sqlpyhelper.migration import migrate_table
|
|
289
|
+
|
|
290
|
+
with SQLPyHelper(db_type="sqlite", database="local.db") as source:
|
|
291
|
+
with SQLPyHelper(db_type="postgres", host="localhost",
|
|
292
|
+
user="user", password="pass",
|
|
293
|
+
database="mydb") as target:
|
|
294
|
+
|
|
295
|
+
stats = migrate_table(
|
|
296
|
+
source=source,
|
|
297
|
+
target=target,
|
|
298
|
+
table="users",
|
|
299
|
+
create_table=True,
|
|
300
|
+
batch_size=1000,
|
|
301
|
+
)
|
|
302
|
+
print(f"Migrated {stats['rows_migrated']} rows")
|
|
303
|
+
"""
|
|
304
|
+
source_db = source.db_type
|
|
305
|
+
target_db = target.db_type
|
|
306
|
+
|
|
307
|
+
logger.info(
|
|
308
|
+
"Starting migration of table '%s' from %s -> %s",
|
|
309
|
+
table,
|
|
310
|
+
source_db,
|
|
311
|
+
target_db,
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
try:
|
|
315
|
+
# Step 1 — fetch column info from source
|
|
316
|
+
columns = _get_column_info(source, table)
|
|
317
|
+
if not columns:
|
|
318
|
+
raise MigrationError(
|
|
319
|
+
f"Table '{table}' not found in source database " f"or has no columns."
|
|
320
|
+
)
|
|
321
|
+
column_names = [col[0] for col in columns]
|
|
322
|
+
logger.info("Found %d columns: %s", len(columns), column_names)
|
|
323
|
+
|
|
324
|
+
# Step 2 — optionally create the table in the target
|
|
325
|
+
if create_table:
|
|
326
|
+
ddl = _build_create_table_sql(table, columns, target_db)
|
|
327
|
+
logger.info("Creating target table: %s", ddl)
|
|
328
|
+
target.execute_query(ddl)
|
|
329
|
+
|
|
330
|
+
# Step 3 — optionally truncate the target table
|
|
331
|
+
if truncate_target:
|
|
332
|
+
if target_db == "sqlite":
|
|
333
|
+
target.execute_query(f"DELETE FROM {table}")
|
|
334
|
+
else:
|
|
335
|
+
target.execute_query(f"TRUNCATE TABLE {table}")
|
|
336
|
+
logger.info("Truncated target table '%s'", table)
|
|
337
|
+
|
|
338
|
+
# Step 4 — fetch all rows from source
|
|
339
|
+
source.execute_query(f"SELECT * FROM {table}")
|
|
340
|
+
all_rows = source.fetch_all()
|
|
341
|
+
total_rows = len(all_rows)
|
|
342
|
+
logger.info("Fetched %d rows from source", total_rows)
|
|
343
|
+
|
|
344
|
+
if total_rows == 0:
|
|
345
|
+
logger.info("Source table '%s' is empty — nothing to migrate", table)
|
|
346
|
+
return {
|
|
347
|
+
"table": table,
|
|
348
|
+
"rows_migrated": 0,
|
|
349
|
+
"batches": 0,
|
|
350
|
+
"source_db": source_db,
|
|
351
|
+
"target_db": target_db,
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
# Step 5 — insert in batches
|
|
355
|
+
insert_sql = _build_insert_sql(table, column_names, target_db)
|
|
356
|
+
batches = 0
|
|
357
|
+
for i in range(0, total_rows, batch_size):
|
|
358
|
+
batch = all_rows[i : i + batch_size]
|
|
359
|
+
target.cursor.executemany(insert_sql, batch)
|
|
360
|
+
target.connection.commit()
|
|
361
|
+
batches += 1
|
|
362
|
+
logger.info(
|
|
363
|
+
"Inserted batch %d (%d/%d rows)",
|
|
364
|
+
batches,
|
|
365
|
+
min(i + batch_size, total_rows),
|
|
366
|
+
total_rows,
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
logger.info("Migration complete: %d rows in %d batches", total_rows, batches)
|
|
370
|
+
|
|
371
|
+
return {
|
|
372
|
+
"table": table,
|
|
373
|
+
"rows_migrated": total_rows,
|
|
374
|
+
"batches": batches,
|
|
375
|
+
"source_db": source_db,
|
|
376
|
+
"target_db": target_db,
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
except MigrationError:
|
|
380
|
+
raise
|
|
381
|
+
except Exception as e:
|
|
382
|
+
raise MigrationError(f"Migration of '{table}' failed: {e}") from e
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tests for sqlpyhelper.migration
|
|
3
|
+
All tests use mocking — no live database required.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from unittest.mock import MagicMock
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from sqlpyhelper.migration import (
|
|
11
|
+
MigrationError,
|
|
12
|
+
_build_create_table_sql,
|
|
13
|
+
_build_insert_sql,
|
|
14
|
+
_map_type,
|
|
15
|
+
_normalise_type,
|
|
16
|
+
migrate_table,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# ---------------------------------------------------------------------------
|
|
20
|
+
# _normalise_type
|
|
21
|
+
# ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class TestNormaliseType:
|
|
25
|
+
def test_integer_variants(self):
|
|
26
|
+
for t in ("int", "integer", "bigint", "smallint", "tinyint", "number"):
|
|
27
|
+
assert _normalise_type(t) == "integer"
|
|
28
|
+
|
|
29
|
+
def test_real_variants(self):
|
|
30
|
+
for t in ("real", "float", "double", "double precision"):
|
|
31
|
+
assert _normalise_type(t) == "real"
|
|
32
|
+
|
|
33
|
+
def test_text_variants(self):
|
|
34
|
+
for t in ("text", "clob", "nvarchar", "longtext"):
|
|
35
|
+
assert _normalise_type(t) == "text"
|
|
36
|
+
|
|
37
|
+
def test_varchar_variants(self):
|
|
38
|
+
for t in ("varchar", "varchar2", "character varying", "char"):
|
|
39
|
+
assert _normalise_type(t) == "varchar"
|
|
40
|
+
|
|
41
|
+
def test_blob_variants(self):
|
|
42
|
+
for t in ("blob", "bytea", "varbinary"):
|
|
43
|
+
assert _normalise_type(t) == "blob"
|
|
44
|
+
|
|
45
|
+
def test_bool_variants(self):
|
|
46
|
+
for t in ("bool", "boolean"):
|
|
47
|
+
assert _normalise_type(t) == "bool"
|
|
48
|
+
|
|
49
|
+
def test_none_returns_text(self):
|
|
50
|
+
assert _normalise_type(None) == "text"
|
|
51
|
+
|
|
52
|
+
def test_unknown_returns_text(self):
|
|
53
|
+
assert _normalise_type("jsonb") == "text"
|
|
54
|
+
|
|
55
|
+
def test_strips_length_spec(self):
|
|
56
|
+
assert _normalise_type("varchar(255)") == "varchar"
|
|
57
|
+
assert _normalise_type("numeric(10,2)") == "numeric"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# ---------------------------------------------------------------------------
|
|
61
|
+
# _map_type
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class TestMapType:
|
|
66
|
+
def test_sqlite_integer(self):
|
|
67
|
+
assert _map_type("integer", "sqlite") == "INTEGER"
|
|
68
|
+
|
|
69
|
+
def test_postgres_integer(self):
|
|
70
|
+
assert _map_type("integer", "postgres") == "INTEGER"
|
|
71
|
+
|
|
72
|
+
def test_mysql_integer(self):
|
|
73
|
+
assert _map_type("integer", "mysql") == "INT"
|
|
74
|
+
|
|
75
|
+
def test_sqlserver_integer(self):
|
|
76
|
+
assert _map_type("integer", "sqlserver") == "INT"
|
|
77
|
+
|
|
78
|
+
def test_oracle_integer(self):
|
|
79
|
+
assert _map_type("integer", "oracle") == "NUMBER"
|
|
80
|
+
|
|
81
|
+
def test_postgres_text(self):
|
|
82
|
+
assert _map_type("text", "postgres") == "TEXT"
|
|
83
|
+
|
|
84
|
+
def test_sqlserver_text(self):
|
|
85
|
+
assert _map_type("text", "sqlserver") == "NVARCHAR(MAX)"
|
|
86
|
+
|
|
87
|
+
def test_oracle_text(self):
|
|
88
|
+
assert _map_type("text", "oracle") == "CLOB"
|
|
89
|
+
|
|
90
|
+
def test_unknown_db_falls_back(self):
|
|
91
|
+
result = _map_type("integer", "unknowndb")
|
|
92
|
+
assert result == "INTEGER"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
# ---------------------------------------------------------------------------
|
|
96
|
+
# _build_create_table_sql
|
|
97
|
+
# ---------------------------------------------------------------------------
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class TestBuildCreateTableSql:
|
|
101
|
+
def test_sqlite_ddl(self):
|
|
102
|
+
columns = [("id", "integer"), ("name", "text")]
|
|
103
|
+
sql = _build_create_table_sql("users", columns, "sqlite")
|
|
104
|
+
assert "CREATE TABLE IF NOT EXISTS users" in sql
|
|
105
|
+
assert "id INTEGER" in sql
|
|
106
|
+
assert "name TEXT" in sql
|
|
107
|
+
|
|
108
|
+
def test_postgres_ddl(self):
|
|
109
|
+
columns = [("id", "integer"), ("name", "varchar")]
|
|
110
|
+
sql = _build_create_table_sql("users", columns, "postgres")
|
|
111
|
+
assert "id INTEGER" in sql
|
|
112
|
+
assert "name TEXT" in sql
|
|
113
|
+
|
|
114
|
+
def test_mysql_ddl(self):
|
|
115
|
+
columns = [("id", "integer"), ("name", "text")]
|
|
116
|
+
sql = _build_create_table_sql("users", columns, "mysql")
|
|
117
|
+
assert "id INT" in sql
|
|
118
|
+
assert "name TEXT" in sql
|
|
119
|
+
|
|
120
|
+
def test_sqlserver_ddl(self):
|
|
121
|
+
columns = [("id", "integer"), ("notes", "text")]
|
|
122
|
+
sql = _build_create_table_sql("orders", columns, "sqlserver")
|
|
123
|
+
assert "CREATE TABLE IF NOT EXISTS orders" in sql
|
|
124
|
+
assert "id INT" in sql
|
|
125
|
+
assert "notes NVARCHAR(MAX)" in sql
|
|
126
|
+
|
|
127
|
+
def test_oracle_ddl(self):
|
|
128
|
+
columns = [("id", "integer"), ("name", "varchar")]
|
|
129
|
+
sql = _build_create_table_sql("users", columns, "oracle")
|
|
130
|
+
assert "id NUMBER" in sql
|
|
131
|
+
assert "name VARCHAR2(255)" in sql
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ---------------------------------------------------------------------------
|
|
135
|
+
# _build_insert_sql
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class TestBuildInsertSql:
|
|
140
|
+
def test_sqlite_uses_question_mark(self):
|
|
141
|
+
sql = _build_insert_sql("users", ["id", "name"], "sqlite")
|
|
142
|
+
assert "?" in sql
|
|
143
|
+
assert "%s" not in sql
|
|
144
|
+
|
|
145
|
+
def test_postgres_uses_percent_s(self):
|
|
146
|
+
sql = _build_insert_sql("users", ["id", "name"], "postgres")
|
|
147
|
+
assert "%s" in sql
|
|
148
|
+
assert "?" not in sql
|
|
149
|
+
|
|
150
|
+
def test_correct_column_count(self):
|
|
151
|
+
sql = _build_insert_sql("users", ["id", "name", "email"], "mysql")
|
|
152
|
+
assert sql.count("%s") == 3
|
|
153
|
+
|
|
154
|
+
def test_table_name_in_sql(self):
|
|
155
|
+
sql = _build_insert_sql("orders", ["id", "item"], "postgres")
|
|
156
|
+
assert "INSERT INTO orders" in sql
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# ---------------------------------------------------------------------------
|
|
160
|
+
# migrate_table
|
|
161
|
+
# ---------------------------------------------------------------------------
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def make_mock_db(db_type="sqlite"):
|
|
165
|
+
"""Create a mock SQLPyHelper instance."""
|
|
166
|
+
db = MagicMock()
|
|
167
|
+
db.db_type = db_type
|
|
168
|
+
db.cursor = MagicMock()
|
|
169
|
+
db.connection = MagicMock()
|
|
170
|
+
return db
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class TestMigrateTable:
|
|
174
|
+
def test_basic_sqlite_to_postgres(self):
|
|
175
|
+
source = make_mock_db("sqlite")
|
|
176
|
+
target = make_mock_db("postgres")
|
|
177
|
+
|
|
178
|
+
# Mock PRAGMA response for SQLite column info
|
|
179
|
+
source.fetch_all.side_effect = [
|
|
180
|
+
[(0, "id", "integer", 0, None, 1), (1, "name", "text", 0, None, 0)],
|
|
181
|
+
[(1, "Alice"), (2, "Bob")],
|
|
182
|
+
]
|
|
183
|
+
|
|
184
|
+
stats = migrate_table(source=source, target=target, table="users")
|
|
185
|
+
|
|
186
|
+
assert stats["rows_migrated"] == 2
|
|
187
|
+
assert stats["source_db"] == "sqlite"
|
|
188
|
+
assert stats["target_db"] == "postgres"
|
|
189
|
+
assert stats["table"] == "users"
|
|
190
|
+
assert stats["batches"] == 1
|
|
191
|
+
|
|
192
|
+
def test_create_table_called_when_flag_true(self):
|
|
193
|
+
source = make_mock_db("sqlite")
|
|
194
|
+
target = make_mock_db("postgres")
|
|
195
|
+
|
|
196
|
+
source.fetch_all.side_effect = [
|
|
197
|
+
[(0, "id", "integer", 0, None, 1)],
|
|
198
|
+
[(1,), (2,)],
|
|
199
|
+
]
|
|
200
|
+
|
|
201
|
+
migrate_table(source=source, target=target, table="users", create_table=True)
|
|
202
|
+
target.execute_query.assert_called()
|
|
203
|
+
first_call = target.execute_query.call_args_list[0][0][0]
|
|
204
|
+
assert "CREATE TABLE IF NOT EXISTS users" in first_call
|
|
205
|
+
|
|
206
|
+
def test_create_table_not_called_when_flag_false(self):
|
|
207
|
+
source = make_mock_db("sqlite")
|
|
208
|
+
target = make_mock_db("postgres")
|
|
209
|
+
|
|
210
|
+
source.fetch_all.side_effect = [
|
|
211
|
+
[(0, "id", "integer", 0, None, 1)],
|
|
212
|
+
[(1,)],
|
|
213
|
+
]
|
|
214
|
+
|
|
215
|
+
migrate_table(source=source, target=target, table="users", create_table=False)
|
|
216
|
+
# execute_query should not be called on target for DDL
|
|
217
|
+
for call in target.execute_query.call_args_list:
|
|
218
|
+
assert "CREATE TABLE" not in str(call)
|
|
219
|
+
|
|
220
|
+
def test_empty_table_returns_zero_rows(self):
|
|
221
|
+
source = make_mock_db("sqlite")
|
|
222
|
+
target = make_mock_db("postgres")
|
|
223
|
+
|
|
224
|
+
source.fetch_all.side_effect = [
|
|
225
|
+
[(0, "id", "integer", 0, None, 1)],
|
|
226
|
+
[],
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
stats = migrate_table(source=source, target=target, table="users")
|
|
230
|
+
assert stats["rows_migrated"] == 0
|
|
231
|
+
assert stats["batches"] == 0
|
|
232
|
+
target.cursor.executemany.assert_not_called()
|
|
233
|
+
|
|
234
|
+
def test_batching(self):
|
|
235
|
+
source = make_mock_db("sqlite")
|
|
236
|
+
target = make_mock_db("postgres")
|
|
237
|
+
|
|
238
|
+
rows = [(i,) for i in range(25)]
|
|
239
|
+
source.fetch_all.side_effect = [
|
|
240
|
+
[(0, "id", "integer", 0, None, 1)],
|
|
241
|
+
rows,
|
|
242
|
+
]
|
|
243
|
+
|
|
244
|
+
stats = migrate_table(
|
|
245
|
+
source=source, target=target, table="users", batch_size=10
|
|
246
|
+
)
|
|
247
|
+
assert stats["rows_migrated"] == 25
|
|
248
|
+
assert stats["batches"] == 3
|
|
249
|
+
assert target.cursor.executemany.call_count == 3
|
|
250
|
+
|
|
251
|
+
def test_truncate_target_sqlite(self):
|
|
252
|
+
source = make_mock_db("sqlite")
|
|
253
|
+
target = make_mock_db("sqlite")
|
|
254
|
+
|
|
255
|
+
source.fetch_all.side_effect = [
|
|
256
|
+
[(0, "id", "integer", 0, None, 1)],
|
|
257
|
+
[(1,)],
|
|
258
|
+
]
|
|
259
|
+
|
|
260
|
+
migrate_table(source=source, target=target, table="users", truncate_target=True)
|
|
261
|
+
delete_calls = [
|
|
262
|
+
str(c) for c in target.execute_query.call_args_list if "DELETE" in str(c)
|
|
263
|
+
]
|
|
264
|
+
assert len(delete_calls) == 1
|
|
265
|
+
|
|
266
|
+
def test_truncate_target_postgres(self):
|
|
267
|
+
source = make_mock_db("sqlite")
|
|
268
|
+
target = make_mock_db("postgres")
|
|
269
|
+
|
|
270
|
+
source.fetch_all.side_effect = [
|
|
271
|
+
[(0, "id", "integer", 0, None, 1)],
|
|
272
|
+
[(1,)],
|
|
273
|
+
]
|
|
274
|
+
|
|
275
|
+
migrate_table(source=source, target=target, table="users", truncate_target=True)
|
|
276
|
+
truncate_calls = [
|
|
277
|
+
str(c) for c in target.execute_query.call_args_list if "TRUNCATE" in str(c)
|
|
278
|
+
]
|
|
279
|
+
assert len(truncate_calls) == 1
|
|
280
|
+
|
|
281
|
+
def test_raises_migration_error_on_empty_columns(self):
|
|
282
|
+
source = make_mock_db("sqlite")
|
|
283
|
+
target = make_mock_db("postgres")
|
|
284
|
+
|
|
285
|
+
source.fetch_all.return_value = []
|
|
286
|
+
|
|
287
|
+
with pytest.raises(MigrationError, match="not found in source database"):
|
|
288
|
+
migrate_table(source=source, target=target, table="nonexistent")
|
|
289
|
+
|
|
290
|
+
def test_raises_migration_error_on_db_failure(self):
|
|
291
|
+
source = make_mock_db("sqlite")
|
|
292
|
+
target = make_mock_db("postgres")
|
|
293
|
+
|
|
294
|
+
source.fetch_all.side_effect = Exception("connection lost")
|
|
295
|
+
|
|
296
|
+
with pytest.raises(MigrationError, match="Migration of 'users' failed"):
|
|
297
|
+
migrate_table(source=source, target=target, table="users")
|
|
298
|
+
|
|
299
|
+
def test_returns_correct_stats_structure(self):
|
|
300
|
+
source = make_mock_db("mysql")
|
|
301
|
+
target = make_mock_db("sqlserver")
|
|
302
|
+
|
|
303
|
+
source.fetch_all.side_effect = [
|
|
304
|
+
[("id", "int"), ("name", "varchar")],
|
|
305
|
+
[(1, "Alice"), (2, "Bob"), (3, "Charlie")],
|
|
306
|
+
]
|
|
307
|
+
|
|
308
|
+
stats = migrate_table(source=source, target=target, table="customers")
|
|
309
|
+
assert set(stats.keys()) == {
|
|
310
|
+
"table",
|
|
311
|
+
"rows_migrated",
|
|
312
|
+
"batches",
|
|
313
|
+
"source_db",
|
|
314
|
+
"target_db",
|
|
315
|
+
}
|
|
316
|
+
assert stats["source_db"] == "mysql"
|
|
317
|
+
assert stats["target_db"] == "sqlserver"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|