meerschaum 2.6.0.dev1__py3-none-any.whl → 2.6.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/api/dash/pages/login.py +17 -17
- meerschaum/api/dash/pipes.py +13 -4
- meerschaum/api/routes/_pipes.py +162 -136
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +1 -0
- meerschaum/connectors/api/_APIConnector.py +1 -0
- meerschaum/connectors/api/_pipes.py +46 -13
- meerschaum/connectors/sql/_SQLConnector.py +4 -3
- meerschaum/connectors/sql/_fetch.py +4 -2
- meerschaum/connectors/sql/_pipes.py +496 -147
- meerschaum/connectors/sql/_sql.py +37 -16
- meerschaum/connectors/valkey/_ValkeyConnector.py +3 -2
- meerschaum/connectors/valkey/_pipes.py +13 -5
- meerschaum/core/Pipe/__init__.py +20 -0
- meerschaum/core/Pipe/_attributes.py +179 -9
- meerschaum/core/Pipe/_clear.py +10 -8
- meerschaum/core/Pipe/_copy.py +2 -0
- meerschaum/core/Pipe/_data.py +57 -28
- meerschaum/core/Pipe/_deduplicate.py +30 -28
- meerschaum/core/Pipe/_dtypes.py +12 -2
- meerschaum/core/Pipe/_fetch.py +11 -9
- meerschaum/core/Pipe/_sync.py +24 -7
- meerschaum/core/Pipe/_verify.py +51 -48
- meerschaum/utils/dataframe.py +16 -8
- meerschaum/utils/dtypes/__init__.py +9 -1
- meerschaum/utils/dtypes/sql.py +32 -6
- meerschaum/utils/misc.py +8 -8
- meerschaum/utils/sql.py +485 -16
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/METADATA +1 -1
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/RECORD +36 -36
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/LICENSE +0 -0
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/NOTICE +0 -0
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/WHEEL +0 -0
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/top_level.txt +0 -0
- {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.2.dist-info}/zip-safe +0 -0
meerschaum/utils/sql.py
CHANGED
@@ -42,6 +42,7 @@ SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
|
|
42
42
|
DROP_IF_EXISTS_FLAVORS = {
|
43
43
|
'timescaledb', 'postgresql', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
|
44
44
|
}
|
45
|
+
SKIP_AUTO_INCREMENT_FLAVORS = {'citus', 'duckdb'}
|
45
46
|
COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'citus'}
|
46
47
|
update_queries = {
|
47
48
|
'default': """
|
@@ -174,7 +175,7 @@ columns_types_queries = {
|
|
174
175
|
p.name "column",
|
175
176
|
p.type "type"
|
176
177
|
FROM sqlite_master m
|
177
|
-
LEFT OUTER JOIN pragma_table_info(
|
178
|
+
LEFT OUTER JOIN pragma_table_info(m.name) p
|
178
179
|
ON m.name <> p.name
|
179
180
|
WHERE m.type = 'table'
|
180
181
|
AND m.name IN ('{table}', '{table_trunc}')
|
@@ -235,6 +236,206 @@ hypertable_queries = {
|
|
235
236
|
'timescaledb': 'SELECT hypertable_size(\'{table_name}\')',
|
236
237
|
'citus': 'SELECT citus_table_size(\'{table_name}\')',
|
237
238
|
}
|
239
|
+
columns_indices_queries = {
|
240
|
+
'default': """
|
241
|
+
SELECT
|
242
|
+
current_database() AS "database",
|
243
|
+
n.nspname AS "schema",
|
244
|
+
t.relname AS "table",
|
245
|
+
c.column_name AS "column",
|
246
|
+
i.relname AS "index",
|
247
|
+
CASE WHEN con.contype = 'p' THEN 'PRIMARY KEY' ELSE 'INDEX' END AS "index_type"
|
248
|
+
FROM pg_class t
|
249
|
+
INNER JOIN pg_index AS ix
|
250
|
+
ON t.oid = ix.indrelid
|
251
|
+
INNER JOIN pg_class AS i
|
252
|
+
ON i.oid = ix.indexrelid
|
253
|
+
INNER JOIN pg_namespace AS n
|
254
|
+
ON n.oid = t.relnamespace
|
255
|
+
INNER JOIN pg_attribute AS a
|
256
|
+
ON a.attnum = ANY(ix.indkey)
|
257
|
+
AND a.attrelid = t.oid
|
258
|
+
INNER JOIN information_schema.columns AS c
|
259
|
+
ON c.column_name = a.attname
|
260
|
+
AND c.table_name = t.relname
|
261
|
+
AND c.table_schema = n.nspname
|
262
|
+
LEFT JOIN pg_constraint AS con
|
263
|
+
ON con.conindid = i.oid
|
264
|
+
AND con.contype = 'p'
|
265
|
+
WHERE
|
266
|
+
t.relname IN ('{table}', '{table_trunc}')
|
267
|
+
AND n.nspname = '{schema}'
|
268
|
+
""",
|
269
|
+
'sqlite': """
|
270
|
+
WITH indexed_columns AS (
|
271
|
+
SELECT
|
272
|
+
'{table}' AS table_name,
|
273
|
+
pi.name AS column_name,
|
274
|
+
i.name AS index_name,
|
275
|
+
'INDEX' AS index_type
|
276
|
+
FROM
|
277
|
+
sqlite_master AS i,
|
278
|
+
pragma_index_info(i.name) AS pi
|
279
|
+
WHERE
|
280
|
+
i.type = 'index'
|
281
|
+
AND i.tbl_name = '{table}'
|
282
|
+
),
|
283
|
+
primary_key_columns AS (
|
284
|
+
SELECT
|
285
|
+
'{table}' AS table_name,
|
286
|
+
ti.name AS column_name,
|
287
|
+
'PRIMARY_KEY' AS index_name,
|
288
|
+
'PRIMARY KEY' AS index_type
|
289
|
+
FROM
|
290
|
+
pragma_table_info('{table}') AS ti
|
291
|
+
WHERE
|
292
|
+
ti.pk > 0
|
293
|
+
)
|
294
|
+
SELECT
|
295
|
+
NULL AS "database",
|
296
|
+
NULL AS "schema",
|
297
|
+
"table_name" AS "table",
|
298
|
+
"column_name" AS "column",
|
299
|
+
"index_name" AS "index",
|
300
|
+
"index_type"
|
301
|
+
FROM indexed_columns
|
302
|
+
UNION ALL
|
303
|
+
SELECT
|
304
|
+
NULL AS "database",
|
305
|
+
NULL AS "schema",
|
306
|
+
table_name AS "table",
|
307
|
+
column_name AS "column",
|
308
|
+
index_name AS "index",
|
309
|
+
index_type
|
310
|
+
FROM primary_key_columns
|
311
|
+
""",
|
312
|
+
'mssql': """
|
313
|
+
SELECT
|
314
|
+
NULL AS [database],
|
315
|
+
s.name AS [schema],
|
316
|
+
t.name AS [table],
|
317
|
+
c.name AS [column],
|
318
|
+
i.name AS [index],
|
319
|
+
CASE
|
320
|
+
WHEN kc.type = 'PK' THEN 'PRIMARY KEY'
|
321
|
+
ELSE 'INDEX'
|
322
|
+
END AS [index_type]
|
323
|
+
FROM
|
324
|
+
sys.schemas s
|
325
|
+
INNER JOIN sys.tables t
|
326
|
+
ON s.schema_id = t.schema_id
|
327
|
+
INNER JOIN sys.indexes i
|
328
|
+
ON t.object_id = i.object_id
|
329
|
+
INNER JOIN sys.index_columns ic
|
330
|
+
ON i.object_id = ic.object_id
|
331
|
+
AND i.index_id = ic.index_id
|
332
|
+
INNER JOIN sys.columns c
|
333
|
+
ON ic.object_id = c.object_id
|
334
|
+
AND ic.column_id = c.column_id
|
335
|
+
LEFT JOIN sys.key_constraints kc
|
336
|
+
ON kc.parent_object_id = i.object_id
|
337
|
+
AND kc.type = 'PK'
|
338
|
+
AND kc.name = i.name
|
339
|
+
WHERE
|
340
|
+
t.name IN ('{table}', '{table_trunc}')
|
341
|
+
AND s.name = 'dbo'
|
342
|
+
AND i.type IN (1, 2) -- 1 = CLUSTERED, 2 = NONCLUSTERED
|
343
|
+
""",
|
344
|
+
'oracle': """
|
345
|
+
SELECT
|
346
|
+
NULL AS "database",
|
347
|
+
ic.table_owner AS "schema",
|
348
|
+
ic.table_name AS "table",
|
349
|
+
ic.column_name AS "column",
|
350
|
+
i.index_name AS "index",
|
351
|
+
CASE
|
352
|
+
WHEN c.constraint_type = 'P' THEN 'PRIMARY KEY'
|
353
|
+
WHEN i.uniqueness = 'UNIQUE' THEN 'UNIQUE INDEX'
|
354
|
+
ELSE 'INDEX'
|
355
|
+
END AS index_type
|
356
|
+
FROM
|
357
|
+
all_ind_columns ic
|
358
|
+
INNER JOIN all_indexes i
|
359
|
+
ON ic.index_name = i.index_name
|
360
|
+
AND ic.table_owner = i.owner
|
361
|
+
LEFT JOIN all_constraints c
|
362
|
+
ON i.index_name = c.constraint_name
|
363
|
+
AND i.table_owner = c.owner
|
364
|
+
AND c.constraint_type = 'P'
|
365
|
+
WHERE ic.table_name IN (
|
366
|
+
'{table}',
|
367
|
+
'{table_trunc}',
|
368
|
+
'{table_upper}',
|
369
|
+
'{table_upper_trunc}'
|
370
|
+
)
|
371
|
+
""",
|
372
|
+
'mysql': """
|
373
|
+
SELECT
|
374
|
+
TABLE_SCHEMA AS `database`,
|
375
|
+
TABLE_SCHEMA AS `schema`,
|
376
|
+
TABLE_NAME AS `table`,
|
377
|
+
COLUMN_NAME AS `column`,
|
378
|
+
INDEX_NAME AS `index`,
|
379
|
+
CASE
|
380
|
+
WHEN NON_UNIQUE = 0 THEN 'PRIMARY KEY'
|
381
|
+
ELSE 'INDEX'
|
382
|
+
END AS `index_type`
|
383
|
+
FROM
|
384
|
+
information_schema.STATISTICS
|
385
|
+
WHERE
|
386
|
+
TABLE_NAME IN ('{table}', '{table_trunc}')
|
387
|
+
""",
|
388
|
+
'mariadb': """
|
389
|
+
SELECT
|
390
|
+
TABLE_SCHEMA AS `database`,
|
391
|
+
TABLE_SCHEMA AS `schema`,
|
392
|
+
TABLE_NAME AS `table`,
|
393
|
+
COLUMN_NAME AS `column`,
|
394
|
+
INDEX_NAME AS `index`,
|
395
|
+
CASE
|
396
|
+
WHEN NON_UNIQUE = 0 THEN 'PRIMARY KEY'
|
397
|
+
ELSE 'INDEX'
|
398
|
+
END AS `index_type`
|
399
|
+
FROM
|
400
|
+
information_schema.STATISTICS
|
401
|
+
WHERE
|
402
|
+
TABLE_NAME IN ('{table}', '{table_trunc}')
|
403
|
+
""",
|
404
|
+
}
|
405
|
+
reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = {
|
406
|
+
'default': """
|
407
|
+
SELECT SETVAL(pg_get_serial_sequence('{table}', '{column}'), {val})
|
408
|
+
FROM {table_name}
|
409
|
+
""",
|
410
|
+
'mssql': """
|
411
|
+
DBCC CHECKIDENT ('{table}', RESEED, {val})
|
412
|
+
""",
|
413
|
+
'mysql': """
|
414
|
+
ALTER TABLE {table_name} AUTO_INCREMENT = {val}
|
415
|
+
""",
|
416
|
+
'mariadb': """
|
417
|
+
ALTER TABLE {table_name} AUTO_INCREMENT = {val}
|
418
|
+
""",
|
419
|
+
'sqlite': """
|
420
|
+
UPDATE sqlite_sequence
|
421
|
+
SET seq = {val}
|
422
|
+
WHERE name = '{table}'
|
423
|
+
""",
|
424
|
+
'oracle': [
|
425
|
+
"""
|
426
|
+
DECLARE
|
427
|
+
max_id NUMBER := {val};
|
428
|
+
current_val NUMBER;
|
429
|
+
BEGIN
|
430
|
+
SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
|
431
|
+
|
432
|
+
WHILE current_val < max_id LOOP
|
433
|
+
SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
|
434
|
+
END LOOP;
|
435
|
+
END;
|
436
|
+
""",
|
437
|
+
],
|
438
|
+
}
|
238
439
|
table_wrappers = {
|
239
440
|
'default' : ('"', '"'),
|
240
441
|
'timescaledb': ('"', '"'),
|
@@ -405,7 +606,7 @@ def dateadd_str(
|
|
405
606
|
da = begin + (f" + INTERVAL '{number} {datepart}'" if number != 0 else '')
|
406
607
|
|
407
608
|
elif flavor in ('mssql',):
|
408
|
-
if begin_time and begin_time.microsecond != 0:
|
609
|
+
if begin_time and begin_time.microsecond != 0 and not dt_is_utc:
|
409
610
|
begin = begin[:-4] + "'"
|
410
611
|
begin = f"CAST({begin} AS {db_type})" if begin != 'now' else 'GETUTCDATE()'
|
411
612
|
da = f"DATEADD({datepart}, {number}, {begin})" if number != 0 else begin
|
@@ -792,7 +993,6 @@ def table_exists(
|
|
792
993
|
Returns
|
793
994
|
-------
|
794
995
|
A `bool` indicating whether or not the table exists on the database.
|
795
|
-
|
796
996
|
"""
|
797
997
|
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
798
998
|
schema = schema or connector.schema
|
@@ -897,6 +1097,7 @@ def get_table_cols_types(
|
|
897
1097
|
connectable: Union[
|
898
1098
|
'mrsm.connectors.sql.SQLConnector',
|
899
1099
|
'sqlalchemy.orm.session.Session',
|
1100
|
+
'sqlalchemy.engine.base.Engine'
|
900
1101
|
]
|
901
1102
|
The connection object used to fetch the columns and types.
|
902
1103
|
|
@@ -1017,6 +1218,164 @@ def get_table_cols_types(
|
|
1017
1218
|
return {}
|
1018
1219
|
|
1019
1220
|
|
1221
|
+
def get_table_cols_indices(
|
1222
|
+
table: str,
|
1223
|
+
connectable: Union[
|
1224
|
+
'mrsm.connectors.sql.SQLConnector',
|
1225
|
+
'sqlalchemy.orm.session.Session',
|
1226
|
+
'sqlalchemy.engine.base.Engine'
|
1227
|
+
],
|
1228
|
+
flavor: Optional[str] = None,
|
1229
|
+
schema: Optional[str] = None,
|
1230
|
+
database: Optional[str] = None,
|
1231
|
+
debug: bool = False,
|
1232
|
+
) -> Dict[str, List[str]]:
|
1233
|
+
"""
|
1234
|
+
Return a dictionary mapping a table's columns to lists of indices.
|
1235
|
+
This is useful for inspecting tables creating during a not-yet-committed session.
|
1236
|
+
|
1237
|
+
NOTE: This may return incorrect columns if the schema is not explicitly stated.
|
1238
|
+
Use this function if you are confident the table name is unique or if you have
|
1239
|
+
and explicit schema.
|
1240
|
+
To use the configured schema, get the columns from `get_sqlalchemy_table()` instead.
|
1241
|
+
|
1242
|
+
Parameters
|
1243
|
+
----------
|
1244
|
+
table: str
|
1245
|
+
The name of the table (unquoted).
|
1246
|
+
|
1247
|
+
connectable: Union[
|
1248
|
+
'mrsm.connectors.sql.SQLConnector',
|
1249
|
+
'sqlalchemy.orm.session.Session',
|
1250
|
+
'sqlalchemy.engine.base.Engine'
|
1251
|
+
]
|
1252
|
+
The connection object used to fetch the columns and types.
|
1253
|
+
|
1254
|
+
flavor: Optional[str], default None
|
1255
|
+
The database dialect flavor to use for the query.
|
1256
|
+
If omitted, default to `connectable.flavor`.
|
1257
|
+
|
1258
|
+
schema: Optional[str], default None
|
1259
|
+
If provided, restrict the query to this schema.
|
1260
|
+
|
1261
|
+
database: Optional[str]. default None
|
1262
|
+
If provided, restrict the query to this database.
|
1263
|
+
|
1264
|
+
Returns
|
1265
|
+
-------
|
1266
|
+
A dictionary mapping column names to a list of indices.
|
1267
|
+
"""
|
1268
|
+
from collections import defaultdict
|
1269
|
+
from meerschaum.connectors import SQLConnector
|
1270
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
1271
|
+
flavor = flavor or getattr(connectable, 'flavor', None)
|
1272
|
+
if not flavor:
|
1273
|
+
raise ValueError("Please provide a database flavor.")
|
1274
|
+
if flavor == 'duckdb' and not isinstance(connectable, SQLConnector):
|
1275
|
+
raise ValueError("You must provide a SQLConnector when using DuckDB.")
|
1276
|
+
if flavor in NO_SCHEMA_FLAVORS:
|
1277
|
+
schema = None
|
1278
|
+
if schema is None:
|
1279
|
+
schema = DEFAULT_SCHEMA_FLAVORS.get(flavor, None)
|
1280
|
+
if flavor in ('sqlite', 'duckdb', 'oracle'):
|
1281
|
+
database = None
|
1282
|
+
table_trunc = truncate_item_name(table, flavor=flavor)
|
1283
|
+
table_lower = table.lower()
|
1284
|
+
table_upper = table.upper()
|
1285
|
+
table_lower_trunc = truncate_item_name(table_lower, flavor=flavor)
|
1286
|
+
table_upper_trunc = truncate_item_name(table_upper, flavor=flavor)
|
1287
|
+
db_prefix = (
|
1288
|
+
"tempdb."
|
1289
|
+
if flavor == 'mssql' and table.startswith('#')
|
1290
|
+
else ""
|
1291
|
+
)
|
1292
|
+
|
1293
|
+
cols_indices_query = sqlalchemy.text(
|
1294
|
+
columns_indices_queries.get(
|
1295
|
+
flavor,
|
1296
|
+
columns_indices_queries['default']
|
1297
|
+
).format(
|
1298
|
+
table=table,
|
1299
|
+
table_trunc=table_trunc,
|
1300
|
+
table_lower=table_lower,
|
1301
|
+
table_lower_trunc=table_lower_trunc,
|
1302
|
+
table_upper=table_upper,
|
1303
|
+
table_upper_trunc=table_upper_trunc,
|
1304
|
+
db_prefix=db_prefix,
|
1305
|
+
schema=schema,
|
1306
|
+
)
|
1307
|
+
)
|
1308
|
+
|
1309
|
+
cols = ['database', 'schema', 'table', 'column', 'index', 'index_type']
|
1310
|
+
result_cols_ix = dict(enumerate(cols))
|
1311
|
+
|
1312
|
+
debug_kwargs = {'debug': debug} if isinstance(connectable, SQLConnector) else {}
|
1313
|
+
if not debug_kwargs and debug:
|
1314
|
+
dprint(cols_indices_query)
|
1315
|
+
|
1316
|
+
try:
|
1317
|
+
result_rows = (
|
1318
|
+
[
|
1319
|
+
row
|
1320
|
+
for row in connectable.execute(cols_indices_query, **debug_kwargs).fetchall()
|
1321
|
+
]
|
1322
|
+
if flavor != 'duckdb'
|
1323
|
+
else [
|
1324
|
+
tuple([doc[col] for col in cols])
|
1325
|
+
for doc in connectable.read(cols_indices_query, debug=debug).to_dict(orient='records')
|
1326
|
+
]
|
1327
|
+
)
|
1328
|
+
cols_types_docs = [
|
1329
|
+
{
|
1330
|
+
result_cols_ix[i]: val
|
1331
|
+
for i, val in enumerate(row)
|
1332
|
+
}
|
1333
|
+
for row in result_rows
|
1334
|
+
]
|
1335
|
+
cols_types_docs_filtered = [
|
1336
|
+
doc
|
1337
|
+
for doc in cols_types_docs
|
1338
|
+
if (
|
1339
|
+
(
|
1340
|
+
not schema
|
1341
|
+
or doc['schema'] == schema
|
1342
|
+
)
|
1343
|
+
and
|
1344
|
+
(
|
1345
|
+
not database
|
1346
|
+
or doc['database'] == database
|
1347
|
+
)
|
1348
|
+
)
|
1349
|
+
]
|
1350
|
+
|
1351
|
+
### NOTE: This may return incorrect columns if the schema is not explicitly stated.
|
1352
|
+
if cols_types_docs and not cols_types_docs_filtered:
|
1353
|
+
cols_types_docs_filtered = cols_types_docs
|
1354
|
+
|
1355
|
+
cols_indices = defaultdict(lambda: [])
|
1356
|
+
for doc in cols_types_docs_filtered:
|
1357
|
+
col = (
|
1358
|
+
doc['column']
|
1359
|
+
if flavor != 'oracle'
|
1360
|
+
else (
|
1361
|
+
doc['column'].lower()
|
1362
|
+
if (doc['column'].isupper() and doc['column'].replace('_', '').isalpha())
|
1363
|
+
else doc['column']
|
1364
|
+
)
|
1365
|
+
)
|
1366
|
+
cols_indices[col].append(
|
1367
|
+
{
|
1368
|
+
'name': doc.get('index', None),
|
1369
|
+
'type': doc.get('index_type', None),
|
1370
|
+
}
|
1371
|
+
)
|
1372
|
+
|
1373
|
+
return dict(cols_indices)
|
1374
|
+
except Exception as e:
|
1375
|
+
warn(f"Failed to fetch columns for table '{table}':\n{e}")
|
1376
|
+
return {}
|
1377
|
+
|
1378
|
+
|
1020
1379
|
def get_update_queries(
|
1021
1380
|
target: str,
|
1022
1381
|
patch: str,
|
@@ -1257,10 +1616,11 @@ def get_null_replacement(typ: str, flavor: str) -> str:
|
|
1257
1616
|
A value which may stand in place of NULL for this type.
|
1258
1617
|
`'None'` is returned if a value cannot be determined.
|
1259
1618
|
"""
|
1619
|
+
from meerschaum.utils.dtypes import are_dtypes_equal
|
1260
1620
|
from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
|
1261
1621
|
if 'int' in typ.lower() or typ.lower() in ('numeric', 'number'):
|
1262
1622
|
return '-987654321'
|
1263
|
-
if 'bool' in typ.lower():
|
1623
|
+
if 'bool' in typ.lower() or typ.lower() == 'bit':
|
1264
1624
|
bool_typ = (
|
1265
1625
|
PD_TO_DB_DTYPES_FLAVORS
|
1266
1626
|
.get('bool', {})
|
@@ -1270,7 +1630,7 @@ def get_null_replacement(typ: str, flavor: str) -> str:
|
|
1270
1630
|
bool_typ = DB_FLAVORS_CAST_DTYPES[flavor].get(bool_typ, bool_typ)
|
1271
1631
|
val_to_cast = (
|
1272
1632
|
-987654321
|
1273
|
-
if flavor in ('mysql', 'mariadb'
|
1633
|
+
if flavor in ('mysql', 'mariadb')
|
1274
1634
|
else 0
|
1275
1635
|
)
|
1276
1636
|
return f'CAST({val_to_cast} AS {bool_typ})'
|
@@ -1278,6 +1638,8 @@ def get_null_replacement(typ: str, flavor: str) -> str:
|
|
1278
1638
|
return dateadd_str(flavor=flavor, begin='1900-01-01')
|
1279
1639
|
if 'float' in typ.lower() or 'double' in typ.lower() or typ.lower() in ('decimal',):
|
1280
1640
|
return '-987654321.0'
|
1641
|
+
if flavor == 'oracle' and typ.lower().split('(', maxsplit=1)[0] == 'char':
|
1642
|
+
return "'-987654321'"
|
1281
1643
|
if typ.lower() in ('uniqueidentifier', 'guid', 'uuid'):
|
1282
1644
|
magic_val = 'DEADBEEF-ABBA-BABE-CAFE-DECAFC0FFEE5'
|
1283
1645
|
if flavor == 'mssql':
|
@@ -1361,7 +1723,7 @@ def get_create_table_query(
|
|
1361
1723
|
schema: Optional[str] = None,
|
1362
1724
|
) -> str:
|
1363
1725
|
"""
|
1364
|
-
NOTE: This function is deprecated. Use `
|
1726
|
+
NOTE: This function is deprecated. Use `get_create_table_queries()` instead.
|
1365
1727
|
|
1366
1728
|
Return a query to create a new table from a `SELECT` query.
|
1367
1729
|
|
@@ -1399,6 +1761,8 @@ def get_create_table_queries(
|
|
1399
1761
|
flavor: str,
|
1400
1762
|
schema: Optional[str] = None,
|
1401
1763
|
primary_key: Optional[str] = None,
|
1764
|
+
autoincrement: bool = False,
|
1765
|
+
datetime_column: Optional[str] = None,
|
1402
1766
|
) -> List[str]:
|
1403
1767
|
"""
|
1404
1768
|
Return a query to create a new table from a `SELECT` query or a `dtypes` dictionary.
|
@@ -1421,6 +1785,14 @@ def get_create_table_queries(
|
|
1421
1785
|
primary_key: Optional[str], default None
|
1422
1786
|
If provided, designate this column as the primary key in the new table.
|
1423
1787
|
|
1788
|
+
autoincrement: bool, default False
|
1789
|
+
If `True` and `primary_key` is provided, create the `primary_key` column
|
1790
|
+
as an auto-incrementing integer column.
|
1791
|
+
|
1792
|
+
datetime_column: Optional[str], default None
|
1793
|
+
If provided, include this column in the primary key.
|
1794
|
+
Applicable to TimescaleDB only.
|
1795
|
+
|
1424
1796
|
Returns
|
1425
1797
|
-------
|
1426
1798
|
A `CREATE TABLE` (or `SELECT INTO`) query for the database flavor.
|
@@ -1439,6 +1811,8 @@ def get_create_table_queries(
|
|
1439
1811
|
flavor,
|
1440
1812
|
schema=schema,
|
1441
1813
|
primary_key=primary_key,
|
1814
|
+
autoincrement=(autoincrement and flavor not in SKIP_AUTO_INCREMENT_FLAVORS),
|
1815
|
+
datetime_column=datetime_column,
|
1442
1816
|
)
|
1443
1817
|
|
1444
1818
|
|
@@ -1448,6 +1822,8 @@ def _get_create_table_query_from_dtypes(
|
|
1448
1822
|
flavor: str,
|
1449
1823
|
schema: Optional[str] = None,
|
1450
1824
|
primary_key: Optional[str] = None,
|
1825
|
+
autoincrement: bool = False,
|
1826
|
+
datetime_column: Optional[str] = None,
|
1451
1827
|
) -> List[str]:
|
1452
1828
|
"""
|
1453
1829
|
Create a new table from a `dtypes` dictionary.
|
@@ -1456,40 +1832,61 @@ def _get_create_table_query_from_dtypes(
|
|
1456
1832
|
if not dtypes and not primary_key:
|
1457
1833
|
raise ValueError(f"Expecting columns for table '{new_table}'.")
|
1458
1834
|
|
1835
|
+
if flavor in SKIP_AUTO_INCREMENT_FLAVORS:
|
1836
|
+
autoincrement = False
|
1837
|
+
|
1459
1838
|
cols_types = (
|
1460
|
-
[(primary_key, get_db_type_from_pd_type(dtypes.get(primary_key, 'int')))]
|
1839
|
+
[(primary_key, get_db_type_from_pd_type(dtypes.get(primary_key, 'int'), flavor=flavor))]
|
1461
1840
|
if primary_key
|
1462
1841
|
else []
|
1463
1842
|
) + [
|
1464
|
-
(col, get_db_type_from_pd_type(typ))
|
1843
|
+
(col, get_db_type_from_pd_type(typ, flavor=flavor))
|
1465
1844
|
for col, typ in dtypes.items()
|
1466
1845
|
if col != primary_key
|
1467
1846
|
]
|
1468
1847
|
|
1469
1848
|
table_name = sql_item_name(new_table, schema=schema, flavor=flavor)
|
1849
|
+
primary_key_name = sql_item_name(primary_key, flavor) if primary_key else None
|
1850
|
+
datetime_column_name = sql_item_name(datetime_column, flavor) if datetime_column else None
|
1470
1851
|
query = f"CREATE TABLE {table_name} ("
|
1471
1852
|
if primary_key:
|
1472
1853
|
col_db_type = cols_types[0][1]
|
1473
|
-
|
1854
|
+
auto_increment_str = (' ' + AUTO_INCREMENT_COLUMN_FLAVORS.get(
|
1474
1855
|
flavor,
|
1475
1856
|
AUTO_INCREMENT_COLUMN_FLAVORS['default']
|
1476
|
-
)) if primary_key not in dtypes else ''
|
1857
|
+
)) if autoincrement or primary_key not in dtypes else ''
|
1477
1858
|
col_name = sql_item_name(primary_key, flavor=flavor, schema=None)
|
1478
1859
|
|
1479
1860
|
if flavor == 'sqlite':
|
1480
|
-
query +=
|
1861
|
+
query += (
|
1862
|
+
f"\n {col_name} "
|
1863
|
+
+ (f"{col_db_type}" if not auto_increment_str else 'INTEGER')
|
1864
|
+
+ f" PRIMARY KEY{auto_increment_str} NOT NULL,"
|
1865
|
+
)
|
1866
|
+
elif flavor == 'oracle':
|
1867
|
+
query += f"\n {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
|
1868
|
+
elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
|
1869
|
+
query += f"\n {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
|
1481
1870
|
else:
|
1482
|
-
query += f"\n {col_name} {col_db_type} PRIMARY KEY{
|
1871
|
+
query += f"\n {col_name} {col_db_type} PRIMARY KEY{auto_increment_str} NOT NULL,"
|
1483
1872
|
|
1484
1873
|
for col, db_type in cols_types:
|
1485
1874
|
if col == primary_key:
|
1486
1875
|
continue
|
1487
1876
|
col_name = sql_item_name(col, schema=None, flavor=flavor)
|
1488
1877
|
query += f"\n {col_name} {db_type},"
|
1878
|
+
if (
|
1879
|
+
flavor == 'timescaledb'
|
1880
|
+
and datetime_column
|
1881
|
+
and primary_key
|
1882
|
+
and datetime_column != primary_key
|
1883
|
+
):
|
1884
|
+
query += f"\n PRIMARY KEY({datetime_column_name}, {primary_key_name}),"
|
1489
1885
|
query = query[:-1]
|
1490
1886
|
query += "\n)"
|
1491
1887
|
|
1492
|
-
|
1888
|
+
queries = [query]
|
1889
|
+
return queries
|
1493
1890
|
|
1494
1891
|
|
1495
1892
|
def _get_create_table_query_from_cte(
|
@@ -1498,6 +1895,8 @@ def _get_create_table_query_from_cte(
|
|
1498
1895
|
flavor: str,
|
1499
1896
|
schema: Optional[str] = None,
|
1500
1897
|
primary_key: Optional[str] = None,
|
1898
|
+
autoincrement: bool = False,
|
1899
|
+
datetime_column: Optional[str] = None,
|
1501
1900
|
) -> List[str]:
|
1502
1901
|
"""
|
1503
1902
|
Create a new table from a CTE query.
|
@@ -1517,9 +1916,10 @@ def _get_create_table_query_from_cte(
|
|
1517
1916
|
if primary_key
|
1518
1917
|
else None
|
1519
1918
|
)
|
1520
|
-
|
1521
|
-
flavor
|
1522
|
-
|
1919
|
+
datetime_column_name = (
|
1920
|
+
sql_item_name(datetime_column, flavor)
|
1921
|
+
if datetime_column
|
1922
|
+
else None
|
1523
1923
|
)
|
1524
1924
|
if flavor in ('mssql',):
|
1525
1925
|
query = query.lstrip()
|
@@ -1566,6 +1966,17 @@ def _get_create_table_query_from_cte(
|
|
1566
1966
|
ALTER TABLE {new_table_name}
|
1567
1967
|
ADD PRIMARY KEY ({primary_key_name})
|
1568
1968
|
"""
|
1969
|
+
elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
|
1970
|
+
create_table_query = f"""
|
1971
|
+
SELECT *
|
1972
|
+
INTO {new_table_name}
|
1973
|
+
FROM ({query}) AS {create_cte_name}
|
1974
|
+
"""
|
1975
|
+
|
1976
|
+
alter_type_query = f"""
|
1977
|
+
ALTER TABLE {new_table_name}
|
1978
|
+
ADD PRIMARY KEY ({datetime_column_name}, {primary_key_name})
|
1979
|
+
"""
|
1569
1980
|
else:
|
1570
1981
|
create_table_query = f"""
|
1571
1982
|
SELECT *
|
@@ -1758,3 +2169,61 @@ def session_execute(
|
|
1758
2169
|
if with_results:
|
1759
2170
|
return (success, msg), results
|
1760
2171
|
return success, msg
|
2172
|
+
|
2173
|
+
|
2174
|
+
def get_reset_autoincrement_queries(
|
2175
|
+
table: str,
|
2176
|
+
column: str,
|
2177
|
+
connector: mrsm.connectors.SQLConnector,
|
2178
|
+
schema: Optional[str] = None,
|
2179
|
+
debug: bool = False,
|
2180
|
+
) -> List[str]:
|
2181
|
+
"""
|
2182
|
+
Return a list of queries to reset a table's auto-increment counter.
|
2183
|
+
"""
|
2184
|
+
if not table_exists(table, connector, schema=schema, debug=debug):
|
2185
|
+
return []
|
2186
|
+
|
2187
|
+
schema = schema or connector.schema
|
2188
|
+
max_id_name = sql_item_name('max_id', connector.flavor)
|
2189
|
+
table_name = sql_item_name(table, connector.flavor, schema)
|
2190
|
+
table_trunc = truncate_item_name(table, connector.flavor)
|
2191
|
+
table_seq_name = sql_item_name(table + '_' + column + '_seq', connector.flavor, schema)
|
2192
|
+
column_name = sql_item_name(column, connector.flavor)
|
2193
|
+
if connector.flavor == 'oracle':
|
2194
|
+
df = connector.read(f"""
|
2195
|
+
SELECT SEQUENCE_NAME
|
2196
|
+
FROM ALL_TAB_IDENTITY_COLS
|
2197
|
+
WHERE TABLE_NAME IN '{table_trunc.upper()}'
|
2198
|
+
""", debug=debug)
|
2199
|
+
if len(df) > 0:
|
2200
|
+
table_seq_name = df['sequence_name'][0]
|
2201
|
+
|
2202
|
+
max_id = connector.value(
|
2203
|
+
f"""
|
2204
|
+
SELECT COALESCE(MAX({column_name}), 0) AS {max_id_name}
|
2205
|
+
FROM {table_name}
|
2206
|
+
""",
|
2207
|
+
debug=debug,
|
2208
|
+
)
|
2209
|
+
if max_id is None:
|
2210
|
+
return []
|
2211
|
+
|
2212
|
+
reset_queries = reset_autoincrement_queries.get(
|
2213
|
+
connector.flavor,
|
2214
|
+
reset_autoincrement_queries['default']
|
2215
|
+
)
|
2216
|
+
if not isinstance(reset_queries, list):
|
2217
|
+
reset_queries = [reset_queries]
|
2218
|
+
|
2219
|
+
return [
|
2220
|
+
query.format(
|
2221
|
+
column=column,
|
2222
|
+
column_name=column_name,
|
2223
|
+
table=table,
|
2224
|
+
table_name=table_name,
|
2225
|
+
table_seq_name=table_seq_name,
|
2226
|
+
val=(max_id),
|
2227
|
+
)
|
2228
|
+
for query in reset_queries
|
2229
|
+
]
|