meerschaum 2.6.0.dev1__py3-none-any.whl → 2.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. meerschaum/api/dash/pages/login.py +17 -17
  2. meerschaum/api/dash/pipes.py +13 -4
  3. meerschaum/api/routes/_pipes.py +162 -136
  4. meerschaum/config/_version.py +1 -1
  5. meerschaum/config/static/__init__.py +1 -0
  6. meerschaum/connectors/api/_APIConnector.py +1 -0
  7. meerschaum/connectors/api/_pipes.py +46 -13
  8. meerschaum/connectors/sql/_SQLConnector.py +4 -3
  9. meerschaum/connectors/sql/_fetch.py +4 -2
  10. meerschaum/connectors/sql/_pipes.py +496 -148
  11. meerschaum/connectors/sql/_sql.py +37 -16
  12. meerschaum/connectors/valkey/_ValkeyConnector.py +3 -2
  13. meerschaum/connectors/valkey/_pipes.py +13 -5
  14. meerschaum/core/Pipe/__init__.py +20 -0
  15. meerschaum/core/Pipe/_attributes.py +179 -9
  16. meerschaum/core/Pipe/_clear.py +10 -8
  17. meerschaum/core/Pipe/_copy.py +2 -0
  18. meerschaum/core/Pipe/_data.py +57 -28
  19. meerschaum/core/Pipe/_deduplicate.py +30 -28
  20. meerschaum/core/Pipe/_dtypes.py +12 -2
  21. meerschaum/core/Pipe/_fetch.py +11 -9
  22. meerschaum/core/Pipe/_sync.py +24 -7
  23. meerschaum/core/Pipe/_verify.py +51 -48
  24. meerschaum/utils/dataframe.py +16 -8
  25. meerschaum/utils/dtypes/__init__.py +9 -1
  26. meerschaum/utils/dtypes/sql.py +32 -6
  27. meerschaum/utils/misc.py +8 -8
  28. meerschaum/utils/sql.py +485 -16
  29. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/METADATA +1 -1
  30. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/RECORD +36 -36
  31. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/LICENSE +0 -0
  32. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/NOTICE +0 -0
  33. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/WHEEL +0 -0
  34. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/entry_points.txt +0 -0
  35. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/top_level.txt +0 -0
  36. {meerschaum-2.6.0.dev1.dist-info → meerschaum-2.6.1.dist-info}/zip-safe +0 -0
meerschaum/utils/sql.py CHANGED
@@ -42,6 +42,7 @@ SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
42
42
  DROP_IF_EXISTS_FLAVORS = {
43
43
  'timescaledb', 'postgresql', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
44
44
  }
45
+ SKIP_AUTO_INCREMENT_FLAVORS = {'citus', 'duckdb'}
45
46
  COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'citus'}
46
47
  update_queries = {
47
48
  'default': """
@@ -174,7 +175,7 @@ columns_types_queries = {
174
175
  p.name "column",
175
176
  p.type "type"
176
177
  FROM sqlite_master m
177
- LEFT OUTER JOIN pragma_table_info((m.name)) p
178
+ LEFT OUTER JOIN pragma_table_info(m.name) p
178
179
  ON m.name <> p.name
179
180
  WHERE m.type = 'table'
180
181
  AND m.name IN ('{table}', '{table_trunc}')
@@ -235,6 +236,206 @@ hypertable_queries = {
235
236
  'timescaledb': 'SELECT hypertable_size(\'{table_name}\')',
236
237
  'citus': 'SELECT citus_table_size(\'{table_name}\')',
237
238
  }
239
+ columns_indices_queries = {
240
+ 'default': """
241
+ SELECT
242
+ current_database() AS "database",
243
+ n.nspname AS "schema",
244
+ t.relname AS "table",
245
+ c.column_name AS "column",
246
+ i.relname AS "index",
247
+ CASE WHEN con.contype = 'p' THEN 'PRIMARY KEY' ELSE 'INDEX' END AS "index_type"
248
+ FROM pg_class t
249
+ INNER JOIN pg_index AS ix
250
+ ON t.oid = ix.indrelid
251
+ INNER JOIN pg_class AS i
252
+ ON i.oid = ix.indexrelid
253
+ INNER JOIN pg_namespace AS n
254
+ ON n.oid = t.relnamespace
255
+ INNER JOIN pg_attribute AS a
256
+ ON a.attnum = ANY(ix.indkey)
257
+ AND a.attrelid = t.oid
258
+ INNER JOIN information_schema.columns AS c
259
+ ON c.column_name = a.attname
260
+ AND c.table_name = t.relname
261
+ AND c.table_schema = n.nspname
262
+ LEFT JOIN pg_constraint AS con
263
+ ON con.conindid = i.oid
264
+ AND con.contype = 'p'
265
+ WHERE
266
+ t.relname IN ('{table}', '{table_trunc}')
267
+ AND n.nspname = '{schema}'
268
+ """,
269
+ 'sqlite': """
270
+ WITH indexed_columns AS (
271
+ SELECT
272
+ '{table}' AS table_name,
273
+ pi.name AS column_name,
274
+ i.name AS index_name,
275
+ 'INDEX' AS index_type
276
+ FROM
277
+ sqlite_master AS i,
278
+ pragma_index_info(i.name) AS pi
279
+ WHERE
280
+ i.type = 'index'
281
+ AND i.tbl_name = '{table}'
282
+ ),
283
+ primary_key_columns AS (
284
+ SELECT
285
+ '{table}' AS table_name,
286
+ ti.name AS column_name,
287
+ 'PRIMARY_KEY' AS index_name,
288
+ 'PRIMARY KEY' AS index_type
289
+ FROM
290
+ pragma_table_info('{table}') AS ti
291
+ WHERE
292
+ ti.pk > 0
293
+ )
294
+ SELECT
295
+ NULL AS "database",
296
+ NULL AS "schema",
297
+ "table_name" AS "table",
298
+ "column_name" AS "column",
299
+ "index_name" AS "index",
300
+ "index_type"
301
+ FROM indexed_columns
302
+ UNION ALL
303
+ SELECT
304
+ NULL AS "database",
305
+ NULL AS "schema",
306
+ table_name AS "table",
307
+ column_name AS "column",
308
+ index_name AS "index",
309
+ index_type
310
+ FROM primary_key_columns
311
+ """,
312
+ 'mssql': """
313
+ SELECT
314
+ NULL AS [database],
315
+ s.name AS [schema],
316
+ t.name AS [table],
317
+ c.name AS [column],
318
+ i.name AS [index],
319
+ CASE
320
+ WHEN kc.type = 'PK' THEN 'PRIMARY KEY'
321
+ ELSE 'INDEX'
322
+ END AS [index_type]
323
+ FROM
324
+ sys.schemas s
325
+ INNER JOIN sys.tables t
326
+ ON s.schema_id = t.schema_id
327
+ INNER JOIN sys.indexes i
328
+ ON t.object_id = i.object_id
329
+ INNER JOIN sys.index_columns ic
330
+ ON i.object_id = ic.object_id
331
+ AND i.index_id = ic.index_id
332
+ INNER JOIN sys.columns c
333
+ ON ic.object_id = c.object_id
334
+ AND ic.column_id = c.column_id
335
+ LEFT JOIN sys.key_constraints kc
336
+ ON kc.parent_object_id = i.object_id
337
+ AND kc.type = 'PK'
338
+ AND kc.name = i.name
339
+ WHERE
340
+ t.name IN ('{table}', '{table_trunc}')
341
+ AND s.name = 'dbo'
342
+ AND i.type IN (1, 2) -- 1 = CLUSTERED, 2 = NONCLUSTERED
343
+ """,
344
+ 'oracle': """
345
+ SELECT
346
+ NULL AS "database",
347
+ ic.table_owner AS "schema",
348
+ ic.table_name AS "table",
349
+ ic.column_name AS "column",
350
+ i.index_name AS "index",
351
+ CASE
352
+ WHEN c.constraint_type = 'P' THEN 'PRIMARY KEY'
353
+ WHEN i.uniqueness = 'UNIQUE' THEN 'UNIQUE INDEX'
354
+ ELSE 'INDEX'
355
+ END AS index_type
356
+ FROM
357
+ all_ind_columns ic
358
+ INNER JOIN all_indexes i
359
+ ON ic.index_name = i.index_name
360
+ AND ic.table_owner = i.owner
361
+ LEFT JOIN all_constraints c
362
+ ON i.index_name = c.constraint_name
363
+ AND i.table_owner = c.owner
364
+ AND c.constraint_type = 'P'
365
+ WHERE ic.table_name IN (
366
+ '{table}',
367
+ '{table_trunc}',
368
+ '{table_upper}',
369
+ '{table_upper_trunc}'
370
+ )
371
+ """,
372
+ 'mysql': """
373
+ SELECT
374
+ TABLE_SCHEMA AS `database`,
375
+ TABLE_SCHEMA AS `schema`,
376
+ TABLE_NAME AS `table`,
377
+ COLUMN_NAME AS `column`,
378
+ INDEX_NAME AS `index`,
379
+ CASE
380
+ WHEN NON_UNIQUE = 0 THEN 'PRIMARY KEY'
381
+ ELSE 'INDEX'
382
+ END AS `index_type`
383
+ FROM
384
+ information_schema.STATISTICS
385
+ WHERE
386
+ TABLE_NAME IN ('{table}', '{table_trunc}')
387
+ """,
388
+ 'mariadb': """
389
+ SELECT
390
+ TABLE_SCHEMA AS `database`,
391
+ TABLE_SCHEMA AS `schema`,
392
+ TABLE_NAME AS `table`,
393
+ COLUMN_NAME AS `column`,
394
+ INDEX_NAME AS `index`,
395
+ CASE
396
+ WHEN NON_UNIQUE = 0 THEN 'PRIMARY KEY'
397
+ ELSE 'INDEX'
398
+ END AS `index_type`
399
+ FROM
400
+ information_schema.STATISTICS
401
+ WHERE
402
+ TABLE_NAME IN ('{table}', '{table_trunc}')
403
+ """,
404
+ }
405
+ reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = {
406
+ 'default': """
407
+ SELECT SETVAL(pg_get_serial_sequence('{table}', '{column}'), {val})
408
+ FROM {table_name}
409
+ """,
410
+ 'mssql': """
411
+ DBCC CHECKIDENT ('{table}', RESEED, {val})
412
+ """,
413
+ 'mysql': """
414
+ ALTER TABLE {table_name} AUTO_INCREMENT = {val}
415
+ """,
416
+ 'mariadb': """
417
+ ALTER TABLE {table_name} AUTO_INCREMENT = {val}
418
+ """,
419
+ 'sqlite': """
420
+ UPDATE sqlite_sequence
421
+ SET seq = {val}
422
+ WHERE name = '{table}'
423
+ """,
424
+ 'oracle': [
425
+ """
426
+ DECLARE
427
+ max_id NUMBER := {val};
428
+ current_val NUMBER;
429
+ BEGIN
430
+ SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
431
+
432
+ WHILE current_val < max_id LOOP
433
+ SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
434
+ END LOOP;
435
+ END;
436
+ """,
437
+ ],
438
+ }
238
439
  table_wrappers = {
239
440
  'default' : ('"', '"'),
240
441
  'timescaledb': ('"', '"'),
@@ -405,7 +606,7 @@ def dateadd_str(
405
606
  da = begin + (f" + INTERVAL '{number} {datepart}'" if number != 0 else '')
406
607
 
407
608
  elif flavor in ('mssql',):
408
- if begin_time and begin_time.microsecond != 0:
609
+ if begin_time and begin_time.microsecond != 0 and not dt_is_utc:
409
610
  begin = begin[:-4] + "'"
410
611
  begin = f"CAST({begin} AS {db_type})" if begin != 'now' else 'GETUTCDATE()'
411
612
  da = f"DATEADD({datepart}, {number}, {begin})" if number != 0 else begin
@@ -792,7 +993,6 @@ def table_exists(
792
993
  Returns
793
994
  -------
794
995
  A `bool` indicating whether or not the table exists on the database.
795
-
796
996
  """
797
997
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
798
998
  schema = schema or connector.schema
@@ -897,6 +1097,7 @@ def get_table_cols_types(
897
1097
  connectable: Union[
898
1098
  'mrsm.connectors.sql.SQLConnector',
899
1099
  'sqlalchemy.orm.session.Session',
1100
+ 'sqlalchemy.engine.base.Engine'
900
1101
  ]
901
1102
  The connection object used to fetch the columns and types.
902
1103
 
@@ -1017,6 +1218,164 @@ def get_table_cols_types(
1017
1218
  return {}
1018
1219
 
1019
1220
 
1221
+ def get_table_cols_indices(
1222
+ table: str,
1223
+ connectable: Union[
1224
+ 'mrsm.connectors.sql.SQLConnector',
1225
+ 'sqlalchemy.orm.session.Session',
1226
+ 'sqlalchemy.engine.base.Engine'
1227
+ ],
1228
+ flavor: Optional[str] = None,
1229
+ schema: Optional[str] = None,
1230
+ database: Optional[str] = None,
1231
+ debug: bool = False,
1232
+ ) -> Dict[str, List[str]]:
1233
+ """
1234
+ Return a dictionary mapping a table's columns to lists of indices.
1235
+ This is useful for inspecting tables creating during a not-yet-committed session.
1236
+
1237
+ NOTE: This may return incorrect columns if the schema is not explicitly stated.
1238
+ Use this function if you are confident the table name is unique or if you have
1239
+ and explicit schema.
1240
+ To use the configured schema, get the columns from `get_sqlalchemy_table()` instead.
1241
+
1242
+ Parameters
1243
+ ----------
1244
+ table: str
1245
+ The name of the table (unquoted).
1246
+
1247
+ connectable: Union[
1248
+ 'mrsm.connectors.sql.SQLConnector',
1249
+ 'sqlalchemy.orm.session.Session',
1250
+ 'sqlalchemy.engine.base.Engine'
1251
+ ]
1252
+ The connection object used to fetch the columns and types.
1253
+
1254
+ flavor: Optional[str], default None
1255
+ The database dialect flavor to use for the query.
1256
+ If omitted, default to `connectable.flavor`.
1257
+
1258
+ schema: Optional[str], default None
1259
+ If provided, restrict the query to this schema.
1260
+
1261
+ database: Optional[str]. default None
1262
+ If provided, restrict the query to this database.
1263
+
1264
+ Returns
1265
+ -------
1266
+ A dictionary mapping column names to a list of indices.
1267
+ """
1268
+ from collections import defaultdict
1269
+ from meerschaum.connectors import SQLConnector
1270
+ sqlalchemy = mrsm.attempt_import('sqlalchemy')
1271
+ flavor = flavor or getattr(connectable, 'flavor', None)
1272
+ if not flavor:
1273
+ raise ValueError("Please provide a database flavor.")
1274
+ if flavor == 'duckdb' and not isinstance(connectable, SQLConnector):
1275
+ raise ValueError("You must provide a SQLConnector when using DuckDB.")
1276
+ if flavor in NO_SCHEMA_FLAVORS:
1277
+ schema = None
1278
+ if schema is None:
1279
+ schema = DEFAULT_SCHEMA_FLAVORS.get(flavor, None)
1280
+ if flavor in ('sqlite', 'duckdb', 'oracle'):
1281
+ database = None
1282
+ table_trunc = truncate_item_name(table, flavor=flavor)
1283
+ table_lower = table.lower()
1284
+ table_upper = table.upper()
1285
+ table_lower_trunc = truncate_item_name(table_lower, flavor=flavor)
1286
+ table_upper_trunc = truncate_item_name(table_upper, flavor=flavor)
1287
+ db_prefix = (
1288
+ "tempdb."
1289
+ if flavor == 'mssql' and table.startswith('#')
1290
+ else ""
1291
+ )
1292
+
1293
+ cols_indices_query = sqlalchemy.text(
1294
+ columns_indices_queries.get(
1295
+ flavor,
1296
+ columns_indices_queries['default']
1297
+ ).format(
1298
+ table=table,
1299
+ table_trunc=table_trunc,
1300
+ table_lower=table_lower,
1301
+ table_lower_trunc=table_lower_trunc,
1302
+ table_upper=table_upper,
1303
+ table_upper_trunc=table_upper_trunc,
1304
+ db_prefix=db_prefix,
1305
+ schema=schema,
1306
+ )
1307
+ )
1308
+
1309
+ cols = ['database', 'schema', 'table', 'column', 'index', 'index_type']
1310
+ result_cols_ix = dict(enumerate(cols))
1311
+
1312
+ debug_kwargs = {'debug': debug} if isinstance(connectable, SQLConnector) else {}
1313
+ if not debug_kwargs and debug:
1314
+ dprint(cols_indices_query)
1315
+
1316
+ try:
1317
+ result_rows = (
1318
+ [
1319
+ row
1320
+ for row in connectable.execute(cols_indices_query, **debug_kwargs).fetchall()
1321
+ ]
1322
+ if flavor != 'duckdb'
1323
+ else [
1324
+ tuple([doc[col] for col in cols])
1325
+ for doc in connectable.read(cols_indices_query, debug=debug).to_dict(orient='records')
1326
+ ]
1327
+ )
1328
+ cols_types_docs = [
1329
+ {
1330
+ result_cols_ix[i]: val
1331
+ for i, val in enumerate(row)
1332
+ }
1333
+ for row in result_rows
1334
+ ]
1335
+ cols_types_docs_filtered = [
1336
+ doc
1337
+ for doc in cols_types_docs
1338
+ if (
1339
+ (
1340
+ not schema
1341
+ or doc['schema'] == schema
1342
+ )
1343
+ and
1344
+ (
1345
+ not database
1346
+ or doc['database'] == database
1347
+ )
1348
+ )
1349
+ ]
1350
+
1351
+ ### NOTE: This may return incorrect columns if the schema is not explicitly stated.
1352
+ if cols_types_docs and not cols_types_docs_filtered:
1353
+ cols_types_docs_filtered = cols_types_docs
1354
+
1355
+ cols_indices = defaultdict(lambda: [])
1356
+ for doc in cols_types_docs_filtered:
1357
+ col = (
1358
+ doc['column']
1359
+ if flavor != 'oracle'
1360
+ else (
1361
+ doc['column'].lower()
1362
+ if (doc['column'].isupper() and doc['column'].replace('_', '').isalpha())
1363
+ else doc['column']
1364
+ )
1365
+ )
1366
+ cols_indices[col].append(
1367
+ {
1368
+ 'name': doc.get('index', None),
1369
+ 'type': doc.get('index_type', None),
1370
+ }
1371
+ )
1372
+
1373
+ return dict(cols_indices)
1374
+ except Exception as e:
1375
+ warn(f"Failed to fetch columns for table '{table}':\n{e}")
1376
+ return {}
1377
+
1378
+
1020
1379
  def get_update_queries(
1021
1380
  target: str,
1022
1381
  patch: str,
@@ -1257,10 +1616,11 @@ def get_null_replacement(typ: str, flavor: str) -> str:
1257
1616
  A value which may stand in place of NULL for this type.
1258
1617
  `'None'` is returned if a value cannot be determined.
1259
1618
  """
1619
+ from meerschaum.utils.dtypes import are_dtypes_equal
1260
1620
  from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
1261
1621
  if 'int' in typ.lower() or typ.lower() in ('numeric', 'number'):
1262
1622
  return '-987654321'
1263
- if 'bool' in typ.lower():
1623
+ if 'bool' in typ.lower() or typ.lower() == 'bit':
1264
1624
  bool_typ = (
1265
1625
  PD_TO_DB_DTYPES_FLAVORS
1266
1626
  .get('bool', {})
@@ -1270,7 +1630,7 @@ def get_null_replacement(typ: str, flavor: str) -> str:
1270
1630
  bool_typ = DB_FLAVORS_CAST_DTYPES[flavor].get(bool_typ, bool_typ)
1271
1631
  val_to_cast = (
1272
1632
  -987654321
1273
- if flavor in ('mysql', 'mariadb', 'sqlite', 'mssql')
1633
+ if flavor in ('mysql', 'mariadb')
1274
1634
  else 0
1275
1635
  )
1276
1636
  return f'CAST({val_to_cast} AS {bool_typ})'
@@ -1278,6 +1638,8 @@ def get_null_replacement(typ: str, flavor: str) -> str:
1278
1638
  return dateadd_str(flavor=flavor, begin='1900-01-01')
1279
1639
  if 'float' in typ.lower() or 'double' in typ.lower() or typ.lower() in ('decimal',):
1280
1640
  return '-987654321.0'
1641
+ if flavor == 'oracle' and typ.lower().split('(', maxsplit=1)[0] == 'char':
1642
+ return "'-987654321'"
1281
1643
  if typ.lower() in ('uniqueidentifier', 'guid', 'uuid'):
1282
1644
  magic_val = 'DEADBEEF-ABBA-BABE-CAFE-DECAFC0FFEE5'
1283
1645
  if flavor == 'mssql':
@@ -1361,7 +1723,7 @@ def get_create_table_query(
1361
1723
  schema: Optional[str] = None,
1362
1724
  ) -> str:
1363
1725
  """
1364
- NOTE: This function is deprecated. Use `get_create_index_queries()` instead.
1726
+ NOTE: This function is deprecated. Use `get_create_table_queries()` instead.
1365
1727
 
1366
1728
  Return a query to create a new table from a `SELECT` query.
1367
1729
 
@@ -1399,6 +1761,8 @@ def get_create_table_queries(
1399
1761
  flavor: str,
1400
1762
  schema: Optional[str] = None,
1401
1763
  primary_key: Optional[str] = None,
1764
+ autoincrement: bool = False,
1765
+ datetime_column: Optional[str] = None,
1402
1766
  ) -> List[str]:
1403
1767
  """
1404
1768
  Return a query to create a new table from a `SELECT` query or a `dtypes` dictionary.
@@ -1421,6 +1785,14 @@ def get_create_table_queries(
1421
1785
  primary_key: Optional[str], default None
1422
1786
  If provided, designate this column as the primary key in the new table.
1423
1787
 
1788
+ autoincrement: bool, default False
1789
+ If `True` and `primary_key` is provided, create the `primary_key` column
1790
+ as an auto-incrementing integer column.
1791
+
1792
+ datetime_column: Optional[str], default None
1793
+ If provided, include this column in the primary key.
1794
+ Applicable to TimescaleDB only.
1795
+
1424
1796
  Returns
1425
1797
  -------
1426
1798
  A `CREATE TABLE` (or `SELECT INTO`) query for the database flavor.
@@ -1439,6 +1811,8 @@ def get_create_table_queries(
1439
1811
  flavor,
1440
1812
  schema=schema,
1441
1813
  primary_key=primary_key,
1814
+ autoincrement=(autoincrement and flavor not in SKIP_AUTO_INCREMENT_FLAVORS),
1815
+ datetime_column=datetime_column,
1442
1816
  )
1443
1817
 
1444
1818
 
@@ -1448,6 +1822,8 @@ def _get_create_table_query_from_dtypes(
1448
1822
  flavor: str,
1449
1823
  schema: Optional[str] = None,
1450
1824
  primary_key: Optional[str] = None,
1825
+ autoincrement: bool = False,
1826
+ datetime_column: Optional[str] = None,
1451
1827
  ) -> List[str]:
1452
1828
  """
1453
1829
  Create a new table from a `dtypes` dictionary.
@@ -1456,40 +1832,61 @@ def _get_create_table_query_from_dtypes(
1456
1832
  if not dtypes and not primary_key:
1457
1833
  raise ValueError(f"Expecting columns for table '{new_table}'.")
1458
1834
 
1835
+ if flavor in SKIP_AUTO_INCREMENT_FLAVORS:
1836
+ autoincrement = False
1837
+
1459
1838
  cols_types = (
1460
- [(primary_key, get_db_type_from_pd_type(dtypes.get(primary_key, 'int')))]
1839
+ [(primary_key, get_db_type_from_pd_type(dtypes.get(primary_key, 'int'), flavor=flavor))]
1461
1840
  if primary_key
1462
1841
  else []
1463
1842
  ) + [
1464
- (col, get_db_type_from_pd_type(typ))
1843
+ (col, get_db_type_from_pd_type(typ, flavor=flavor))
1465
1844
  for col, typ in dtypes.items()
1466
1845
  if col != primary_key
1467
1846
  ]
1468
1847
 
1469
1848
  table_name = sql_item_name(new_table, schema=schema, flavor=flavor)
1849
+ primary_key_name = sql_item_name(primary_key, flavor) if primary_key else None
1850
+ datetime_column_name = sql_item_name(datetime_column, flavor) if datetime_column else None
1470
1851
  query = f"CREATE TABLE {table_name} ("
1471
1852
  if primary_key:
1472
1853
  col_db_type = cols_types[0][1]
1473
- auto_increment = (' ' + AUTO_INCREMENT_COLUMN_FLAVORS.get(
1854
+ auto_increment_str = (' ' + AUTO_INCREMENT_COLUMN_FLAVORS.get(
1474
1855
  flavor,
1475
1856
  AUTO_INCREMENT_COLUMN_FLAVORS['default']
1476
- )) if primary_key not in dtypes else ''
1857
+ )) if autoincrement or primary_key not in dtypes else ''
1477
1858
  col_name = sql_item_name(primary_key, flavor=flavor, schema=None)
1478
1859
 
1479
1860
  if flavor == 'sqlite':
1480
- query += f"\n {col_name} INTEGER PRIMARY KEY{auto_increment} NOT NULL,"
1861
+ query += (
1862
+ f"\n {col_name} "
1863
+ + (f"{col_db_type}" if not auto_increment_str else 'INTEGER')
1864
+ + f" PRIMARY KEY{auto_increment_str} NOT NULL,"
1865
+ )
1866
+ elif flavor == 'oracle':
1867
+ query += f"\n {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
1868
+ elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
1869
+ query += f"\n {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
1481
1870
  else:
1482
- query += f"\n {col_name} {col_db_type} PRIMARY KEY{auto_increment} NOT NULL,"
1871
+ query += f"\n {col_name} {col_db_type} PRIMARY KEY{auto_increment_str} NOT NULL,"
1483
1872
 
1484
1873
  for col, db_type in cols_types:
1485
1874
  if col == primary_key:
1486
1875
  continue
1487
1876
  col_name = sql_item_name(col, schema=None, flavor=flavor)
1488
1877
  query += f"\n {col_name} {db_type},"
1878
+ if (
1879
+ flavor == 'timescaledb'
1880
+ and datetime_column
1881
+ and primary_key
1882
+ and datetime_column != primary_key
1883
+ ):
1884
+ query += f"\n PRIMARY KEY({datetime_column_name}, {primary_key_name}),"
1489
1885
  query = query[:-1]
1490
1886
  query += "\n)"
1491
1887
 
1492
- return [query]
1888
+ queries = [query]
1889
+ return queries
1493
1890
 
1494
1891
 
1495
1892
  def _get_create_table_query_from_cte(
@@ -1498,6 +1895,8 @@ def _get_create_table_query_from_cte(
1498
1895
  flavor: str,
1499
1896
  schema: Optional[str] = None,
1500
1897
  primary_key: Optional[str] = None,
1898
+ autoincrement: bool = False,
1899
+ datetime_column: Optional[str] = None,
1501
1900
  ) -> List[str]:
1502
1901
  """
1503
1902
  Create a new table from a CTE query.
@@ -1517,9 +1916,10 @@ def _get_create_table_query_from_cte(
1517
1916
  if primary_key
1518
1917
  else None
1519
1918
  )
1520
- auto_increment = AUTO_INCREMENT_COLUMN_FLAVORS.get(
1521
- flavor,
1522
- AUTO_INCREMENT_COLUMN_FLAVORS['default']
1919
+ datetime_column_name = (
1920
+ sql_item_name(datetime_column, flavor)
1921
+ if datetime_column
1922
+ else None
1523
1923
  )
1524
1924
  if flavor in ('mssql',):
1525
1925
  query = query.lstrip()
@@ -1566,6 +1966,17 @@ def _get_create_table_query_from_cte(
1566
1966
  ALTER TABLE {new_table_name}
1567
1967
  ADD PRIMARY KEY ({primary_key_name})
1568
1968
  """
1969
+ elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
1970
+ create_table_query = f"""
1971
+ SELECT *
1972
+ INTO {new_table_name}
1973
+ FROM ({query}) AS {create_cte_name}
1974
+ """
1975
+
1976
+ alter_type_query = f"""
1977
+ ALTER TABLE {new_table_name}
1978
+ ADD PRIMARY KEY ({datetime_column_name}, {primary_key_name})
1979
+ """
1569
1980
  else:
1570
1981
  create_table_query = f"""
1571
1982
  SELECT *
@@ -1758,3 +2169,61 @@ def session_execute(
1758
2169
  if with_results:
1759
2170
  return (success, msg), results
1760
2171
  return success, msg
2172
+
2173
+
2174
+ def get_reset_autoincrement_queries(
2175
+ table: str,
2176
+ column: str,
2177
+ connector: mrsm.connectors.SQLConnector,
2178
+ schema: Optional[str] = None,
2179
+ debug: bool = False,
2180
+ ) -> List[str]:
2181
+ """
2182
+ Return a list of queries to reset a table's auto-increment counter.
2183
+ """
2184
+ if not table_exists(table, connector, schema=schema, debug=debug):
2185
+ return []
2186
+
2187
+ schema = schema or connector.schema
2188
+ max_id_name = sql_item_name('max_id', connector.flavor)
2189
+ table_name = sql_item_name(table, connector.flavor, schema)
2190
+ table_trunc = truncate_item_name(table, connector.flavor)
2191
+ table_seq_name = sql_item_name(table + '_' + column + '_seq', connector.flavor, schema)
2192
+ column_name = sql_item_name(column, connector.flavor)
2193
+ if connector.flavor == 'oracle':
2194
+ df = connector.read(f"""
2195
+ SELECT SEQUENCE_NAME
2196
+ FROM ALL_TAB_IDENTITY_COLS
2197
+ WHERE TABLE_NAME IN '{table_trunc.upper()}'
2198
+ """, debug=debug)
2199
+ if len(df) > 0:
2200
+ table_seq_name = df['sequence_name'][0]
2201
+
2202
+ max_id = connector.value(
2203
+ f"""
2204
+ SELECT COALESCE(MAX({column_name}), 0) AS {max_id_name}
2205
+ FROM {table_name}
2206
+ """,
2207
+ debug=debug,
2208
+ )
2209
+ if max_id is None:
2210
+ return []
2211
+
2212
+ reset_queries = reset_autoincrement_queries.get(
2213
+ connector.flavor,
2214
+ reset_autoincrement_queries['default']
2215
+ )
2216
+ if not isinstance(reset_queries, list):
2217
+ reset_queries = [reset_queries]
2218
+
2219
+ return [
2220
+ query.format(
2221
+ column=column,
2222
+ column_name=column_name,
2223
+ table=table,
2224
+ table_name=table_name,
2225
+ table_seq_name=table_seq_name,
2226
+ val=(max_id),
2227
+ )
2228
+ for query in reset_queries
2229
+ ]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: meerschaum
3
- Version: 2.6.0.dev1
3
+ Version: 2.6.1
4
4
  Summary: Sync Time-Series Pipes with Meerschaum
5
5
  Home-page: https://meerschaum.io
6
6
  Author: Bennett Meares