meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +1 -1
  2. meerschaum/actions/delete.py +65 -69
  3. meerschaum/actions/edit.py +22 -2
  4. meerschaum/actions/install.py +1 -2
  5. meerschaum/actions/sync.py +2 -3
  6. meerschaum/api/routes/_pipes.py +7 -8
  7. meerschaum/config/_default.py +1 -1
  8. meerschaum/config/_paths.py +2 -1
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/api/_pipes.py +18 -21
  11. meerschaum/connectors/sql/_create_engine.py +3 -3
  12. meerschaum/connectors/sql/_instance.py +11 -12
  13. meerschaum/connectors/sql/_pipes.py +143 -91
  14. meerschaum/connectors/sql/_sql.py +43 -8
  15. meerschaum/connectors/valkey/_pipes.py +12 -1
  16. meerschaum/core/Pipe/__init__.py +23 -13
  17. meerschaum/core/Pipe/_attributes.py +25 -1
  18. meerschaum/core/Pipe/_dtypes.py +23 -16
  19. meerschaum/core/Pipe/_sync.py +59 -31
  20. meerschaum/core/Pipe/_verify.py +8 -7
  21. meerschaum/jobs/_Job.py +4 -1
  22. meerschaum/plugins/_Plugin.py +11 -14
  23. meerschaum/utils/daemon/Daemon.py +22 -15
  24. meerschaum/utils/dataframe.py +178 -16
  25. meerschaum/utils/dtypes/__init__.py +149 -14
  26. meerschaum/utils/dtypes/sql.py +41 -7
  27. meerschaum/utils/misc.py +8 -8
  28. meerschaum/utils/packages/_packages.py +1 -1
  29. meerschaum/utils/schedule.py +8 -3
  30. meerschaum/utils/sql.py +180 -100
  31. meerschaum/utils/venv/_Venv.py +4 -4
  32. meerschaum/utils/venv/__init__.py +53 -20
  33. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/METADATA +2 -2
  34. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/RECORD +40 -40
  35. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/LICENSE +0 -0
  36. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/NOTICE +0 -0
  37. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/WHEEL +0 -0
  38. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/entry_points.txt +0 -0
  39. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/top_level.txt +0 -0
  40. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/zip-safe +0 -0
meerschaum/utils/sql.py CHANGED
@@ -7,6 +7,7 @@ Flavor-specific SQL tools.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
+
10
11
  from datetime import datetime, timezone, timedelta
11
12
  import meerschaum as mrsm
12
13
  from meerschaum.utils.typing import Optional, Dict, Any, Union, List, Iterable, Tuple
@@ -50,10 +51,12 @@ update_queries = {
50
51
  {sets_subquery_none}
51
52
  FROM {target_table_name} AS t
52
53
  INNER JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
53
- ON {and_subquery_t}
54
+ ON
55
+ {and_subquery_t}
54
56
  WHERE
55
57
  {and_subquery_f}
56
- AND {date_bounds_subquery}
58
+ AND
59
+ {date_bounds_subquery}
57
60
  """,
58
61
  'timescaledb-upsert': """
59
62
  INSERT INTO {target_table_name} ({patch_cols_str})
@@ -82,9 +85,11 @@ update_queries = {
82
85
  'mysql': """
83
86
  UPDATE {target_table_name} AS f
84
87
  JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
85
- ON {and_subquery_f}
88
+ ON
89
+ {and_subquery_f}
86
90
  {sets_subquery_f}
87
- WHERE {date_bounds_subquery}
91
+ WHERE
92
+ {date_bounds_subquery}
88
93
  """,
89
94
  'mysql-upsert': """
90
95
  INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
@@ -96,9 +101,11 @@ update_queries = {
96
101
  'mariadb': """
97
102
  UPDATE {target_table_name} AS f
98
103
  JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
99
- ON {and_subquery_f}
104
+ ON
105
+ {and_subquery_f}
100
106
  {sets_subquery_f}
101
- WHERE {date_bounds_subquery}
107
+ WHERE
108
+ {date_bounds_subquery}
102
109
  """,
103
110
  'mariadb-upsert': """
104
111
  INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
@@ -108,34 +115,56 @@ update_queries = {
108
115
  {cols_equal_values}
109
116
  """,
110
117
  'mssql': """
118
+ {with_temp_date_bounds}
111
119
  MERGE {target_table_name} f
112
- USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
113
- ON {and_subquery_f}
114
- AND {date_bounds_subquery}
120
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
121
+ ON
122
+ {and_subquery_f}
123
+ AND
124
+ {date_bounds_subquery}
115
125
  WHEN MATCHED THEN
116
126
  UPDATE
117
127
  {sets_subquery_none};
118
128
  """,
119
- 'mssql-upsert': """
129
+ 'mssql-upsert': [
130
+ "{identity_insert_on}",
131
+ """
132
+ {with_temp_date_bounds}
120
133
  MERGE {target_table_name} f
121
- USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
122
- ON {and_subquery_f}
123
- AND {date_bounds_subquery}
124
- {when_matched_update_sets_subquery_none}
134
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
135
+ ON
136
+ {and_subquery_f}
137
+ AND
138
+ {date_bounds_subquery}{when_matched_update_sets_subquery_none}
125
139
  WHEN NOT MATCHED THEN
126
140
  INSERT ({patch_cols_str})
127
141
  VALUES ({patch_cols_prefixed_str});
128
- """,
142
+ """,
143
+ "{identity_insert_off}",
144
+ ],
129
145
  'oracle': """
130
146
  MERGE INTO {target_table_name} f
131
- USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
147
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
132
148
  ON (
133
149
  {and_subquery_f}
134
- AND {date_bounds_subquery}
150
+ AND
151
+ {date_bounds_subquery}
135
152
  )
136
- WHEN MATCHED THEN
137
- UPDATE
138
- {sets_subquery_none}
153
+ WHEN MATCHED THEN
154
+ UPDATE
155
+ {sets_subquery_none}
156
+ """,
157
+ 'oracle-upsert': """
158
+ MERGE INTO {target_table_name} f
159
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
160
+ ON (
161
+ {and_subquery_f}
162
+ AND
163
+ {date_bounds_subquery}
164
+ ){when_matched_update_sets_subquery_none}
165
+ WHEN NOT MATCHED THEN
166
+ INSERT ({patch_cols_str})
167
+ VALUES ({patch_cols_prefixed_str})
139
168
  """,
140
169
  'sqlite-upsert': """
141
170
  INSERT INTO {target_table_name} ({patch_cols_str})
@@ -323,7 +352,11 @@ columns_indices_queries = {
323
352
  CASE
324
353
  WHEN kc.type = 'PK' THEN 'PRIMARY KEY'
325
354
  ELSE 'INDEX'
326
- END AS [index_type]
355
+ END AS [index_type],
356
+ CASE
357
+ WHEN i.type = 1 THEN CAST(1 AS BIT)
358
+ ELSE CAST(0 AS BIT)
359
+ END AS [clustered]
327
360
  FROM
328
361
  sys.schemas s
329
362
  INNER JOIN sys.tables t
@@ -425,20 +458,10 @@ reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = {
425
458
  SET seq = {val}
426
459
  WHERE name = '{table}'
427
460
  """,
428
- 'oracle': [
429
- """
430
- DECLARE
431
- max_id NUMBER := {val};
432
- current_val NUMBER;
433
- BEGIN
434
- SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
435
-
436
- WHILE current_val < max_id LOOP
437
- SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
438
- END LOOP;
439
- END;
440
- """,
441
- ],
461
+ 'oracle': (
462
+ "ALTER TABLE {table_name} MODIFY {column_name} "
463
+ "GENERATED BY DEFAULT ON NULL AS IDENTITY (START WITH {val_plus_1})"
464
+ ),
442
465
  }
443
466
  table_wrappers = {
444
467
  'default' : ('"', '"'),
@@ -499,7 +522,8 @@ def dateadd_str(
499
522
  flavor: str = 'postgresql',
500
523
  datepart: str = 'day',
501
524
  number: Union[int, float] = 0,
502
- begin: Union[str, datetime, int] = 'now'
525
+ begin: Union[str, datetime, int] = 'now',
526
+ db_type: Optional[str] = None,
503
527
  ) -> str:
504
528
  """
505
529
  Generate a `DATEADD` clause depending on database flavor.
@@ -538,6 +562,10 @@ def dateadd_str(
538
562
  begin: Union[str, datetime], default `'now'`
539
563
  Base datetime to which to add dateparts.
540
564
 
565
+ db_type: Optional[str], default None
566
+ If provided, cast the datetime string as the type.
567
+ Otherwise, infer this from the input datetime value.
568
+
541
569
  Returns
542
570
  -------
543
571
  The appropriate `DATEADD` string for the corresponding database flavor.
@@ -549,7 +577,7 @@ def dateadd_str(
549
577
  ... begin = datetime(2022, 1, 1, 0, 0),
550
578
  ... number = 1,
551
579
  ... )
552
- "DATEADD(day, 1, CAST('2022-01-01 00:00:00' AS DATETIME))"
580
+ "DATEADD(day, 1, CAST('2022-01-01 00:00:00' AS DATETIME2))"
553
581
  >>> dateadd_str(
554
582
  ... flavor = 'postgresql',
555
583
  ... begin = datetime(2022, 1, 1, 0, 0),
@@ -592,7 +620,7 @@ def dateadd_str(
592
620
  )
593
621
 
594
622
  dt_is_utc = begin_time.tzinfo is not None if begin_time is not None else '+' in str(begin)
595
- db_type = get_db_type_from_pd_type(
623
+ db_type = db_type or get_db_type_from_pd_type(
596
624
  ('datetime64[ns, UTC]' if dt_is_utc else 'datetime64[ns]'),
597
625
  flavor=flavor,
598
626
  )
@@ -717,7 +745,7 @@ def get_distinct_col_count(
717
745
  result = connector.value(_meta_query, debug=debug)
718
746
  try:
719
747
  return int(result)
720
- except Exception as e:
748
+ except Exception:
721
749
  return None
722
750
 
723
751
 
@@ -727,12 +755,15 @@ def sql_item_name(item: str, flavor: str, schema: Optional[str] = None) -> str:
727
755
 
728
756
  Parameters
729
757
  ----------
730
- item: str :
758
+ item: str
731
759
  The database item (table, view, etc.) in need of quotes.
732
760
 
733
- flavor: str :
761
+ flavor: str
734
762
  The database flavor (`'postgresql'`, `'mssql'`, `'sqllite'`, etc.).
735
763
 
764
+ schema: Optional[str], default None
765
+ If provided, prefix the table name with the schema.
766
+
736
767
  Returns
737
768
  -------
738
769
  A `str` which contains the input `item` wrapped in the corresponding escape characters.
@@ -764,6 +795,8 @@ def sql_item_name(item: str, flavor: str, schema: Optional[str] = None) -> str:
764
795
  ### NOTE: SQLite does not support schemas.
765
796
  if flavor == 'sqlite':
766
797
  schema = None
798
+ elif flavor == 'mssql' and str(item).startswith('#'):
799
+ schema = None
767
800
 
768
801
  schema_prefix = (
769
802
  (wrappers[0] + schema + wrappers[1] + '.')
@@ -1066,7 +1099,7 @@ def get_sqlalchemy_table(
1066
1099
  connector.metadata,
1067
1100
  **table_kwargs
1068
1101
  )
1069
- except sqlalchemy.exc.NoSuchTableError as e:
1102
+ except sqlalchemy.exc.NoSuchTableError:
1070
1103
  warn(f"Table '{truncated_table_name}' does not exist in '{connector}'.")
1071
1104
  return None
1072
1105
  return tables[truncated_table_name]
@@ -1119,6 +1152,7 @@ def get_table_cols_types(
1119
1152
  -------
1120
1153
  A dictionary mapping column names to data types.
1121
1154
  """
1155
+ import textwrap
1122
1156
  from meerschaum.connectors import SQLConnector
1123
1157
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
1124
1158
  flavor = flavor or getattr(connectable, 'flavor', None)
@@ -1144,7 +1178,7 @@ def get_table_cols_types(
1144
1178
  )
1145
1179
 
1146
1180
  cols_types_query = sqlalchemy.text(
1147
- columns_types_queries.get(
1181
+ textwrap.dedent(columns_types_queries.get(
1148
1182
  flavor,
1149
1183
  columns_types_queries['default']
1150
1184
  ).format(
@@ -1155,7 +1189,7 @@ def get_table_cols_types(
1155
1189
  table_upper=table_upper,
1156
1190
  table_upper_trunc=table_upper_trunc,
1157
1191
  db_prefix=db_prefix,
1158
- )
1192
+ )).lstrip().rstrip()
1159
1193
  )
1160
1194
 
1161
1195
  cols = ['database', 'schema', 'table', 'column', 'type']
@@ -1269,6 +1303,7 @@ def get_table_cols_indices(
1269
1303
  -------
1270
1304
  A dictionary mapping column names to a list of indices.
1271
1305
  """
1306
+ import textwrap
1272
1307
  from collections import defaultdict
1273
1308
  from meerschaum.connectors import SQLConnector
1274
1309
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
@@ -1295,7 +1330,7 @@ def get_table_cols_indices(
1295
1330
  )
1296
1331
 
1297
1332
  cols_indices_query = sqlalchemy.text(
1298
- columns_indices_queries.get(
1333
+ textwrap.dedent(columns_indices_queries.get(
1299
1334
  flavor,
1300
1335
  columns_indices_queries['default']
1301
1336
  ).format(
@@ -1307,10 +1342,12 @@ def get_table_cols_indices(
1307
1342
  table_upper_trunc=table_upper_trunc,
1308
1343
  db_prefix=db_prefix,
1309
1344
  schema=schema,
1310
- )
1345
+ )).lstrip().rstrip()
1311
1346
  )
1312
1347
 
1313
1348
  cols = ['database', 'schema', 'table', 'column', 'index', 'index_type']
1349
+ if flavor == 'mssql':
1350
+ cols.append('clustered')
1314
1351
  result_cols_ix = dict(enumerate(cols))
1315
1352
 
1316
1353
  debug_kwargs = {'debug': debug} if isinstance(connectable, SQLConnector) else {}
@@ -1351,7 +1388,6 @@ def get_table_cols_indices(
1351
1388
  )
1352
1389
  )
1353
1390
  ]
1354
-
1355
1391
  ### NOTE: This may return incorrect columns if the schema is not explicitly stated.
1356
1392
  if cols_types_docs and not cols_types_docs_filtered:
1357
1393
  cols_types_docs_filtered = cols_types_docs
@@ -1367,12 +1403,13 @@ def get_table_cols_indices(
1367
1403
  else doc['column']
1368
1404
  )
1369
1405
  )
1370
- cols_indices[col].append(
1371
- {
1372
- 'name': doc.get('index', None),
1373
- 'type': doc.get('index_type', None),
1374
- }
1375
- )
1406
+ index_doc = {
1407
+ 'name': doc.get('index', None),
1408
+ 'type': doc.get('index_type', None)
1409
+ }
1410
+ if flavor == 'mssql':
1411
+ index_doc['clustered'] = doc.get('clustered', None)
1412
+ cols_indices[col].append(index_doc)
1376
1413
 
1377
1414
  return dict(cols_indices)
1378
1415
  except Exception as e:
@@ -1393,6 +1430,7 @@ def get_update_queries(
1393
1430
  datetime_col: Optional[str] = None,
1394
1431
  schema: Optional[str] = None,
1395
1432
  patch_schema: Optional[str] = None,
1433
+ identity_insert: bool = False,
1396
1434
  debug: bool = False,
1397
1435
  ) -> List[str]:
1398
1436
  """
@@ -1430,6 +1468,10 @@ def get_update_queries(
1430
1468
  If provided, use this schema when quoting the patch table.
1431
1469
  Defaults to `schema`.
1432
1470
 
1471
+ identity_insert: bool, default False
1472
+ If `True`, include `SET IDENTITY_INSERT` queries before and after the update queries.
1473
+ Only applies for MSSQL upserts.
1474
+
1433
1475
  debug: bool, default False
1434
1476
  Verbosity toggle.
1435
1477
 
@@ -1437,9 +1479,11 @@ def get_update_queries(
1437
1479
  -------
1438
1480
  A list of query strings to perform the update operation.
1439
1481
  """
1482
+ import textwrap
1440
1483
  from meerschaum.connectors import SQLConnector
1441
1484
  from meerschaum.utils.debug import dprint
1442
- from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
1485
+ from meerschaum.utils.dtypes import are_dtypes_equal
1486
+ from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES, get_pd_type_from_db_type
1443
1487
  flavor = flavor or (connectable.flavor if isinstance(connectable, SQLConnector) else None)
1444
1488
  if not flavor:
1445
1489
  raise ValueError("Provide a flavor if using a SQLAlchemy session.")
@@ -1532,21 +1576,35 @@ def get_update_queries(
1532
1576
  def sets_subquery(l_prefix: str, r_prefix: str):
1533
1577
  if not value_cols:
1534
1578
  return ''
1579
+
1580
+ cast_func_cols = {
1581
+ c_name: (
1582
+ ('', '', '')
1583
+ if (
1584
+ flavor == 'oracle'
1585
+ and are_dtypes_equal(get_pd_type_from_db_type(c_type), 'bytes')
1586
+ )
1587
+ else (
1588
+ ('CAST(', f" AS {c_type.replace('_', ' ')}", ')')
1589
+ if flavor != 'sqlite'
1590
+ else ('', '', '')
1591
+ )
1592
+ )
1593
+ for c_name, c_type in value_cols
1594
+ }
1535
1595
  return 'SET ' + ',\n'.join([
1536
1596
  (
1537
1597
  l_prefix + sql_item_name(c_name, flavor, None)
1538
1598
  + ' = '
1539
- + ('CAST(' if flavor != 'sqlite' else '')
1540
- + r_prefix
1541
- + sql_item_name(c_name, flavor, None)
1542
- + (' AS ' if flavor != 'sqlite' else '')
1543
- + (c_type.replace('_', ' ') if flavor != 'sqlite' else '')
1544
- + (')' if flavor != 'sqlite' else '')
1599
+ + cast_func_cols[c_name][0]
1600
+ + r_prefix + sql_item_name(c_name, flavor, None)
1601
+ + cast_func_cols[c_name][1]
1602
+ + cast_func_cols[c_name][2]
1545
1603
  ) for c_name, c_type in value_cols
1546
1604
  ])
1547
1605
 
1548
1606
  def and_subquery(l_prefix: str, r_prefix: str):
1549
- return '\nAND\n'.join([
1607
+ return '\n AND\n '.join([
1550
1608
  (
1551
1609
  "COALESCE("
1552
1610
  + l_prefix
@@ -1554,7 +1612,7 @@ def get_update_queries(
1554
1612
  + ", "
1555
1613
  + get_null_replacement(c_type, flavor)
1556
1614
  + ")"
1557
- + ' = '
1615
+ + '\n =\n '
1558
1616
  + "COALESCE("
1559
1617
  + r_prefix
1560
1618
  + sql_item_name(c_name, flavor, None)
@@ -1564,22 +1622,39 @@ def get_update_queries(
1564
1622
  ) for c_name, c_type in join_cols_types
1565
1623
  ])
1566
1624
 
1625
+ skip_query_val = ""
1567
1626
  target_table_name = sql_item_name(target, flavor, schema)
1568
1627
  patch_table_name = sql_item_name(patch, flavor, patch_schema)
1569
1628
  dt_col_name = sql_item_name(datetime_col, flavor, None) if datetime_col else None
1629
+ date_bounds_table = patch_table_name if flavor != 'mssql' else '[date_bounds]'
1630
+ min_dt_col_name = f"MIN({dt_col_name})" if flavor != 'mssql' else '[Min_dt]'
1631
+ max_dt_col_name = f"MAX({dt_col_name})" if flavor != 'mssql' else '[Max_dt]'
1570
1632
  date_bounds_subquery = (
1571
- f"""
1572
- f.{dt_col_name} >= (SELECT MIN({dt_col_name}) FROM {patch_table_name})
1573
- AND f.{dt_col_name} <= (SELECT MAX({dt_col_name}) FROM {patch_table_name})
1574
- """
1633
+ f"""f.{dt_col_name} >= (SELECT {min_dt_col_name} FROM {date_bounds_table})
1634
+ AND
1635
+ f.{dt_col_name} <= (SELECT {max_dt_col_name} FROM {date_bounds_table})"""
1575
1636
  if datetime_col
1576
1637
  else "1 = 1"
1577
1638
  )
1639
+ with_temp_date_bounds = f"""WITH [date_bounds] AS (
1640
+ SELECT MIN({dt_col_name}) AS {min_dt_col_name}, MAX({dt_col_name}) AS {max_dt_col_name}
1641
+ FROM {patch_table_name}
1642
+ )""" if datetime_col else ""
1643
+ identity_insert_on = (
1644
+ f"SET IDENTITY_INSERT {target_table_name} ON"
1645
+ if identity_insert
1646
+ else skip_query_val
1647
+ )
1648
+ identity_insert_off = (
1649
+ f"SET IDENTITY_INSERT {target_table_name} OFF"
1650
+ if identity_insert
1651
+ else skip_query_val
1652
+ )
1578
1653
 
1579
1654
  ### NOTE: MSSQL upserts must exclude the update portion if only upserting indices.
1580
1655
  when_matched_update_sets_subquery_none = "" if not value_cols else (
1581
- "WHEN MATCHED THEN"
1582
- f" UPDATE {sets_subquery('', 'p.')}"
1656
+ "\n WHEN MATCHED THEN\n"
1657
+ f" UPDATE {sets_subquery('', 'p.')}"
1583
1658
  )
1584
1659
 
1585
1660
  cols_equal_values = '\n,'.join(
@@ -1595,8 +1670,8 @@ def get_update_queries(
1595
1670
  )
1596
1671
  ignore = "IGNORE " if not value_cols else ""
1597
1672
 
1598
- return [
1599
- base_query.format(
1673
+ formatted_queries = [
1674
+ textwrap.dedent(base_query.format(
1600
1675
  sets_subquery_none=sets_subquery('', 'p.'),
1601
1676
  sets_subquery_none_excluded=sets_subquery('', 'EXCLUDED.'),
1602
1677
  sets_subquery_f=sets_subquery('f.', 'p.'),
@@ -1614,10 +1689,16 @@ def get_update_queries(
1614
1689
  cols_equal_values=cols_equal_values,
1615
1690
  on_duplicate_key_update=on_duplicate_key_update,
1616
1691
  ignore=ignore,
1617
- )
1692
+ with_temp_date_bounds=with_temp_date_bounds,
1693
+ identity_insert_on=identity_insert_on,
1694
+ identity_insert_off=identity_insert_off,
1695
+ )).lstrip().rstrip()
1618
1696
  for base_query in base_queries
1619
1697
  ]
1620
1698
 
1699
+ ### NOTE: Allow for skipping some queries.
1700
+ return [query for query in formatted_queries if query]
1701
+
1621
1702
 
1622
1703
  def get_null_replacement(typ: str, flavor: str) -> str:
1623
1704
  """
@@ -1655,11 +1736,14 @@ def get_null_replacement(typ: str, flavor: str) -> str:
1655
1736
  )
1656
1737
  return f'CAST({val_to_cast} AS {bool_typ})'
1657
1738
  if 'time' in typ.lower() or 'date' in typ.lower():
1658
- return dateadd_str(flavor=flavor, begin='1900-01-01')
1739
+ db_type = typ if typ.isupper() else None
1740
+ return dateadd_str(flavor=flavor, begin='1900-01-01', db_type=db_type)
1659
1741
  if 'float' in typ.lower() or 'double' in typ.lower() or typ.lower() in ('decimal',):
1660
1742
  return '-987654321.0'
1661
1743
  if flavor == 'oracle' and typ.lower().split('(', maxsplit=1)[0] == 'char':
1662
1744
  return "'-987654321'"
1745
+ if flavor == 'oracle' and typ.lower() in ('blob', 'bytes'):
1746
+ return '00'
1663
1747
  if typ.lower() in ('uniqueidentifier', 'guid', 'uuid'):
1664
1748
  magic_val = 'DEADBEEF-ABBA-BABE-CAFE-DECAFC0FFEE5'
1665
1749
  if flavor == 'mssql':
@@ -1867,7 +1951,17 @@ def _get_create_table_query_from_dtypes(
1867
1951
 
1868
1952
  table_name = sql_item_name(new_table, schema=schema, flavor=flavor)
1869
1953
  primary_key_name = sql_item_name(primary_key, flavor) if primary_key else None
1954
+ primary_key_constraint_name = (
1955
+ sql_item_name(f'PK_{new_table}', flavor, None)
1956
+ if primary_key
1957
+ else None
1958
+ )
1870
1959
  datetime_column_name = sql_item_name(datetime_column, flavor) if datetime_column else None
1960
+ primary_key_clustered = (
1961
+ "CLUSTERED"
1962
+ if not datetime_column or datetime_column == primary_key
1963
+ else "NONCLUSTERED"
1964
+ )
1871
1965
  query = f"CREATE TABLE {table_name} ("
1872
1966
  if primary_key:
1873
1967
  col_db_type = cols_types[0][1]
@@ -1887,6 +1981,8 @@ def _get_create_table_query_from_dtypes(
1887
1981
  query += f"\n {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
1888
1982
  elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
1889
1983
  query += f"\n {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
1984
+ elif flavor == 'mssql':
1985
+ query += f"\n {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
1890
1986
  else:
1891
1987
  query += f"\n {col_name} {col_db_type} PRIMARY KEY{auto_increment_str} NOT NULL,"
1892
1988
 
@@ -1902,6 +1998,10 @@ def _get_create_table_query_from_dtypes(
1902
1998
  and datetime_column != primary_key
1903
1999
  ):
1904
2000
  query += f"\n PRIMARY KEY({datetime_column_name}, {primary_key_name}),"
2001
+
2002
+ if flavor == 'mssql' and primary_key:
2003
+ query += f"\n CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name}),"
2004
+
1905
2005
  query = query[:-1]
1906
2006
  query += "\n)"
1907
2007
 
@@ -1922,12 +2022,11 @@ def _get_create_table_query_from_cte(
1922
2022
  Create a new table from a CTE query.
1923
2023
  """
1924
2024
  import textwrap
1925
- from meerschaum.utils.dtypes.sql import AUTO_INCREMENT_COLUMN_FLAVORS
1926
2025
  create_cte = 'create_query'
1927
2026
  create_cte_name = sql_item_name(create_cte, flavor, None)
1928
2027
  new_table_name = sql_item_name(new_table, flavor, schema)
1929
2028
  primary_key_constraint_name = (
1930
- sql_item_name(f'pk_{new_table}', flavor, None)
2029
+ sql_item_name(f'PK_{new_table}', flavor, None)
1931
2030
  if primary_key
1932
2031
  else None
1933
2032
  )
@@ -1936,6 +2035,7 @@ def _get_create_table_query_from_cte(
1936
2035
  if primary_key
1937
2036
  else None
1938
2037
  )
2038
+ primary_key_clustered = "CLUSTERED" if not datetime_column else "NONCLUSTERED"
1939
2039
  datetime_column_name = (
1940
2040
  sql_item_name(datetime_column, flavor)
1941
2041
  if datetime_column
@@ -1943,7 +2043,7 @@ def _get_create_table_query_from_cte(
1943
2043
  )
1944
2044
  if flavor in ('mssql',):
1945
2045
  query = query.lstrip()
1946
- if 'with ' in query.lower():
2046
+ if query.lower().startswith('with '):
1947
2047
  final_select_ix = query.lower().rfind('select')
1948
2048
  create_table_query = (
1949
2049
  query[:final_select_ix].rstrip() + ',\n'
@@ -1961,7 +2061,7 @@ def _get_create_table_query_from_cte(
1961
2061
 
1962
2062
  alter_type_query = f"""
1963
2063
  ALTER TABLE {new_table_name}
1964
- ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY ({primary_key_name})
2064
+ ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name})
1965
2065
  """
1966
2066
  elif flavor in (None,):
1967
2067
  create_table_query = f"""
@@ -2009,11 +2109,11 @@ def _get_create_table_query_from_cte(
2009
2109
  ADD PRIMARY KEY ({primary_key_name})
2010
2110
  """
2011
2111
 
2012
- create_table_query = textwrap.dedent(create_table_query)
2112
+ create_table_query = textwrap.dedent(create_table_query).lstrip().rstrip()
2013
2113
  if not primary_key:
2014
2114
  return [create_table_query]
2015
2115
 
2016
- alter_type_query = textwrap.dedent(alter_type_query)
2116
+ alter_type_query = textwrap.dedent(alter_type_query).lstrip().rstrip()
2017
2117
 
2018
2118
  return [
2019
2119
  create_table_query,
@@ -2225,29 +2325,8 @@ def get_reset_autoincrement_queries(
2225
2325
  schema = schema or connector.schema
2226
2326
  max_id_name = sql_item_name('max_id', connector.flavor)
2227
2327
  table_name = sql_item_name(table, connector.flavor, schema)
2228
- table_trunc = truncate_item_name(table, connector.flavor)
2229
2328
  table_seq_name = sql_item_name(table + '_' + column + '_seq', connector.flavor, schema)
2230
2329
  column_name = sql_item_name(column, connector.flavor)
2231
- if connector.flavor == 'oracle':
2232
- potential_table_names = set([
2233
- f"'{table_trunc.upper()}'",
2234
- f"'{table_trunc}'",
2235
- f"'{table_name}'",
2236
- f"'{table_name.upper()}'",
2237
- ])
2238
- df = connector.read(
2239
- """
2240
- SELECT SEQUENCE_NAME
2241
- FROM ALL_TAB_IDENTITY_COLS
2242
- WHERE TABLE_NAME IN ("""
2243
- + ", ".join([name for name in potential_table_names])
2244
- + """)
2245
- """,
2246
- debug=debug
2247
- )
2248
- if len(df) > 0:
2249
- table_seq_name = df['sequence_name'][0]
2250
-
2251
2330
  max_id = connector.value(
2252
2331
  f"""
2253
2332
  SELECT COALESCE(MAX({column_name}), 0) AS {max_id_name}
@@ -2272,7 +2351,8 @@ def get_reset_autoincrement_queries(
2272
2351
  table=table,
2273
2352
  table_name=table_name,
2274
2353
  table_seq_name=table_seq_name,
2275
- val=(max_id),
2354
+ val=max_id,
2355
+ val_plus_1=(max_id + 1),
2276
2356
  )
2277
2357
  for query in reset_queries
2278
2358
  ]
@@ -34,10 +34,10 @@ class Venv:
34
34
  """
35
35
 
36
36
  def __init__(
37
- self,
38
- venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
39
- debug: bool = False,
40
- ) -> None:
37
+ self,
38
+ venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
39
+ debug: bool = False,
40
+ ) -> None:
41
41
  from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
42
42
  ### For some weird threading issue,
43
43
  ### we can't use `isinstance` here.