meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +1 -1
  2. meerschaum/actions/delete.py +65 -69
  3. meerschaum/actions/edit.py +22 -2
  4. meerschaum/actions/install.py +1 -2
  5. meerschaum/actions/sync.py +2 -3
  6. meerschaum/api/routes/_pipes.py +7 -8
  7. meerschaum/config/_default.py +1 -1
  8. meerschaum/config/_paths.py +2 -1
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/api/_pipes.py +18 -21
  11. meerschaum/connectors/sql/_create_engine.py +3 -3
  12. meerschaum/connectors/sql/_instance.py +11 -12
  13. meerschaum/connectors/sql/_pipes.py +143 -91
  14. meerschaum/connectors/sql/_sql.py +43 -8
  15. meerschaum/connectors/valkey/_pipes.py +12 -1
  16. meerschaum/core/Pipe/__init__.py +23 -13
  17. meerschaum/core/Pipe/_attributes.py +25 -1
  18. meerschaum/core/Pipe/_dtypes.py +23 -16
  19. meerschaum/core/Pipe/_sync.py +59 -31
  20. meerschaum/core/Pipe/_verify.py +8 -7
  21. meerschaum/jobs/_Job.py +4 -1
  22. meerschaum/plugins/_Plugin.py +11 -14
  23. meerschaum/utils/daemon/Daemon.py +22 -15
  24. meerschaum/utils/dataframe.py +178 -16
  25. meerschaum/utils/dtypes/__init__.py +149 -14
  26. meerschaum/utils/dtypes/sql.py +41 -7
  27. meerschaum/utils/misc.py +8 -8
  28. meerschaum/utils/packages/_packages.py +1 -1
  29. meerschaum/utils/schedule.py +8 -3
  30. meerschaum/utils/sql.py +180 -100
  31. meerschaum/utils/venv/_Venv.py +4 -4
  32. meerschaum/utils/venv/__init__.py +53 -20
  33. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/METADATA +2 -2
  34. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/RECORD +40 -40
  35. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/LICENSE +0 -0
  36. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/NOTICE +0 -0
  37. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/WHEEL +0 -0
  38. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/entry_points.txt +0 -0
  39. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/top_level.txt +0 -0
  40. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/zip-safe +0 -0
meerschaum/utils/sql.py CHANGED
@@ -7,6 +7,7 @@ Flavor-specific SQL tools.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
+
10
11
  from datetime import datetime, timezone, timedelta
11
12
  import meerschaum as mrsm
12
13
  from meerschaum.utils.typing import Optional, Dict, Any, Union, List, Iterable, Tuple
@@ -50,10 +51,12 @@ update_queries = {
50
51
  {sets_subquery_none}
51
52
  FROM {target_table_name} AS t
52
53
  INNER JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
53
- ON {and_subquery_t}
54
+ ON
55
+ {and_subquery_t}
54
56
  WHERE
55
57
  {and_subquery_f}
56
- AND {date_bounds_subquery}
58
+ AND
59
+ {date_bounds_subquery}
57
60
  """,
58
61
  'timescaledb-upsert': """
59
62
  INSERT INTO {target_table_name} ({patch_cols_str})
@@ -82,9 +85,11 @@ update_queries = {
82
85
  'mysql': """
83
86
  UPDATE {target_table_name} AS f
84
87
  JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
85
- ON {and_subquery_f}
88
+ ON
89
+ {and_subquery_f}
86
90
  {sets_subquery_f}
87
- WHERE {date_bounds_subquery}
91
+ WHERE
92
+ {date_bounds_subquery}
88
93
  """,
89
94
  'mysql-upsert': """
90
95
  INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
@@ -96,9 +101,11 @@ update_queries = {
96
101
  'mariadb': """
97
102
  UPDATE {target_table_name} AS f
98
103
  JOIN (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) AS p
99
- ON {and_subquery_f}
104
+ ON
105
+ {and_subquery_f}
100
106
  {sets_subquery_f}
101
- WHERE {date_bounds_subquery}
107
+ WHERE
108
+ {date_bounds_subquery}
102
109
  """,
103
110
  'mariadb-upsert': """
104
111
  INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
@@ -108,34 +115,56 @@ update_queries = {
108
115
  {cols_equal_values}
109
116
  """,
110
117
  'mssql': """
118
+ {with_temp_date_bounds}
111
119
  MERGE {target_table_name} f
112
- USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
113
- ON {and_subquery_f}
114
- AND {date_bounds_subquery}
120
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
121
+ ON
122
+ {and_subquery_f}
123
+ AND
124
+ {date_bounds_subquery}
115
125
  WHEN MATCHED THEN
116
126
  UPDATE
117
127
  {sets_subquery_none};
118
128
  """,
119
- 'mssql-upsert': """
129
+ 'mssql-upsert': [
130
+ "{identity_insert_on}",
131
+ """
132
+ {with_temp_date_bounds}
120
133
  MERGE {target_table_name} f
121
- USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
122
- ON {and_subquery_f}
123
- AND {date_bounds_subquery}
124
- {when_matched_update_sets_subquery_none}
134
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
135
+ ON
136
+ {and_subquery_f}
137
+ AND
138
+ {date_bounds_subquery}{when_matched_update_sets_subquery_none}
125
139
  WHEN NOT MATCHED THEN
126
140
  INSERT ({patch_cols_str})
127
141
  VALUES ({patch_cols_prefixed_str});
128
- """,
142
+ """,
143
+ "{identity_insert_off}",
144
+ ],
129
145
  'oracle': """
130
146
  MERGE INTO {target_table_name} f
131
- USING (SELECT DISTINCT {patch_cols_str} FROM {patch_table_name}) p
147
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
132
148
  ON (
133
149
  {and_subquery_f}
134
- AND {date_bounds_subquery}
150
+ AND
151
+ {date_bounds_subquery}
135
152
  )
136
- WHEN MATCHED THEN
137
- UPDATE
138
- {sets_subquery_none}
153
+ WHEN MATCHED THEN
154
+ UPDATE
155
+ {sets_subquery_none}
156
+ """,
157
+ 'oracle-upsert': """
158
+ MERGE INTO {target_table_name} f
159
+ USING (SELECT {patch_cols_str} FROM {patch_table_name}) p
160
+ ON (
161
+ {and_subquery_f}
162
+ AND
163
+ {date_bounds_subquery}
164
+ ){when_matched_update_sets_subquery_none}
165
+ WHEN NOT MATCHED THEN
166
+ INSERT ({patch_cols_str})
167
+ VALUES ({patch_cols_prefixed_str})
139
168
  """,
140
169
  'sqlite-upsert': """
141
170
  INSERT INTO {target_table_name} ({patch_cols_str})
@@ -323,7 +352,11 @@ columns_indices_queries = {
323
352
  CASE
324
353
  WHEN kc.type = 'PK' THEN 'PRIMARY KEY'
325
354
  ELSE 'INDEX'
326
- END AS [index_type]
355
+ END AS [index_type],
356
+ CASE
357
+ WHEN i.type = 1 THEN CAST(1 AS BIT)
358
+ ELSE CAST(0 AS BIT)
359
+ END AS [clustered]
327
360
  FROM
328
361
  sys.schemas s
329
362
  INNER JOIN sys.tables t
@@ -425,20 +458,10 @@ reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = {
425
458
  SET seq = {val}
426
459
  WHERE name = '{table}'
427
460
  """,
428
- 'oracle': [
429
- """
430
- DECLARE
431
- max_id NUMBER := {val};
432
- current_val NUMBER;
433
- BEGIN
434
- SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
435
-
436
- WHILE current_val < max_id LOOP
437
- SELECT {table_seq_name}.NEXTVAL INTO current_val FROM dual;
438
- END LOOP;
439
- END;
440
- """,
441
- ],
461
+ 'oracle': (
462
+ "ALTER TABLE {table_name} MODIFY {column_name} "
463
+ "GENERATED BY DEFAULT ON NULL AS IDENTITY (START WITH {val_plus_1})"
464
+ ),
442
465
  }
443
466
  table_wrappers = {
444
467
  'default' : ('"', '"'),
@@ -499,7 +522,8 @@ def dateadd_str(
499
522
  flavor: str = 'postgresql',
500
523
  datepart: str = 'day',
501
524
  number: Union[int, float] = 0,
502
- begin: Union[str, datetime, int] = 'now'
525
+ begin: Union[str, datetime, int] = 'now',
526
+ db_type: Optional[str] = None,
503
527
  ) -> str:
504
528
  """
505
529
  Generate a `DATEADD` clause depending on database flavor.
@@ -538,6 +562,10 @@ def dateadd_str(
538
562
  begin: Union[str, datetime], default `'now'`
539
563
  Base datetime to which to add dateparts.
540
564
 
565
+ db_type: Optional[str], default None
566
+ If provided, cast the datetime string as the type.
567
+ Otherwise, infer this from the input datetime value.
568
+
541
569
  Returns
542
570
  -------
543
571
  The appropriate `DATEADD` string for the corresponding database flavor.
@@ -549,7 +577,7 @@ def dateadd_str(
549
577
  ... begin = datetime(2022, 1, 1, 0, 0),
550
578
  ... number = 1,
551
579
  ... )
552
- "DATEADD(day, 1, CAST('2022-01-01 00:00:00' AS DATETIME))"
580
+ "DATEADD(day, 1, CAST('2022-01-01 00:00:00' AS DATETIME2))"
553
581
  >>> dateadd_str(
554
582
  ... flavor = 'postgresql',
555
583
  ... begin = datetime(2022, 1, 1, 0, 0),
@@ -592,7 +620,7 @@ def dateadd_str(
592
620
  )
593
621
 
594
622
  dt_is_utc = begin_time.tzinfo is not None if begin_time is not None else '+' in str(begin)
595
- db_type = get_db_type_from_pd_type(
623
+ db_type = db_type or get_db_type_from_pd_type(
596
624
  ('datetime64[ns, UTC]' if dt_is_utc else 'datetime64[ns]'),
597
625
  flavor=flavor,
598
626
  )
@@ -717,7 +745,7 @@ def get_distinct_col_count(
717
745
  result = connector.value(_meta_query, debug=debug)
718
746
  try:
719
747
  return int(result)
720
- except Exception as e:
748
+ except Exception:
721
749
  return None
722
750
 
723
751
 
@@ -727,12 +755,15 @@ def sql_item_name(item: str, flavor: str, schema: Optional[str] = None) -> str:
727
755
 
728
756
  Parameters
729
757
  ----------
730
- item: str :
758
+ item: str
731
759
  The database item (table, view, etc.) in need of quotes.
732
760
 
733
- flavor: str :
761
+ flavor: str
734
762
  The database flavor (`'postgresql'`, `'mssql'`, `'sqllite'`, etc.).
735
763
 
764
+ schema: Optional[str], default None
765
+ If provided, prefix the table name with the schema.
766
+
736
767
  Returns
737
768
  -------
738
769
  A `str` which contains the input `item` wrapped in the corresponding escape characters.
@@ -764,6 +795,8 @@ def sql_item_name(item: str, flavor: str, schema: Optional[str] = None) -> str:
764
795
  ### NOTE: SQLite does not support schemas.
765
796
  if flavor == 'sqlite':
766
797
  schema = None
798
+ elif flavor == 'mssql' and str(item).startswith('#'):
799
+ schema = None
767
800
 
768
801
  schema_prefix = (
769
802
  (wrappers[0] + schema + wrappers[1] + '.')
@@ -1066,7 +1099,7 @@ def get_sqlalchemy_table(
1066
1099
  connector.metadata,
1067
1100
  **table_kwargs
1068
1101
  )
1069
- except sqlalchemy.exc.NoSuchTableError as e:
1102
+ except sqlalchemy.exc.NoSuchTableError:
1070
1103
  warn(f"Table '{truncated_table_name}' does not exist in '{connector}'.")
1071
1104
  return None
1072
1105
  return tables[truncated_table_name]
@@ -1119,6 +1152,7 @@ def get_table_cols_types(
1119
1152
  -------
1120
1153
  A dictionary mapping column names to data types.
1121
1154
  """
1155
+ import textwrap
1122
1156
  from meerschaum.connectors import SQLConnector
1123
1157
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
1124
1158
  flavor = flavor or getattr(connectable, 'flavor', None)
@@ -1144,7 +1178,7 @@ def get_table_cols_types(
1144
1178
  )
1145
1179
 
1146
1180
  cols_types_query = sqlalchemy.text(
1147
- columns_types_queries.get(
1181
+ textwrap.dedent(columns_types_queries.get(
1148
1182
  flavor,
1149
1183
  columns_types_queries['default']
1150
1184
  ).format(
@@ -1155,7 +1189,7 @@ def get_table_cols_types(
1155
1189
  table_upper=table_upper,
1156
1190
  table_upper_trunc=table_upper_trunc,
1157
1191
  db_prefix=db_prefix,
1158
- )
1192
+ )).lstrip().rstrip()
1159
1193
  )
1160
1194
 
1161
1195
  cols = ['database', 'schema', 'table', 'column', 'type']
@@ -1269,6 +1303,7 @@ def get_table_cols_indices(
1269
1303
  -------
1270
1304
  A dictionary mapping column names to a list of indices.
1271
1305
  """
1306
+ import textwrap
1272
1307
  from collections import defaultdict
1273
1308
  from meerschaum.connectors import SQLConnector
1274
1309
  sqlalchemy = mrsm.attempt_import('sqlalchemy')
@@ -1295,7 +1330,7 @@ def get_table_cols_indices(
1295
1330
  )
1296
1331
 
1297
1332
  cols_indices_query = sqlalchemy.text(
1298
- columns_indices_queries.get(
1333
+ textwrap.dedent(columns_indices_queries.get(
1299
1334
  flavor,
1300
1335
  columns_indices_queries['default']
1301
1336
  ).format(
@@ -1307,10 +1342,12 @@ def get_table_cols_indices(
1307
1342
  table_upper_trunc=table_upper_trunc,
1308
1343
  db_prefix=db_prefix,
1309
1344
  schema=schema,
1310
- )
1345
+ )).lstrip().rstrip()
1311
1346
  )
1312
1347
 
1313
1348
  cols = ['database', 'schema', 'table', 'column', 'index', 'index_type']
1349
+ if flavor == 'mssql':
1350
+ cols.append('clustered')
1314
1351
  result_cols_ix = dict(enumerate(cols))
1315
1352
 
1316
1353
  debug_kwargs = {'debug': debug} if isinstance(connectable, SQLConnector) else {}
@@ -1351,7 +1388,6 @@ def get_table_cols_indices(
1351
1388
  )
1352
1389
  )
1353
1390
  ]
1354
-
1355
1391
  ### NOTE: This may return incorrect columns if the schema is not explicitly stated.
1356
1392
  if cols_types_docs and not cols_types_docs_filtered:
1357
1393
  cols_types_docs_filtered = cols_types_docs
@@ -1367,12 +1403,13 @@ def get_table_cols_indices(
1367
1403
  else doc['column']
1368
1404
  )
1369
1405
  )
1370
- cols_indices[col].append(
1371
- {
1372
- 'name': doc.get('index', None),
1373
- 'type': doc.get('index_type', None),
1374
- }
1375
- )
1406
+ index_doc = {
1407
+ 'name': doc.get('index', None),
1408
+ 'type': doc.get('index_type', None)
1409
+ }
1410
+ if flavor == 'mssql':
1411
+ index_doc['clustered'] = doc.get('clustered', None)
1412
+ cols_indices[col].append(index_doc)
1376
1413
 
1377
1414
  return dict(cols_indices)
1378
1415
  except Exception as e:
@@ -1393,6 +1430,7 @@ def get_update_queries(
1393
1430
  datetime_col: Optional[str] = None,
1394
1431
  schema: Optional[str] = None,
1395
1432
  patch_schema: Optional[str] = None,
1433
+ identity_insert: bool = False,
1396
1434
  debug: bool = False,
1397
1435
  ) -> List[str]:
1398
1436
  """
@@ -1430,6 +1468,10 @@ def get_update_queries(
1430
1468
  If provided, use this schema when quoting the patch table.
1431
1469
  Defaults to `schema`.
1432
1470
 
1471
+ identity_insert: bool, default False
1472
+ If `True`, include `SET IDENTITY_INSERT` queries before and after the update queries.
1473
+ Only applies for MSSQL upserts.
1474
+
1433
1475
  debug: bool, default False
1434
1476
  Verbosity toggle.
1435
1477
 
@@ -1437,9 +1479,11 @@ def get_update_queries(
1437
1479
  -------
1438
1480
  A list of query strings to perform the update operation.
1439
1481
  """
1482
+ import textwrap
1440
1483
  from meerschaum.connectors import SQLConnector
1441
1484
  from meerschaum.utils.debug import dprint
1442
- from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
1485
+ from meerschaum.utils.dtypes import are_dtypes_equal
1486
+ from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES, get_pd_type_from_db_type
1443
1487
  flavor = flavor or (connectable.flavor if isinstance(connectable, SQLConnector) else None)
1444
1488
  if not flavor:
1445
1489
  raise ValueError("Provide a flavor if using a SQLAlchemy session.")
@@ -1532,21 +1576,35 @@ def get_update_queries(
1532
1576
  def sets_subquery(l_prefix: str, r_prefix: str):
1533
1577
  if not value_cols:
1534
1578
  return ''
1579
+
1580
+ cast_func_cols = {
1581
+ c_name: (
1582
+ ('', '', '')
1583
+ if (
1584
+ flavor == 'oracle'
1585
+ and are_dtypes_equal(get_pd_type_from_db_type(c_type), 'bytes')
1586
+ )
1587
+ else (
1588
+ ('CAST(', f" AS {c_type.replace('_', ' ')}", ')')
1589
+ if flavor != 'sqlite'
1590
+ else ('', '', '')
1591
+ )
1592
+ )
1593
+ for c_name, c_type in value_cols
1594
+ }
1535
1595
  return 'SET ' + ',\n'.join([
1536
1596
  (
1537
1597
  l_prefix + sql_item_name(c_name, flavor, None)
1538
1598
  + ' = '
1539
- + ('CAST(' if flavor != 'sqlite' else '')
1540
- + r_prefix
1541
- + sql_item_name(c_name, flavor, None)
1542
- + (' AS ' if flavor != 'sqlite' else '')
1543
- + (c_type.replace('_', ' ') if flavor != 'sqlite' else '')
1544
- + (')' if flavor != 'sqlite' else '')
1599
+ + cast_func_cols[c_name][0]
1600
+ + r_prefix + sql_item_name(c_name, flavor, None)
1601
+ + cast_func_cols[c_name][1]
1602
+ + cast_func_cols[c_name][2]
1545
1603
  ) for c_name, c_type in value_cols
1546
1604
  ])
1547
1605
 
1548
1606
  def and_subquery(l_prefix: str, r_prefix: str):
1549
- return '\nAND\n'.join([
1607
+ return '\n AND\n '.join([
1550
1608
  (
1551
1609
  "COALESCE("
1552
1610
  + l_prefix
@@ -1554,7 +1612,7 @@ def get_update_queries(
1554
1612
  + ", "
1555
1613
  + get_null_replacement(c_type, flavor)
1556
1614
  + ")"
1557
- + ' = '
1615
+ + '\n =\n '
1558
1616
  + "COALESCE("
1559
1617
  + r_prefix
1560
1618
  + sql_item_name(c_name, flavor, None)
@@ -1564,22 +1622,39 @@ def get_update_queries(
1564
1622
  ) for c_name, c_type in join_cols_types
1565
1623
  ])
1566
1624
 
1625
+ skip_query_val = ""
1567
1626
  target_table_name = sql_item_name(target, flavor, schema)
1568
1627
  patch_table_name = sql_item_name(patch, flavor, patch_schema)
1569
1628
  dt_col_name = sql_item_name(datetime_col, flavor, None) if datetime_col else None
1629
+ date_bounds_table = patch_table_name if flavor != 'mssql' else '[date_bounds]'
1630
+ min_dt_col_name = f"MIN({dt_col_name})" if flavor != 'mssql' else '[Min_dt]'
1631
+ max_dt_col_name = f"MAX({dt_col_name})" if flavor != 'mssql' else '[Max_dt]'
1570
1632
  date_bounds_subquery = (
1571
- f"""
1572
- f.{dt_col_name} >= (SELECT MIN({dt_col_name}) FROM {patch_table_name})
1573
- AND f.{dt_col_name} <= (SELECT MAX({dt_col_name}) FROM {patch_table_name})
1574
- """
1633
+ f"""f.{dt_col_name} >= (SELECT {min_dt_col_name} FROM {date_bounds_table})
1634
+ AND
1635
+ f.{dt_col_name} <= (SELECT {max_dt_col_name} FROM {date_bounds_table})"""
1575
1636
  if datetime_col
1576
1637
  else "1 = 1"
1577
1638
  )
1639
+ with_temp_date_bounds = f"""WITH [date_bounds] AS (
1640
+ SELECT MIN({dt_col_name}) AS {min_dt_col_name}, MAX({dt_col_name}) AS {max_dt_col_name}
1641
+ FROM {patch_table_name}
1642
+ )""" if datetime_col else ""
1643
+ identity_insert_on = (
1644
+ f"SET IDENTITY_INSERT {target_table_name} ON"
1645
+ if identity_insert
1646
+ else skip_query_val
1647
+ )
1648
+ identity_insert_off = (
1649
+ f"SET IDENTITY_INSERT {target_table_name} OFF"
1650
+ if identity_insert
1651
+ else skip_query_val
1652
+ )
1578
1653
 
1579
1654
  ### NOTE: MSSQL upserts must exclude the update portion if only upserting indices.
1580
1655
  when_matched_update_sets_subquery_none = "" if not value_cols else (
1581
- "WHEN MATCHED THEN"
1582
- f" UPDATE {sets_subquery('', 'p.')}"
1656
+ "\n WHEN MATCHED THEN\n"
1657
+ f" UPDATE {sets_subquery('', 'p.')}"
1583
1658
  )
1584
1659
 
1585
1660
  cols_equal_values = '\n,'.join(
@@ -1595,8 +1670,8 @@ def get_update_queries(
1595
1670
  )
1596
1671
  ignore = "IGNORE " if not value_cols else ""
1597
1672
 
1598
- return [
1599
- base_query.format(
1673
+ formatted_queries = [
1674
+ textwrap.dedent(base_query.format(
1600
1675
  sets_subquery_none=sets_subquery('', 'p.'),
1601
1676
  sets_subquery_none_excluded=sets_subquery('', 'EXCLUDED.'),
1602
1677
  sets_subquery_f=sets_subquery('f.', 'p.'),
@@ -1614,10 +1689,16 @@ def get_update_queries(
1614
1689
  cols_equal_values=cols_equal_values,
1615
1690
  on_duplicate_key_update=on_duplicate_key_update,
1616
1691
  ignore=ignore,
1617
- )
1692
+ with_temp_date_bounds=with_temp_date_bounds,
1693
+ identity_insert_on=identity_insert_on,
1694
+ identity_insert_off=identity_insert_off,
1695
+ )).lstrip().rstrip()
1618
1696
  for base_query in base_queries
1619
1697
  ]
1620
1698
 
1699
+ ### NOTE: Allow for skipping some queries.
1700
+ return [query for query in formatted_queries if query]
1701
+
1621
1702
 
1622
1703
  def get_null_replacement(typ: str, flavor: str) -> str:
1623
1704
  """
@@ -1655,11 +1736,14 @@ def get_null_replacement(typ: str, flavor: str) -> str:
1655
1736
  )
1656
1737
  return f'CAST({val_to_cast} AS {bool_typ})'
1657
1738
  if 'time' in typ.lower() or 'date' in typ.lower():
1658
- return dateadd_str(flavor=flavor, begin='1900-01-01')
1739
+ db_type = typ if typ.isupper() else None
1740
+ return dateadd_str(flavor=flavor, begin='1900-01-01', db_type=db_type)
1659
1741
  if 'float' in typ.lower() or 'double' in typ.lower() or typ.lower() in ('decimal',):
1660
1742
  return '-987654321.0'
1661
1743
  if flavor == 'oracle' and typ.lower().split('(', maxsplit=1)[0] == 'char':
1662
1744
  return "'-987654321'"
1745
+ if flavor == 'oracle' and typ.lower() in ('blob', 'bytes'):
1746
+ return '00'
1663
1747
  if typ.lower() in ('uniqueidentifier', 'guid', 'uuid'):
1664
1748
  magic_val = 'DEADBEEF-ABBA-BABE-CAFE-DECAFC0FFEE5'
1665
1749
  if flavor == 'mssql':
@@ -1867,7 +1951,17 @@ def _get_create_table_query_from_dtypes(
1867
1951
 
1868
1952
  table_name = sql_item_name(new_table, schema=schema, flavor=flavor)
1869
1953
  primary_key_name = sql_item_name(primary_key, flavor) if primary_key else None
1954
+ primary_key_constraint_name = (
1955
+ sql_item_name(f'PK_{new_table}', flavor, None)
1956
+ if primary_key
1957
+ else None
1958
+ )
1870
1959
  datetime_column_name = sql_item_name(datetime_column, flavor) if datetime_column else None
1960
+ primary_key_clustered = (
1961
+ "CLUSTERED"
1962
+ if not datetime_column or datetime_column == primary_key
1963
+ else "NONCLUSTERED"
1964
+ )
1871
1965
  query = f"CREATE TABLE {table_name} ("
1872
1966
  if primary_key:
1873
1967
  col_db_type = cols_types[0][1]
@@ -1887,6 +1981,8 @@ def _get_create_table_query_from_dtypes(
1887
1981
  query += f"\n {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
1888
1982
  elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
1889
1983
  query += f"\n {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
1984
+ elif flavor == 'mssql':
1985
+ query += f"\n {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
1890
1986
  else:
1891
1987
  query += f"\n {col_name} {col_db_type} PRIMARY KEY{auto_increment_str} NOT NULL,"
1892
1988
 
@@ -1902,6 +1998,10 @@ def _get_create_table_query_from_dtypes(
1902
1998
  and datetime_column != primary_key
1903
1999
  ):
1904
2000
  query += f"\n PRIMARY KEY({datetime_column_name}, {primary_key_name}),"
2001
+
2002
+ if flavor == 'mssql' and primary_key:
2003
+ query += f"\n CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name}),"
2004
+
1905
2005
  query = query[:-1]
1906
2006
  query += "\n)"
1907
2007
 
@@ -1922,12 +2022,11 @@ def _get_create_table_query_from_cte(
1922
2022
  Create a new table from a CTE query.
1923
2023
  """
1924
2024
  import textwrap
1925
- from meerschaum.utils.dtypes.sql import AUTO_INCREMENT_COLUMN_FLAVORS
1926
2025
  create_cte = 'create_query'
1927
2026
  create_cte_name = sql_item_name(create_cte, flavor, None)
1928
2027
  new_table_name = sql_item_name(new_table, flavor, schema)
1929
2028
  primary_key_constraint_name = (
1930
- sql_item_name(f'pk_{new_table}', flavor, None)
2029
+ sql_item_name(f'PK_{new_table}', flavor, None)
1931
2030
  if primary_key
1932
2031
  else None
1933
2032
  )
@@ -1936,6 +2035,7 @@ def _get_create_table_query_from_cte(
1936
2035
  if primary_key
1937
2036
  else None
1938
2037
  )
2038
+ primary_key_clustered = "CLUSTERED" if not datetime_column else "NONCLUSTERED"
1939
2039
  datetime_column_name = (
1940
2040
  sql_item_name(datetime_column, flavor)
1941
2041
  if datetime_column
@@ -1943,7 +2043,7 @@ def _get_create_table_query_from_cte(
1943
2043
  )
1944
2044
  if flavor in ('mssql',):
1945
2045
  query = query.lstrip()
1946
- if 'with ' in query.lower():
2046
+ if query.lower().startswith('with '):
1947
2047
  final_select_ix = query.lower().rfind('select')
1948
2048
  create_table_query = (
1949
2049
  query[:final_select_ix].rstrip() + ',\n'
@@ -1961,7 +2061,7 @@ def _get_create_table_query_from_cte(
1961
2061
 
1962
2062
  alter_type_query = f"""
1963
2063
  ALTER TABLE {new_table_name}
1964
- ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY ({primary_key_name})
2064
+ ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY {primary_key_clustered} ({primary_key_name})
1965
2065
  """
1966
2066
  elif flavor in (None,):
1967
2067
  create_table_query = f"""
@@ -2009,11 +2109,11 @@ def _get_create_table_query_from_cte(
2009
2109
  ADD PRIMARY KEY ({primary_key_name})
2010
2110
  """
2011
2111
 
2012
- create_table_query = textwrap.dedent(create_table_query)
2112
+ create_table_query = textwrap.dedent(create_table_query).lstrip().rstrip()
2013
2113
  if not primary_key:
2014
2114
  return [create_table_query]
2015
2115
 
2016
- alter_type_query = textwrap.dedent(alter_type_query)
2116
+ alter_type_query = textwrap.dedent(alter_type_query).lstrip().rstrip()
2017
2117
 
2018
2118
  return [
2019
2119
  create_table_query,
@@ -2225,29 +2325,8 @@ def get_reset_autoincrement_queries(
2225
2325
  schema = schema or connector.schema
2226
2326
  max_id_name = sql_item_name('max_id', connector.flavor)
2227
2327
  table_name = sql_item_name(table, connector.flavor, schema)
2228
- table_trunc = truncate_item_name(table, connector.flavor)
2229
2328
  table_seq_name = sql_item_name(table + '_' + column + '_seq', connector.flavor, schema)
2230
2329
  column_name = sql_item_name(column, connector.flavor)
2231
- if connector.flavor == 'oracle':
2232
- potential_table_names = set([
2233
- f"'{table_trunc.upper()}'",
2234
- f"'{table_trunc}'",
2235
- f"'{table_name}'",
2236
- f"'{table_name.upper()}'",
2237
- ])
2238
- df = connector.read(
2239
- """
2240
- SELECT SEQUENCE_NAME
2241
- FROM ALL_TAB_IDENTITY_COLS
2242
- WHERE TABLE_NAME IN ("""
2243
- + ", ".join([name for name in potential_table_names])
2244
- + """)
2245
- """,
2246
- debug=debug
2247
- )
2248
- if len(df) > 0:
2249
- table_seq_name = df['sequence_name'][0]
2250
-
2251
2330
  max_id = connector.value(
2252
2331
  f"""
2253
2332
  SELECT COALESCE(MAX({column_name}), 0) AS {max_id_name}
@@ -2272,7 +2351,8 @@ def get_reset_autoincrement_queries(
2272
2351
  table=table,
2273
2352
  table_name=table_name,
2274
2353
  table_seq_name=table_seq_name,
2275
- val=(max_id),
2354
+ val=max_id,
2355
+ val_plus_1=(max_id + 1),
2276
2356
  )
2277
2357
  for query in reset_queries
2278
2358
  ]
@@ -34,10 +34,10 @@ class Venv:
34
34
  """
35
35
 
36
36
  def __init__(
37
- self,
38
- venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
39
- debug: bool = False,
40
- ) -> None:
37
+ self,
38
+ venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
39
+ debug: bool = False,
40
+ ) -> None:
41
41
  from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
42
42
  ### For some weird threading issue,
43
43
  ### we can't use `isinstance` here.