meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +1 -1
  2. meerschaum/actions/delete.py +65 -69
  3. meerschaum/actions/edit.py +22 -2
  4. meerschaum/actions/install.py +1 -2
  5. meerschaum/actions/sync.py +2 -3
  6. meerschaum/api/routes/_pipes.py +7 -8
  7. meerschaum/config/_default.py +1 -1
  8. meerschaum/config/_paths.py +2 -1
  9. meerschaum/config/_version.py +1 -1
  10. meerschaum/connectors/api/_pipes.py +18 -21
  11. meerschaum/connectors/sql/_create_engine.py +3 -3
  12. meerschaum/connectors/sql/_instance.py +11 -12
  13. meerschaum/connectors/sql/_pipes.py +143 -91
  14. meerschaum/connectors/sql/_sql.py +43 -8
  15. meerschaum/connectors/valkey/_pipes.py +12 -1
  16. meerschaum/core/Pipe/__init__.py +23 -13
  17. meerschaum/core/Pipe/_attributes.py +25 -1
  18. meerschaum/core/Pipe/_dtypes.py +23 -16
  19. meerschaum/core/Pipe/_sync.py +59 -31
  20. meerschaum/core/Pipe/_verify.py +8 -7
  21. meerschaum/jobs/_Job.py +4 -1
  22. meerschaum/plugins/_Plugin.py +11 -14
  23. meerschaum/utils/daemon/Daemon.py +22 -15
  24. meerschaum/utils/dataframe.py +178 -16
  25. meerschaum/utils/dtypes/__init__.py +149 -14
  26. meerschaum/utils/dtypes/sql.py +41 -7
  27. meerschaum/utils/misc.py +8 -8
  28. meerschaum/utils/packages/_packages.py +1 -1
  29. meerschaum/utils/schedule.py +8 -3
  30. meerschaum/utils/sql.py +180 -100
  31. meerschaum/utils/venv/_Venv.py +4 -4
  32. meerschaum/utils/venv/__init__.py +53 -20
  33. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/METADATA +2 -2
  34. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/RECORD +40 -40
  35. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/LICENSE +0 -0
  36. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/NOTICE +0 -0
  37. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/WHEEL +0 -0
  38. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/entry_points.txt +0 -0
  39. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/top_level.txt +0 -0
  40. {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/zip-safe +0 -0
@@ -13,9 +13,8 @@ NUMERIC_PRECISION_FLAVORS: Dict[str, Tuple[int, int]] = {
13
13
  'mariadb': (38, 20),
14
14
  'mysql': (38, 20),
15
15
  'mssql': (28, 10),
16
- 'duckdb': (15, 3),
17
- 'sqlite': (15, 4),
18
16
  }
17
+ NUMERIC_AS_TEXT_FLAVORS = {'sqlite', 'duckdb'}
19
18
  TIMEZONE_NAIVE_FLAVORS = {'oracle', 'mysql', 'mariadb'}
20
19
 
21
20
  ### MySQL doesn't allow for casting as BIGINT, so this is a workaround.
@@ -102,6 +101,10 @@ DB_TO_PD_DTYPES: Dict[str, Union[str, Dict[str, str]]] = {
102
101
  'JSONB': 'json',
103
102
  'UUID': 'uuid',
104
103
  'UNIQUEIDENTIFIER': 'uuid',
104
+ 'BYTEA': 'bytes',
105
+ 'BLOB': 'bytes',
106
+ 'VARBINARY': 'bytes',
107
+ 'VARBINARY(MAX)': 'bytes',
105
108
  'substrings': {
106
109
  'CHAR': 'string[pyarrow]',
107
110
  'TIMESTAMP': 'datetime64[ns]',
@@ -114,6 +117,9 @@ DB_TO_PD_DTYPES: Dict[str, Union[str, Dict[str, str]]] = {
114
117
  'INT': 'int64[pyarrow]',
115
118
  'BOOL': 'bool[pyarrow]',
116
119
  'JSON': 'json',
120
+ 'BYTE': 'bytes',
121
+ 'LOB': 'bytes',
122
+ 'BINARY': 'bytes',
117
123
  },
118
124
  'default': 'object',
119
125
  }
@@ -256,8 +262,8 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
256
262
  'mysql': f'DECIMAL{NUMERIC_PRECISION_FLAVORS["mysql"]}',
257
263
  'mssql': f'NUMERIC{NUMERIC_PRECISION_FLAVORS["mssql"]}',
258
264
  'oracle': 'NUMBER',
259
- 'sqlite': f'DECIMAL{NUMERIC_PRECISION_FLAVORS["sqlite"]}',
260
- 'duckdb': 'NUMERIC',
265
+ 'sqlite': 'TEXT',
266
+ 'duckdb': 'TEXT',
261
267
  'citus': 'NUMERIC',
262
268
  'cockroachdb': 'NUMERIC',
263
269
  'default': 'NUMERIC',
@@ -276,6 +282,19 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
276
282
  'cockroachdb': 'UUID',
277
283
  'default': 'TEXT',
278
284
  },
285
+ 'bytes': {
286
+ 'timescaledb': 'BYTEA',
287
+ 'postgresql': 'BYTEA',
288
+ 'mariadb': 'BLOB',
289
+ 'mysql': 'BLOB',
290
+ 'mssql': 'VARBINARY(MAX)',
291
+ 'oracle': 'BLOB',
292
+ 'sqlite': 'BLOB',
293
+ 'duckdb': 'BLOB',
294
+ 'citus': 'BYTEA',
295
+ 'cockroachdb': 'BYTEA',
296
+ 'default': 'BLOB',
297
+ },
279
298
  }
280
299
  PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
281
300
  'int': {
@@ -402,7 +421,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
402
421
  'mysql': 'Numeric',
403
422
  'mssql': 'Numeric',
404
423
  'oracle': 'Numeric',
405
- 'sqlite': 'Numeric',
424
+ 'sqlite': 'UnicodeText',
406
425
  'duckdb': 'Numeric',
407
426
  'citus': 'Numeric',
408
427
  'cockroachdb': 'Numeric',
@@ -421,6 +440,19 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
421
440
  'cockroachdb': 'Uuid',
422
441
  'default': 'Uuid',
423
442
  },
443
+ 'bytes': {
444
+ 'timescaledb': 'LargeBinary',
445
+ 'postgresql': 'LargeBinary',
446
+ 'mariadb': 'LargeBinary',
447
+ 'mysql': 'LargeBinary',
448
+ 'mssql': 'LargeBinary',
449
+ 'oracle': 'LargeBinary',
450
+ 'sqlite': 'LargeBinary',
451
+ 'duckdb': 'LargeBinary',
452
+ 'citus': 'LargeBinary',
453
+ 'cockroachdb': 'LargeBinary',
454
+ 'default': 'LargeBinary',
455
+ },
424
456
  }
425
457
 
426
458
  AUTO_INCREMENT_COLUMN_FLAVORS: Dict[str, str] = {
@@ -502,7 +534,7 @@ def get_db_type_from_pd_type(
502
534
  """
503
535
  from meerschaum.utils.warnings import warn
504
536
  from meerschaum.utils.packages import attempt_import
505
- from meerschaum.utils.dtypes import are_dtypes_equal
537
+ from meerschaum.utils.dtypes import are_dtypes_equal, MRSM_ALIAS_DTYPES
506
538
  from meerschaum.utils.misc import parse_arguments_str
507
539
  sqlalchemy_types = attempt_import('sqlalchemy.types')
508
540
 
@@ -512,6 +544,9 @@ def get_db_type_from_pd_type(
512
544
  else PD_TO_SQLALCHEMY_DTYPES_FLAVORS
513
545
  )
514
546
 
547
+ if pd_type in MRSM_ALIAS_DTYPES:
548
+ pd_type = MRSM_ALIAS_DTYPES[pd_type]
549
+
515
550
  ### Check whether we are able to match this type (e.g. pyarrow support).
516
551
  found_db_type = False
517
552
  if pd_type not in types_registry:
@@ -568,7 +603,6 @@ def get_db_type_from_pd_type(
568
603
  return cls(*cls_args, **cls_kwargs)
569
604
 
570
605
  if 'numeric' in db_type.lower():
571
- numeric_type_str = PD_TO_DB_DTYPES_FLAVORS['numeric'].get(flavor, 'NUMERIC')
572
606
  if flavor not in NUMERIC_PRECISION_FLAVORS:
573
607
  return sqlalchemy_types.Numeric
574
608
  precision, scale = NUMERIC_PRECISION_FLAVORS[flavor]
meerschaum/utils/misc.py CHANGED
@@ -177,14 +177,14 @@ def string_to_dict(
177
177
  keys = _keys[:-1]
178
178
  try:
179
179
  val = ast.literal_eval(_keys[-1])
180
- except Exception as e:
180
+ except Exception:
181
181
  val = str(_keys[-1])
182
182
 
183
183
  c = params_dict
184
184
  for _k in keys[:-1]:
185
185
  try:
186
186
  k = ast.literal_eval(_k)
187
- except Exception as e:
187
+ except Exception:
188
188
  k = str(_k)
189
189
  if k not in c:
190
190
  c[k] = {}
@@ -196,12 +196,12 @@ def string_to_dict(
196
196
 
197
197
 
198
198
  def parse_config_substitution(
199
- value: str,
200
- leading_key: str = 'MRSM',
201
- begin_key: str = '{',
202
- end_key: str = '}',
203
- delimeter: str = ':'
204
- ) -> List[Any]:
199
+ value: str,
200
+ leading_key: str = 'MRSM',
201
+ begin_key: str = '{',
202
+ end_key: str = '}',
203
+ delimeter: str = ':'
204
+ ) -> List[Any]:
205
205
  """
206
206
  Parse Meerschaum substitution syntax
207
207
  E.g. MRSM{value1:value2} => ['value1', 'value2']
@@ -79,7 +79,7 @@ packages: Dict[str, Dict[str, str]] = {
79
79
  },
80
80
  'drivers-extras': {
81
81
  'pyodbc' : 'pyodbc>=4.0.30',
82
- 'cx_Oracle' : 'cx_Oracle>=8.3.0',
82
+ 'oracledb' : 'oracledb>=2.5.0',
83
83
  },
84
84
  'cli': {
85
85
  'pgcli' : 'pgcli>=3.1.0',
@@ -132,7 +132,7 @@ def schedule_function(
132
132
 
133
133
  try:
134
134
  loop.run_until_complete(run_scheduler())
135
- except (KeyboardInterrupt, SystemExit) as e:
135
+ except (KeyboardInterrupt, SystemExit):
136
136
  loop.run_until_complete(_stop_scheduler())
137
137
 
138
138
  return True, "Success"
@@ -159,13 +159,13 @@ def parse_schedule(schedule: str, now: Optional[datetime] = None):
159
159
  )
160
160
 
161
161
  starting_ts = parse_start_time(schedule, now=now)
162
- schedule = schedule.split(STARTING_KEYWORD)[0].strip()
162
+ schedule = schedule.split(STARTING_KEYWORD, maxsplit=1)[0].strip()
163
163
  for alias_keyword, true_keyword in SCHEDULE_ALIASES.items():
164
164
  schedule = schedule.replace(alias_keyword, true_keyword)
165
165
 
166
166
  ### TODO Allow for combining `and` + `or` logic.
167
167
  if '&' in schedule and '|' in schedule:
168
- raise ValueError(f"Cannot accept both 'and' + 'or' logic in the schedule frequency.")
168
+ raise ValueError("Cannot accept both 'and' + 'or' logic in the schedule frequency.")
169
169
 
170
170
  join_str = '|' if '|' in schedule else '&'
171
171
  join_trigger = (
@@ -300,6 +300,11 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
300
300
  try:
301
301
  if starting_str == 'now':
302
302
  starting_ts = now
303
+ elif starting_str.startswith('in '):
304
+ delta_vals = starting_str.replace('in ', '').split(' ', maxsplit=1)
305
+ delta_unit = delta_vals[-1].rstrip('s') + 's'
306
+ delta_num = float(delta_vals[0])
307
+ starting_ts = now + timedelta(**{delta_unit: delta_num})
303
308
  elif 'tomorrow' in starting_str or 'today' in starting_str:
304
309
  today = round_time(now, timedelta(days=1))
305
310
  tomorrow = today + timedelta(days=1)