meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +1 -1
- meerschaum/actions/delete.py +65 -69
- meerschaum/actions/edit.py +22 -2
- meerschaum/actions/install.py +1 -2
- meerschaum/actions/sync.py +2 -3
- meerschaum/api/routes/_pipes.py +7 -8
- meerschaum/config/_default.py +1 -1
- meerschaum/config/_paths.py +2 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_pipes.py +18 -21
- meerschaum/connectors/sql/_create_engine.py +3 -3
- meerschaum/connectors/sql/_instance.py +11 -12
- meerschaum/connectors/sql/_pipes.py +143 -91
- meerschaum/connectors/sql/_sql.py +43 -8
- meerschaum/connectors/valkey/_pipes.py +12 -1
- meerschaum/core/Pipe/__init__.py +23 -13
- meerschaum/core/Pipe/_attributes.py +25 -1
- meerschaum/core/Pipe/_dtypes.py +23 -16
- meerschaum/core/Pipe/_sync.py +59 -31
- meerschaum/core/Pipe/_verify.py +8 -7
- meerschaum/jobs/_Job.py +4 -1
- meerschaum/plugins/_Plugin.py +11 -14
- meerschaum/utils/daemon/Daemon.py +22 -15
- meerschaum/utils/dataframe.py +178 -16
- meerschaum/utils/dtypes/__init__.py +149 -14
- meerschaum/utils/dtypes/sql.py +41 -7
- meerschaum/utils/misc.py +8 -8
- meerschaum/utils/packages/_packages.py +1 -1
- meerschaum/utils/schedule.py +8 -3
- meerschaum/utils/sql.py +180 -100
- meerschaum/utils/venv/_Venv.py +4 -4
- meerschaum/utils/venv/__init__.py +53 -20
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/METADATA +2 -2
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/RECORD +40 -40
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/WHEEL +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.6.16.dist-info → meerschaum-2.7.0.dist-info}/zip-safe +0 -0
meerschaum/utils/dtypes/sql.py
CHANGED
@@ -13,9 +13,8 @@ NUMERIC_PRECISION_FLAVORS: Dict[str, Tuple[int, int]] = {
|
|
13
13
|
'mariadb': (38, 20),
|
14
14
|
'mysql': (38, 20),
|
15
15
|
'mssql': (28, 10),
|
16
|
-
'duckdb': (15, 3),
|
17
|
-
'sqlite': (15, 4),
|
18
16
|
}
|
17
|
+
NUMERIC_AS_TEXT_FLAVORS = {'sqlite', 'duckdb'}
|
19
18
|
TIMEZONE_NAIVE_FLAVORS = {'oracle', 'mysql', 'mariadb'}
|
20
19
|
|
21
20
|
### MySQL doesn't allow for casting as BIGINT, so this is a workaround.
|
@@ -102,6 +101,10 @@ DB_TO_PD_DTYPES: Dict[str, Union[str, Dict[str, str]]] = {
|
|
102
101
|
'JSONB': 'json',
|
103
102
|
'UUID': 'uuid',
|
104
103
|
'UNIQUEIDENTIFIER': 'uuid',
|
104
|
+
'BYTEA': 'bytes',
|
105
|
+
'BLOB': 'bytes',
|
106
|
+
'VARBINARY': 'bytes',
|
107
|
+
'VARBINARY(MAX)': 'bytes',
|
105
108
|
'substrings': {
|
106
109
|
'CHAR': 'string[pyarrow]',
|
107
110
|
'TIMESTAMP': 'datetime64[ns]',
|
@@ -114,6 +117,9 @@ DB_TO_PD_DTYPES: Dict[str, Union[str, Dict[str, str]]] = {
|
|
114
117
|
'INT': 'int64[pyarrow]',
|
115
118
|
'BOOL': 'bool[pyarrow]',
|
116
119
|
'JSON': 'json',
|
120
|
+
'BYTE': 'bytes',
|
121
|
+
'LOB': 'bytes',
|
122
|
+
'BINARY': 'bytes',
|
117
123
|
},
|
118
124
|
'default': 'object',
|
119
125
|
}
|
@@ -256,8 +262,8 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
256
262
|
'mysql': f'DECIMAL{NUMERIC_PRECISION_FLAVORS["mysql"]}',
|
257
263
|
'mssql': f'NUMERIC{NUMERIC_PRECISION_FLAVORS["mssql"]}',
|
258
264
|
'oracle': 'NUMBER',
|
259
|
-
'sqlite':
|
260
|
-
'duckdb': '
|
265
|
+
'sqlite': 'TEXT',
|
266
|
+
'duckdb': 'TEXT',
|
261
267
|
'citus': 'NUMERIC',
|
262
268
|
'cockroachdb': 'NUMERIC',
|
263
269
|
'default': 'NUMERIC',
|
@@ -276,6 +282,19 @@ PD_TO_DB_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
276
282
|
'cockroachdb': 'UUID',
|
277
283
|
'default': 'TEXT',
|
278
284
|
},
|
285
|
+
'bytes': {
|
286
|
+
'timescaledb': 'BYTEA',
|
287
|
+
'postgresql': 'BYTEA',
|
288
|
+
'mariadb': 'BLOB',
|
289
|
+
'mysql': 'BLOB',
|
290
|
+
'mssql': 'VARBINARY(MAX)',
|
291
|
+
'oracle': 'BLOB',
|
292
|
+
'sqlite': 'BLOB',
|
293
|
+
'duckdb': 'BLOB',
|
294
|
+
'citus': 'BYTEA',
|
295
|
+
'cockroachdb': 'BYTEA',
|
296
|
+
'default': 'BLOB',
|
297
|
+
},
|
279
298
|
}
|
280
299
|
PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
281
300
|
'int': {
|
@@ -402,7 +421,7 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
402
421
|
'mysql': 'Numeric',
|
403
422
|
'mssql': 'Numeric',
|
404
423
|
'oracle': 'Numeric',
|
405
|
-
'sqlite': '
|
424
|
+
'sqlite': 'UnicodeText',
|
406
425
|
'duckdb': 'Numeric',
|
407
426
|
'citus': 'Numeric',
|
408
427
|
'cockroachdb': 'Numeric',
|
@@ -421,6 +440,19 @@ PD_TO_SQLALCHEMY_DTYPES_FLAVORS: Dict[str, Dict[str, str]] = {
|
|
421
440
|
'cockroachdb': 'Uuid',
|
422
441
|
'default': 'Uuid',
|
423
442
|
},
|
443
|
+
'bytes': {
|
444
|
+
'timescaledb': 'LargeBinary',
|
445
|
+
'postgresql': 'LargeBinary',
|
446
|
+
'mariadb': 'LargeBinary',
|
447
|
+
'mysql': 'LargeBinary',
|
448
|
+
'mssql': 'LargeBinary',
|
449
|
+
'oracle': 'LargeBinary',
|
450
|
+
'sqlite': 'LargeBinary',
|
451
|
+
'duckdb': 'LargeBinary',
|
452
|
+
'citus': 'LargeBinary',
|
453
|
+
'cockroachdb': 'LargeBinary',
|
454
|
+
'default': 'LargeBinary',
|
455
|
+
},
|
424
456
|
}
|
425
457
|
|
426
458
|
AUTO_INCREMENT_COLUMN_FLAVORS: Dict[str, str] = {
|
@@ -502,7 +534,7 @@ def get_db_type_from_pd_type(
|
|
502
534
|
"""
|
503
535
|
from meerschaum.utils.warnings import warn
|
504
536
|
from meerschaum.utils.packages import attempt_import
|
505
|
-
from meerschaum.utils.dtypes import are_dtypes_equal
|
537
|
+
from meerschaum.utils.dtypes import are_dtypes_equal, MRSM_ALIAS_DTYPES
|
506
538
|
from meerschaum.utils.misc import parse_arguments_str
|
507
539
|
sqlalchemy_types = attempt_import('sqlalchemy.types')
|
508
540
|
|
@@ -512,6 +544,9 @@ def get_db_type_from_pd_type(
|
|
512
544
|
else PD_TO_SQLALCHEMY_DTYPES_FLAVORS
|
513
545
|
)
|
514
546
|
|
547
|
+
if pd_type in MRSM_ALIAS_DTYPES:
|
548
|
+
pd_type = MRSM_ALIAS_DTYPES[pd_type]
|
549
|
+
|
515
550
|
### Check whether we are able to match this type (e.g. pyarrow support).
|
516
551
|
found_db_type = False
|
517
552
|
if pd_type not in types_registry:
|
@@ -568,7 +603,6 @@ def get_db_type_from_pd_type(
|
|
568
603
|
return cls(*cls_args, **cls_kwargs)
|
569
604
|
|
570
605
|
if 'numeric' in db_type.lower():
|
571
|
-
numeric_type_str = PD_TO_DB_DTYPES_FLAVORS['numeric'].get(flavor, 'NUMERIC')
|
572
606
|
if flavor not in NUMERIC_PRECISION_FLAVORS:
|
573
607
|
return sqlalchemy_types.Numeric
|
574
608
|
precision, scale = NUMERIC_PRECISION_FLAVORS[flavor]
|
meerschaum/utils/misc.py
CHANGED
@@ -177,14 +177,14 @@ def string_to_dict(
|
|
177
177
|
keys = _keys[:-1]
|
178
178
|
try:
|
179
179
|
val = ast.literal_eval(_keys[-1])
|
180
|
-
except Exception
|
180
|
+
except Exception:
|
181
181
|
val = str(_keys[-1])
|
182
182
|
|
183
183
|
c = params_dict
|
184
184
|
for _k in keys[:-1]:
|
185
185
|
try:
|
186
186
|
k = ast.literal_eval(_k)
|
187
|
-
except Exception
|
187
|
+
except Exception:
|
188
188
|
k = str(_k)
|
189
189
|
if k not in c:
|
190
190
|
c[k] = {}
|
@@ -196,12 +196,12 @@ def string_to_dict(
|
|
196
196
|
|
197
197
|
|
198
198
|
def parse_config_substitution(
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
199
|
+
value: str,
|
200
|
+
leading_key: str = 'MRSM',
|
201
|
+
begin_key: str = '{',
|
202
|
+
end_key: str = '}',
|
203
|
+
delimeter: str = ':'
|
204
|
+
) -> List[Any]:
|
205
205
|
"""
|
206
206
|
Parse Meerschaum substitution syntax
|
207
207
|
E.g. MRSM{value1:value2} => ['value1', 'value2']
|
meerschaum/utils/schedule.py
CHANGED
@@ -132,7 +132,7 @@ def schedule_function(
|
|
132
132
|
|
133
133
|
try:
|
134
134
|
loop.run_until_complete(run_scheduler())
|
135
|
-
except (KeyboardInterrupt, SystemExit)
|
135
|
+
except (KeyboardInterrupt, SystemExit):
|
136
136
|
loop.run_until_complete(_stop_scheduler())
|
137
137
|
|
138
138
|
return True, "Success"
|
@@ -159,13 +159,13 @@ def parse_schedule(schedule: str, now: Optional[datetime] = None):
|
|
159
159
|
)
|
160
160
|
|
161
161
|
starting_ts = parse_start_time(schedule, now=now)
|
162
|
-
schedule = schedule.split(STARTING_KEYWORD)[0].strip()
|
162
|
+
schedule = schedule.split(STARTING_KEYWORD, maxsplit=1)[0].strip()
|
163
163
|
for alias_keyword, true_keyword in SCHEDULE_ALIASES.items():
|
164
164
|
schedule = schedule.replace(alias_keyword, true_keyword)
|
165
165
|
|
166
166
|
### TODO Allow for combining `and` + `or` logic.
|
167
167
|
if '&' in schedule and '|' in schedule:
|
168
|
-
raise ValueError(
|
168
|
+
raise ValueError("Cannot accept both 'and' + 'or' logic in the schedule frequency.")
|
169
169
|
|
170
170
|
join_str = '|' if '|' in schedule else '&'
|
171
171
|
join_trigger = (
|
@@ -300,6 +300,11 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
|
|
300
300
|
try:
|
301
301
|
if starting_str == 'now':
|
302
302
|
starting_ts = now
|
303
|
+
elif starting_str.startswith('in '):
|
304
|
+
delta_vals = starting_str.replace('in ', '').split(' ', maxsplit=1)
|
305
|
+
delta_unit = delta_vals[-1].rstrip('s') + 's'
|
306
|
+
delta_num = float(delta_vals[0])
|
307
|
+
starting_ts = now + timedelta(**{delta_unit: delta_num})
|
303
308
|
elif 'tomorrow' in starting_str or 'today' in starting_str:
|
304
309
|
today = round_time(now, timedelta(days=1))
|
305
310
|
tomorrow = today + timedelta(days=1)
|