meerschaum 2.1.4__py3-none-any.whl → 2.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__main__.py +1 -1
- meerschaum/_internal/arguments/_parser.py +5 -4
- meerschaum/_internal/docs/index.py +2 -2
- meerschaum/_internal/term/TermPageHandler.py +2 -2
- meerschaum/actions/register.py +5 -1
- meerschaum/actions/show.py +12 -1
- meerschaum/actions/sync.py +49 -21
- meerschaum/actions/tag.py +101 -0
- meerschaum/api/__init__.py +1 -1
- meerschaum/api/dash/callbacks/dashboard.py +1 -1
- meerschaum/api/routes/_login.py +4 -3
- meerschaum/config/_formatting.py +1 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/APIConnector.py +6 -2
- meerschaum/connectors/sql/_pipes.py +43 -30
- meerschaum/connectors/sql/tables/__init__.py +0 -16
- meerschaum/core/Pipe/__init__.py +1 -2
- meerschaum/core/Pipe/_data.py +5 -4
- meerschaum/utils/__init__.py +3 -1
- meerschaum/utils/{get_pipes.py → _get_pipes.py} +5 -16
- meerschaum/utils/daemon/Daemon.py +4 -2
- meerschaum/utils/dataframe.py +3 -3
- meerschaum/utils/interactive.py +2 -16
- meerschaum/utils/misc.py +27 -28
- meerschaum/utils/packages/__init__.py +7 -1
- meerschaum/utils/packages/_packages.py +0 -1
- meerschaum/utils/pool.py +14 -17
- meerschaum/utils/sql.py +41 -19
- meerschaum/utils/typing.py +11 -0
- meerschaum/utils/venv/__init__.py +2 -1
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/METADATA +2 -3
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/RECORD +38 -37
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/WHEEL +1 -1
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/LICENSE +0 -0
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/NOTICE +0 -0
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/top_level.txt +0 -0
- {meerschaum-2.1.4.dist-info → meerschaum-2.1.6.dist-info}/zip-safe +0 -0
@@ -8,6 +8,7 @@ Return a dictionary (or list) of pipe objects. See documentation below for more
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
|
11
|
+
import meerschaum as mrsm
|
11
12
|
from meerschaum.utils.typing import (
|
12
13
|
Sequence, Optional, Union, Mapping, Any, InstanceConnector, PipesDict, List, Dict, Tuple
|
13
14
|
)
|
@@ -27,7 +28,7 @@ def get_pipes(
|
|
27
28
|
wait: bool = False,
|
28
29
|
debug: bool = False,
|
29
30
|
**kw: Any
|
30
|
-
) -> Union[PipesDict, List[
|
31
|
+
) -> Union[PipesDict, List[mrsm.Pipe]]:
|
31
32
|
"""
|
32
33
|
Return a dictionary or list of `meerschaum.Pipe` objects.
|
33
34
|
|
@@ -182,8 +183,8 @@ def get_pipes(
|
|
182
183
|
|
183
184
|
pipes[ck][mk][lk] = Pipe(
|
184
185
|
ck, mk, lk,
|
185
|
-
mrsm_instance=connector,
|
186
|
-
debug=debug,
|
186
|
+
mrsm_instance = connector,
|
187
|
+
debug = debug,
|
187
188
|
**filter_keywords(Pipe, **kw)
|
188
189
|
)
|
189
190
|
|
@@ -278,6 +279,7 @@ def fetch_pipes_keys(
|
|
278
279
|
metric_keys: Optional[List[str]] = None,
|
279
280
|
location_keys: Optional[List[str]] = None,
|
280
281
|
params: Optional[Dict[str, Any]] = None,
|
282
|
+
tags: Optional[List[str]] = None,
|
281
283
|
debug: bool = False,
|
282
284
|
**kw
|
283
285
|
) -> List[Tuple[str, str, str]]:
|
@@ -321,22 +323,9 @@ def fetch_pipes_keys(
|
|
321
323
|
result.append((ck, mk, lk))
|
322
324
|
return result
|
323
325
|
|
324
|
-
def _all(**kw):
|
325
|
-
"""
|
326
|
-
Fetch all available metrics and locations and create every combination.
|
327
|
-
Connector keys are required.
|
328
|
-
**NOTE**: Not implemented!
|
329
|
-
"""
|
330
|
-
error(
|
331
|
-
"Need to implement metrics and locations logic in SQL and API.",
|
332
|
-
NotImplementedError
|
333
|
-
)
|
334
|
-
|
335
326
|
_method_functions = {
|
336
327
|
'registered' : _registered,
|
337
328
|
'explicit' : _explicit,
|
338
|
-
'all' : _all,
|
339
|
-
### TODO implement 'all'
|
340
329
|
}
|
341
330
|
if method not in _method_functions:
|
342
331
|
error(f"Method '{method}' is not supported!", NotImplementedError)
|
@@ -15,7 +15,7 @@ import signal
|
|
15
15
|
import sys
|
16
16
|
import time
|
17
17
|
import traceback
|
18
|
-
from datetime import datetime
|
18
|
+
from datetime import datetime, timezone
|
19
19
|
from meerschaum.utils.typing import Optional, Dict, Any, SuccessTuple, Callable, List, Union
|
20
20
|
from meerschaum.config import get_config
|
21
21
|
from meerschaum.config._paths import DAEMON_RESOURCES_PATH, LOGS_RESOURCES_PATH
|
@@ -209,7 +209,9 @@ class Daemon:
|
|
209
209
|
if process_key not in ('began', 'ended', 'paused'):
|
210
210
|
raise ValueError(f"Invalid key '{process_key}'.")
|
211
211
|
|
212
|
-
self.properties['process'][process_key] =
|
212
|
+
self.properties['process'][process_key] = (
|
213
|
+
datetime.now(timezone.utc).replace(tzinfo=None).isoformat()
|
214
|
+
)
|
213
215
|
if write_properties:
|
214
216
|
self.write_properties()
|
215
217
|
|
meerschaum/utils/dataframe.py
CHANGED
@@ -9,7 +9,7 @@ Utility functions for working with DataFrames.
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from meerschaum.utils.typing import (
|
11
11
|
Optional, Dict, Any, List, Hashable, Generator,
|
12
|
-
Iterator, Iterable,
|
12
|
+
Iterator, Iterable, Union,
|
13
13
|
)
|
14
14
|
|
15
15
|
|
@@ -840,8 +840,8 @@ def df_from_literal(
|
|
840
840
|
)
|
841
841
|
val = literal
|
842
842
|
|
843
|
-
import datetime
|
844
|
-
now = datetime.
|
843
|
+
from datetime import datetime, timezone
|
844
|
+
now = datetime.now(timezone.utc).replace(tzinfo=None)
|
845
845
|
|
846
846
|
pd = import_pandas()
|
847
847
|
return pd.DataFrame({dt_name: [now], val_name: [val]})
|
meerschaum/utils/interactive.py
CHANGED
@@ -13,24 +13,10 @@ def select_pipes(
|
|
13
13
|
force: bool = False,
|
14
14
|
debug: bool = False,
|
15
15
|
) -> List[Pipe]:
|
16
|
-
"""Prompt the user for the keys to identify a list of pipes.
|
17
|
-
|
18
|
-
Parameters
|
19
|
-
----------
|
20
|
-
yes: bool :
|
21
|
-
(Default value = False)
|
22
|
-
force: bool :
|
23
|
-
(Default value = False)
|
24
|
-
debug: bool :
|
25
|
-
(Default value = False)
|
26
|
-
|
27
|
-
Returns
|
28
|
-
-------
|
29
|
-
|
30
|
-
"""
|
16
|
+
"""Prompt the user for the keys to identify a list of pipes."""
|
31
17
|
from meerschaum.utils.misc import get_connector_labels
|
32
18
|
from meerschaum.utils.prompt import prompt, choose, yes_no
|
33
|
-
from meerschaum.utils
|
19
|
+
from meerschaum.utils import get_pipes
|
34
20
|
from meerschaum.utils.formatting._shell import clear_screen
|
35
21
|
from meerschaum.utils.formatting import pprint_pipes
|
36
22
|
from meerschaum.config import get_config
|
meerschaum/utils/misc.py
CHANGED
@@ -6,7 +6,7 @@ Miscellaneous functions go here
|
|
6
6
|
"""
|
7
7
|
|
8
8
|
from __future__ import annotations
|
9
|
-
from datetime import timedelta
|
9
|
+
from datetime import timedelta, datetime, timezone
|
10
10
|
from meerschaum.utils.typing import (
|
11
11
|
Union,
|
12
12
|
Any,
|
@@ -386,10 +386,10 @@ def flatten_pipes_dict(pipes_dict: PipesDict) -> List[Pipe]:
|
|
386
386
|
|
387
387
|
|
388
388
|
def round_time(
|
389
|
-
dt: Optional[
|
390
|
-
date_delta: Optional[
|
389
|
+
dt: Optional[datetime] = None,
|
390
|
+
date_delta: Optional[timedelta] = None,
|
391
391
|
to: 'str' = 'down'
|
392
|
-
) ->
|
392
|
+
) -> datetime:
|
393
393
|
"""
|
394
394
|
Round a datetime object to a multiple of a timedelta.
|
395
395
|
http://stackoverflow.com/questions/3463930/how-to-round-the-minute-of-a-datetime-object-python
|
@@ -398,10 +398,10 @@ def round_time(
|
|
398
398
|
|
399
399
|
Parameters
|
400
400
|
----------
|
401
|
-
dt:
|
401
|
+
dt: Optional[datetime], default None
|
402
402
|
If `None`, grab the current UTC datetime.
|
403
403
|
|
404
|
-
date_delta:
|
404
|
+
date_delta: Optional[timedelta], default None
|
405
405
|
If `None`, use a delta of 1 minute.
|
406
406
|
|
407
407
|
to: 'str', default 'down'
|
@@ -409,36 +409,35 @@ def round_time(
|
|
409
409
|
|
410
410
|
Returns
|
411
411
|
-------
|
412
|
-
A rounded `datetime
|
412
|
+
A rounded `datetime` object.
|
413
413
|
|
414
414
|
Examples
|
415
415
|
--------
|
416
|
-
>>> round_time(datetime
|
416
|
+
>>> round_time(datetime(2022, 1, 1, 12, 15, 57, 200))
|
417
417
|
datetime.datetime(2022, 1, 1, 12, 15)
|
418
|
-
>>> round_time(datetime
|
418
|
+
>>> round_time(datetime(2022, 1, 1, 12, 15, 57, 200), to='up')
|
419
419
|
datetime.datetime(2022, 1, 1, 12, 16)
|
420
|
-
>>> round_time(datetime
|
420
|
+
>>> round_time(datetime(2022, 1, 1, 12, 15, 57, 200), timedelta(hours=1))
|
421
421
|
datetime.datetime(2022, 1, 1, 12, 0)
|
422
422
|
>>> round_time(
|
423
|
-
... datetime
|
424
|
-
...
|
425
|
-
... to='closest'
|
423
|
+
... datetime(2022, 1, 1, 12, 15, 57, 200),
|
424
|
+
... timedelta(hours=1),
|
425
|
+
... to = 'closest'
|
426
426
|
... )
|
427
427
|
datetime.datetime(2022, 1, 1, 12, 0)
|
428
428
|
>>> round_time(
|
429
|
-
... datetime
|
429
|
+
... datetime(2022, 1, 1, 12, 45, 57, 200),
|
430
430
|
... datetime.timedelta(hours=1),
|
431
|
-
... to='closest'
|
431
|
+
... to = 'closest'
|
432
432
|
... )
|
433
433
|
datetime.datetime(2022, 1, 1, 13, 0)
|
434
434
|
|
435
435
|
"""
|
436
|
-
import datetime
|
437
436
|
if date_delta is None:
|
438
|
-
date_delta =
|
437
|
+
date_delta = timedelta(minutes=1)
|
439
438
|
round_to = date_delta.total_seconds()
|
440
439
|
if dt is None:
|
441
|
-
dt = datetime.
|
440
|
+
dt = datetime.now(timezone.utc).replace(tzinfo=None)
|
442
441
|
seconds = (dt.replace(tzinfo=None) - dt.min.replace(tzinfo=None)).seconds
|
443
442
|
|
444
443
|
if seconds % round_to == 0 and dt.microsecond == 0:
|
@@ -451,7 +450,7 @@ def round_time(
|
|
451
450
|
else:
|
452
451
|
rounding = (seconds + round_to / 2) // round_to * round_to
|
453
452
|
|
454
|
-
return dt +
|
453
|
+
return dt + timedelta(0, rounding - seconds, - dt.microsecond)
|
455
454
|
|
456
455
|
|
457
456
|
def timed_input(
|
@@ -857,22 +856,22 @@ def get_connector_labels(
|
|
857
856
|
return sorted(possibilities)
|
858
857
|
|
859
858
|
|
860
|
-
def json_serialize_datetime(dt:
|
859
|
+
def json_serialize_datetime(dt: datetime) -> Union[str, None]:
|
861
860
|
"""
|
862
|
-
Serialize a datetime
|
861
|
+
Serialize a datetime object into JSON (ISO format string).
|
863
862
|
|
864
863
|
Examples
|
865
864
|
--------
|
866
|
-
>>> import json
|
867
|
-
>>>
|
865
|
+
>>> import json
|
866
|
+
>>> from datetime import datetime
|
867
|
+
>>> json.dumps({'a': datetime(2022, 1, 1)}, default=json_serialize_datetime)
|
868
868
|
'{"a": "2022-01-01T00:00:00Z"}'
|
869
869
|
|
870
870
|
"""
|
871
|
-
|
872
|
-
|
873
|
-
if
|
874
|
-
|
875
|
-
return None
|
871
|
+
if not isinstance(dt, datetime):
|
872
|
+
return None
|
873
|
+
tz_suffix = 'Z' if dt.tzinfo is None else ''
|
874
|
+
return dt.isoformat() + tz_suffix
|
876
875
|
|
877
876
|
|
878
877
|
def wget(
|
@@ -1544,7 +1544,13 @@ def reload_meerschaum(debug: bool = False) -> SuccessTuple:
|
|
1544
1544
|
"""
|
1545
1545
|
Reload the currently loaded Meercshaum modules, refreshing plugins and shell configuration.
|
1546
1546
|
"""
|
1547
|
-
reload_package(
|
1547
|
+
reload_package(
|
1548
|
+
'meerschaum',
|
1549
|
+
skip_submodules = [
|
1550
|
+
'meerschaum._internal.shell',
|
1551
|
+
'meerschaum.utils.pool',
|
1552
|
+
]
|
1553
|
+
)
|
1548
1554
|
|
1549
1555
|
from meerschaum.plugins import reload_plugins
|
1550
1556
|
from meerschaum._internal.shell.Shell import _insert_shell_actions
|
meerschaum/utils/pool.py
CHANGED
@@ -11,7 +11,7 @@ from meerschaum.utils.typing import Optional, Callable, List, Any
|
|
11
11
|
from meerschaum.utils.threading import Lock, RLock
|
12
12
|
import signal
|
13
13
|
|
14
|
-
pools =
|
14
|
+
pools = {}
|
15
15
|
_locks = {
|
16
16
|
'pools': Lock(),
|
17
17
|
}
|
@@ -28,10 +28,10 @@ def get_pool(
|
|
28
28
|
):
|
29
29
|
"""If the requested pool does not exist, instantiate it here.
|
30
30
|
Pools are joined and closed on exit."""
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
31
|
+
from multiprocessing import cpu_count
|
32
|
+
if workers is None:
|
33
|
+
workers = cpu_count()
|
34
|
+
pool_key = pool_class_name + f'-{workers}'
|
35
35
|
|
36
36
|
def build_pool(workers):
|
37
37
|
from meerschaum.utils.warnings import warn
|
@@ -49,9 +49,6 @@ def get_pool(
|
|
49
49
|
'ThreadPool'
|
50
50
|
)
|
51
51
|
|
52
|
-
if workers is None:
|
53
|
-
from multiprocessing import cpu_count
|
54
|
-
workers = cpu_count()
|
55
52
|
try:
|
56
53
|
pool = Pool(workers, initializer=initializer, initargs=initargs)
|
57
54
|
except Exception as e:
|
@@ -59,23 +56,24 @@ def get_pool(
|
|
59
56
|
pool = None
|
60
57
|
|
61
58
|
with _locks['pools']:
|
62
|
-
pools[
|
59
|
+
pools[pool_key] = pool
|
63
60
|
|
64
|
-
if
|
61
|
+
if pools.get(pool_key, None) is None:
|
65
62
|
build_pool(workers)
|
66
63
|
|
67
64
|
if (
|
68
|
-
pools[
|
69
|
-
and pools[
|
65
|
+
pools[pool_key] is not None
|
66
|
+
and pools[pool_key]._state not in ('RUN', 0)
|
70
67
|
):
|
71
68
|
try:
|
72
|
-
pools[
|
69
|
+
pools[pool_key].close()
|
70
|
+
pools[pool_key].terminate()
|
73
71
|
except Exception as e:
|
74
72
|
pass
|
75
|
-
del pools[
|
73
|
+
del pools[pool_key]
|
76
74
|
build_pool(workers)
|
77
75
|
|
78
|
-
return pools[
|
76
|
+
return pools[pool_key]
|
79
77
|
|
80
78
|
|
81
79
|
def get_pools():
|
@@ -94,7 +92,6 @@ def get_pool_executor(workers: Optional[int] = None):
|
|
94
92
|
from concurrent.futures import ThreadPoolExecutor
|
95
93
|
workers = cpu_count() if workers is None else workers
|
96
94
|
except Exception as e:
|
97
|
-
|
95
|
+
return None
|
98
96
|
|
99
97
|
return ThreadPoolExecutor(max_workers=workers) if ThreadPoolExecutor is not None else None
|
100
|
-
|
meerschaum/utils/sql.py
CHANGED
@@ -7,6 +7,7 @@ Flavor-specific SQL tools.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
+
from datetime import datetime, timezone, timedelta
|
10
11
|
import meerschaum as mrsm
|
11
12
|
from meerschaum.utils.typing import Optional, Dict, Any, Union, List, Iterable, Tuple
|
12
13
|
### Preserve legacy imports.
|
@@ -37,6 +38,7 @@ version_queries = {
|
|
37
38
|
'oracle': "SELECT version from PRODUCT_COMPONENT_VERSION WHERE rownum = 1",
|
38
39
|
}
|
39
40
|
SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
|
41
|
+
COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'citus'}
|
40
42
|
update_queries = {
|
41
43
|
'default': """
|
42
44
|
UPDATE {target_table_name} AS f
|
@@ -52,25 +54,25 @@ update_queries = {
|
|
52
54
|
INSERT INTO {target_table_name} ({patch_cols_str})
|
53
55
|
SELECT {patch_cols_str}
|
54
56
|
FROM {patch_table_name}
|
55
|
-
ON CONFLICT ({join_cols_str}) DO
|
57
|
+
ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
|
56
58
|
""",
|
57
59
|
'postgresql-upsert': """
|
58
60
|
INSERT INTO {target_table_name} ({patch_cols_str})
|
59
61
|
SELECT {patch_cols_str}
|
60
62
|
FROM {patch_table_name}
|
61
|
-
ON CONFLICT ({join_cols_str}) DO
|
63
|
+
ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
|
62
64
|
""",
|
63
65
|
'citus-upsert': """
|
64
66
|
INSERT INTO {target_table_name} ({patch_cols_str})
|
65
67
|
SELECT {patch_cols_str}
|
66
68
|
FROM {patch_table_name}
|
67
|
-
ON CONFLICT ({join_cols_str}) DO
|
69
|
+
ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
|
68
70
|
""",
|
69
71
|
'cockroachdb-upsert': """
|
70
72
|
INSERT INTO {target_table_name} ({patch_cols_str})
|
71
73
|
SELECT {patch_cols_str}
|
72
74
|
FROM {patch_table_name}
|
73
|
-
ON CONFLICT ({join_cols_str}) DO
|
75
|
+
ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
|
74
76
|
""",
|
75
77
|
'mysql': """
|
76
78
|
UPDATE {target_table_name} AS f
|
@@ -121,7 +123,7 @@ update_queries = {
|
|
121
123
|
SELECT {patch_cols_str}
|
122
124
|
FROM {patch_table_name}
|
123
125
|
WHERE true
|
124
|
-
ON CONFLICT ({join_cols_str}) DO
|
126
|
+
ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
|
125
127
|
""",
|
126
128
|
'sqlite_delete_insert': [
|
127
129
|
"""
|
@@ -274,7 +276,7 @@ def dateadd_str(
|
|
274
276
|
flavor: str = 'postgresql',
|
275
277
|
datepart: str = 'day',
|
276
278
|
number: Union[int, float] = 0,
|
277
|
-
begin: Union[str, datetime
|
279
|
+
begin: Union[str, datetime, int] = 'now'
|
278
280
|
) -> str:
|
279
281
|
"""
|
280
282
|
Generate a `DATEADD` clause depending on database flavor.
|
@@ -310,7 +312,7 @@ def dateadd_str(
|
|
310
312
|
number: Union[int, float], default `0`
|
311
313
|
How many units to add to the date part.
|
312
314
|
|
313
|
-
begin: Union[str, datetime
|
315
|
+
begin: Union[str, datetime], default `'now'`
|
314
316
|
Base datetime to which to add dateparts.
|
315
317
|
|
316
318
|
Returns
|
@@ -321,13 +323,13 @@ def dateadd_str(
|
|
321
323
|
--------
|
322
324
|
>>> dateadd_str(
|
323
325
|
... flavor = 'mssql',
|
324
|
-
... begin = datetime
|
326
|
+
... begin = datetime(2022, 1, 1, 0, 0),
|
325
327
|
... number = 1,
|
326
328
|
... )
|
327
329
|
"DATEADD(day, 1, CAST('2022-01-01 00:00:00' AS DATETIME))"
|
328
330
|
>>> dateadd_str(
|
329
331
|
... flavor = 'postgresql',
|
330
|
-
... begin = datetime
|
332
|
+
... begin = datetime(2022, 1, 1, 0, 0),
|
331
333
|
... number = 1,
|
332
334
|
... )
|
333
335
|
"CAST('2022-01-01 00:00:00' AS TIMESTAMP) + INTERVAL '1 day'"
|
@@ -336,7 +338,6 @@ def dateadd_str(
|
|
336
338
|
from meerschaum.utils.debug import dprint
|
337
339
|
from meerschaum.utils.packages import attempt_import
|
338
340
|
from meerschaum.utils.warnings import error
|
339
|
-
import datetime
|
340
341
|
dateutil_parser = attempt_import('dateutil.parser')
|
341
342
|
if 'int' in str(type(begin)).lower():
|
342
343
|
return str(begin)
|
@@ -346,7 +347,7 @@ def dateadd_str(
|
|
346
347
|
_original_begin = begin
|
347
348
|
begin_time = None
|
348
349
|
### Sanity check: make sure `begin` is a valid datetime before we inject anything.
|
349
|
-
if not isinstance(begin, datetime
|
350
|
+
if not isinstance(begin, datetime):
|
350
351
|
try:
|
351
352
|
begin_time = dateutil_parser.parse(begin)
|
352
353
|
except Exception:
|
@@ -360,7 +361,13 @@ def dateadd_str(
|
|
360
361
|
clean(str(begin))
|
361
362
|
### If begin is a valid datetime, wrap it in quotes.
|
362
363
|
else:
|
363
|
-
begin
|
364
|
+
if isinstance(begin, datetime) and begin.tzinfo is not None:
|
365
|
+
begin = begin.astimezone(timezone.utc)
|
366
|
+
begin = (
|
367
|
+
f"'{begin.replace(tzinfo=None)}'"
|
368
|
+
if isinstance(begin, datetime)
|
369
|
+
else f"'{begin}'"
|
370
|
+
)
|
364
371
|
|
365
372
|
da = ""
|
366
373
|
if flavor in ('postgresql', 'timescaledb', 'cockroachdb', 'citus'):
|
@@ -390,7 +397,7 @@ def dateadd_str(
|
|
390
397
|
elif flavor == 'oracle':
|
391
398
|
if begin == 'now':
|
392
399
|
begin = str(
|
393
|
-
datetime.
|
400
|
+
datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y:%m:%d %M:%S.%f')
|
394
401
|
)
|
395
402
|
elif begin_time:
|
396
403
|
begin = str(begin_time.strftime('%Y-%m-%d %H:%M:%S.%f'))
|
@@ -956,10 +963,6 @@ def get_table_cols_types(
|
|
956
963
|
)
|
957
964
|
)
|
958
965
|
]
|
959
|
-
if debug:
|
960
|
-
dprint(f"schema={schema}, database={database}")
|
961
|
-
for doc in cols_types_docs:
|
962
|
-
print(doc)
|
963
966
|
|
964
967
|
### NOTE: This may return incorrect columns if the schema is not explicitly stated.
|
965
968
|
if cols_types_docs and not cols_types_docs_filtered:
|
@@ -1082,7 +1085,7 @@ def get_update_queries(
|
|
1082
1085
|
for col in patch_table_columns
|
1083
1086
|
]
|
1084
1087
|
)
|
1085
|
-
join_cols_str = ','.join(
|
1088
|
+
join_cols_str = ', '.join(
|
1086
1089
|
[
|
1087
1090
|
sql_item_name(col, flavor)
|
1088
1091
|
for col in join_cols
|
@@ -1107,10 +1110,27 @@ def get_update_queries(
|
|
1107
1110
|
if debug:
|
1108
1111
|
dprint(f"value_cols: {value_cols}")
|
1109
1112
|
|
1110
|
-
if not
|
1113
|
+
if not join_cols_types:
|
1114
|
+
return []
|
1115
|
+
if not value_cols and not upsert:
|
1111
1116
|
return []
|
1112
1117
|
|
1118
|
+
coalesce_join_cols_str = ', '.join(
|
1119
|
+
[
|
1120
|
+
'COALESCE('
|
1121
|
+
+ sql_item_name(c_name, flavor)
|
1122
|
+
+ ', '
|
1123
|
+
+ get_null_replacement(c_type, flavor)
|
1124
|
+
+ ')'
|
1125
|
+
for c_name, c_type in join_cols_types
|
1126
|
+
]
|
1127
|
+
)
|
1128
|
+
|
1129
|
+
update_or_nothing = ('UPDATE' if value_cols else 'NOTHING')
|
1130
|
+
|
1113
1131
|
def sets_subquery(l_prefix: str, r_prefix: str):
|
1132
|
+
if not value_cols:
|
1133
|
+
return ''
|
1114
1134
|
return 'SET ' + ',\n'.join([
|
1115
1135
|
(
|
1116
1136
|
l_prefix + sql_item_name(c_name, flavor, None)
|
@@ -1167,6 +1187,8 @@ def get_update_queries(
|
|
1167
1187
|
patch_cols_str = patch_cols_str,
|
1168
1188
|
date_bounds_subquery = date_bounds_subquery,
|
1169
1189
|
join_cols_str = join_cols_str,
|
1190
|
+
coalesce_join_cols_str = coalesce_join_cols_str,
|
1191
|
+
update_or_nothing = update_or_nothing,
|
1170
1192
|
)
|
1171
1193
|
for base_query in base_queries
|
1172
1194
|
]
|
meerschaum/utils/typing.py
CHANGED
@@ -87,3 +87,14 @@ PipesDict = Dict[
|
|
87
87
|
]
|
88
88
|
]
|
89
89
|
WebState = Dict[str, str]
|
90
|
+
|
91
|
+
def is_success_tuple(x: Any) -> bool:
|
92
|
+
"""
|
93
|
+
Determine whether an object is a `SuccessTuple`.
|
94
|
+
"""
|
95
|
+
return (
|
96
|
+
isinstance(x, tuple)
|
97
|
+
and len(x) == 2
|
98
|
+
and isinstance(x[0], bool)
|
99
|
+
and isinstance(x[1], str)
|
100
|
+
)
|
@@ -368,10 +368,11 @@ def init_venv(
|
|
368
368
|
docker_home_venv_path = pathlib.Path('/home/meerschaum/venvs/mrsm')
|
369
369
|
|
370
370
|
runtime_env_var = STATIC_CONFIG['environment']['runtime']
|
371
|
+
work_dir_env_var = STATIC_CONFIG['environment']['work_dir']
|
371
372
|
if (
|
372
373
|
not force
|
373
374
|
and venv == 'mrsm'
|
374
|
-
and os.environ.get(
|
375
|
+
and os.environ.get(work_dir_env_var, None) is not None
|
375
376
|
and docker_home_venv_path.exists()
|
376
377
|
):
|
377
378
|
shutil.move(docker_home_venv_path, venv_path)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: meerschaum
|
3
|
-
Version: 2.1.
|
3
|
+
Version: 2.1.6
|
4
4
|
Summary: Sync Time-Series Pipes with Meerschaum
|
5
5
|
Home-page: https://meerschaum.io
|
6
6
|
Author: Bennett Meares
|
@@ -149,7 +149,6 @@ Requires-Dist: mypy >=0.812.0 ; extra == 'dev-tools'
|
|
149
149
|
Requires-Dist: pytest >=6.2.2 ; extra == 'dev-tools'
|
150
150
|
Requires-Dist: pytest-xdist >=3.2.1 ; extra == 'dev-tools'
|
151
151
|
Requires-Dist: heartrate >=0.2.1 ; extra == 'dev-tools'
|
152
|
-
Requires-Dist: py-heat >=0.0.6 ; extra == 'dev-tools'
|
153
152
|
Provides-Extra: docs
|
154
153
|
Requires-Dist: mkdocs >=1.1.2 ; extra == 'docs'
|
155
154
|
Requires-Dist: mkdocs-material >=6.2.5 ; extra == 'docs'
|
@@ -307,7 +306,7 @@ Requires-Dist: docker-compose >=1.27.4 ; extra == 'stack'
|
|
307
306
|
| PyPI | GitHub | Info | Stats |
|
308
307
|
|---|---|---|---|
|
309
308
|
|  |  |  |  |
|
310
|
-
|  |  | [ |  | [](https://artifacthub.io/packages/search?repo=meerschaum) |  |
|
311
310
|
|
312
311
|
<p align="center">
|
313
312
|
<img src="https://meerschaum.io/files/images/demo.gif" alt="Meerschaum demo" height="450px">
|