meerschaum 2.8.3__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parser.py +5 -0
- meerschaum/actions/drop.py +1 -1
- meerschaum/actions/start.py +14 -6
- meerschaum/actions/sync.py +9 -0
- meerschaum/api/__init__.py +9 -3
- meerschaum/api/_chunks.py +67 -0
- meerschaum/api/dash/callbacks/custom.py +23 -2
- meerschaum/api/dash/callbacks/dashboard.py +41 -3
- meerschaum/api/dash/components.py +27 -19
- meerschaum/api/dash/pages/dashboard.py +11 -9
- meerschaum/api/dash/pages/plugins.py +31 -27
- meerschaum/api/dash/webterm.py +6 -3
- meerschaum/api/resources/static/css/dash.css +1 -1
- meerschaum/api/resources/templates/termpage.html +4 -0
- meerschaum/api/routes/_pipes.py +193 -81
- meerschaum/config/_default.py +3 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_APIConnector.py +12 -1
- meerschaum/connectors/api/_pipes.py +27 -15
- meerschaum/connectors/api/_plugins.py +51 -45
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/parse.py +1 -2
- meerschaum/connectors/sql/_SQLConnector.py +1 -1
- meerschaum/core/Pipe/_data.py +1 -2
- meerschaum/core/Pipe/_verify.py +23 -8
- meerschaum/jobs/systemd.py +1 -1
- meerschaum/plugins/_Plugin.py +21 -5
- meerschaum/plugins/__init__.py +6 -4
- meerschaum/utils/formatting/_shell.py +1 -4
- meerschaum/utils/packages/_packages.py +2 -1
- meerschaum/utils/process.py +27 -3
- meerschaum/utils/schedule.py +3 -3
- meerschaum/utils/sql.py +1 -1
- meerschaum/utils/venv/__init__.py +6 -1
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/METADATA +4 -1
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/RECORD +42 -41
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/WHEEL +1 -1
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/LICENSE +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/NOTICE +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dev1.dist-info}/zip-safe +0 -0
@@ -7,21 +7,25 @@ Manage plugins via the API connector
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
|
10
|
+
|
11
|
+
import meerschaum as mrsm
|
12
|
+
from meerschaum.utils.typing import Union, Any, Optional, SuccessTuple, List, Dict
|
13
|
+
|
11
14
|
|
12
15
|
def plugin_r_url(
|
13
|
-
|
14
|
-
|
16
|
+
plugin: Union[mrsm.core.Plugin, str],
|
17
|
+
) -> str:
|
15
18
|
"""Generate a relative URL path from a Plugin."""
|
16
19
|
from meerschaum.config.static import STATIC_CONFIG
|
17
20
|
return f"{STATIC_CONFIG['api']['endpoints']['plugins']}/{plugin}"
|
18
21
|
|
22
|
+
|
19
23
|
def register_plugin(
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
24
|
+
self,
|
25
|
+
plugin: mrsm.core.Plugin,
|
26
|
+
make_archive: bool = True,
|
27
|
+
debug: bool = False,
|
28
|
+
) -> SuccessTuple:
|
25
29
|
"""Register a plugin and upload its archive."""
|
26
30
|
import json
|
27
31
|
archive_path = plugin.make_tar(debug=debug) if make_archive else plugin.archive_path
|
@@ -34,27 +38,30 @@ def register_plugin(
|
|
34
38
|
r_url = plugin_r_url(plugin)
|
35
39
|
try:
|
36
40
|
response = self.post(r_url, files=files, params=metadata, debug=debug)
|
37
|
-
except Exception
|
41
|
+
except Exception:
|
38
42
|
return False, f"Failed to register plugin '{plugin}'."
|
39
43
|
finally:
|
40
44
|
file_pointer.close()
|
41
45
|
|
42
46
|
try:
|
43
47
|
success, msg = json.loads(response.text)
|
44
|
-
except Exception
|
48
|
+
except Exception:
|
45
49
|
return False, response.text
|
46
50
|
|
47
51
|
return success, msg
|
48
52
|
|
53
|
+
|
49
54
|
def install_plugin(
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
55
|
+
self,
|
56
|
+
name: str,
|
57
|
+
skip_deps: bool = False,
|
58
|
+
force: bool = False,
|
59
|
+
debug: bool = False
|
60
|
+
) -> SuccessTuple:
|
56
61
|
"""Download and attempt to install a plugin from the API."""
|
57
|
-
import os
|
62
|
+
import os
|
63
|
+
import pathlib
|
64
|
+
import json
|
58
65
|
from meerschaum.core import Plugin
|
59
66
|
from meerschaum.config._paths import PLUGINS_TEMP_RESOURCES_PATH
|
60
67
|
from meerschaum.utils.debug import dprint
|
@@ -75,41 +82,39 @@ def install_plugin(
|
|
75
82
|
success, msg = tuple(j)
|
76
83
|
elif isinstance(j, dict) and 'detail' in j:
|
77
84
|
success, msg = False, fail_msg
|
78
|
-
except Exception
|
85
|
+
except Exception:
|
79
86
|
success, msg = False, fail_msg
|
80
87
|
return success, msg
|
81
88
|
plugin = Plugin(name, archive_path=archive_path, repo_connector=self)
|
82
89
|
return plugin.install(skip_deps=skip_deps, force=force, debug=debug)
|
83
90
|
|
91
|
+
|
84
92
|
def get_plugins(
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
93
|
+
self,
|
94
|
+
user_id: Optional[int] = None,
|
95
|
+
search_term: Optional[str] = None,
|
96
|
+
debug: bool = False
|
97
|
+
) -> List[str]:
|
90
98
|
"""Return a list of registered plugin names.
|
91
99
|
|
92
100
|
Parameters
|
93
101
|
----------
|
94
|
-
user_id
|
102
|
+
user_id: Optional[int], default None
|
95
103
|
If specified, return all plugins from a certain user.
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
(Default value = None)
|
100
|
-
debug : bool :
|
101
|
-
(Default value = False)
|
104
|
+
|
105
|
+
search_term: Optional[str], default None
|
106
|
+
If specified, return plugins beginning with this string.
|
102
107
|
|
103
108
|
Returns
|
104
109
|
-------
|
105
|
-
|
110
|
+
A list of plugin names.
|
106
111
|
"""
|
107
112
|
import json
|
108
|
-
from meerschaum.utils.warnings import
|
113
|
+
from meerschaum.utils.warnings import error
|
109
114
|
from meerschaum.config.static import STATIC_CONFIG
|
110
115
|
response = self.get(
|
111
116
|
STATIC_CONFIG['api']['endpoints']['plugins'],
|
112
|
-
params = {'user_id'
|
117
|
+
params = {'user_id': user_id, 'search_term': search_term},
|
113
118
|
use_token = True,
|
114
119
|
debug = debug
|
115
120
|
)
|
@@ -120,11 +125,12 @@ def get_plugins(
|
|
120
125
|
error(response.text)
|
121
126
|
return plugins
|
122
127
|
|
128
|
+
|
123
129
|
def get_plugin_attributes(
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
130
|
+
self,
|
131
|
+
plugin: mrsm.core.Plugin,
|
132
|
+
debug: bool = False
|
133
|
+
) -> Dict[str, Any]:
|
128
134
|
"""
|
129
135
|
Return a plugin's attributes.
|
130
136
|
"""
|
@@ -136,7 +142,7 @@ def get_plugin_attributes(
|
|
136
142
|
if isinstance(attributes, str) and attributes and attributes[0] == '{':
|
137
143
|
try:
|
138
144
|
attributes = json.loads(attributes)
|
139
|
-
except Exception
|
145
|
+
except Exception:
|
140
146
|
pass
|
141
147
|
if not isinstance(attributes, dict):
|
142
148
|
error(response.text)
|
@@ -145,23 +151,23 @@ def get_plugin_attributes(
|
|
145
151
|
return {}
|
146
152
|
return attributes
|
147
153
|
|
154
|
+
|
148
155
|
def delete_plugin(
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
156
|
+
self,
|
157
|
+
plugin: mrsm.core.Plugin,
|
158
|
+
debug: bool = False
|
159
|
+
) -> SuccessTuple:
|
153
160
|
"""Delete a plugin from an API repository."""
|
154
161
|
import json
|
155
162
|
r_url = plugin_r_url(plugin)
|
156
163
|
try:
|
157
164
|
response = self.delete(r_url, debug=debug)
|
158
|
-
except Exception
|
165
|
+
except Exception:
|
159
166
|
return False, f"Failed to delete plugin '{plugin}'."
|
160
167
|
|
161
168
|
try:
|
162
169
|
success, msg = json.loads(response.text)
|
163
|
-
except Exception
|
170
|
+
except Exception:
|
164
171
|
return False, response.text
|
165
172
|
|
166
173
|
return success, msg
|
167
|
-
|
meerschaum/connectors/parse.py
CHANGED
@@ -184,7 +184,7 @@ class SQLConnector(Connector):
|
|
184
184
|
### ensure flavor and label are set accordingly
|
185
185
|
if 'flavor' not in self.__dict__:
|
186
186
|
if flavor is None and 'uri' not in self.__dict__:
|
187
|
-
raise
|
187
|
+
raise ValueError(
|
188
188
|
f" Missing flavor. Provide flavor as a key for '{self}'."
|
189
189
|
)
|
190
190
|
self.flavor = flavor or self.parse_uri(self.__dict__['uri']).get('flavor', None)
|
meerschaum/core/Pipe/_data.py
CHANGED
@@ -88,9 +88,8 @@ def get_data(
|
|
88
88
|
limit: Optional[int], default None
|
89
89
|
If provided, cap the dataframe to this many rows.
|
90
90
|
|
91
|
-
fresh: bool, default
|
91
|
+
fresh: bool, default False
|
92
92
|
If `True`, skip local cache and directly query the instance connector.
|
93
|
-
Defaults to `True`.
|
94
93
|
|
95
94
|
debug: bool, default False
|
96
95
|
Verbosity toggle.
|
meerschaum/core/Pipe/_verify.py
CHANGED
@@ -119,7 +119,7 @@ def verify(
|
|
119
119
|
begin, end = self.parse_date_bounds(begin, end)
|
120
120
|
cannot_determine_bounds = bounded and begin is None and end is None
|
121
121
|
|
122
|
-
if cannot_determine_bounds:
|
122
|
+
if cannot_determine_bounds and not check_rowcounts_only:
|
123
123
|
warn(f"Cannot determine sync bounds for {self}. Syncing instead...", stack=False)
|
124
124
|
sync_success, sync_msg = self.sync(
|
125
125
|
begin=begin,
|
@@ -187,7 +187,9 @@ def verify(
|
|
187
187
|
max_chunks_syncs = mrsm.get_config('pipes', 'verify', 'max_chunks_syncs')
|
188
188
|
|
189
189
|
info(
|
190
|
-
f"Verifying {self}:\n
|
190
|
+
f"Verifying {self}:\n "
|
191
|
+
+ ("Syncing" if not check_rowcounts_only else "Checking")
|
192
|
+
+ f" {len(chunk_bounds)} chunk"
|
191
193
|
+ ('s' if len(chunk_bounds) != 1 else '')
|
192
194
|
+ f" ({'un' if not bounded else ''}bounded)"
|
193
195
|
+ f" of size '{interval_str(chunk_interval)}'"
|
@@ -293,6 +295,17 @@ def verify(
|
|
293
295
|
_batch_begin = batch_chunk_bounds[0][0]
|
294
296
|
_batch_end = batch_chunk_bounds[-1][-1]
|
295
297
|
batch_message_header = f"{_batch_begin} - {_batch_end}"
|
298
|
+
|
299
|
+
if check_rowcounts_only:
|
300
|
+
info(f"Checking row-counts for batch bounds:\n {batch_message_header}")
|
301
|
+
_, (batch_init_success, batch_init_msg) = process_chunk_bounds(
|
302
|
+
(_batch_begin, _batch_end)
|
303
|
+
)
|
304
|
+
mrsm.pprint((batch_init_success, batch_init_msg))
|
305
|
+
if batch_init_success and 'up-to-date' in batch_init_msg:
|
306
|
+
info("Entire batch is up-to-date.")
|
307
|
+
return batch_init_success, batch_init_msg
|
308
|
+
|
296
309
|
batch_bounds_success_tuples = dict(pool.map(process_chunk_bounds, batch_chunk_bounds))
|
297
310
|
bounds_success_tuples.update(batch_bounds_success_tuples)
|
298
311
|
batch_bounds_success_bools = {
|
@@ -404,7 +417,7 @@ def verify(
|
|
404
417
|
retry_failed_batch = False
|
405
418
|
|
406
419
|
batch_msg_to_print = (
|
407
|
-
f"{make_header('Completed batch ' + batch_counter_str + '
|
420
|
+
f"{make_header('Completed batch ' + batch_counter_str + ':')}\n{batch_msg}"
|
408
421
|
)
|
409
422
|
mrsm.pprint((batch_success, batch_msg_to_print))
|
410
423
|
|
@@ -480,11 +493,13 @@ def get_chunks_success_message(
|
|
480
493
|
header = (header + "\n") if header else ""
|
481
494
|
stats_msg = items_str(
|
482
495
|
(
|
483
|
-
(
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
496
|
+
(
|
497
|
+
([f'inserted {num_inserted:,}'] if num_inserted else [])
|
498
|
+
+ ([f'updated {num_updated:,}'] if num_updated else [])
|
499
|
+
+ ([f'upserted {num_upserted:,}'] if num_upserted else [])
|
500
|
+
+ ([f'checked {num_checked:,}'] if num_checked else [])
|
501
|
+
) or ['synced 0']
|
502
|
+
),
|
488
503
|
quotes=False,
|
489
504
|
and_=False,
|
490
505
|
)
|
meerschaum/jobs/systemd.py
CHANGED
meerschaum/plugins/_Plugin.py
CHANGED
@@ -450,7 +450,7 @@ class Plugin:
|
|
450
450
|
success, msg = False, (
|
451
451
|
f"Failed to run post-install setup for plugin '{self}'." + '\n' +
|
452
452
|
f"Check `setup()` in '{self.__file__}' for more information " +
|
453
|
-
|
453
|
+
"(no error message provided)."
|
454
454
|
)
|
455
455
|
else:
|
456
456
|
success, msg = True, success_msg
|
@@ -458,7 +458,7 @@ class Plugin:
|
|
458
458
|
success = True
|
459
459
|
msg = (
|
460
460
|
f"Post-install for plugin '{self}' returned None. " +
|
461
|
-
|
461
|
+
"Assuming plugin successfully installed."
|
462
462
|
)
|
463
463
|
warn(msg)
|
464
464
|
else:
|
@@ -469,7 +469,7 @@ class Plugin:
|
|
469
469
|
)
|
470
470
|
|
471
471
|
_ongoing_installations.remove(self.full_name)
|
472
|
-
|
472
|
+
_ = self.module
|
473
473
|
return success, msg
|
474
474
|
|
475
475
|
|
@@ -716,13 +716,14 @@ class Plugin:
|
|
716
716
|
return required
|
717
717
|
|
718
718
|
|
719
|
-
def get_required_plugins(self, debug: bool=False) -> List[
|
719
|
+
def get_required_plugins(self, debug: bool=False) -> List[mrsm.plugins.Plugin]:
|
720
720
|
"""
|
721
721
|
Return a list of required Plugin objects.
|
722
722
|
"""
|
723
723
|
from meerschaum.utils.warnings import warn
|
724
724
|
from meerschaum.config import get_config
|
725
725
|
from meerschaum.config.static import STATIC_CONFIG
|
726
|
+
from meerschaum.connectors.parse import is_valid_connector_keys
|
726
727
|
plugins = []
|
727
728
|
_deps = self.get_dependencies(debug=debug)
|
728
729
|
sep = STATIC_CONFIG['plugins']['repo_separator']
|
@@ -731,11 +732,13 @@ class Plugin:
|
|
731
732
|
if _d.startswith('plugin:') and len(_d) > len('plugin:')
|
732
733
|
]
|
733
734
|
default_repo_keys = get_config('meerschaum', 'default_repository')
|
735
|
+
skipped_repo_keys = set()
|
736
|
+
|
734
737
|
for _plugin_name in plugin_names:
|
735
738
|
if sep in _plugin_name:
|
736
739
|
try:
|
737
740
|
_plugin_name, _repo_keys = _plugin_name.split(sep)
|
738
|
-
except Exception
|
741
|
+
except Exception:
|
739
742
|
_repo_keys = default_repo_keys
|
740
743
|
warn(
|
741
744
|
f"Invalid repo keys for required plugin '{_plugin_name}'.\n "
|
@@ -744,7 +747,20 @@ class Plugin:
|
|
744
747
|
)
|
745
748
|
else:
|
746
749
|
_repo_keys = default_repo_keys
|
750
|
+
|
751
|
+
if _repo_keys in skipped_repo_keys:
|
752
|
+
continue
|
753
|
+
|
754
|
+
if not is_valid_connector_keys(_repo_keys):
|
755
|
+
warn(
|
756
|
+
f"Invalid connector '{_repo_keys}'.\n"
|
757
|
+
f" Skipping required plugins from repository '{_repo_keys}'",
|
758
|
+
stack=False,
|
759
|
+
)
|
760
|
+
continue
|
761
|
+
|
747
762
|
plugins.append(Plugin(_plugin_name, repo=_repo_keys))
|
763
|
+
|
748
764
|
return plugins
|
749
765
|
|
750
766
|
|
meerschaum/plugins/__init__.py
CHANGED
@@ -166,10 +166,11 @@ def post_sync_hook(
|
|
166
166
|
|
167
167
|
_plugin_endpoints_to_pages = {}
|
168
168
|
def web_page(
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
169
|
+
page: Union[str, None, Callable[[Any], Any]] = None,
|
170
|
+
login_required: bool = True,
|
171
|
+
skip_navbar: bool = False,
|
172
|
+
**kwargs
|
173
|
+
) -> Any:
|
173
174
|
"""
|
174
175
|
Quickly add pages to the dash application.
|
175
176
|
|
@@ -200,6 +201,7 @@ def web_page(
|
|
200
201
|
_plugin_endpoints_to_pages[page_str] = {
|
201
202
|
'function': _func,
|
202
203
|
'login_required': login_required,
|
204
|
+
'skip_navbar': skip_navbar,
|
203
205
|
}
|
204
206
|
return wrapper
|
205
207
|
|
@@ -66,10 +66,7 @@ def clear_screen(debug: bool = False) -> bool:
|
|
66
66
|
get_console().clear()
|
67
67
|
print("", end="", flush=True)
|
68
68
|
return True
|
69
|
-
clear_string =
|
70
|
-
'\033c' if platform.system() != 'Windows'
|
71
|
-
else '\033[2J'
|
72
|
-
)
|
69
|
+
clear_string = '\033[2J'
|
73
70
|
reset_string = '\033[0m'
|
74
71
|
print(clear_string + reset_string, end="")
|
75
72
|
print("", end="", flush=True)
|
@@ -58,7 +58,7 @@ packages: Dict[str, Dict[str, str]] = {
|
|
58
58
|
'_internal' : {
|
59
59
|
'apscheduler' : (
|
60
60
|
f"{_MRSM_PACKAGE_ARCHIVES_PREFIX}"
|
61
|
-
"APScheduler-4.0.0a5.
|
61
|
+
"APScheduler-4.0.0a5.post87+mrsm-py3-none-any.whl>=4.0.0a5"
|
62
62
|
),
|
63
63
|
'dataclass_wizard' : 'dataclass-wizard>=0.28.0',
|
64
64
|
},
|
@@ -135,6 +135,7 @@ packages: Dict[str, Dict[str, str]] = {
|
|
135
135
|
packages['sql'] = {
|
136
136
|
'numpy' : 'numpy>=1.18.5',
|
137
137
|
'pandas' : 'pandas[parquet]>=2.0.1',
|
138
|
+
'geopandas' : 'geopandas>=1.0.1',
|
138
139
|
'pyarrow' : 'pyarrow>=16.1.0',
|
139
140
|
'dask' : 'dask[complete]>=2024.12.1',
|
140
141
|
'partd' : 'partd>=1.4.2',
|
meerschaum/utils/process.py
CHANGED
@@ -9,15 +9,25 @@ See `meerschaum.utils.pool` for multiprocessing and
|
|
9
9
|
"""
|
10
10
|
|
11
11
|
from __future__ import annotations
|
12
|
-
|
12
|
+
|
13
|
+
import os
|
14
|
+
import signal
|
15
|
+
import subprocess
|
16
|
+
import sys
|
17
|
+
import platform
|
18
|
+
|
19
|
+
import meerschaum as mrsm
|
13
20
|
from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
|
14
21
|
from meerschaum.config.static import STATIC_CONFIG
|
15
22
|
|
16
23
|
_child_processes = []
|
17
24
|
def signal_handler(sig, frame):
|
18
25
|
for child in _child_processes:
|
19
|
-
|
20
|
-
|
26
|
+
try:
|
27
|
+
child.send_signal(sig)
|
28
|
+
child.wait()
|
29
|
+
except Exception:
|
30
|
+
pass
|
21
31
|
|
22
32
|
def run_process(
|
23
33
|
*args,
|
@@ -223,3 +233,17 @@ def poll_process(
|
|
223
233
|
watchdog_thread.cancel()
|
224
234
|
|
225
235
|
return proc.poll()
|
236
|
+
|
237
|
+
|
238
|
+
def _stop_process(
|
239
|
+
proc: subprocess.Popen,
|
240
|
+
timeout_seconds: int = 8,
|
241
|
+
):
|
242
|
+
"""
|
243
|
+
Stop a `subproccess.Popen` object.
|
244
|
+
"""
|
245
|
+
proc.terminate()
|
246
|
+
try:
|
247
|
+
proc.wait(timeout=timeout_seconds)
|
248
|
+
except subprocess.TimeoutExpired:
|
249
|
+
proc.kill()
|
meerschaum/utils/schedule.py
CHANGED
@@ -95,7 +95,7 @@ def schedule_function(
|
|
95
95
|
A `SuccessTuple` upon exit.
|
96
96
|
"""
|
97
97
|
import asyncio
|
98
|
-
from meerschaum.utils.misc import filter_keywords
|
98
|
+
from meerschaum.utils.misc import filter_keywords
|
99
99
|
|
100
100
|
global _scheduler
|
101
101
|
kw['debug'] = debug
|
@@ -103,7 +103,7 @@ def schedule_function(
|
|
103
103
|
|
104
104
|
_ = mrsm.attempt_import('attrs', lazy=False)
|
105
105
|
apscheduler = mrsm.attempt_import('apscheduler', lazy=False)
|
106
|
-
now =
|
106
|
+
now = datetime.now(timezone.utc)
|
107
107
|
trigger = parse_schedule(schedule, now=now)
|
108
108
|
_scheduler = apscheduler.AsyncScheduler(identity='mrsm-scheduler')
|
109
109
|
try:
|
@@ -296,7 +296,7 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
|
|
296
296
|
dateutil_parser = mrsm.attempt_import('dateutil.parser')
|
297
297
|
starting_parts = schedule.split(STARTING_KEYWORD)
|
298
298
|
starting_str = ('now' if len(starting_parts) == 1 else starting_parts[-1]).strip()
|
299
|
-
now = now or
|
299
|
+
now = now or datetime.now(timezone.utc)
|
300
300
|
try:
|
301
301
|
if starting_str == 'now':
|
302
302
|
starting_ts = now
|
meerschaum/utils/sql.py
CHANGED
@@ -1548,7 +1548,7 @@ def get_update_queries(
|
|
1548
1548
|
from meerschaum.utils.debug import dprint
|
1549
1549
|
from meerschaum.utils.dtypes import are_dtypes_equal
|
1550
1550
|
from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES, get_pd_type_from_db_type
|
1551
|
-
flavor = flavor or (connectable
|
1551
|
+
flavor = flavor or getattr(connectable, 'flavor', None)
|
1552
1552
|
if not flavor:
|
1553
1553
|
raise ValueError("Provide a flavor if using a SQLAlchemy session.")
|
1554
1554
|
if (
|
@@ -410,6 +410,8 @@ def init_venv(
|
|
410
410
|
pass
|
411
411
|
|
412
412
|
def wait_for_lock():
|
413
|
+
if platform.system() == 'Windows':
|
414
|
+
return
|
413
415
|
max_lock_seconds = 30.0
|
414
416
|
sleep_message_seconds = 5.0
|
415
417
|
step_sleep_seconds = 0.1
|
@@ -595,7 +597,7 @@ def venv_exec(
|
|
595
597
|
as_proc: bool = False,
|
596
598
|
capture_output: bool = True,
|
597
599
|
debug: bool = False,
|
598
|
-
) -> Union[bool, Tuple[int, bytes, bytes]]:
|
600
|
+
) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
|
599
601
|
"""
|
600
602
|
Execute Python code in a subprocess via a virtual environment's interpeter.
|
601
603
|
Return `True` if the code successfully executes, `False` on failure.
|
@@ -630,6 +632,8 @@ def venv_exec(
|
|
630
632
|
import subprocess
|
631
633
|
import platform
|
632
634
|
from meerschaum.utils.debug import dprint
|
635
|
+
from meerschaum.utils.process import _child_processes
|
636
|
+
|
633
637
|
executable = venv_executable(venv=venv)
|
634
638
|
cmd_list = [executable, '-c', code]
|
635
639
|
if env is None:
|
@@ -656,6 +660,7 @@ def venv_exec(
|
|
656
660
|
**group_kwargs
|
657
661
|
)
|
658
662
|
if as_proc:
|
663
|
+
_child_processes.append(process)
|
659
664
|
return process
|
660
665
|
stdout, stderr = process.communicate()
|
661
666
|
exit_code = process.returncode
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: meerschaum
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.9.0.dev1
|
4
4
|
Summary: Sync Time-Series Pipes with Meerschaum
|
5
5
|
Home-page: https://meerschaum.io
|
6
6
|
Author: Bennett Meares
|
@@ -126,6 +126,7 @@ Requires-Dist: importlib-metadata>=4.12.0; extra == "extras"
|
|
126
126
|
Provides-Extra: sql
|
127
127
|
Requires-Dist: numpy>=1.18.5; extra == "sql"
|
128
128
|
Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
|
129
|
+
Requires-Dist: geopandas>=1.0.1; extra == "sql"
|
129
130
|
Requires-Dist: pyarrow>=16.1.0; extra == "sql"
|
130
131
|
Requires-Dist: dask[complete]>=2024.12.1; extra == "sql"
|
131
132
|
Requires-Dist: partd>=1.4.2; extra == "sql"
|
@@ -186,6 +187,7 @@ Requires-Dist: httpcore>=1.0.6; extra == "api"
|
|
186
187
|
Requires-Dist: valkey>=6.0.0; extra == "api"
|
187
188
|
Requires-Dist: numpy>=1.18.5; extra == "api"
|
188
189
|
Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
|
190
|
+
Requires-Dist: geopandas>=1.0.1; extra == "api"
|
189
191
|
Requires-Dist: pyarrow>=16.1.0; extra == "api"
|
190
192
|
Requires-Dist: dask[complete]>=2024.12.1; extra == "api"
|
191
193
|
Requires-Dist: partd>=1.4.2; extra == "api"
|
@@ -291,6 +293,7 @@ Requires-Dist: pywebview>=3.6.3; extra == "full"
|
|
291
293
|
Requires-Dist: pycparser>=2.21.0; extra == "full"
|
292
294
|
Requires-Dist: numpy>=1.18.5; extra == "full"
|
293
295
|
Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
|
296
|
+
Requires-Dist: geopandas>=1.0.1; extra == "full"
|
294
297
|
Requires-Dist: pyarrow>=16.1.0; extra == "full"
|
295
298
|
Requires-Dist: dask[complete]>=2024.12.1; extra == "full"
|
296
299
|
Requires-Dist: partd>=1.4.2; extra == "full"
|