meerschaum 2.2.5.dev3__py3-none-any.whl → 2.2.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +4 -1
- meerschaum/__main__.py +10 -5
- meerschaum/_internal/arguments/_parser.py +13 -2
- meerschaum/_internal/docs/index.py +523 -26
- meerschaum/_internal/entry.py +13 -13
- meerschaum/_internal/shell/Shell.py +26 -22
- meerschaum/_internal/shell/updates.py +175 -0
- meerschaum/_internal/term/__init__.py +2 -2
- meerschaum/actions/bootstrap.py +13 -14
- meerschaum/actions/python.py +11 -8
- meerschaum/actions/register.py +149 -37
- meerschaum/actions/show.py +79 -71
- meerschaum/actions/stop.py +11 -11
- meerschaum/actions/sync.py +3 -3
- meerschaum/actions/upgrade.py +28 -36
- meerschaum/api/dash/callbacks/login.py +21 -13
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/routes/_login.py +5 -5
- meerschaum/api/routes/_pipes.py +20 -20
- meerschaum/config/__init__.py +8 -1
- meerschaum/config/_formatting.py +1 -0
- meerschaum/config/_paths.py +24 -2
- meerschaum/config/_shell.py +78 -66
- meerschaum/config/_version.py +1 -1
- meerschaum/config/paths.py +21 -2
- meerschaum/config/static/__init__.py +2 -0
- meerschaum/connectors/Connector.py +7 -2
- meerschaum/connectors/__init__.py +7 -5
- meerschaum/connectors/api/APIConnector.py +7 -2
- meerschaum/connectors/api/_actions.py +23 -31
- meerschaum/connectors/api/_misc.py +1 -1
- meerschaum/connectors/api/_request.py +13 -9
- meerschaum/connectors/api/_uri.py +5 -5
- meerschaum/core/Pipe/__init__.py +7 -3
- meerschaum/core/Pipe/_data.py +23 -15
- meerschaum/core/Pipe/_deduplicate.py +1 -1
- meerschaum/core/Pipe/_dtypes.py +5 -0
- meerschaum/core/Pipe/_fetch.py +18 -16
- meerschaum/core/Pipe/_sync.py +23 -15
- meerschaum/plugins/_Plugin.py +6 -6
- meerschaum/plugins/__init__.py +1 -1
- meerschaum/utils/daemon/Daemon.py +88 -129
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +14 -5
- meerschaum/utils/daemon/RotatingFile.py +23 -17
- meerschaum/utils/daemon/__init__.py +28 -21
- meerschaum/utils/dataframe.py +12 -4
- meerschaum/utils/debug.py +9 -15
- meerschaum/utils/formatting/__init__.py +92 -46
- meerschaum/utils/formatting/_jobs.py +47 -9
- meerschaum/utils/misc.py +117 -11
- meerschaum/utils/packages/__init__.py +28 -16
- meerschaum/utils/prompt.py +5 -0
- meerschaum/utils/schedule.py +21 -15
- meerschaum/utils/typing.py +1 -0
- meerschaum/utils/venv/__init__.py +5 -1
- meerschaum/utils/warnings.py +8 -1
- meerschaum/utils/yaml.py +2 -2
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/METADATA +1 -1
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/RECORD +65 -64
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/zip-safe +0 -0
meerschaum/core/Pipe/_fetch.py
CHANGED
@@ -8,11 +8,14 @@ Functions for fetching new data into the Pipe
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from datetime import timedelta, datetime
|
11
|
+
|
11
12
|
import meerschaum as mrsm
|
12
|
-
from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
|
13
|
+
from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator, TYPE_CHECKING
|
13
14
|
from meerschaum.config import get_config
|
14
15
|
from meerschaum.utils.warnings import warn
|
15
|
-
|
16
|
+
|
17
|
+
if TYPE_CHECKING:
|
18
|
+
pd = mrsm.attempt_import('pandas')
|
16
19
|
|
17
20
|
def fetch(
|
18
21
|
self,
|
@@ -55,6 +58,7 @@ def fetch(
|
|
55
58
|
|
56
59
|
from meerschaum.connectors import custom_types, get_connector_plugin
|
57
60
|
from meerschaum.utils.debug import dprint, _checkpoint
|
61
|
+
from meerschaum.utils.misc import filter_arguments
|
58
62
|
|
59
63
|
_chunk_hook = kw.pop('chunk_hook', None)
|
60
64
|
kw['workers'] = self.get_num_workers(kw.get('workers', None))
|
@@ -72,24 +76,22 @@ def fetch(
|
|
72
76
|
chunk_message = '\n' + chunk_label + '\n' + chunk_message
|
73
77
|
return chunk_success, chunk_message
|
74
78
|
|
75
|
-
|
76
79
|
with mrsm.Venv(get_connector_plugin(self.connector)):
|
77
|
-
|
80
|
+
_args, _kwargs = filter_arguments(
|
81
|
+
self.connector.fetch,
|
78
82
|
self,
|
79
|
-
|
80
|
-
self
|
81
|
-
begin
|
82
|
-
|
83
|
-
begin,
|
84
|
-
check_existing=check_existing,
|
85
|
-
debug=debug,
|
86
|
-
),
|
87
|
-
end=end,
|
88
|
-
chunk_hook=_chunk_hook,
|
83
|
+
begin=_determine_begin(
|
84
|
+
self,
|
85
|
+
begin,
|
86
|
+
check_existing=check_existing,
|
89
87
|
debug=debug,
|
90
|
-
|
91
|
-
|
88
|
+
),
|
89
|
+
end=end,
|
90
|
+
chunk_hook=_chunk_hook,
|
91
|
+
debug=debug,
|
92
|
+
**kw
|
92
93
|
)
|
94
|
+
df = self.connector.fetch(*_args, **_kwargs)
|
93
95
|
return df
|
94
96
|
|
95
97
|
|
meerschaum/core/Pipe/_sync.py
CHANGED
@@ -14,7 +14,9 @@ import threading
|
|
14
14
|
import multiprocessing
|
15
15
|
import functools
|
16
16
|
from datetime import datetime, timedelta
|
17
|
+
from typing import TYPE_CHECKING
|
17
18
|
|
19
|
+
import meerschaum as mrsm
|
18
20
|
from meerschaum.utils.typing import (
|
19
21
|
Union,
|
20
22
|
Optional,
|
@@ -26,13 +28,16 @@ from meerschaum.utils.typing import (
|
|
26
28
|
List,
|
27
29
|
Iterable,
|
28
30
|
Generator,
|
29
|
-
Iterator,
|
30
31
|
)
|
31
32
|
from meerschaum.utils.warnings import warn, error
|
32
33
|
|
34
|
+
if TYPE_CHECKING:
|
35
|
+
pd = mrsm.attempt_import('pandas')
|
36
|
+
|
33
37
|
class InferFetch:
|
34
38
|
MRSM_INFER_FETCH: bool = True
|
35
39
|
|
40
|
+
|
36
41
|
def sync(
|
37
42
|
self,
|
38
43
|
df: Union[
|
@@ -125,7 +130,7 @@ def sync(
|
|
125
130
|
from meerschaum.utils.formatting import get_console
|
126
131
|
from meerschaum.utils.venv import Venv
|
127
132
|
from meerschaum.connectors import get_connector_plugin
|
128
|
-
from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
|
133
|
+
from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords, filter_arguments
|
129
134
|
from meerschaum.utils.pool import get_pool
|
130
135
|
from meerschaum.config import get_config
|
131
136
|
|
@@ -189,6 +194,9 @@ def sync(
|
|
189
194
|
if hasattr(df, 'MRSM_INFER_FETCH'):
|
190
195
|
try:
|
191
196
|
if p.connector is None:
|
197
|
+
if ':' not in p.connector_keys:
|
198
|
+
return True, f"{p} does not support fetching; nothing to do."
|
199
|
+
|
192
200
|
msg = f"{p} does not have a valid connector."
|
193
201
|
if p.connector_keys.startswith('plugin:'):
|
194
202
|
msg += f"\n Perhaps {p.connector_keys} has a syntax error?"
|
@@ -210,28 +218,28 @@ def sync(
|
|
210
218
|
):
|
211
219
|
with Venv(get_connector_plugin(self.instance_connector)):
|
212
220
|
p._exists = None
|
213
|
-
|
221
|
+
_args, _kwargs = filter_arguments(
|
222
|
+
p.instance_connector.sync_pipe_inplace,
|
214
223
|
p,
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
224
|
+
debug=debug,
|
225
|
+
**kw
|
226
|
+
)
|
227
|
+
return self.instance_connector.sync_pipe_inplace(
|
228
|
+
*_args,
|
229
|
+
**_kwargs
|
220
230
|
)
|
221
|
-
|
222
231
|
|
223
232
|
### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
|
224
233
|
try:
|
225
234
|
if getattr(p.connector, 'sync', None) is not None:
|
226
235
|
with Venv(get_connector_plugin(p.connector), debug=debug):
|
227
|
-
|
236
|
+
_args, _kwargs = filter_arguments(
|
237
|
+
p.connector.sync,
|
228
238
|
p,
|
229
|
-
|
230
|
-
|
231
|
-
debug=debug,
|
232
|
-
**kw
|
233
|
-
)
|
239
|
+
debug=debug,
|
240
|
+
**kw
|
234
241
|
)
|
242
|
+
return_tuple = p.connector.sync(*_args, **_kwargs)
|
235
243
|
p._exists = None
|
236
244
|
if not isinstance(return_tuple, tuple):
|
237
245
|
return_tuple = (
|
meerschaum/plugins/_Plugin.py
CHANGED
@@ -654,17 +654,18 @@ class Plugin:
|
|
654
654
|
import ast, re
|
655
655
|
### NOTE: This technically would break
|
656
656
|
### if `required` was the very first line of the file.
|
657
|
-
req_start_match = re.search(r'
|
657
|
+
req_start_match = re.search(r'required(:\s*)?.*=', text)
|
658
658
|
if not req_start_match:
|
659
659
|
return []
|
660
660
|
req_start = req_start_match.start()
|
661
|
+
equals_sign = req_start + text[req_start:].find('=')
|
661
662
|
|
662
663
|
### Dependencies may have brackets within the strings, so push back the index.
|
663
|
-
first_opening_brace =
|
664
|
+
first_opening_brace = equals_sign + 1 + text[equals_sign:].find('[')
|
664
665
|
if first_opening_brace == -1:
|
665
666
|
return []
|
666
667
|
|
667
|
-
next_closing_brace =
|
668
|
+
next_closing_brace = equals_sign + 1 + text[equals_sign:].find(']')
|
668
669
|
if next_closing_brace == -1:
|
669
670
|
return []
|
670
671
|
|
@@ -681,12 +682,11 @@ class Plugin:
|
|
681
682
|
|
682
683
|
req_end = end_ix + 1
|
683
684
|
req_text = (
|
684
|
-
text[
|
685
|
-
.lstrip()
|
686
|
-
.replace('required', '', 1)
|
685
|
+
text[(first_opening_brace-1):req_end]
|
687
686
|
.lstrip()
|
688
687
|
.replace('=', '', 1)
|
689
688
|
.lstrip()
|
689
|
+
.rstrip()
|
690
690
|
)
|
691
691
|
try:
|
692
692
|
required = ast.literal_eval(req_text)
|
meerschaum/plugins/__init__.py
CHANGED
@@ -27,7 +27,7 @@ _locks = {
|
|
27
27
|
'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
|
28
28
|
}
|
29
29
|
__all__ = (
|
30
|
-
"Plugin", "make_action", "api_plugin", "import_plugins",
|
30
|
+
"Plugin", "make_action", "api_plugin", "dash_plugin", "import_plugins",
|
31
31
|
"reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
|
32
32
|
"pre_sync_hook", "post_sync_hook",
|
33
33
|
)
|