meerschaum 2.2.5.dev3__py3-none-any.whl → 2.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +4 -1
- meerschaum/_internal/arguments/_parser.py +13 -2
- meerschaum/_internal/docs/index.py +523 -26
- meerschaum/_internal/term/__init__.py +2 -2
- meerschaum/actions/bootstrap.py +13 -14
- meerschaum/actions/python.py +11 -8
- meerschaum/actions/register.py +130 -32
- meerschaum/actions/show.py +79 -71
- meerschaum/actions/stop.py +11 -11
- meerschaum/api/dash/callbacks/login.py +21 -13
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/routes/_login.py +5 -5
- meerschaum/config/__init__.py +8 -1
- meerschaum/config/_paths.py +20 -2
- meerschaum/config/_version.py +1 -1
- meerschaum/config/paths.py +21 -2
- meerschaum/config/static/__init__.py +1 -0
- meerschaum/connectors/Connector.py +7 -2
- meerschaum/connectors/__init__.py +7 -5
- meerschaum/connectors/api/APIConnector.py +7 -2
- meerschaum/connectors/api/_actions.py +23 -31
- meerschaum/connectors/api/_uri.py +5 -5
- meerschaum/core/Pipe/__init__.py +7 -3
- meerschaum/core/Pipe/_data.py +23 -15
- meerschaum/core/Pipe/_deduplicate.py +1 -1
- meerschaum/core/Pipe/_dtypes.py +5 -0
- meerschaum/core/Pipe/_fetch.py +18 -16
- meerschaum/core/Pipe/_sync.py +20 -15
- meerschaum/plugins/_Plugin.py +6 -6
- meerschaum/plugins/__init__.py +1 -1
- meerschaum/utils/daemon/RotatingFile.py +15 -16
- meerschaum/utils/dataframe.py +12 -4
- meerschaum/utils/debug.py +9 -15
- meerschaum/utils/formatting/__init__.py +13 -12
- meerschaum/utils/misc.py +117 -11
- meerschaum/utils/packages/__init__.py +7 -1
- meerschaum/utils/typing.py +1 -0
- meerschaum/utils/venv/__init__.py +5 -1
- meerschaum/utils/warnings.py +8 -1
- meerschaum/utils/yaml.py +2 -2
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/METADATA +1 -1
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/RECORD +48 -48
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.6.dist-info}/zip-safe +0 -0
meerschaum/config/__init__.py
CHANGED
@@ -26,7 +26,14 @@ from meerschaum.config._paths import (
|
|
26
26
|
from meerschaum.config._patch import (
|
27
27
|
apply_patch_to_config,
|
28
28
|
)
|
29
|
-
__all__ = (
|
29
|
+
__all__ = (
|
30
|
+
'get_plugin_config',
|
31
|
+
'write_plugin_config',
|
32
|
+
'get_config',
|
33
|
+
'write_config',
|
34
|
+
'set_config',
|
35
|
+
'paths',
|
36
|
+
)
|
30
37
|
__pdoc__ = {'static': False, 'resources': False, 'stack': False, }
|
31
38
|
_locks = {'config': RLock()}
|
32
39
|
|
meerschaum/config/_paths.py
CHANGED
@@ -34,12 +34,29 @@ if ENVIRONMENT_ROOT_DIR in os.environ:
|
|
34
34
|
f"Invalid root directory '{str(_ROOT_DIR_PATH)}' set for " +
|
35
35
|
f"environment variable '{ENVIRONMENT_ROOT_DIR}'.\n" +
|
36
36
|
f"Please enter a valid path for {ENVIRONMENT_ROOT_DIR}.",
|
37
|
-
file
|
37
|
+
file=sys.stderr,
|
38
38
|
)
|
39
39
|
sys.exit(1)
|
40
40
|
else:
|
41
41
|
_ROOT_DIR_PATH = DEFAULT_ROOT_DIR_PATH
|
42
42
|
|
43
|
+
|
44
|
+
ENVIRONMENT_CONFIG_DIR = STATIC_CONFIG['environment']['config_dir']
|
45
|
+
if ENVIRONMENT_CONFIG_DIR in os.environ:
|
46
|
+
_CONFIG_DIR_PATH = Path(os.environ[ENVIRONMENT_CONFIG_DIR]).resolve()
|
47
|
+
if not _CONFIG_DIR_PATH.exists():
|
48
|
+
print(
|
49
|
+
(
|
50
|
+
f"Invalid configuration directory '{_CONFIG_DIR_PATH}' set"
|
51
|
+
+ f" for environment variable '{ENVIRONMENT_CONFIG_DIR}'\n"
|
52
|
+
+ f"Please enter a valid path for {ENVIRONMENT_CONFIG_DIR}."
|
53
|
+
),
|
54
|
+
file=sys.stderr,
|
55
|
+
)
|
56
|
+
sys.exit(1)
|
57
|
+
else:
|
58
|
+
_CONFIG_DIR_PATH = _ROOT_DIR_PATH / 'config'
|
59
|
+
|
43
60
|
ENVIRONMENT_PLUGINS_DIR = STATIC_CONFIG['environment']['plugins']
|
44
61
|
if ENVIRONMENT_PLUGINS_DIR in os.environ:
|
45
62
|
try:
|
@@ -100,7 +117,7 @@ paths = {
|
|
100
117
|
'PACKAGE_ROOT_PATH' : Path(__file__).parent.parent.resolve().as_posix(),
|
101
118
|
'ROOT_DIR_PATH' : _ROOT_DIR_PATH.as_posix(),
|
102
119
|
'VIRTENV_RESOURCES_PATH' : _VENVS_DIR_PATH.as_posix(),
|
103
|
-
'CONFIG_DIR_PATH' : (
|
120
|
+
'CONFIG_DIR_PATH' : _CONFIG_DIR_PATH.as_posix(),
|
104
121
|
'DEFAULT_CONFIG_DIR_PATH' : ('{ROOT_DIR_PATH}', 'default_config'),
|
105
122
|
'PATCH_DIR_PATH' : ('{ROOT_DIR_PATH}', 'patch_config'),
|
106
123
|
'PERMANENT_PATCH_DIR_PATH' : ('{ROOT_DIR_PATH}', 'permanent_patch_config'),
|
@@ -168,6 +185,7 @@ def set_root(root: Union[Path, str]):
|
|
168
185
|
if isinstance(path_parts, tuple) and path_parts[0] == '{ROOT_DIR_PATH}':
|
169
186
|
globals()[path_name] = __getattr__(path_name)
|
170
187
|
|
188
|
+
|
171
189
|
def __getattr__(name: str) -> Path:
|
172
190
|
if name not in paths:
|
173
191
|
if name not in globals():
|
meerschaum/config/_version.py
CHANGED
meerschaum/config/paths.py
CHANGED
@@ -6,5 +6,24 @@
|
|
6
6
|
External API for importing Meerschaum paths.
|
7
7
|
"""
|
8
8
|
|
9
|
-
|
10
|
-
|
9
|
+
import pathlib
|
10
|
+
import inspect
|
11
|
+
import meerschaum.config._paths as _paths
|
12
|
+
|
13
|
+
|
14
|
+
def __getattr__(*args, **kwargs):
|
15
|
+
return _paths.__getattr__(*args, **kwargs)
|
16
|
+
|
17
|
+
|
18
|
+
_globals_dict = inspect.getmembers(
|
19
|
+
_paths,
|
20
|
+
lambda member: not inspect.isroutine(member)
|
21
|
+
)
|
22
|
+
_all_caps_globals = [
|
23
|
+
name
|
24
|
+
for name, value in _globals_dict
|
25
|
+
if ('PATH' in name or 'FILE' in name) and not name.startswith('_')
|
26
|
+
and isinstance(value, pathlib.Path)
|
27
|
+
]
|
28
|
+
|
29
|
+
__all__ = tuple(_all_caps_globals + list(_paths.paths.keys()))
|
@@ -18,7 +18,7 @@ class InvalidAttributesError(Exception):
|
|
18
18
|
|
19
19
|
class Connector(metaclass=abc.ABCMeta):
|
20
20
|
"""
|
21
|
-
The base connector class to hold connection attributes
|
21
|
+
The base connector class to hold connection attributes.
|
22
22
|
"""
|
23
23
|
def __init__(
|
24
24
|
self,
|
@@ -27,6 +27,8 @@ class Connector(metaclass=abc.ABCMeta):
|
|
27
27
|
**kw: Any
|
28
28
|
):
|
29
29
|
"""
|
30
|
+
Set the given keyword arguments as attributes.
|
31
|
+
|
30
32
|
Parameters
|
31
33
|
----------
|
32
34
|
type: str
|
@@ -35,9 +37,12 @@ class Connector(metaclass=abc.ABCMeta):
|
|
35
37
|
label: str
|
36
38
|
The `label` for the connector.
|
37
39
|
|
40
|
+
|
41
|
+
Examples
|
42
|
+
--------
|
38
43
|
Run `mrsm edit config` and to edit connectors in the YAML file:
|
39
44
|
|
40
|
-
```
|
45
|
+
```yaml
|
41
46
|
meerschaum:
|
42
47
|
connections:
|
43
48
|
{type}:
|
@@ -24,6 +24,7 @@ from meerschaum.connectors.api.APIConnector import APIConnector
|
|
24
24
|
from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
|
25
25
|
|
26
26
|
__all__ = (
|
27
|
+
"make_connector",
|
27
28
|
"Connector",
|
28
29
|
"SQLConnector",
|
29
30
|
"APIConnector",
|
@@ -290,13 +291,14 @@ def make_connector(
|
|
290
291
|
--------
|
291
292
|
>>> import meerschaum as mrsm
|
292
293
|
>>> from meerschaum.connectors import make_connector, Connector
|
294
|
+
>>>
|
295
|
+
>>> @make_connector
|
293
296
|
>>> class FooConnector(Connector):
|
294
|
-
...
|
295
|
-
... super().__init__('foo', label, **kw)
|
297
|
+
... REQUIRED_ATTRIBUTES: list[str] = ['username', 'password']
|
296
298
|
...
|
297
|
-
>>>
|
298
|
-
>>>
|
299
|
-
|
299
|
+
>>> conn = mrsm.get_connector('foo:bar', username='dog', password='cat')
|
300
|
+
>>> print(conn.username, conn.password)
|
301
|
+
dog cat
|
300
302
|
>>>
|
301
303
|
"""
|
302
304
|
import re
|
@@ -83,12 +83,16 @@ class APIConnector(Connector):
|
|
83
83
|
if 'uri' in kw:
|
84
84
|
from_uri_params = self.from_uri(kw['uri'], as_dict=True)
|
85
85
|
label = label or from_uri_params.get('label', None)
|
86
|
-
from_uri_params.pop('label', None)
|
86
|
+
_ = from_uri_params.pop('label', None)
|
87
87
|
kw.update(from_uri_params)
|
88
88
|
|
89
89
|
super().__init__('api', label=label, **kw)
|
90
90
|
if 'protocol' not in self.__dict__:
|
91
|
-
self.protocol =
|
91
|
+
self.protocol = (
|
92
|
+
'https' if self.__dict__.get('uri', '').startswith('https')
|
93
|
+
else 'http'
|
94
|
+
)
|
95
|
+
|
92
96
|
if 'uri' not in self.__dict__:
|
93
97
|
self.verify_attributes(required_attributes)
|
94
98
|
else:
|
@@ -97,6 +101,7 @@ class APIConnector(Connector):
|
|
97
101
|
if 'host' not in conn_attrs:
|
98
102
|
raise Exception(f"Invalid URI for '{self}'.")
|
99
103
|
self.__dict__.update(conn_attrs)
|
104
|
+
|
100
105
|
self.url = (
|
101
106
|
self.protocol + '://' +
|
102
107
|
self.host
|
@@ -9,44 +9,31 @@ Functions to interact with /mrsm/actions
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from meerschaum.utils.typing import SuccessTuple, Optional, List
|
11
11
|
|
12
|
-
def get_actions(
|
13
|
-
|
14
|
-
) -> list:
|
15
|
-
"""Get available actions from the API server"""
|
12
|
+
def get_actions(self) -> list:
|
13
|
+
"""Get available actions from the API instance."""
|
16
14
|
from meerschaum.config.static import STATIC_CONFIG
|
17
15
|
return self.get(STATIC_CONFIG['api']['endpoints']['actions'])
|
18
16
|
|
19
17
|
|
20
18
|
def do_action(
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
19
|
+
self,
|
20
|
+
action: Optional[List[str]] = None,
|
21
|
+
sysargs: Optional[List[str]] = None,
|
22
|
+
debug: bool = False,
|
23
|
+
**kw
|
24
|
+
) -> SuccessTuple:
|
27
25
|
"""Execute a Meerschaum action remotely.
|
28
|
-
|
29
|
-
If sysargs is provided, parse those instead. Otherwise infer everything from keyword arguments.
|
30
|
-
|
31
|
-
NOTE: The first index of `action` should NOT be removed!
|
32
|
-
Example: action = ['show', 'config']
|
33
|
-
|
34
|
-
Returns: tuple (succeeded : bool, message : str)
|
35
26
|
|
36
|
-
|
37
|
-
|
38
|
-
action: Optional[List[str]] :
|
39
|
-
(Default value = None)
|
40
|
-
sysargs: Optional[List[str]] :
|
41
|
-
(Default value = None)
|
42
|
-
debug: bool :
|
43
|
-
(Default value = False)
|
44
|
-
**kw :
|
45
|
-
|
46
|
-
|
47
|
-
Returns
|
48
|
-
-------
|
27
|
+
If `sysargs` are provided, parse those instead.
|
28
|
+
Otherwise infer everything from keyword arguments.
|
49
29
|
|
30
|
+
Examples
|
31
|
+
--------
|
32
|
+
>>> conn = mrsm.get_connector('api:main')
|
33
|
+
>>> conn.do_action(['show', 'pipes'])
|
34
|
+
(True, "Success")
|
35
|
+
>>> conn.do_action(['show', 'arguments'], name='test')
|
36
|
+
(True, "Success")
|
50
37
|
"""
|
51
38
|
import sys, json
|
52
39
|
from meerschaum.utils.debug import dprint
|
@@ -63,7 +50,12 @@ def do_action(
|
|
63
50
|
else:
|
64
51
|
json_dict = kw
|
65
52
|
json_dict['action'] = action
|
66
|
-
|
53
|
+
if 'noask' not in kw:
|
54
|
+
json_dict['noask'] = True
|
55
|
+
if 'yes' not in kw:
|
56
|
+
json_dict['yes'] = True
|
57
|
+
if debug:
|
58
|
+
json_dict['debug'] = debug
|
67
59
|
|
68
60
|
root_action = json_dict['action'][0]
|
69
61
|
del json_dict['action'][0]
|
@@ -11,11 +11,11 @@ from meerschaum.utils.warnings import warn, error
|
|
11
11
|
|
12
12
|
@classmethod
|
13
13
|
def from_uri(
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
14
|
+
cls,
|
15
|
+
uri: str,
|
16
|
+
label: Optional[str] = None,
|
17
|
+
as_dict: bool = False,
|
18
|
+
) -> Union[
|
19
19
|
'meerschaum.connectors.APIConnector',
|
20
20
|
Dict[str, Union[str, int]],
|
21
21
|
]:
|
meerschaum/core/Pipe/__init__.py
CHANGED
@@ -50,6 +50,7 @@ with correct credentials, as well as a network connection and valid permissions.
|
|
50
50
|
"""
|
51
51
|
|
52
52
|
from __future__ import annotations
|
53
|
+
import sys
|
53
54
|
import copy
|
54
55
|
from meerschaum.utils.typing import Optional, Dict, Any, Union, InstanceConnector, List
|
55
56
|
from meerschaum.utils.formatting._pipes import pipe_repr
|
@@ -433,13 +434,16 @@ class Pipe:
|
|
433
434
|
+ str(self.instance_keys) + sep
|
434
435
|
)
|
435
436
|
|
436
|
-
def __repr__(self, **kw) -> str:
|
437
|
-
|
437
|
+
def __repr__(self, ansi: bool=True, **kw) -> str:
|
438
|
+
if not hasattr(sys, 'ps1'):
|
439
|
+
ansi = False
|
440
|
+
|
441
|
+
return pipe_repr(self, ansi=ansi, **kw)
|
438
442
|
|
439
443
|
def __pt_repr__(self):
|
440
444
|
from meerschaum.utils.packages import attempt_import
|
441
445
|
prompt_toolkit_formatted_text = attempt_import('prompt_toolkit.formatted_text', lazy=False)
|
442
|
-
return prompt_toolkit_formatted_text.ANSI(self
|
446
|
+
return prompt_toolkit_formatted_text.ANSI(pipe_repr(self, ansi=True))
|
443
447
|
|
444
448
|
def __getstate__(self) -> Dict[str, Any]:
|
445
449
|
"""
|
meerschaum/core/Pipe/_data.py
CHANGED
@@ -8,24 +8,32 @@ Retrieve Pipes' data from instances.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from datetime import datetime, timedelta
|
11
|
-
|
11
|
+
|
12
|
+
import meerschaum as mrsm
|
13
|
+
from meerschaum.utils.typing import (
|
14
|
+
Optional, Dict, Any, Union, List, Tuple, Iterator, TYPE_CHECKING,
|
15
|
+
)
|
12
16
|
from meerschaum.config import get_config
|
13
17
|
|
18
|
+
if TYPE_CHECKING:
|
19
|
+
pd = mrsm.attempt_import('pandas')
|
20
|
+
|
21
|
+
|
14
22
|
def get_data(
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
23
|
+
self,
|
24
|
+
select_columns: Optional[List[str]] = None,
|
25
|
+
omit_columns: Optional[List[str]] = None,
|
26
|
+
begin: Union[datetime, int, None] = None,
|
27
|
+
end: Union[datetime, int, None] = None,
|
28
|
+
params: Optional[Dict[str, Any]] = None,
|
29
|
+
as_iterator: bool = False,
|
30
|
+
as_chunks: bool = False,
|
31
|
+
as_dask: bool = False,
|
32
|
+
chunk_interval: Union[timedelta, int, None] = None,
|
33
|
+
fresh: bool = False,
|
34
|
+
debug: bool = False,
|
35
|
+
**kw: Any
|
36
|
+
) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
|
29
37
|
"""
|
30
38
|
Get a pipe's data from the instance connector.
|
31
39
|
|
@@ -8,7 +8,7 @@ Delete duplicate rows within a pipe's table.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from datetime import datetime, timedelta
|
11
|
-
from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple
|
11
|
+
from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple, Union
|
12
12
|
|
13
13
|
|
14
14
|
def deduplicate(
|
meerschaum/core/Pipe/_dtypes.py
CHANGED
@@ -8,7 +8,12 @@ Enforce data types for a pipe's underlying table.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from io import StringIO
|
11
|
+
import meerschaum as mrsm
|
11
12
|
from meerschaum.utils.typing import Dict, Any, Optional
|
13
|
+
from typing import TYPE_CHECKING
|
14
|
+
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
pd = mrsm.attempt_import('pandas')
|
12
17
|
|
13
18
|
def enforce_dtypes(
|
14
19
|
self,
|
meerschaum/core/Pipe/_fetch.py
CHANGED
@@ -8,11 +8,14 @@ Functions for fetching new data into the Pipe
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from datetime import timedelta, datetime
|
11
|
+
|
11
12
|
import meerschaum as mrsm
|
12
|
-
from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
|
13
|
+
from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator, TYPE_CHECKING
|
13
14
|
from meerschaum.config import get_config
|
14
15
|
from meerschaum.utils.warnings import warn
|
15
|
-
|
16
|
+
|
17
|
+
if TYPE_CHECKING:
|
18
|
+
pd = mrsm.attempt_import('pandas')
|
16
19
|
|
17
20
|
def fetch(
|
18
21
|
self,
|
@@ -55,6 +58,7 @@ def fetch(
|
|
55
58
|
|
56
59
|
from meerschaum.connectors import custom_types, get_connector_plugin
|
57
60
|
from meerschaum.utils.debug import dprint, _checkpoint
|
61
|
+
from meerschaum.utils.misc import filter_arguments
|
58
62
|
|
59
63
|
_chunk_hook = kw.pop('chunk_hook', None)
|
60
64
|
kw['workers'] = self.get_num_workers(kw.get('workers', None))
|
@@ -72,24 +76,22 @@ def fetch(
|
|
72
76
|
chunk_message = '\n' + chunk_label + '\n' + chunk_message
|
73
77
|
return chunk_success, chunk_message
|
74
78
|
|
75
|
-
|
76
79
|
with mrsm.Venv(get_connector_plugin(self.connector)):
|
77
|
-
|
80
|
+
_args, _kwargs = filter_arguments(
|
81
|
+
self.connector.fetch,
|
78
82
|
self,
|
79
|
-
|
80
|
-
self
|
81
|
-
begin
|
82
|
-
|
83
|
-
begin,
|
84
|
-
check_existing=check_existing,
|
85
|
-
debug=debug,
|
86
|
-
),
|
87
|
-
end=end,
|
88
|
-
chunk_hook=_chunk_hook,
|
83
|
+
begin=_determine_begin(
|
84
|
+
self,
|
85
|
+
begin,
|
86
|
+
check_existing=check_existing,
|
89
87
|
debug=debug,
|
90
|
-
|
91
|
-
|
88
|
+
),
|
89
|
+
end=end,
|
90
|
+
chunk_hook=_chunk_hook,
|
91
|
+
debug=debug,
|
92
|
+
**kw
|
92
93
|
)
|
94
|
+
df = self.connector.fetch(*_args, **_kwargs)
|
93
95
|
return df
|
94
96
|
|
95
97
|
|
meerschaum/core/Pipe/_sync.py
CHANGED
@@ -14,7 +14,9 @@ import threading
|
|
14
14
|
import multiprocessing
|
15
15
|
import functools
|
16
16
|
from datetime import datetime, timedelta
|
17
|
+
from typing import TYPE_CHECKING
|
17
18
|
|
19
|
+
import meerschaum as mrsm
|
18
20
|
from meerschaum.utils.typing import (
|
19
21
|
Union,
|
20
22
|
Optional,
|
@@ -26,13 +28,16 @@ from meerschaum.utils.typing import (
|
|
26
28
|
List,
|
27
29
|
Iterable,
|
28
30
|
Generator,
|
29
|
-
Iterator,
|
30
31
|
)
|
31
32
|
from meerschaum.utils.warnings import warn, error
|
32
33
|
|
34
|
+
if TYPE_CHECKING:
|
35
|
+
pd = mrsm.attempt_import('pandas')
|
36
|
+
|
33
37
|
class InferFetch:
|
34
38
|
MRSM_INFER_FETCH: bool = True
|
35
39
|
|
40
|
+
|
36
41
|
def sync(
|
37
42
|
self,
|
38
43
|
df: Union[
|
@@ -125,7 +130,7 @@ def sync(
|
|
125
130
|
from meerschaum.utils.formatting import get_console
|
126
131
|
from meerschaum.utils.venv import Venv
|
127
132
|
from meerschaum.connectors import get_connector_plugin
|
128
|
-
from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
|
133
|
+
from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords, filter_arguments
|
129
134
|
from meerschaum.utils.pool import get_pool
|
130
135
|
from meerschaum.config import get_config
|
131
136
|
|
@@ -210,28 +215,28 @@ def sync(
|
|
210
215
|
):
|
211
216
|
with Venv(get_connector_plugin(self.instance_connector)):
|
212
217
|
p._exists = None
|
213
|
-
|
218
|
+
_args, _kwargs = filter_arguments(
|
219
|
+
p.instance_connector.sync_pipe_inplace,
|
214
220
|
p,
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
221
|
+
debug=debug,
|
222
|
+
**kw
|
223
|
+
)
|
224
|
+
return self.instance_connector.sync_pipe_inplace(
|
225
|
+
*_args,
|
226
|
+
**_kwargs
|
220
227
|
)
|
221
|
-
|
222
228
|
|
223
229
|
### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
|
224
230
|
try:
|
225
231
|
if getattr(p.connector, 'sync', None) is not None:
|
226
232
|
with Venv(get_connector_plugin(p.connector), debug=debug):
|
227
|
-
|
233
|
+
_args, _kwargs = filter_arguments(
|
234
|
+
p.connector.sync,
|
228
235
|
p,
|
229
|
-
|
230
|
-
|
231
|
-
debug=debug,
|
232
|
-
**kw
|
233
|
-
)
|
236
|
+
debug=debug,
|
237
|
+
**kw
|
234
238
|
)
|
239
|
+
return_tuple = p.connector.sync(*_args, **_kwargs)
|
235
240
|
p._exists = None
|
236
241
|
if not isinstance(return_tuple, tuple):
|
237
242
|
return_tuple = (
|
meerschaum/plugins/_Plugin.py
CHANGED
@@ -654,17 +654,18 @@ class Plugin:
|
|
654
654
|
import ast, re
|
655
655
|
### NOTE: This technically would break
|
656
656
|
### if `required` was the very first line of the file.
|
657
|
-
req_start_match = re.search(r'
|
657
|
+
req_start_match = re.search(r'required(:\s*)?.*=', text)
|
658
658
|
if not req_start_match:
|
659
659
|
return []
|
660
660
|
req_start = req_start_match.start()
|
661
|
+
equals_sign = req_start + text[req_start:].find('=')
|
661
662
|
|
662
663
|
### Dependencies may have brackets within the strings, so push back the index.
|
663
|
-
first_opening_brace =
|
664
|
+
first_opening_brace = equals_sign + 1 + text[equals_sign:].find('[')
|
664
665
|
if first_opening_brace == -1:
|
665
666
|
return []
|
666
667
|
|
667
|
-
next_closing_brace =
|
668
|
+
next_closing_brace = equals_sign + 1 + text[equals_sign:].find(']')
|
668
669
|
if next_closing_brace == -1:
|
669
670
|
return []
|
670
671
|
|
@@ -681,12 +682,11 @@ class Plugin:
|
|
681
682
|
|
682
683
|
req_end = end_ix + 1
|
683
684
|
req_text = (
|
684
|
-
text[
|
685
|
-
.lstrip()
|
686
|
-
.replace('required', '', 1)
|
685
|
+
text[(first_opening_brace-1):req_end]
|
687
686
|
.lstrip()
|
688
687
|
.replace('=', '', 1)
|
689
688
|
.lstrip()
|
689
|
+
.rstrip()
|
690
690
|
)
|
691
691
|
try:
|
692
692
|
required = ast.literal_eval(req_text)
|
meerschaum/plugins/__init__.py
CHANGED
@@ -27,7 +27,7 @@ _locks = {
|
|
27
27
|
'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
|
28
28
|
}
|
29
29
|
__all__ = (
|
30
|
-
"Plugin", "make_action", "api_plugin", "import_plugins",
|
30
|
+
"Plugin", "make_action", "api_plugin", "dash_plugin", "import_plugins",
|
31
31
|
"reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
|
32
32
|
"pre_sync_hook", "post_sync_hook",
|
33
33
|
)
|
@@ -32,14 +32,14 @@ class RotatingFile(io.IOBase):
|
|
32
32
|
SEEK_BACK_ATTEMPTS: int = 5
|
33
33
|
|
34
34
|
def __init__(
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
35
|
+
self,
|
36
|
+
file_path: pathlib.Path,
|
37
|
+
num_files_to_keep: Optional[int] = None,
|
38
|
+
max_file_size: Optional[int] = None,
|
39
|
+
redirect_streams: bool = False,
|
40
|
+
write_timestamps: bool = False,
|
41
|
+
timestamp_format: str = '%Y-%m-%d %H:%M',
|
42
|
+
):
|
43
43
|
"""
|
44
44
|
Create a file-like object which manages other files.
|
45
45
|
|
@@ -79,11 +79,7 @@ class RotatingFile(io.IOBase):
|
|
79
79
|
self.redirect_streams = redirect_streams
|
80
80
|
self.write_timestamps = write_timestamps
|
81
81
|
self.timestamp_format = timestamp_format
|
82
|
-
self.subfile_regex_pattern = re.compile(
|
83
|
-
r'^'
|
84
|
-
+ self.file_path.name
|
85
|
-
+ r'(?:\.\d+)?$'
|
86
|
-
)
|
82
|
+
self.subfile_regex_pattern = re.compile(r'(.*)\.log(?:\.\d+)?$')
|
87
83
|
|
88
84
|
### When subfiles are opened, map from their index to the file objects.
|
89
85
|
self.subfile_objects = {}
|
@@ -173,7 +169,7 @@ class RotatingFile(io.IOBase):
|
|
173
169
|
latest_index = (
|
174
170
|
self.get_index_from_subfile_name(existing_subfile_paths[-1].name)
|
175
171
|
if existing_subfile_paths
|
176
|
-
else
|
172
|
+
else 0
|
177
173
|
)
|
178
174
|
return latest_index
|
179
175
|
|
@@ -222,9 +218,12 @@ class RotatingFile(io.IOBase):
|
|
222
218
|
[
|
223
219
|
(file_name, self.get_index_from_subfile_name(file_name))
|
224
220
|
for file_name in os.listdir(self.file_path.parent)
|
225
|
-
if
|
221
|
+
if (
|
222
|
+
file_name.startswith(self.file_path.name)
|
223
|
+
and re.match(self.subfile_regex_pattern, file_name)
|
224
|
+
)
|
226
225
|
],
|
227
|
-
key
|
226
|
+
key=lambda x: x[1],
|
228
227
|
)
|
229
228
|
return [
|
230
229
|
(self.file_path.parent / file_name)
|