meerschaum 2.2.5.dev2__py3-none-any.whl → 2.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +4 -1
- meerschaum/_internal/arguments/_parser.py +13 -3
- meerschaum/_internal/docs/index.py +513 -110
- meerschaum/_internal/term/__init__.py +2 -2
- meerschaum/actions/bootstrap.py +13 -14
- meerschaum/actions/python.py +11 -8
- meerschaum/actions/register.py +130 -32
- meerschaum/actions/show.py +92 -75
- meerschaum/actions/stack.py +12 -12
- meerschaum/actions/stop.py +11 -11
- meerschaum/api/__init__.py +0 -1
- meerschaum/api/dash/__init__.py +0 -1
- meerschaum/api/dash/callbacks/custom.py +1 -1
- meerschaum/api/dash/callbacks/login.py +21 -13
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/dash/plugins.py +5 -6
- meerschaum/api/routes/_login.py +5 -5
- meerschaum/config/__init__.py +8 -1
- meerschaum/config/_paths.py +20 -2
- meerschaum/config/_version.py +1 -1
- meerschaum/config/paths.py +21 -2
- meerschaum/config/static/__init__.py +1 -0
- meerschaum/connectors/Connector.py +7 -2
- meerschaum/connectors/__init__.py +7 -5
- meerschaum/connectors/api/APIConnector.py +7 -2
- meerschaum/connectors/api/_actions.py +23 -31
- meerschaum/connectors/api/_uri.py +5 -5
- meerschaum/core/Pipe/__init__.py +7 -3
- meerschaum/core/Pipe/_data.py +23 -15
- meerschaum/core/Pipe/_deduplicate.py +1 -1
- meerschaum/core/Pipe/_dtypes.py +5 -0
- meerschaum/core/Pipe/_fetch.py +18 -16
- meerschaum/core/Pipe/_sync.py +20 -15
- meerschaum/plugins/_Plugin.py +6 -6
- meerschaum/plugins/__init__.py +1 -1
- meerschaum/utils/daemon/RotatingFile.py +15 -16
- meerschaum/utils/dataframe.py +12 -4
- meerschaum/utils/debug.py +9 -15
- meerschaum/utils/formatting/__init__.py +13 -12
- meerschaum/utils/misc.py +117 -11
- meerschaum/utils/packages/__init__.py +7 -1
- meerschaum/utils/typing.py +1 -0
- meerschaum/utils/venv/__init__.py +5 -1
- meerschaum/utils/warnings.py +9 -1
- meerschaum/utils/yaml.py +2 -2
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/METADATA +1 -1
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/RECORD +53 -54
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/WHEEL +1 -1
- meerschaum/actions/backup.py +0 -43
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.5.dev2.dist-info → meerschaum-2.2.6.dist-info}/zip-safe +0 -0
meerschaum/api/__init__.py
CHANGED
meerschaum/api/dash/__init__.py
CHANGED
@@ -17,6 +17,7 @@ from dash.dependencies import Input, Output, State
|
|
17
17
|
from meerschaum.api.dash import dash_app, debug, pipes, _get_pipes, active_sessions
|
18
18
|
from meerschaum.api.dash.connectors import get_web_connector
|
19
19
|
from meerschaum.api.routes._login import login
|
20
|
+
from meerschaum.api.dash.components import alert_from_success_tuple
|
20
21
|
from fastapi_login.exceptions import InvalidCredentialsException
|
21
22
|
from fastapi.exceptions import HTTPException
|
22
23
|
dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
|
@@ -39,6 +40,7 @@ def show_registration_disabled_collapse(n_clicks, is_open):
|
|
39
40
|
Output('session-store', 'data'),
|
40
41
|
Output('username-input', 'className'),
|
41
42
|
Output('location', 'pathname'),
|
43
|
+
Output('login-alert-div', 'children'),
|
42
44
|
Input('username-input', 'n_submit'),
|
43
45
|
Input('password-input', 'n_submit'),
|
44
46
|
Input('login-button', 'n_clicks'),
|
@@ -48,14 +50,14 @@ def show_registration_disabled_collapse(n_clicks, is_open):
|
|
48
50
|
State('location', 'pathname'),
|
49
51
|
)
|
50
52
|
def login_button_click(
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
53
|
+
username_submit,
|
54
|
+
password_submit,
|
55
|
+
n_clicks,
|
56
|
+
username,
|
57
|
+
password,
|
58
|
+
location_href,
|
59
|
+
location_pathname,
|
60
|
+
):
|
59
61
|
"""
|
60
62
|
When the user submits the login form, check the login.
|
61
63
|
On successful login, set the session id.
|
@@ -63,12 +65,18 @@ def login_button_click(
|
|
63
65
|
form_class = 'form-control'
|
64
66
|
ctx = dash.callback_context
|
65
67
|
if not username or not password or not ctx.triggered:
|
66
|
-
|
68
|
+
raise PreventUpdate
|
69
|
+
|
67
70
|
try:
|
68
|
-
|
69
|
-
session_data = {
|
71
|
+
_ = login({'username': username, 'password': password})
|
72
|
+
session_data = {
|
73
|
+
'session-id': str(uuid.uuid4()),
|
74
|
+
'location.href': location_href,
|
75
|
+
}
|
70
76
|
active_sessions[session_data['session-id']] = {'username': username}
|
77
|
+
alerts = []
|
71
78
|
except HTTPException:
|
72
79
|
form_class += ' is-invalid'
|
73
|
-
session_data =
|
74
|
-
|
80
|
+
session_data = dash.no_update
|
81
|
+
alerts = alert_from_success_tuple((False, "Invalid login."))
|
82
|
+
return session_data, form_class, dash.no_update, alerts
|
@@ -31,7 +31,7 @@ registration_div = html.Div(
|
|
31
31
|
else [
|
32
32
|
dcc.Markdown("""
|
33
33
|
#### **Web registration is disabled for security.**
|
34
|
-
You can
|
34
|
+
You can register users via the CLI with `mrsm register user`.
|
35
35
|
"""),
|
36
36
|
dbc.Button(
|
37
37
|
'More information.',
|
@@ -80,6 +80,7 @@ layout = dbc.Container([
|
|
80
80
|
html.Br(),
|
81
81
|
dbc.Container([
|
82
82
|
dcc.Location(id='location-login', refresh=True),
|
83
|
+
html.Div(id="login-alert-div"),
|
83
84
|
html.Div([
|
84
85
|
dbc.Container(
|
85
86
|
html.Img(
|
@@ -129,4 +130,3 @@ layout = dbc.Container([
|
|
129
130
|
]),
|
130
131
|
], className='jumbotron')
|
131
132
|
])
|
132
|
-
|
meerschaum/api/dash/plugins.py
CHANGED
@@ -7,7 +7,7 @@ Functions for interacting with plugins via the web interface.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
from meerschaum.utils.typing import List, Tuple, SuccessTuple
|
10
|
+
from meerschaum.utils.typing import List, Tuple, SuccessTuple, Optional, WebState, Dict, Any
|
11
11
|
from meerschaum.utils.packages import import_dcc, import_html
|
12
12
|
from meerschaum.api import get_api_connector, endpoints, CHECK_UPDATE
|
13
13
|
html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
|
@@ -17,10 +17,10 @@ from meerschaum.api.dash import dash_app, debug, active_sessions
|
|
17
17
|
|
18
18
|
|
19
19
|
def get_plugins_cards(
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
20
|
+
state: Optional[WebState] = None,
|
21
|
+
search_term: Optional[str] = None,
|
22
|
+
session_data: Optional[Dict[str, Any]] = None,
|
23
|
+
) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
|
24
24
|
"""
|
25
25
|
Return the cards and alerts for plugins.
|
26
26
|
"""
|
@@ -96,4 +96,3 @@ def is_plugin_owner(plugin_name: str, session_data: Dict['str', Any]) -> bool:
|
|
96
96
|
_username is not None
|
97
97
|
and _username == _plugin_username
|
98
98
|
)
|
99
|
-
|
meerschaum/api/routes/_login.py
CHANGED
@@ -32,16 +32,16 @@ def load_user(
|
|
32
32
|
|
33
33
|
|
34
34
|
@app.post(endpoints['login'], tags=['Users'])
|
35
|
-
|
35
|
+
def login(
|
36
36
|
data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
|
37
|
-
# data: dict[str, str],
|
38
|
-
# request: Request
|
39
37
|
) -> Dict[str, Any]:
|
40
38
|
"""
|
41
39
|
Login and set the session token.
|
42
40
|
"""
|
43
41
|
username, password = (
|
44
|
-
(data
|
42
|
+
(data['username'], data['password'])
|
43
|
+
if isinstance(data, dict)
|
44
|
+
else (data.username, data.password)
|
45
45
|
) if not no_auth else ('no-auth', 'no-auth')
|
46
46
|
|
47
47
|
user = User(username, password)
|
@@ -62,7 +62,7 @@ async def login(
|
|
62
62
|
return {
|
63
63
|
'access_token': access_token,
|
64
64
|
'token_type': 'bearer',
|
65
|
-
'expires'
|
65
|
+
'expires': expires_dt,
|
66
66
|
}
|
67
67
|
|
68
68
|
|
meerschaum/config/__init__.py
CHANGED
@@ -26,7 +26,14 @@ from meerschaum.config._paths import (
|
|
26
26
|
from meerschaum.config._patch import (
|
27
27
|
apply_patch_to_config,
|
28
28
|
)
|
29
|
-
__all__ = (
|
29
|
+
__all__ = (
|
30
|
+
'get_plugin_config',
|
31
|
+
'write_plugin_config',
|
32
|
+
'get_config',
|
33
|
+
'write_config',
|
34
|
+
'set_config',
|
35
|
+
'paths',
|
36
|
+
)
|
30
37
|
__pdoc__ = {'static': False, 'resources': False, 'stack': False, }
|
31
38
|
_locks = {'config': RLock()}
|
32
39
|
|
meerschaum/config/_paths.py
CHANGED
@@ -34,12 +34,29 @@ if ENVIRONMENT_ROOT_DIR in os.environ:
|
|
34
34
|
f"Invalid root directory '{str(_ROOT_DIR_PATH)}' set for " +
|
35
35
|
f"environment variable '{ENVIRONMENT_ROOT_DIR}'.\n" +
|
36
36
|
f"Please enter a valid path for {ENVIRONMENT_ROOT_DIR}.",
|
37
|
-
file
|
37
|
+
file=sys.stderr,
|
38
38
|
)
|
39
39
|
sys.exit(1)
|
40
40
|
else:
|
41
41
|
_ROOT_DIR_PATH = DEFAULT_ROOT_DIR_PATH
|
42
42
|
|
43
|
+
|
44
|
+
ENVIRONMENT_CONFIG_DIR = STATIC_CONFIG['environment']['config_dir']
|
45
|
+
if ENVIRONMENT_CONFIG_DIR in os.environ:
|
46
|
+
_CONFIG_DIR_PATH = Path(os.environ[ENVIRONMENT_CONFIG_DIR]).resolve()
|
47
|
+
if not _CONFIG_DIR_PATH.exists():
|
48
|
+
print(
|
49
|
+
(
|
50
|
+
f"Invalid configuration directory '{_CONFIG_DIR_PATH}' set"
|
51
|
+
+ f" for environment variable '{ENVIRONMENT_CONFIG_DIR}'\n"
|
52
|
+
+ f"Please enter a valid path for {ENVIRONMENT_CONFIG_DIR}."
|
53
|
+
),
|
54
|
+
file=sys.stderr,
|
55
|
+
)
|
56
|
+
sys.exit(1)
|
57
|
+
else:
|
58
|
+
_CONFIG_DIR_PATH = _ROOT_DIR_PATH / 'config'
|
59
|
+
|
43
60
|
ENVIRONMENT_PLUGINS_DIR = STATIC_CONFIG['environment']['plugins']
|
44
61
|
if ENVIRONMENT_PLUGINS_DIR in os.environ:
|
45
62
|
try:
|
@@ -100,7 +117,7 @@ paths = {
|
|
100
117
|
'PACKAGE_ROOT_PATH' : Path(__file__).parent.parent.resolve().as_posix(),
|
101
118
|
'ROOT_DIR_PATH' : _ROOT_DIR_PATH.as_posix(),
|
102
119
|
'VIRTENV_RESOURCES_PATH' : _VENVS_DIR_PATH.as_posix(),
|
103
|
-
'CONFIG_DIR_PATH' : (
|
120
|
+
'CONFIG_DIR_PATH' : _CONFIG_DIR_PATH.as_posix(),
|
104
121
|
'DEFAULT_CONFIG_DIR_PATH' : ('{ROOT_DIR_PATH}', 'default_config'),
|
105
122
|
'PATCH_DIR_PATH' : ('{ROOT_DIR_PATH}', 'patch_config'),
|
106
123
|
'PERMANENT_PATCH_DIR_PATH' : ('{ROOT_DIR_PATH}', 'permanent_patch_config'),
|
@@ -168,6 +185,7 @@ def set_root(root: Union[Path, str]):
|
|
168
185
|
if isinstance(path_parts, tuple) and path_parts[0] == '{ROOT_DIR_PATH}':
|
169
186
|
globals()[path_name] = __getattr__(path_name)
|
170
187
|
|
188
|
+
|
171
189
|
def __getattr__(name: str) -> Path:
|
172
190
|
if name not in paths:
|
173
191
|
if name not in globals():
|
meerschaum/config/_version.py
CHANGED
meerschaum/config/paths.py
CHANGED
@@ -6,5 +6,24 @@
|
|
6
6
|
External API for importing Meerschaum paths.
|
7
7
|
"""
|
8
8
|
|
9
|
-
|
10
|
-
|
9
|
+
import pathlib
|
10
|
+
import inspect
|
11
|
+
import meerschaum.config._paths as _paths
|
12
|
+
|
13
|
+
|
14
|
+
def __getattr__(*args, **kwargs):
|
15
|
+
return _paths.__getattr__(*args, **kwargs)
|
16
|
+
|
17
|
+
|
18
|
+
_globals_dict = inspect.getmembers(
|
19
|
+
_paths,
|
20
|
+
lambda member: not inspect.isroutine(member)
|
21
|
+
)
|
22
|
+
_all_caps_globals = [
|
23
|
+
name
|
24
|
+
for name, value in _globals_dict
|
25
|
+
if ('PATH' in name or 'FILE' in name) and not name.startswith('_')
|
26
|
+
and isinstance(value, pathlib.Path)
|
27
|
+
]
|
28
|
+
|
29
|
+
__all__ = tuple(_all_caps_globals + list(_paths.paths.keys()))
|
@@ -18,7 +18,7 @@ class InvalidAttributesError(Exception):
|
|
18
18
|
|
19
19
|
class Connector(metaclass=abc.ABCMeta):
|
20
20
|
"""
|
21
|
-
The base connector class to hold connection attributes
|
21
|
+
The base connector class to hold connection attributes.
|
22
22
|
"""
|
23
23
|
def __init__(
|
24
24
|
self,
|
@@ -27,6 +27,8 @@ class Connector(metaclass=abc.ABCMeta):
|
|
27
27
|
**kw: Any
|
28
28
|
):
|
29
29
|
"""
|
30
|
+
Set the given keyword arguments as attributes.
|
31
|
+
|
30
32
|
Parameters
|
31
33
|
----------
|
32
34
|
type: str
|
@@ -35,9 +37,12 @@ class Connector(metaclass=abc.ABCMeta):
|
|
35
37
|
label: str
|
36
38
|
The `label` for the connector.
|
37
39
|
|
40
|
+
|
41
|
+
Examples
|
42
|
+
--------
|
38
43
|
Run `mrsm edit config` and to edit connectors in the YAML file:
|
39
44
|
|
40
|
-
```
|
45
|
+
```yaml
|
41
46
|
meerschaum:
|
42
47
|
connections:
|
43
48
|
{type}:
|
@@ -24,6 +24,7 @@ from meerschaum.connectors.api.APIConnector import APIConnector
|
|
24
24
|
from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
|
25
25
|
|
26
26
|
__all__ = (
|
27
|
+
"make_connector",
|
27
28
|
"Connector",
|
28
29
|
"SQLConnector",
|
29
30
|
"APIConnector",
|
@@ -290,13 +291,14 @@ def make_connector(
|
|
290
291
|
--------
|
291
292
|
>>> import meerschaum as mrsm
|
292
293
|
>>> from meerschaum.connectors import make_connector, Connector
|
294
|
+
>>>
|
295
|
+
>>> @make_connector
|
293
296
|
>>> class FooConnector(Connector):
|
294
|
-
...
|
295
|
-
... super().__init__('foo', label, **kw)
|
297
|
+
... REQUIRED_ATTRIBUTES: list[str] = ['username', 'password']
|
296
298
|
...
|
297
|
-
>>>
|
298
|
-
>>>
|
299
|
-
|
299
|
+
>>> conn = mrsm.get_connector('foo:bar', username='dog', password='cat')
|
300
|
+
>>> print(conn.username, conn.password)
|
301
|
+
dog cat
|
300
302
|
>>>
|
301
303
|
"""
|
302
304
|
import re
|
@@ -83,12 +83,16 @@ class APIConnector(Connector):
|
|
83
83
|
if 'uri' in kw:
|
84
84
|
from_uri_params = self.from_uri(kw['uri'], as_dict=True)
|
85
85
|
label = label or from_uri_params.get('label', None)
|
86
|
-
from_uri_params.pop('label', None)
|
86
|
+
_ = from_uri_params.pop('label', None)
|
87
87
|
kw.update(from_uri_params)
|
88
88
|
|
89
89
|
super().__init__('api', label=label, **kw)
|
90
90
|
if 'protocol' not in self.__dict__:
|
91
|
-
self.protocol =
|
91
|
+
self.protocol = (
|
92
|
+
'https' if self.__dict__.get('uri', '').startswith('https')
|
93
|
+
else 'http'
|
94
|
+
)
|
95
|
+
|
92
96
|
if 'uri' not in self.__dict__:
|
93
97
|
self.verify_attributes(required_attributes)
|
94
98
|
else:
|
@@ -97,6 +101,7 @@ class APIConnector(Connector):
|
|
97
101
|
if 'host' not in conn_attrs:
|
98
102
|
raise Exception(f"Invalid URI for '{self}'.")
|
99
103
|
self.__dict__.update(conn_attrs)
|
104
|
+
|
100
105
|
self.url = (
|
101
106
|
self.protocol + '://' +
|
102
107
|
self.host
|
@@ -9,44 +9,31 @@ Functions to interact with /mrsm/actions
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from meerschaum.utils.typing import SuccessTuple, Optional, List
|
11
11
|
|
12
|
-
def get_actions(
|
13
|
-
|
14
|
-
) -> list:
|
15
|
-
"""Get available actions from the API server"""
|
12
|
+
def get_actions(self) -> list:
|
13
|
+
"""Get available actions from the API instance."""
|
16
14
|
from meerschaum.config.static import STATIC_CONFIG
|
17
15
|
return self.get(STATIC_CONFIG['api']['endpoints']['actions'])
|
18
16
|
|
19
17
|
|
20
18
|
def do_action(
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
19
|
+
self,
|
20
|
+
action: Optional[List[str]] = None,
|
21
|
+
sysargs: Optional[List[str]] = None,
|
22
|
+
debug: bool = False,
|
23
|
+
**kw
|
24
|
+
) -> SuccessTuple:
|
27
25
|
"""Execute a Meerschaum action remotely.
|
28
|
-
|
29
|
-
If sysargs is provided, parse those instead. Otherwise infer everything from keyword arguments.
|
30
|
-
|
31
|
-
NOTE: The first index of `action` should NOT be removed!
|
32
|
-
Example: action = ['show', 'config']
|
33
|
-
|
34
|
-
Returns: tuple (succeeded : bool, message : str)
|
35
26
|
|
36
|
-
|
37
|
-
|
38
|
-
action: Optional[List[str]] :
|
39
|
-
(Default value = None)
|
40
|
-
sysargs: Optional[List[str]] :
|
41
|
-
(Default value = None)
|
42
|
-
debug: bool :
|
43
|
-
(Default value = False)
|
44
|
-
**kw :
|
45
|
-
|
46
|
-
|
47
|
-
Returns
|
48
|
-
-------
|
27
|
+
If `sysargs` are provided, parse those instead.
|
28
|
+
Otherwise infer everything from keyword arguments.
|
49
29
|
|
30
|
+
Examples
|
31
|
+
--------
|
32
|
+
>>> conn = mrsm.get_connector('api:main')
|
33
|
+
>>> conn.do_action(['show', 'pipes'])
|
34
|
+
(True, "Success")
|
35
|
+
>>> conn.do_action(['show', 'arguments'], name='test')
|
36
|
+
(True, "Success")
|
50
37
|
"""
|
51
38
|
import sys, json
|
52
39
|
from meerschaum.utils.debug import dprint
|
@@ -63,7 +50,12 @@ def do_action(
|
|
63
50
|
else:
|
64
51
|
json_dict = kw
|
65
52
|
json_dict['action'] = action
|
66
|
-
|
53
|
+
if 'noask' not in kw:
|
54
|
+
json_dict['noask'] = True
|
55
|
+
if 'yes' not in kw:
|
56
|
+
json_dict['yes'] = True
|
57
|
+
if debug:
|
58
|
+
json_dict['debug'] = debug
|
67
59
|
|
68
60
|
root_action = json_dict['action'][0]
|
69
61
|
del json_dict['action'][0]
|
@@ -11,11 +11,11 @@ from meerschaum.utils.warnings import warn, error
|
|
11
11
|
|
12
12
|
@classmethod
|
13
13
|
def from_uri(
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
14
|
+
cls,
|
15
|
+
uri: str,
|
16
|
+
label: Optional[str] = None,
|
17
|
+
as_dict: bool = False,
|
18
|
+
) -> Union[
|
19
19
|
'meerschaum.connectors.APIConnector',
|
20
20
|
Dict[str, Union[str, int]],
|
21
21
|
]:
|
meerschaum/core/Pipe/__init__.py
CHANGED
@@ -50,6 +50,7 @@ with correct credentials, as well as a network connection and valid permissions.
|
|
50
50
|
"""
|
51
51
|
|
52
52
|
from __future__ import annotations
|
53
|
+
import sys
|
53
54
|
import copy
|
54
55
|
from meerschaum.utils.typing import Optional, Dict, Any, Union, InstanceConnector, List
|
55
56
|
from meerschaum.utils.formatting._pipes import pipe_repr
|
@@ -433,13 +434,16 @@ class Pipe:
|
|
433
434
|
+ str(self.instance_keys) + sep
|
434
435
|
)
|
435
436
|
|
436
|
-
def __repr__(self, **kw) -> str:
|
437
|
-
|
437
|
+
def __repr__(self, ansi: bool=True, **kw) -> str:
|
438
|
+
if not hasattr(sys, 'ps1'):
|
439
|
+
ansi = False
|
440
|
+
|
441
|
+
return pipe_repr(self, ansi=ansi, **kw)
|
438
442
|
|
439
443
|
def __pt_repr__(self):
|
440
444
|
from meerschaum.utils.packages import attempt_import
|
441
445
|
prompt_toolkit_formatted_text = attempt_import('prompt_toolkit.formatted_text', lazy=False)
|
442
|
-
return prompt_toolkit_formatted_text.ANSI(self
|
446
|
+
return prompt_toolkit_formatted_text.ANSI(pipe_repr(self, ansi=True))
|
443
447
|
|
444
448
|
def __getstate__(self) -> Dict[str, Any]:
|
445
449
|
"""
|
meerschaum/core/Pipe/_data.py
CHANGED
@@ -8,24 +8,32 @@ Retrieve Pipes' data from instances.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from datetime import datetime, timedelta
|
11
|
-
|
11
|
+
|
12
|
+
import meerschaum as mrsm
|
13
|
+
from meerschaum.utils.typing import (
|
14
|
+
Optional, Dict, Any, Union, List, Tuple, Iterator, TYPE_CHECKING,
|
15
|
+
)
|
12
16
|
from meerschaum.config import get_config
|
13
17
|
|
18
|
+
if TYPE_CHECKING:
|
19
|
+
pd = mrsm.attempt_import('pandas')
|
20
|
+
|
21
|
+
|
14
22
|
def get_data(
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
23
|
+
self,
|
24
|
+
select_columns: Optional[List[str]] = None,
|
25
|
+
omit_columns: Optional[List[str]] = None,
|
26
|
+
begin: Union[datetime, int, None] = None,
|
27
|
+
end: Union[datetime, int, None] = None,
|
28
|
+
params: Optional[Dict[str, Any]] = None,
|
29
|
+
as_iterator: bool = False,
|
30
|
+
as_chunks: bool = False,
|
31
|
+
as_dask: bool = False,
|
32
|
+
chunk_interval: Union[timedelta, int, None] = None,
|
33
|
+
fresh: bool = False,
|
34
|
+
debug: bool = False,
|
35
|
+
**kw: Any
|
36
|
+
) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
|
29
37
|
"""
|
30
38
|
Get a pipe's data from the instance connector.
|
31
39
|
|
@@ -8,7 +8,7 @@ Delete duplicate rows within a pipe's table.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from datetime import datetime, timedelta
|
11
|
-
from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple
|
11
|
+
from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple, Union
|
12
12
|
|
13
13
|
|
14
14
|
def deduplicate(
|
meerschaum/core/Pipe/_dtypes.py
CHANGED
@@ -8,7 +8,12 @@ Enforce data types for a pipe's underlying table.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from io import StringIO
|
11
|
+
import meerschaum as mrsm
|
11
12
|
from meerschaum.utils.typing import Dict, Any, Optional
|
13
|
+
from typing import TYPE_CHECKING
|
14
|
+
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
pd = mrsm.attempt_import('pandas')
|
12
17
|
|
13
18
|
def enforce_dtypes(
|
14
19
|
self,
|
meerschaum/core/Pipe/_fetch.py
CHANGED
@@ -8,11 +8,14 @@ Functions for fetching new data into the Pipe
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
from datetime import timedelta, datetime
|
11
|
+
|
11
12
|
import meerschaum as mrsm
|
12
|
-
from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
|
13
|
+
from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator, TYPE_CHECKING
|
13
14
|
from meerschaum.config import get_config
|
14
15
|
from meerschaum.utils.warnings import warn
|
15
|
-
|
16
|
+
|
17
|
+
if TYPE_CHECKING:
|
18
|
+
pd = mrsm.attempt_import('pandas')
|
16
19
|
|
17
20
|
def fetch(
|
18
21
|
self,
|
@@ -55,6 +58,7 @@ def fetch(
|
|
55
58
|
|
56
59
|
from meerschaum.connectors import custom_types, get_connector_plugin
|
57
60
|
from meerschaum.utils.debug import dprint, _checkpoint
|
61
|
+
from meerschaum.utils.misc import filter_arguments
|
58
62
|
|
59
63
|
_chunk_hook = kw.pop('chunk_hook', None)
|
60
64
|
kw['workers'] = self.get_num_workers(kw.get('workers', None))
|
@@ -72,24 +76,22 @@ def fetch(
|
|
72
76
|
chunk_message = '\n' + chunk_label + '\n' + chunk_message
|
73
77
|
return chunk_success, chunk_message
|
74
78
|
|
75
|
-
|
76
79
|
with mrsm.Venv(get_connector_plugin(self.connector)):
|
77
|
-
|
80
|
+
_args, _kwargs = filter_arguments(
|
81
|
+
self.connector.fetch,
|
78
82
|
self,
|
79
|
-
|
80
|
-
self
|
81
|
-
begin
|
82
|
-
|
83
|
-
begin,
|
84
|
-
check_existing=check_existing,
|
85
|
-
debug=debug,
|
86
|
-
),
|
87
|
-
end=end,
|
88
|
-
chunk_hook=_chunk_hook,
|
83
|
+
begin=_determine_begin(
|
84
|
+
self,
|
85
|
+
begin,
|
86
|
+
check_existing=check_existing,
|
89
87
|
debug=debug,
|
90
|
-
|
91
|
-
|
88
|
+
),
|
89
|
+
end=end,
|
90
|
+
chunk_hook=_chunk_hook,
|
91
|
+
debug=debug,
|
92
|
+
**kw
|
92
93
|
)
|
94
|
+
df = self.connector.fetch(*_args, **_kwargs)
|
93
95
|
return df
|
94
96
|
|
95
97
|
|