meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +2 -5
- meerschaum/_internal/docs/index.py +3 -2
- meerschaum/_internal/entry.py +13 -7
- meerschaum/_internal/shell/Shell.py +38 -44
- meerschaum/_internal/term/TermPageHandler.py +2 -3
- meerschaum/_internal/term/__init__.py +13 -11
- meerschaum/actions/api.py +10 -7
- meerschaum/actions/bootstrap.py +38 -11
- meerschaum/actions/copy.py +3 -3
- meerschaum/actions/delete.py +4 -1
- meerschaum/actions/register.py +1 -3
- meerschaum/actions/stack.py +24 -19
- meerschaum/actions/start.py +38 -40
- meerschaum/actions/sync.py +53 -52
- meerschaum/api/__init__.py +48 -14
- meerschaum/api/_events.py +15 -10
- meerschaum/api/_oauth2.py +2 -2
- meerschaum/api/_websockets.py +5 -4
- meerschaum/api/dash/__init__.py +7 -16
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/dashboard.py +52 -58
- meerschaum/api/dash/callbacks/jobs.py +15 -16
- meerschaum/api/dash/callbacks/login.py +16 -10
- meerschaum/api/dash/callbacks/pipes.py +41 -0
- meerschaum/api/dash/callbacks/plugins.py +1 -1
- meerschaum/api/dash/callbacks/register.py +15 -11
- meerschaum/api/dash/components.py +54 -59
- meerschaum/api/dash/jobs.py +5 -9
- meerschaum/api/dash/pages/__init__.py +1 -0
- meerschaum/api/dash/pages/pipes.py +19 -0
- meerschaum/api/dash/pipes.py +86 -58
- meerschaum/api/dash/plugins.py +6 -4
- meerschaum/api/dash/sessions.py +176 -0
- meerschaum/api/dash/users.py +3 -41
- meerschaum/api/dash/webterm.py +12 -17
- meerschaum/api/resources/static/js/terminado.js +1 -1
- meerschaum/api/routes/_actions.py +4 -20
- meerschaum/api/routes/_jobs.py +8 -7
- meerschaum/api/routes/_login.py +4 -4
- meerschaum/api/routes/_pipes.py +3 -3
- meerschaum/api/routes/_webterm.py +5 -6
- meerschaum/config/_default.py +15 -2
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +64 -19
- meerschaum/config/static/__init__.py +4 -0
- meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
- meerschaum/connectors/__init__.py +24 -14
- meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
- meerschaum/connectors/api/__init__.py +2 -1
- meerschaum/connectors/parse.py +18 -16
- meerschaum/connectors/poll.py +30 -24
- meerschaum/connectors/sql/__init__.py +3 -1
- meerschaum/connectors/sql/_pipes.py +172 -197
- meerschaum/connectors/sql/_plugins.py +45 -43
- meerschaum/connectors/sql/_users.py +46 -38
- meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
- meerschaum/connectors/valkey/__init__.py +10 -0
- meerschaum/connectors/valkey/_fetch.py +75 -0
- meerschaum/connectors/valkey/_pipes.py +844 -0
- meerschaum/connectors/valkey/_plugins.py +265 -0
- meerschaum/connectors/valkey/_users.py +305 -0
- meerschaum/core/Pipe/__init__.py +3 -0
- meerschaum/core/Pipe/_attributes.py +1 -2
- meerschaum/core/Pipe/_clear.py +16 -13
- meerschaum/core/Pipe/_copy.py +106 -0
- meerschaum/core/Pipe/_data.py +165 -101
- meerschaum/core/Pipe/_drop.py +4 -4
- meerschaum/core/Pipe/_dtypes.py +14 -14
- meerschaum/core/Pipe/_edit.py +15 -14
- meerschaum/core/Pipe/_sync.py +134 -53
- meerschaum/core/Pipe/_verify.py +11 -11
- meerschaum/core/User/_User.py +14 -12
- meerschaum/jobs/_Job.py +1 -6
- meerschaum/jobs/__init__.py +7 -2
- meerschaum/plugins/_Plugin.py +17 -13
- meerschaum/utils/_get_pipes.py +14 -20
- meerschaum/utils/dataframe.py +291 -101
- meerschaum/utils/dtypes/__init__.py +31 -6
- meerschaum/utils/dtypes/sql.py +4 -4
- meerschaum/utils/formatting/_shell.py +5 -6
- meerschaum/utils/misc.py +3 -3
- meerschaum/utils/packages/__init__.py +14 -9
- meerschaum/utils/packages/_packages.py +2 -0
- meerschaum/utils/schedule.py +1 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/RECORD +93 -84
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
- meerschaum/api/dash/actions.py +0 -255
- /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
@@ -53,6 +53,9 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
53
53
|
'internal_schema': '_mrsm_internal',
|
54
54
|
'instance_schema': 'mrsm',
|
55
55
|
},
|
56
|
+
'valkey': {
|
57
|
+
'colon': '-_',
|
58
|
+
},
|
56
59
|
'environment': {
|
57
60
|
'config': 'MRSM_CONFIG',
|
58
61
|
'config_dir': 'MRSM_CONFIG_DIR',
|
@@ -69,6 +72,7 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
69
72
|
'uid': 'MRSM_UID',
|
70
73
|
'gid': 'MRSM_GID',
|
71
74
|
'noask': 'MRSM_NOASK',
|
75
|
+
'noninteractive': 'MRSM_NONINTERACTIVE',
|
72
76
|
'id': 'MRSM_SERVER_ID',
|
73
77
|
'daemon_id': 'MRSM_DAEMON_ID',
|
74
78
|
'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
|
@@ -53,17 +53,23 @@ class Connector(metaclass=abc.ABCMeta):
|
|
53
53
|
"""
|
54
54
|
self._original_dict = copy.deepcopy(self.__dict__)
|
55
55
|
self._set_attributes(type=type, label=label, **kw)
|
56
|
-
|
56
|
+
|
57
|
+
### NOTE: Override `REQUIRED_ATTRIBUTES` if `uri` is set.
|
58
|
+
self.verify_attributes(
|
59
|
+
['uri']
|
60
|
+
if 'uri' in self.__dict__
|
61
|
+
else getattr(self, 'REQUIRED_ATTRIBUTES', None)
|
62
|
+
)
|
57
63
|
|
58
64
|
def _reset_attributes(self):
|
59
65
|
self.__dict__ = self._original_dict
|
60
66
|
|
61
67
|
def _set_attributes(
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
68
|
+
self,
|
69
|
+
*args,
|
70
|
+
inherit_default: bool = True,
|
71
|
+
**kw: Any
|
72
|
+
):
|
67
73
|
from meerschaum.config.static import STATIC_CONFIG
|
68
74
|
from meerschaum.utils.warnings import error
|
69
75
|
|
@@ -114,12 +120,11 @@ class Connector(metaclass=abc.ABCMeta):
|
|
114
120
|
### finally, update __dict__ with _attributes.
|
115
121
|
self.__dict__.update(self._attributes)
|
116
122
|
|
117
|
-
|
118
123
|
def verify_attributes(
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
124
|
+
self,
|
125
|
+
required_attributes: Optional[List[str]] = None,
|
126
|
+
debug: bool = False,
|
127
|
+
) -> None:
|
123
128
|
"""
|
124
129
|
Ensure that the required attributes have been met.
|
125
130
|
|
@@ -147,6 +152,7 @@ class Connector(metaclass=abc.ABCMeta):
|
|
147
152
|
from meerschaum.utils.misc import items_str
|
148
153
|
if required_attributes is None:
|
149
154
|
required_attributes = ['label']
|
155
|
+
|
150
156
|
missing_attributes = set()
|
151
157
|
for a in required_attributes:
|
152
158
|
if a not in self.__dict__:
|
@@ -158,8 +164,8 @@ class Connector(metaclass=abc.ABCMeta):
|
|
158
164
|
+ f"for connector '{self.type}:{self.label}'."
|
159
165
|
),
|
160
166
|
InvalidAttributesError,
|
161
|
-
silent
|
162
|
-
stack
|
167
|
+
silent=True,
|
168
|
+
stack=False
|
163
169
|
)
|
164
170
|
|
165
171
|
|
@@ -14,13 +14,13 @@ For ease of use, you can also import from the root `meerschaum` module:
|
|
14
14
|
from __future__ import annotations
|
15
15
|
|
16
16
|
import meerschaum as mrsm
|
17
|
-
from meerschaum.utils.typing import Any,
|
18
|
-
from meerschaum.utils.threading import
|
19
|
-
from meerschaum.utils.warnings import
|
17
|
+
from meerschaum.utils.typing import Any, Union, List, Dict
|
18
|
+
from meerschaum.utils.threading import RLock
|
19
|
+
from meerschaum.utils.warnings import warn
|
20
20
|
|
21
|
-
from meerschaum.connectors.
|
22
|
-
from meerschaum.connectors.sql.
|
23
|
-
from meerschaum.connectors.api.
|
21
|
+
from meerschaum.connectors._Connector import Connector, InvalidAttributesError
|
22
|
+
from meerschaum.connectors.sql._SQLConnector import SQLConnector
|
23
|
+
from meerschaum.connectors.api._APIConnector import APIConnector
|
24
24
|
from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
|
25
25
|
|
26
26
|
__all__ = (
|
@@ -31,6 +31,9 @@ __all__ = (
|
|
31
31
|
"get_connector",
|
32
32
|
"is_connected",
|
33
33
|
"poll",
|
34
|
+
"api",
|
35
|
+
"sql",
|
36
|
+
"valkey",
|
34
37
|
)
|
35
38
|
|
36
39
|
### store connectors partitioned by
|
@@ -39,6 +42,7 @@ connectors: Dict[str, Dict[str, Connector]] = {
|
|
39
42
|
'api' : {},
|
40
43
|
'sql' : {},
|
41
44
|
'plugin' : {},
|
45
|
+
'valkey' : {},
|
42
46
|
}
|
43
47
|
instance_types: List[str] = ['sql', 'api']
|
44
48
|
_locks: Dict[str, RLock] = {
|
@@ -53,7 +57,10 @@ attributes: Dict[str, Dict[str, Any]] = {
|
|
53
57
|
'required': [
|
54
58
|
'host',
|
55
59
|
'username',
|
56
|
-
'password'
|
60
|
+
'password',
|
61
|
+
],
|
62
|
+
'optional': [
|
63
|
+
'port',
|
57
64
|
],
|
58
65
|
'default': {
|
59
66
|
'protocol': 'http',
|
@@ -164,13 +171,15 @@ def get_connector(
|
|
164
171
|
|
165
172
|
if 'sql' not in types:
|
166
173
|
from meerschaum.connectors.plugin import PluginConnector
|
174
|
+
from meerschaum.connectors.valkey import ValkeyConnector
|
167
175
|
with _locks['types']:
|
168
176
|
types.update({
|
169
|
-
'api'
|
170
|
-
'sql'
|
177
|
+
'api': APIConnector,
|
178
|
+
'sql': SQLConnector,
|
171
179
|
'plugin': PluginConnector,
|
180
|
+
'valkey': ValkeyConnector,
|
172
181
|
})
|
173
|
-
|
182
|
+
|
174
183
|
### determine if we need to call the constructor
|
175
184
|
if not refresh:
|
176
185
|
### see if any user-supplied arguments differ from the existing instance
|
@@ -260,11 +269,11 @@ def is_connected(keys: str, **kw) -> bool:
|
|
260
269
|
|
261
270
|
try:
|
262
271
|
typ, label = keys.split(':')
|
263
|
-
except Exception
|
272
|
+
except Exception:
|
264
273
|
return False
|
265
274
|
if typ not in instance_types:
|
266
275
|
return False
|
267
|
-
if not
|
276
|
+
if label not in connectors.get(typ, {}):
|
268
277
|
return False
|
269
278
|
|
270
279
|
from meerschaum.connectors.parse import parse_instance_keys
|
@@ -273,7 +282,7 @@ def is_connected(keys: str, **kw) -> bool:
|
|
273
282
|
with warnings.catch_warnings():
|
274
283
|
warnings.filterwarnings('ignore')
|
275
284
|
return conn.test_connection(**kw)
|
276
|
-
except Exception
|
285
|
+
except Exception:
|
277
286
|
return False
|
278
287
|
|
279
288
|
|
@@ -340,7 +349,7 @@ def load_plugin_connectors():
|
|
340
349
|
to_import.append(plugin.name)
|
341
350
|
if not to_import:
|
342
351
|
return
|
343
|
-
import_plugins(*to_import)
|
352
|
+
import_plugins(*to_import)
|
344
353
|
|
345
354
|
|
346
355
|
def get_connector_plugin(
|
@@ -378,3 +387,4 @@ def _load_builtin_custom_connectors():
|
|
378
387
|
Import custom connectors decorated with `@make_connector` or `@make_executor`.
|
379
388
|
"""
|
380
389
|
import meerschaum.jobs.systemd
|
390
|
+
import meerschaum.connectors.valkey
|
@@ -7,7 +7,7 @@ Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_inst
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from datetime import datetime, timedelta, timezone
|
10
|
-
from meerschaum.utils.typing import Optional
|
10
|
+
from meerschaum.utils.typing import Optional, List
|
11
11
|
from meerschaum.connectors import Connector
|
12
12
|
from meerschaum.utils.warnings import warn, error
|
13
13
|
from meerschaum.utils.packages import attempt_import
|
@@ -24,6 +24,8 @@ class APIConnector(Connector):
|
|
24
24
|
IS_INSTANCE: bool = True
|
25
25
|
IS_THREAD_SAFE: bool = False
|
26
26
|
|
27
|
+
OPTIONAL_ATTRIBUTES: List[str] = ['port']
|
28
|
+
|
27
29
|
from ._request import (
|
28
30
|
make_request,
|
29
31
|
get,
|
meerschaum/connectors/parse.py
CHANGED
@@ -7,7 +7,10 @@ Utility functions for parsing connector keys.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
|
10
|
+
|
11
|
+
import meerschaum as mrsm
|
12
|
+
from meerschaum.utils.typing import Any, Union, Optional, Dict, Tuple
|
13
|
+
|
11
14
|
|
12
15
|
def parse_connector_keys(
|
13
16
|
keys: str,
|
@@ -15,20 +18,20 @@ def parse_connector_keys(
|
|
15
18
|
as_tuple: bool = False,
|
16
19
|
**kw: Any
|
17
20
|
) -> (
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
],
|
27
|
-
str
|
21
|
+
Union[
|
22
|
+
mrsm.connectors.Connector,
|
23
|
+
Dict[str, Any],
|
24
|
+
Tuple[
|
25
|
+
Union[
|
26
|
+
mrsm.connectors.Connector,
|
27
|
+
Dict[str, Any],
|
28
|
+
None,
|
28
29
|
],
|
29
|
-
|
30
|
-
]
|
31
|
-
|
30
|
+
str
|
31
|
+
],
|
32
|
+
None
|
33
|
+
]
|
34
|
+
):
|
32
35
|
"""
|
33
36
|
Convenience function for parsing connector keys and returning Connector objects.
|
34
37
|
|
@@ -95,13 +98,12 @@ def parse_instance_keys(
|
|
95
98
|
"""
|
96
99
|
Parse the Meerschaum instance value into a Connector object.
|
97
100
|
"""
|
98
|
-
from meerschaum.utils.warnings import warn
|
99
101
|
from meerschaum.config import get_config
|
100
102
|
|
101
103
|
if keys is None:
|
102
104
|
keys = get_config('meerschaum', 'instance')
|
103
105
|
keys = str(keys)
|
104
|
-
|
106
|
+
|
105
107
|
return parse_connector_keys(keys, construct=construct, as_tuple=as_tuple, **kw)
|
106
108
|
|
107
109
|
|
meerschaum/connectors/poll.py
CHANGED
@@ -9,16 +9,16 @@ Poll database and API connections.
|
|
9
9
|
from meerschaum.utils.typing import InstanceConnector, Union, Optional, Dict, Any
|
10
10
|
|
11
11
|
def retry_connect(
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
12
|
+
connector: Union[InstanceConnector, None] = None,
|
13
|
+
max_retries: int = 50,
|
14
|
+
retry_wait: int = 3,
|
15
|
+
workers: int = 1,
|
16
|
+
warn: bool = True,
|
17
|
+
print_on_connect: bool = False,
|
18
|
+
enforce_chaining: bool = True,
|
19
|
+
enforce_login: bool = True,
|
20
|
+
debug: bool = False,
|
21
|
+
) -> bool:
|
22
22
|
"""
|
23
23
|
Keep trying to connect to the database.
|
24
24
|
|
@@ -85,16 +85,16 @@ def retry_connect(
|
|
85
85
|
|
86
86
|
|
87
87
|
def _wrap_retry_connect(
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
88
|
+
connector_meta: Dict[str, Any],
|
89
|
+
max_retries: int = 50,
|
90
|
+
retry_wait: int = 3,
|
91
|
+
workers: int = 1,
|
92
|
+
print_on_connect: bool = False,
|
93
|
+
warn: bool = True,
|
94
|
+
enforce_chaining: bool = True,
|
95
|
+
enforce_login: bool = True,
|
96
|
+
debug: bool = False,
|
97
|
+
) -> bool:
|
98
98
|
"""
|
99
99
|
Keep trying to connect to the database.
|
100
100
|
|
@@ -144,8 +144,6 @@ def _wrap_retry_connect(
|
|
144
144
|
import time
|
145
145
|
|
146
146
|
connector = get_connector(**connector_meta)
|
147
|
-
if connector.type not in instance_types:
|
148
|
-
return None
|
149
147
|
|
150
148
|
if not hasattr(connector, 'test_connection'):
|
151
149
|
return True
|
@@ -157,7 +155,15 @@ def _wrap_retry_connect(
|
|
157
155
|
dprint(f"Trying to connect to '{connector}'...")
|
158
156
|
dprint(f"Attempt ({retries + 1} / {max_retries})")
|
159
157
|
|
160
|
-
if connector.type
|
158
|
+
if connector.type not in ('sql', 'api'):
|
159
|
+
try:
|
160
|
+
connected = connector.test_connection()
|
161
|
+
except Exception as e:
|
162
|
+
if warn:
|
163
|
+
print(e)
|
164
|
+
connected = False
|
165
|
+
|
166
|
+
elif connector.type == 'sql':
|
161
167
|
|
162
168
|
def _connect(_connector):
|
163
169
|
### Test queries like `SELECT 1`.
|
@@ -185,7 +191,7 @@ def _wrap_retry_connect(
|
|
185
191
|
_warn(
|
186
192
|
f"Meerschaum instance '{connector}' does not allow chaining " +
|
187
193
|
"and cannot be used as the parent for this instance.",
|
188
|
-
stack
|
194
|
+
stack=False,
|
189
195
|
)
|
190
196
|
return False
|
191
197
|
|