meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +2 -5
  2. meerschaum/_internal/docs/index.py +3 -2
  3. meerschaum/_internal/entry.py +13 -7
  4. meerschaum/_internal/shell/Shell.py +38 -44
  5. meerschaum/_internal/term/TermPageHandler.py +2 -3
  6. meerschaum/_internal/term/__init__.py +13 -11
  7. meerschaum/actions/api.py +10 -7
  8. meerschaum/actions/bootstrap.py +38 -11
  9. meerschaum/actions/copy.py +3 -3
  10. meerschaum/actions/delete.py +4 -1
  11. meerschaum/actions/register.py +1 -3
  12. meerschaum/actions/stack.py +24 -19
  13. meerschaum/actions/start.py +38 -40
  14. meerschaum/actions/sync.py +53 -52
  15. meerschaum/api/__init__.py +48 -14
  16. meerschaum/api/_events.py +15 -10
  17. meerschaum/api/_oauth2.py +2 -2
  18. meerschaum/api/_websockets.py +5 -4
  19. meerschaum/api/dash/__init__.py +7 -16
  20. meerschaum/api/dash/callbacks/__init__.py +1 -0
  21. meerschaum/api/dash/callbacks/dashboard.py +52 -58
  22. meerschaum/api/dash/callbacks/jobs.py +15 -16
  23. meerschaum/api/dash/callbacks/login.py +16 -10
  24. meerschaum/api/dash/callbacks/pipes.py +41 -0
  25. meerschaum/api/dash/callbacks/plugins.py +1 -1
  26. meerschaum/api/dash/callbacks/register.py +15 -11
  27. meerschaum/api/dash/components.py +54 -59
  28. meerschaum/api/dash/jobs.py +5 -9
  29. meerschaum/api/dash/pages/__init__.py +1 -0
  30. meerschaum/api/dash/pages/pipes.py +19 -0
  31. meerschaum/api/dash/pipes.py +86 -58
  32. meerschaum/api/dash/plugins.py +6 -4
  33. meerschaum/api/dash/sessions.py +176 -0
  34. meerschaum/api/dash/users.py +3 -41
  35. meerschaum/api/dash/webterm.py +12 -17
  36. meerschaum/api/resources/static/js/terminado.js +1 -1
  37. meerschaum/api/routes/_actions.py +4 -20
  38. meerschaum/api/routes/_jobs.py +8 -7
  39. meerschaum/api/routes/_login.py +4 -4
  40. meerschaum/api/routes/_pipes.py +3 -3
  41. meerschaum/api/routes/_webterm.py +5 -6
  42. meerschaum/config/_default.py +15 -2
  43. meerschaum/config/_version.py +1 -1
  44. meerschaum/config/stack/__init__.py +64 -19
  45. meerschaum/config/static/__init__.py +4 -0
  46. meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
  47. meerschaum/connectors/__init__.py +24 -14
  48. meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
  49. meerschaum/connectors/api/__init__.py +2 -1
  50. meerschaum/connectors/parse.py +18 -16
  51. meerschaum/connectors/poll.py +30 -24
  52. meerschaum/connectors/sql/__init__.py +3 -1
  53. meerschaum/connectors/sql/_pipes.py +172 -197
  54. meerschaum/connectors/sql/_plugins.py +45 -43
  55. meerschaum/connectors/sql/_users.py +46 -38
  56. meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
  57. meerschaum/connectors/valkey/__init__.py +10 -0
  58. meerschaum/connectors/valkey/_fetch.py +75 -0
  59. meerschaum/connectors/valkey/_pipes.py +844 -0
  60. meerschaum/connectors/valkey/_plugins.py +265 -0
  61. meerschaum/connectors/valkey/_users.py +305 -0
  62. meerschaum/core/Pipe/__init__.py +3 -0
  63. meerschaum/core/Pipe/_attributes.py +1 -2
  64. meerschaum/core/Pipe/_clear.py +16 -13
  65. meerschaum/core/Pipe/_copy.py +106 -0
  66. meerschaum/core/Pipe/_data.py +165 -101
  67. meerschaum/core/Pipe/_drop.py +4 -4
  68. meerschaum/core/Pipe/_dtypes.py +14 -14
  69. meerschaum/core/Pipe/_edit.py +15 -14
  70. meerschaum/core/Pipe/_sync.py +134 -53
  71. meerschaum/core/Pipe/_verify.py +11 -11
  72. meerschaum/core/User/_User.py +14 -12
  73. meerschaum/jobs/_Job.py +1 -6
  74. meerschaum/jobs/__init__.py +7 -2
  75. meerschaum/plugins/_Plugin.py +17 -13
  76. meerschaum/utils/_get_pipes.py +14 -20
  77. meerschaum/utils/dataframe.py +291 -101
  78. meerschaum/utils/dtypes/__init__.py +31 -6
  79. meerschaum/utils/dtypes/sql.py +4 -4
  80. meerschaum/utils/formatting/_shell.py +5 -6
  81. meerschaum/utils/misc.py +3 -3
  82. meerschaum/utils/packages/__init__.py +14 -9
  83. meerschaum/utils/packages/_packages.py +2 -0
  84. meerschaum/utils/schedule.py +1 -0
  85. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
  86. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/RECORD +93 -84
  87. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
  88. meerschaum/api/dash/actions.py +0 -255
  89. /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
  90. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
  91. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
  92. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
  93. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
  94. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
@@ -53,6 +53,9 @@ STATIC_CONFIG: Dict[str, Any] = {
53
53
  'internal_schema': '_mrsm_internal',
54
54
  'instance_schema': 'mrsm',
55
55
  },
56
+ 'valkey': {
57
+ 'colon': '-_',
58
+ },
56
59
  'environment': {
57
60
  'config': 'MRSM_CONFIG',
58
61
  'config_dir': 'MRSM_CONFIG_DIR',
@@ -69,6 +72,7 @@ STATIC_CONFIG: Dict[str, Any] = {
69
72
  'uid': 'MRSM_UID',
70
73
  'gid': 'MRSM_GID',
71
74
  'noask': 'MRSM_NOASK',
75
+ 'noninteractive': 'MRSM_NONINTERACTIVE',
72
76
  'id': 'MRSM_SERVER_ID',
73
77
  'daemon_id': 'MRSM_DAEMON_ID',
74
78
  'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
@@ -53,17 +53,23 @@ class Connector(metaclass=abc.ABCMeta):
53
53
  """
54
54
  self._original_dict = copy.deepcopy(self.__dict__)
55
55
  self._set_attributes(type=type, label=label, **kw)
56
- self.verify_attributes(getattr(self, 'REQUIRED_ATTRIBUTES', None))
56
+
57
+ ### NOTE: Override `REQUIRED_ATTRIBUTES` if `uri` is set.
58
+ self.verify_attributes(
59
+ ['uri']
60
+ if 'uri' in self.__dict__
61
+ else getattr(self, 'REQUIRED_ATTRIBUTES', None)
62
+ )
57
63
 
58
64
  def _reset_attributes(self):
59
65
  self.__dict__ = self._original_dict
60
66
 
61
67
  def _set_attributes(
62
- self,
63
- *args,
64
- inherit_default: bool = True,
65
- **kw: Any
66
- ):
68
+ self,
69
+ *args,
70
+ inherit_default: bool = True,
71
+ **kw: Any
72
+ ):
67
73
  from meerschaum.config.static import STATIC_CONFIG
68
74
  from meerschaum.utils.warnings import error
69
75
 
@@ -114,12 +120,11 @@ class Connector(metaclass=abc.ABCMeta):
114
120
  ### finally, update __dict__ with _attributes.
115
121
  self.__dict__.update(self._attributes)
116
122
 
117
-
118
123
  def verify_attributes(
119
- self,
120
- required_attributes: Optional[List[str]] = None,
121
- debug: bool = False
122
- ) -> None:
124
+ self,
125
+ required_attributes: Optional[List[str]] = None,
126
+ debug: bool = False,
127
+ ) -> None:
123
128
  """
124
129
  Ensure that the required attributes have been met.
125
130
 
@@ -147,6 +152,7 @@ class Connector(metaclass=abc.ABCMeta):
147
152
  from meerschaum.utils.misc import items_str
148
153
  if required_attributes is None:
149
154
  required_attributes = ['label']
155
+
150
156
  missing_attributes = set()
151
157
  for a in required_attributes:
152
158
  if a not in self.__dict__:
@@ -158,8 +164,8 @@ class Connector(metaclass=abc.ABCMeta):
158
164
  + f"for connector '{self.type}:{self.label}'."
159
165
  ),
160
166
  InvalidAttributesError,
161
- silent = True,
162
- stack = False
167
+ silent=True,
168
+ stack=False
163
169
  )
164
170
 
165
171
 
@@ -14,13 +14,13 @@ For ease of use, you can also import from the root `meerschaum` module:
14
14
  from __future__ import annotations
15
15
 
16
16
  import meerschaum as mrsm
17
- from meerschaum.utils.typing import Any, SuccessTuple, Union, Optional, List, Dict
18
- from meerschaum.utils.threading import Lock, RLock
19
- from meerschaum.utils.warnings import error, warn
17
+ from meerschaum.utils.typing import Any, Union, List, Dict
18
+ from meerschaum.utils.threading import RLock
19
+ from meerschaum.utils.warnings import warn
20
20
 
21
- from meerschaum.connectors.Connector import Connector, InvalidAttributesError
22
- from meerschaum.connectors.sql.SQLConnector import SQLConnector
23
- from meerschaum.connectors.api.APIConnector import APIConnector
21
+ from meerschaum.connectors._Connector import Connector, InvalidAttributesError
22
+ from meerschaum.connectors.sql._SQLConnector import SQLConnector
23
+ from meerschaum.connectors.api._APIConnector import APIConnector
24
24
  from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
25
25
 
26
26
  __all__ = (
@@ -31,6 +31,9 @@ __all__ = (
31
31
  "get_connector",
32
32
  "is_connected",
33
33
  "poll",
34
+ "api",
35
+ "sql",
36
+ "valkey",
34
37
  )
35
38
 
36
39
  ### store connectors partitioned by
@@ -39,6 +42,7 @@ connectors: Dict[str, Dict[str, Connector]] = {
39
42
  'api' : {},
40
43
  'sql' : {},
41
44
  'plugin' : {},
45
+ 'valkey' : {},
42
46
  }
43
47
  instance_types: List[str] = ['sql', 'api']
44
48
  _locks: Dict[str, RLock] = {
@@ -53,7 +57,10 @@ attributes: Dict[str, Dict[str, Any]] = {
53
57
  'required': [
54
58
  'host',
55
59
  'username',
56
- 'password'
60
+ 'password',
61
+ ],
62
+ 'optional': [
63
+ 'port',
57
64
  ],
58
65
  'default': {
59
66
  'protocol': 'http',
@@ -164,13 +171,15 @@ def get_connector(
164
171
 
165
172
  if 'sql' not in types:
166
173
  from meerschaum.connectors.plugin import PluginConnector
174
+ from meerschaum.connectors.valkey import ValkeyConnector
167
175
  with _locks['types']:
168
176
  types.update({
169
- 'api' : APIConnector,
170
- 'sql' : SQLConnector,
177
+ 'api': APIConnector,
178
+ 'sql': SQLConnector,
171
179
  'plugin': PluginConnector,
180
+ 'valkey': ValkeyConnector,
172
181
  })
173
-
182
+
174
183
  ### determine if we need to call the constructor
175
184
  if not refresh:
176
185
  ### see if any user-supplied arguments differ from the existing instance
@@ -260,11 +269,11 @@ def is_connected(keys: str, **kw) -> bool:
260
269
 
261
270
  try:
262
271
  typ, label = keys.split(':')
263
- except Exception as e:
272
+ except Exception:
264
273
  return False
265
274
  if typ not in instance_types:
266
275
  return False
267
- if not (label in connectors.get(typ, {})):
276
+ if label not in connectors.get(typ, {}):
268
277
  return False
269
278
 
270
279
  from meerschaum.connectors.parse import parse_instance_keys
@@ -273,7 +282,7 @@ def is_connected(keys: str, **kw) -> bool:
273
282
  with warnings.catch_warnings():
274
283
  warnings.filterwarnings('ignore')
275
284
  return conn.test_connection(**kw)
276
- except Exception as e:
285
+ except Exception:
277
286
  return False
278
287
 
279
288
 
@@ -340,7 +349,7 @@ def load_plugin_connectors():
340
349
  to_import.append(plugin.name)
341
350
  if not to_import:
342
351
  return
343
- import_plugins(*to_import)
352
+ import_plugins(*to_import)
344
353
 
345
354
 
346
355
  def get_connector_plugin(
@@ -378,3 +387,4 @@ def _load_builtin_custom_connectors():
378
387
  Import custom connectors decorated with `@make_connector` or `@make_executor`.
379
388
  """
380
389
  import meerschaum.jobs.systemd
390
+ import meerschaum.connectors.valkey
@@ -7,7 +7,7 @@ Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_inst
7
7
  """
8
8
 
9
9
  from datetime import datetime, timedelta, timezone
10
- from meerschaum.utils.typing import Optional
10
+ from meerschaum.utils.typing import Optional, List
11
11
  from meerschaum.connectors import Connector
12
12
  from meerschaum.utils.warnings import warn, error
13
13
  from meerschaum.utils.packages import attempt_import
@@ -24,6 +24,8 @@ class APIConnector(Connector):
24
24
  IS_INSTANCE: bool = True
25
25
  IS_THREAD_SAFE: bool = False
26
26
 
27
+ OPTIONAL_ATTRIBUTES: List[str] = ['port']
28
+
27
29
  from ._request import (
28
30
  make_request,
29
31
  get,
@@ -6,5 +6,6 @@
6
6
  Interact with the Meerschaum API (send commands, pull data, etc.)
7
7
  """
8
8
 
9
- from meerschaum.connectors.api.APIConnector import APIConnector
9
+ from meerschaum.connectors.api._APIConnector import APIConnector
10
10
 
11
+ __all__ = ('APIConnector',)
@@ -7,7 +7,10 @@ Utility functions for parsing connector keys.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from meerschaum.utils.typing import Mapping, Any, SuccessTuple, Union, Optional, Dict, Tuple
10
+
11
+ import meerschaum as mrsm
12
+ from meerschaum.utils.typing import Any, Union, Optional, Dict, Tuple
13
+
11
14
 
12
15
  def parse_connector_keys(
13
16
  keys: str,
@@ -15,20 +18,20 @@ def parse_connector_keys(
15
18
  as_tuple: bool = False,
16
19
  **kw: Any
17
20
  ) -> (
18
- Union[
19
- meerschaum.connectors.Connector,
20
- Dict[str, Any],
21
- Tuple[
22
- Union[
23
- meerschaum.connectors.Connector,
24
- Dict[str, Any],
25
- None
26
- ],
27
- str
21
+ Union[
22
+ mrsm.connectors.Connector,
23
+ Dict[str, Any],
24
+ Tuple[
25
+ Union[
26
+ mrsm.connectors.Connector,
27
+ Dict[str, Any],
28
+ None,
28
29
  ],
29
- None
30
- ]
31
- ):
30
+ str
31
+ ],
32
+ None
33
+ ]
34
+ ):
32
35
  """
33
36
  Convenience function for parsing connector keys and returning Connector objects.
34
37
 
@@ -95,13 +98,12 @@ def parse_instance_keys(
95
98
  """
96
99
  Parse the Meerschaum instance value into a Connector object.
97
100
  """
98
- from meerschaum.utils.warnings import warn
99
101
  from meerschaum.config import get_config
100
102
 
101
103
  if keys is None:
102
104
  keys = get_config('meerschaum', 'instance')
103
105
  keys = str(keys)
104
-
106
+
105
107
  return parse_connector_keys(keys, construct=construct, as_tuple=as_tuple, **kw)
106
108
 
107
109
 
@@ -9,16 +9,16 @@ Poll database and API connections.
9
9
  from meerschaum.utils.typing import InstanceConnector, Union, Optional, Dict, Any
10
10
 
11
11
  def retry_connect(
12
- connector: Union[InstanceConnector, None] = None,
13
- max_retries: int = 50,
14
- retry_wait: int = 3,
15
- workers: int = 1,
16
- warn: bool = True,
17
- print_on_connect: bool = False,
18
- enforce_chaining: bool = True,
19
- enforce_login: bool = True,
20
- debug: bool = False,
21
- ) -> bool:
12
+ connector: Union[InstanceConnector, None] = None,
13
+ max_retries: int = 50,
14
+ retry_wait: int = 3,
15
+ workers: int = 1,
16
+ warn: bool = True,
17
+ print_on_connect: bool = False,
18
+ enforce_chaining: bool = True,
19
+ enforce_login: bool = True,
20
+ debug: bool = False,
21
+ ) -> bool:
22
22
  """
23
23
  Keep trying to connect to the database.
24
24
 
@@ -85,16 +85,16 @@ def retry_connect(
85
85
 
86
86
 
87
87
  def _wrap_retry_connect(
88
- connector_meta: Dict[str, Any],
89
- max_retries: int = 50,
90
- retry_wait: int = 3,
91
- workers: int = 1,
92
- print_on_connect: bool = False,
93
- warn: bool = True,
94
- enforce_chaining: bool = True,
95
- enforce_login: bool = True,
96
- debug: bool = False,
97
- ) -> bool:
88
+ connector_meta: Dict[str, Any],
89
+ max_retries: int = 50,
90
+ retry_wait: int = 3,
91
+ workers: int = 1,
92
+ print_on_connect: bool = False,
93
+ warn: bool = True,
94
+ enforce_chaining: bool = True,
95
+ enforce_login: bool = True,
96
+ debug: bool = False,
97
+ ) -> bool:
98
98
  """
99
99
  Keep trying to connect to the database.
100
100
 
@@ -144,8 +144,6 @@ def _wrap_retry_connect(
144
144
  import time
145
145
 
146
146
  connector = get_connector(**connector_meta)
147
- if connector.type not in instance_types:
148
- return None
149
147
 
150
148
  if not hasattr(connector, 'test_connection'):
151
149
  return True
@@ -157,7 +155,15 @@ def _wrap_retry_connect(
157
155
  dprint(f"Trying to connect to '{connector}'...")
158
156
  dprint(f"Attempt ({retries + 1} / {max_retries})")
159
157
 
160
- if connector.type == 'sql':
158
+ if connector.type not in ('sql', 'api'):
159
+ try:
160
+ connected = connector.test_connection()
161
+ except Exception as e:
162
+ if warn:
163
+ print(e)
164
+ connected = False
165
+
166
+ elif connector.type == 'sql':
161
167
 
162
168
  def _connect(_connector):
163
169
  ### Test queries like `SELECT 1`.
@@ -185,7 +191,7 @@ def _wrap_retry_connect(
185
191
  _warn(
186
192
  f"Meerschaum instance '{connector}' does not allow chaining " +
187
193
  "and cannot be used as the parent for this instance.",
188
- stack = False
194
+ stack=False,
189
195
  )
190
196
  return False
191
197
 
@@ -6,4 +6,6 @@
6
6
  Subpackage for SQLConnector subclass
7
7
  """
8
8
 
9
- from meerschaum.connectors.sql.SQLConnector import SQLConnector
9
+ from meerschaum.connectors.sql._SQLConnector import SQLConnector
10
+
11
+ __all__ = ('SQLConnector',)