meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +5 -2
- meerschaum/_internal/__init__.py +1 -0
- meerschaum/_internal/arguments/_parse_arguments.py +4 -4
- meerschaum/_internal/arguments/_parser.py +19 -2
- meerschaum/_internal/docs/index.py +49 -2
- meerschaum/_internal/entry.py +6 -6
- meerschaum/_internal/shell/Shell.py +1 -1
- meerschaum/_internal/static.py +356 -0
- meerschaum/actions/api.py +12 -2
- meerschaum/actions/bootstrap.py +7 -7
- meerschaum/actions/edit.py +142 -18
- meerschaum/actions/register.py +137 -6
- meerschaum/actions/show.py +117 -29
- meerschaum/actions/stop.py +4 -1
- meerschaum/actions/sync.py +1 -1
- meerschaum/actions/tag.py +9 -8
- meerschaum/actions/verify.py +5 -8
- meerschaum/api/__init__.py +11 -3
- meerschaum/api/_events.py +39 -2
- meerschaum/api/_oauth2.py +118 -8
- meerschaum/api/_tokens.py +102 -0
- meerschaum/api/dash/__init__.py +0 -3
- meerschaum/api/dash/callbacks/custom.py +2 -2
- meerschaum/api/dash/callbacks/dashboard.py +103 -19
- meerschaum/api/dash/callbacks/plugins.py +0 -1
- meerschaum/api/dash/callbacks/register.py +1 -1
- meerschaum/api/dash/callbacks/settings/__init__.py +1 -0
- meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
- meerschaum/api/dash/callbacks/settings/tokens.py +388 -0
- meerschaum/api/dash/components.py +30 -8
- meerschaum/api/dash/keys.py +19 -93
- meerschaum/api/dash/pages/dashboard.py +1 -20
- meerschaum/api/dash/pages/settings/__init__.py +1 -0
- meerschaum/api/dash/pages/settings/password_reset.py +1 -1
- meerschaum/api/dash/pages/settings/tokens.py +55 -0
- meerschaum/api/dash/pipes.py +94 -59
- meerschaum/api/dash/sessions.py +12 -0
- meerschaum/api/dash/tokens.py +606 -0
- meerschaum/api/dash/websockets.py +1 -1
- meerschaum/api/dash/webterm.py +4 -0
- meerschaum/api/models/__init__.py +23 -3
- meerschaum/api/models/_actions.py +22 -0
- meerschaum/api/models/_pipes.py +85 -7
- meerschaum/api/models/_tokens.py +81 -0
- meerschaum/api/resources/templates/termpage.html +12 -0
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +3 -4
- meerschaum/api/routes/_connectors.py +3 -7
- meerschaum/api/routes/_jobs.py +14 -35
- meerschaum/api/routes/_login.py +49 -12
- meerschaum/api/routes/_misc.py +5 -10
- meerschaum/api/routes/_pipes.py +173 -140
- meerschaum/api/routes/_plugins.py +38 -28
- meerschaum/api/routes/_tokens.py +236 -0
- meerschaum/api/routes/_users.py +47 -35
- meerschaum/api/routes/_version.py +3 -3
- meerschaum/config/__init__.py +43 -20
- meerschaum/config/_default.py +43 -6
- meerschaum/config/_edit.py +28 -24
- meerschaum/config/_environment.py +1 -1
- meerschaum/config/_patch.py +6 -6
- meerschaum/config/_paths.py +5 -1
- meerschaum/config/_read_config.py +65 -34
- meerschaum/config/_sync.py +6 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +31 -11
- meerschaum/config/static.py +18 -0
- meerschaum/connectors/_Connector.py +10 -4
- meerschaum/connectors/__init__.py +4 -20
- meerschaum/connectors/api/_APIConnector.py +34 -6
- meerschaum/connectors/api/_actions.py +2 -2
- meerschaum/connectors/api/_jobs.py +1 -1
- meerschaum/connectors/api/_login.py +33 -7
- meerschaum/connectors/api/_misc.py +2 -2
- meerschaum/connectors/api/_pipes.py +16 -31
- meerschaum/connectors/api/_plugins.py +2 -2
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/api/_tokens.py +146 -0
- meerschaum/connectors/api/_users.py +70 -58
- meerschaum/connectors/instance/_InstanceConnector.py +83 -0
- meerschaum/connectors/instance/__init__.py +10 -0
- meerschaum/connectors/instance/_pipes.py +442 -0
- meerschaum/connectors/instance/_plugins.py +151 -0
- meerschaum/connectors/instance/_tokens.py +296 -0
- meerschaum/connectors/instance/_users.py +181 -0
- meerschaum/connectors/parse.py +4 -1
- meerschaum/connectors/sql/_SQLConnector.py +8 -5
- meerschaum/connectors/sql/_cli.py +12 -11
- meerschaum/connectors/sql/_create_engine.py +9 -168
- meerschaum/connectors/sql/_fetch.py +2 -18
- meerschaum/connectors/sql/_pipes.py +156 -190
- meerschaum/connectors/sql/_plugins.py +29 -0
- meerschaum/connectors/sql/_sql.py +46 -21
- meerschaum/connectors/sql/_users.py +29 -2
- meerschaum/connectors/sql/tables/__init__.py +1 -1
- meerschaum/connectors/valkey/_ValkeyConnector.py +2 -4
- meerschaum/connectors/valkey/_pipes.py +53 -26
- meerschaum/connectors/valkey/_plugins.py +2 -26
- meerschaum/core/Pipe/__init__.py +59 -19
- meerschaum/core/Pipe/_attributes.py +412 -90
- meerschaum/core/Pipe/_bootstrap.py +54 -24
- meerschaum/core/Pipe/_data.py +96 -18
- meerschaum/core/Pipe/_dtypes.py +48 -18
- meerschaum/core/Pipe/_edit.py +14 -4
- meerschaum/core/Pipe/_fetch.py +1 -1
- meerschaum/core/Pipe/_show.py +5 -5
- meerschaum/core/Pipe/_sync.py +118 -193
- meerschaum/core/Pipe/_verify.py +4 -4
- meerschaum/{plugins → core/Plugin}/_Plugin.py +9 -11
- meerschaum/core/Plugin/__init__.py +1 -1
- meerschaum/core/Token/_Token.py +220 -0
- meerschaum/core/Token/__init__.py +12 -0
- meerschaum/core/User/_User.py +34 -8
- meerschaum/core/User/__init__.py +9 -1
- meerschaum/core/__init__.py +1 -0
- meerschaum/jobs/_Job.py +3 -2
- meerschaum/jobs/__init__.py +3 -2
- meerschaum/jobs/systemd.py +1 -1
- meerschaum/models/__init__.py +35 -0
- meerschaum/models/pipes.py +247 -0
- meerschaum/models/tokens.py +38 -0
- meerschaum/models/users.py +26 -0
- meerschaum/plugins/__init__.py +22 -7
- meerschaum/plugins/bootstrap.py +2 -1
- meerschaum/utils/_get_pipes.py +68 -27
- meerschaum/utils/daemon/Daemon.py +2 -1
- meerschaum/utils/daemon/__init__.py +30 -2
- meerschaum/utils/dataframe.py +473 -81
- meerschaum/utils/debug.py +15 -15
- meerschaum/utils/dtypes/__init__.py +473 -34
- meerschaum/utils/dtypes/sql.py +368 -28
- meerschaum/utils/formatting/__init__.py +1 -1
- meerschaum/utils/formatting/_pipes.py +5 -4
- meerschaum/utils/formatting/_shell.py +11 -9
- meerschaum/utils/misc.py +246 -148
- meerschaum/utils/packages/__init__.py +10 -27
- meerschaum/utils/packages/_packages.py +41 -34
- meerschaum/utils/pipes.py +181 -0
- meerschaum/utils/process.py +1 -1
- meerschaum/utils/prompt.py +3 -1
- meerschaum/utils/schedule.py +2 -1
- meerschaum/utils/sql.py +121 -44
- meerschaum/utils/typing.py +1 -4
- meerschaum/utils/venv/_Venv.py +2 -2
- meerschaum/utils/venv/__init__.py +5 -7
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/METADATA +92 -96
- meerschaum-3.0.0rc2.dist-info/RECORD +283 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/WHEEL +1 -1
- meerschaum-3.0.0rc2.dist-info/licenses/NOTICE +2 -0
- meerschaum/api/models/_interfaces.py +0 -15
- meerschaum/api/models/_locations.py +0 -15
- meerschaum/api/models/_metrics.py +0 -15
- meerschaum/config/static/__init__.py +0 -186
- meerschaum-2.9.5.dist-info/RECORD +0 -263
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/top_level.txt +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/zip-safe +0 -0
@@ -8,11 +8,12 @@ Fetch and manipulate Pipes' attributes
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
|
11
|
+
import uuid
|
11
12
|
from datetime import timezone
|
12
13
|
|
13
14
|
import meerschaum as mrsm
|
14
15
|
from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
|
15
|
-
from meerschaum.utils.warnings import warn
|
16
|
+
from meerschaum.utils.warnings import warn, dprint
|
16
17
|
|
17
18
|
|
18
19
|
@property
|
@@ -48,21 +49,89 @@ def attributes(self) -> Dict[str, Any]:
|
|
48
49
|
return self._attributes
|
49
50
|
|
50
51
|
|
52
|
+
def get_parameters(
|
53
|
+
self,
|
54
|
+
apply_symlinks: bool = True,
|
55
|
+
refresh: bool = False,
|
56
|
+
debug: bool = False,
|
57
|
+
_visited: 'Optional[set[mrsm.Pipe]]' = None,
|
58
|
+
) -> Dict[str, Any]:
|
59
|
+
"""
|
60
|
+
Return the `parameters` dictionary of the pipe.
|
61
|
+
|
62
|
+
Parameters
|
63
|
+
----------
|
64
|
+
apply_symlinks: bool, default True
|
65
|
+
If `True`, resolve references to parameters from other pipes.
|
66
|
+
|
67
|
+
refresh: bool, default False
|
68
|
+
If `True`, pull the latest attributes for the pipe.
|
69
|
+
|
70
|
+
Returns
|
71
|
+
-------
|
72
|
+
The pipe's parameters dictionary.
|
73
|
+
"""
|
74
|
+
from meerschaum.config._patch import apply_patch_to_config
|
75
|
+
from meerschaum.config._read_config import search_and_substitute_config
|
76
|
+
|
77
|
+
if _visited is None:
|
78
|
+
_visited = {self}
|
79
|
+
|
80
|
+
if refresh:
|
81
|
+
self._invalidate_cache(hard=True)
|
82
|
+
|
83
|
+
raw_parameters = self.attributes.get('parameters', {})
|
84
|
+
ref_keys = raw_parameters.get('reference')
|
85
|
+
if not apply_symlinks:
|
86
|
+
return raw_parameters
|
87
|
+
|
88
|
+
if ref_keys:
|
89
|
+
try:
|
90
|
+
if debug:
|
91
|
+
dprint(f"Building reference pipe from keys: {ref_keys}")
|
92
|
+
ref_pipe = mrsm.Pipe(**ref_keys)
|
93
|
+
if ref_pipe in _visited:
|
94
|
+
warn(f"Circular reference detected in {self}: chain involves {ref_pipe}.")
|
95
|
+
return search_and_substitute_config(raw_parameters)
|
96
|
+
|
97
|
+
_visited.add(ref_pipe)
|
98
|
+
base_params = ref_pipe.get_parameters(_visited=_visited, debug=debug)
|
99
|
+
except Exception as e:
|
100
|
+
warn(f"Failed to resolve reference pipe for {self}: {e}")
|
101
|
+
base_params = {}
|
102
|
+
|
103
|
+
params_to_apply = {k: v for k, v in raw_parameters.items() if k != 'reference'}
|
104
|
+
parameters = apply_patch_to_config(base_params, params_to_apply)
|
105
|
+
else:
|
106
|
+
parameters = raw_parameters
|
107
|
+
|
108
|
+
from meerschaum.utils.pipes import replace_pipes_syntax
|
109
|
+
self._symlinks = {}
|
110
|
+
|
111
|
+
def recursive_replace(obj: Any, path: tuple) -> Any:
|
112
|
+
if isinstance(obj, dict):
|
113
|
+
return {k: recursive_replace(v, path + (k,)) for k, v in obj.items()}
|
114
|
+
if isinstance(obj, list):
|
115
|
+
return [recursive_replace(elem, path + (i,)) for i, elem in enumerate(obj)]
|
116
|
+
if isinstance(obj, str):
|
117
|
+
substituted_val = replace_pipes_syntax(obj)
|
118
|
+
if substituted_val != obj:
|
119
|
+
self._symlinks[path] = {
|
120
|
+
'original': obj,
|
121
|
+
'substituted': substituted_val,
|
122
|
+
}
|
123
|
+
return substituted_val
|
124
|
+
return obj
|
125
|
+
|
126
|
+
return search_and_substitute_config(recursive_replace(parameters, tuple()))
|
127
|
+
|
128
|
+
|
51
129
|
@property
|
52
130
|
def parameters(self) -> Optional[Dict[str, Any]]:
|
53
131
|
"""
|
54
132
|
Return the parameters dictionary of the pipe.
|
55
133
|
"""
|
56
|
-
|
57
|
-
self.attributes['parameters'] = {}
|
58
|
-
_parameters = self.attributes['parameters']
|
59
|
-
dt_col = _parameters.get('columns', {}).get('datetime', None)
|
60
|
-
dt_typ = _parameters.get('dtypes', {}).get(dt_col, None) if dt_col else None
|
61
|
-
if dt_col and not dt_typ:
|
62
|
-
if 'dtypes' not in _parameters:
|
63
|
-
self.attributes['parameters']['dtypes'] = {}
|
64
|
-
self.attributes['parameters']['dtypes'][dt_col] = 'datetime'
|
65
|
-
return self.attributes['parameters']
|
134
|
+
return self.get_parameters()
|
66
135
|
|
67
136
|
|
68
137
|
@parameters.setter
|
@@ -71,7 +140,9 @@ def parameters(self, parameters: Dict[str, Any]) -> None:
|
|
71
140
|
Set the parameters dictionary of the in-memory pipe.
|
72
141
|
Call `meerschaum.Pipe.edit()` to persist changes.
|
73
142
|
"""
|
74
|
-
self.
|
143
|
+
self._attributes['parameters'] = parameters
|
144
|
+
if '_parameters' in self.__dict__:
|
145
|
+
del self.__dict__['_parameters']
|
75
146
|
|
76
147
|
|
77
148
|
@property
|
@@ -79,12 +150,9 @@ def columns(self) -> Union[Dict[str, str], None]:
|
|
79
150
|
"""
|
80
151
|
Return the `columns` dictionary defined in `meerschaum.Pipe.parameters`.
|
81
152
|
"""
|
82
|
-
|
83
|
-
self.parameters['columns'] = {}
|
84
|
-
cols = self.parameters['columns']
|
153
|
+
cols = self.parameters.get('columns', {})
|
85
154
|
if not isinstance(cols, dict):
|
86
|
-
|
87
|
-
self.parameters['columns'] = cols
|
155
|
+
return {}
|
88
156
|
return {col_ix: col for col_ix, col in cols.items() if col}
|
89
157
|
|
90
158
|
|
@@ -99,7 +167,7 @@ def columns(self, _columns: Union[Dict[str, str], List[str]]) -> None:
|
|
99
167
|
if not isinstance(_columns, dict):
|
100
168
|
warn(f"{self}.columns must be a dictionary, received {type(_columns)}.")
|
101
169
|
return
|
102
|
-
self.
|
170
|
+
self.update_parameters({'columns': _columns}, persist=False)
|
103
171
|
|
104
172
|
|
105
173
|
@property
|
@@ -112,14 +180,12 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
|
|
112
180
|
if 'indexes' in self.parameters
|
113
181
|
else 'indices'
|
114
182
|
)
|
115
|
-
|
116
|
-
|
117
|
-
_indices = self.parameters[indices_key]
|
183
|
+
|
184
|
+
_indices = self.parameters.get(indices_key, {})
|
118
185
|
_columns = self.columns
|
119
186
|
dt_col = _columns.get('datetime', None)
|
120
187
|
if not isinstance(_indices, dict):
|
121
188
|
_indices = {}
|
122
|
-
self.parameters[indices_key] = _indices
|
123
189
|
unique_cols = list(set((
|
124
190
|
[dt_col]
|
125
191
|
if dt_col
|
@@ -158,7 +224,7 @@ def indices(self, _indices: Union[Dict[str, Union[str, List[str]]], List[str]])
|
|
158
224
|
if 'indexes' in self.parameters
|
159
225
|
else 'indices'
|
160
226
|
)
|
161
|
-
self.
|
227
|
+
self.update_parameters({indices_key: _indices}, persist=False)
|
162
228
|
|
163
229
|
|
164
230
|
@indexes.setter
|
@@ -174,41 +240,30 @@ def tags(self) -> Union[List[str], None]:
|
|
174
240
|
"""
|
175
241
|
If defined, return the `tags` list defined in `meerschaum.Pipe.parameters`.
|
176
242
|
"""
|
177
|
-
|
178
|
-
self.parameters['tags'] = []
|
179
|
-
return self.parameters['tags']
|
243
|
+
return self.parameters.get('tags', [])
|
180
244
|
|
181
245
|
|
182
246
|
@tags.setter
|
183
|
-
def tags(self, _tags: List[str
|
247
|
+
def tags(self, _tags: List[str]) -> None:
|
184
248
|
"""
|
185
249
|
Override the tags list of the in-memory pipe.
|
186
250
|
Call `meerschaum.Pipe.edit` to persist changes.
|
187
251
|
"""
|
188
252
|
from meerschaum.utils.warnings import error
|
189
|
-
from meerschaum.
|
253
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
190
254
|
negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
|
191
255
|
for t in _tags:
|
192
256
|
if t.startswith(negation_prefix):
|
193
257
|
error(f"Tags cannot begin with '{negation_prefix}'.")
|
194
|
-
self.
|
258
|
+
self.update_parameters({'tags': _tags}, persist=False)
|
195
259
|
|
196
260
|
|
197
261
|
@property
|
198
|
-
def dtypes(self) ->
|
262
|
+
def dtypes(self) -> Dict[str, Any]:
|
199
263
|
"""
|
200
264
|
If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
|
201
265
|
"""
|
202
|
-
|
203
|
-
from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
|
204
|
-
configured_dtypes = self.parameters.get('dtypes', {})
|
205
|
-
remote_dtypes = self.infer_dtypes(persist=False)
|
206
|
-
patched_dtypes = apply_patch_to_config(remote_dtypes, configured_dtypes)
|
207
|
-
return {
|
208
|
-
col: MRSM_ALIAS_DTYPES.get(typ, typ)
|
209
|
-
for col, typ in patched_dtypes.items()
|
210
|
-
if col and typ
|
211
|
-
}
|
266
|
+
return self.get_dtypes(refresh=False)
|
212
267
|
|
213
268
|
|
214
269
|
@dtypes.setter
|
@@ -217,7 +272,60 @@ def dtypes(self, _dtypes: Dict[str, Any]) -> None:
|
|
217
272
|
Override the dtypes dictionary of the in-memory pipe.
|
218
273
|
Call `meerschaum.Pipe.edit()` to persist changes.
|
219
274
|
"""
|
220
|
-
self.
|
275
|
+
self.update_parameters({'dtypes': _dtypes}, persist=False)
|
276
|
+
_ = self.__dict__.pop('_remote_dtypes', None)
|
277
|
+
_ = self.__dict__.pop('_remote_dtypes_timestamp', None)
|
278
|
+
|
279
|
+
|
280
|
+
def get_dtypes(
|
281
|
+
self,
|
282
|
+
infer: bool = True,
|
283
|
+
refresh: bool = False,
|
284
|
+
debug: bool = False,
|
285
|
+
) -> Dict[str, Any]:
|
286
|
+
"""
|
287
|
+
If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
|
288
|
+
|
289
|
+
|
290
|
+
Parameters
|
291
|
+
----------
|
292
|
+
infer: bool, default True
|
293
|
+
If `True`, include the implicit existing dtypes.
|
294
|
+
Else only return the explicitly configured dtypes (e.g. `Pipe.parameters['dtypes']`).
|
295
|
+
|
296
|
+
refresh: bool, default False
|
297
|
+
If `True`, invalidate any cache and return the latest known dtypes.
|
298
|
+
|
299
|
+
Returns
|
300
|
+
-------
|
301
|
+
A dictionary mapping column names to dtypes.
|
302
|
+
"""
|
303
|
+
import time
|
304
|
+
from meerschaum.config._patch import apply_patch_to_config
|
305
|
+
from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
|
306
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
307
|
+
parameters = self.get_parameters(refresh=refresh, debug=debug)
|
308
|
+
configured_dtypes = parameters.get('dtypes', {})
|
309
|
+
if debug:
|
310
|
+
dprint(f"Configured dtypes for {self}:")
|
311
|
+
mrsm.pprint(configured_dtypes)
|
312
|
+
|
313
|
+
remote_dtypes = self.infer_dtypes(persist=False, refresh=refresh, debug=debug)
|
314
|
+
patched_dtypes = apply_patch_to_config((remote_dtypes or {}), (configured_dtypes or {}))
|
315
|
+
|
316
|
+
dt_col = parameters.get('columns', {}).get('datetime', None)
|
317
|
+
primary_col = parameters.get('columns', {}).get('primary', None)
|
318
|
+
_dtypes = {
|
319
|
+
col: MRSM_ALIAS_DTYPES.get(typ, typ)
|
320
|
+
for col, typ in patched_dtypes.items()
|
321
|
+
if col and typ
|
322
|
+
}
|
323
|
+
if dt_col and dt_col not in configured_dtypes:
|
324
|
+
_dtypes[dt_col] = 'datetime'
|
325
|
+
if primary_col and parameters.get('autoincrement', False) and primary_col not in _dtypes:
|
326
|
+
_dtypes[primary_col] = 'int'
|
327
|
+
|
328
|
+
return _dtypes
|
221
329
|
|
222
330
|
|
223
331
|
@property
|
@@ -225,9 +333,7 @@ def upsert(self) -> bool:
|
|
225
333
|
"""
|
226
334
|
Return whether `upsert` is set for the pipe.
|
227
335
|
"""
|
228
|
-
|
229
|
-
self.parameters['upsert'] = False
|
230
|
-
return self.parameters['upsert']
|
336
|
+
return self.parameters.get('upsert', False)
|
231
337
|
|
232
338
|
|
233
339
|
@upsert.setter
|
@@ -235,7 +341,7 @@ def upsert(self, _upsert: bool) -> None:
|
|
235
341
|
"""
|
236
342
|
Set the `upsert` parameter for the pipe.
|
237
343
|
"""
|
238
|
-
self.
|
344
|
+
self.update_parameters({'upsert': _upsert}, persist=False)
|
239
345
|
|
240
346
|
|
241
347
|
@property
|
@@ -243,9 +349,7 @@ def static(self) -> bool:
|
|
243
349
|
"""
|
244
350
|
Return whether `static` is set for the pipe.
|
245
351
|
"""
|
246
|
-
|
247
|
-
self.parameters['static'] = False
|
248
|
-
return self.parameters['static']
|
352
|
+
return self.parameters.get('static', False)
|
249
353
|
|
250
354
|
|
251
355
|
@static.setter
|
@@ -253,7 +357,7 @@ def static(self, _static: bool) -> None:
|
|
253
357
|
"""
|
254
358
|
Set the `static` parameter for the pipe.
|
255
359
|
"""
|
256
|
-
self.
|
360
|
+
self.update_parameters({'static': _static}, persist=False)
|
257
361
|
|
258
362
|
|
259
363
|
@property
|
@@ -261,10 +365,7 @@ def autoincrement(self) -> bool:
|
|
261
365
|
"""
|
262
366
|
Return the `autoincrement` parameter for the pipe.
|
263
367
|
"""
|
264
|
-
|
265
|
-
self.parameters['autoincrement'] = False
|
266
|
-
|
267
|
-
return self.parameters['autoincrement']
|
368
|
+
return self.parameters.get('autoincrement', False)
|
268
369
|
|
269
370
|
|
270
371
|
@autoincrement.setter
|
@@ -272,7 +373,23 @@ def autoincrement(self, _autoincrement: bool) -> None:
|
|
272
373
|
"""
|
273
374
|
Set the `autoincrement` parameter for the pipe.
|
274
375
|
"""
|
275
|
-
self.
|
376
|
+
self.update_parameters({'autoincrement': _autoincrement}, persist=False)
|
377
|
+
|
378
|
+
|
379
|
+
@property
|
380
|
+
def autotime(self) -> bool:
|
381
|
+
"""
|
382
|
+
Return the `autotime` parameter for the pipe.
|
383
|
+
"""
|
384
|
+
return self.parameters.get('autotime', False)
|
385
|
+
|
386
|
+
|
387
|
+
@autotime.setter
|
388
|
+
def autotime(self, _autotime: bool) -> None:
|
389
|
+
"""
|
390
|
+
Set the `autotime` parameter for the pipe.
|
391
|
+
"""
|
392
|
+
self.update_parameters({'autotime': _autotime}, persist=False)
|
276
393
|
|
277
394
|
|
278
395
|
@property
|
@@ -280,18 +397,22 @@ def tzinfo(self) -> Union[None, timezone]:
|
|
280
397
|
"""
|
281
398
|
Return `timezone.utc` if the pipe is timezone-aware.
|
282
399
|
"""
|
283
|
-
|
284
|
-
|
285
|
-
return None
|
400
|
+
if '_tzinfo' in self.__dict__:
|
401
|
+
return self.__dict__['_tzinfo']
|
286
402
|
|
287
|
-
|
288
|
-
|
289
|
-
|
403
|
+
_tzinfo = None
|
404
|
+
dt_col = self.columns.get('datetime', None)
|
405
|
+
dt_typ = str(self.dtypes.get(dt_col, 'datetime')) if dt_col else None
|
406
|
+
if self.autotime:
|
407
|
+
ts_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
|
408
|
+
ts_typ = self.dtypes.get(ts_col, 'datetime')
|
409
|
+
dt_typ = ts_typ
|
290
410
|
|
291
|
-
if dt_typ == '
|
292
|
-
|
411
|
+
if dt_typ and 'utc' in dt_typ.lower() or dt_typ == 'datetime':
|
412
|
+
_tzinfo = timezone.utc
|
293
413
|
|
294
|
-
|
414
|
+
self._tzinfo = _tzinfo
|
415
|
+
return _tzinfo
|
295
416
|
|
296
417
|
|
297
418
|
@property
|
@@ -299,10 +420,7 @@ def enforce(self) -> bool:
|
|
299
420
|
"""
|
300
421
|
Return the `enforce` parameter for the pipe.
|
301
422
|
"""
|
302
|
-
|
303
|
-
self.parameters['enforce'] = True
|
304
|
-
|
305
|
-
return self.parameters['enforce']
|
423
|
+
return self.parameters.get('enforce', True)
|
306
424
|
|
307
425
|
|
308
426
|
@enforce.setter
|
@@ -310,7 +428,7 @@ def enforce(self, _enforce: bool) -> None:
|
|
310
428
|
"""
|
311
429
|
Set the `enforce` parameter for the pipe.
|
312
430
|
"""
|
313
|
-
self.
|
431
|
+
self.update_parameters({'enforce': _enforce}, persist=False)
|
314
432
|
|
315
433
|
|
316
434
|
@property
|
@@ -318,10 +436,7 @@ def null_indices(self) -> bool:
|
|
318
436
|
"""
|
319
437
|
Return the `null_indices` parameter for the pipe.
|
320
438
|
"""
|
321
|
-
|
322
|
-
self.parameters['null_indices'] = True
|
323
|
-
|
324
|
-
return self.parameters['null_indices']
|
439
|
+
return self.parameters.get('null_indices', True)
|
325
440
|
|
326
441
|
|
327
442
|
@null_indices.setter
|
@@ -329,7 +444,23 @@ def null_indices(self, _null_indices: bool) -> None:
|
|
329
444
|
"""
|
330
445
|
Set the `null_indices` parameter for the pipe.
|
331
446
|
"""
|
332
|
-
self.
|
447
|
+
self.update_parameters({'null_indices': _null_indices}, persist=False)
|
448
|
+
|
449
|
+
|
450
|
+
@property
|
451
|
+
def mixed_numerics(self) -> bool:
|
452
|
+
"""
|
453
|
+
Return the `mixed_numerics` parameter for the pipe.
|
454
|
+
"""
|
455
|
+
return self.parameters.get('mixed_numerics', True)
|
456
|
+
|
457
|
+
|
458
|
+
@mixed_numerics.setter
|
459
|
+
def mixed_numerics(self, _mixed_numerics: bool) -> None:
|
460
|
+
"""
|
461
|
+
Set the `mixed_numerics` parameter for the pipe.
|
462
|
+
"""
|
463
|
+
self.update_parameters({'mixed_numerics': _mixed_numerics}, persist=False)
|
333
464
|
|
334
465
|
|
335
466
|
def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
|
@@ -357,7 +488,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
|
|
357
488
|
>>> pipe.get_columns('value', error=True)
|
358
489
|
Exception: 🛑 Missing 'value' column for Pipe('test', 'test').
|
359
490
|
"""
|
360
|
-
from meerschaum.utils.warnings import error as _error
|
491
|
+
from meerschaum.utils.warnings import error as _error
|
361
492
|
if not args:
|
362
493
|
args = tuple(self.columns.keys())
|
363
494
|
col_names = []
|
@@ -367,7 +498,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
|
|
367
498
|
col_name = self.columns[col]
|
368
499
|
if col_name is None and error:
|
369
500
|
_error(f"Please define the name of the '{col}' column for {self}.")
|
370
|
-
except Exception
|
501
|
+
except Exception:
|
371
502
|
col_name = None
|
372
503
|
if col_name is None and error:
|
373
504
|
_error(f"Missing '{col}'" + f" column for {self}.")
|
@@ -409,16 +540,18 @@ def get_columns_types(
|
|
409
540
|
"""
|
410
541
|
import time
|
411
542
|
from meerschaum.connectors import get_connector_plugin
|
412
|
-
from meerschaum.
|
413
|
-
from meerschaum.utils.warnings import dprint
|
543
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
414
544
|
|
415
545
|
now = time.perf_counter()
|
416
|
-
cache_seconds =
|
417
|
-
|
418
|
-
|
546
|
+
cache_seconds = (
|
547
|
+
mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
|
548
|
+
if self.static
|
549
|
+
else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
|
550
|
+
)
|
419
551
|
if refresh:
|
420
552
|
_ = self.__dict__.pop('_columns_types_timestamp', None)
|
421
553
|
_ = self.__dict__.pop('_columns_types', None)
|
554
|
+
|
422
555
|
_columns_types = self.__dict__.get('_columns_types', None)
|
423
556
|
if _columns_types:
|
424
557
|
columns_types_timestamp = self.__dict__.get('_columns_types_timestamp', None)
|
@@ -454,14 +587,13 @@ def get_columns_indices(
|
|
454
587
|
"""
|
455
588
|
import time
|
456
589
|
from meerschaum.connectors import get_connector_plugin
|
457
|
-
from meerschaum.
|
458
|
-
from meerschaum.utils.warnings import dprint
|
590
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
459
591
|
|
460
592
|
now = time.perf_counter()
|
461
593
|
cache_seconds = (
|
462
|
-
|
594
|
+
mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
|
463
595
|
if self.static
|
464
|
-
else
|
596
|
+
else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
|
465
597
|
)
|
466
598
|
if refresh:
|
467
599
|
_ = self.__dict__.pop('_columns_indices_timestamp', None)
|
@@ -509,7 +641,7 @@ def get_id(self, **kw: Any) -> Union[int, None]:
|
|
509
641
|
|
510
642
|
|
511
643
|
@property
|
512
|
-
def id(self) -> Union[int, None]:
|
644
|
+
def id(self) -> Union[int, str, uuid.UUID, None]:
|
513
645
|
"""
|
514
646
|
Fetch and cache a pipe's ID.
|
515
647
|
"""
|
@@ -534,12 +666,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
|
|
534
666
|
-------
|
535
667
|
Either a string or `None`.
|
536
668
|
"""
|
537
|
-
from meerschaum.utils.debug import dprint
|
538
669
|
if debug:
|
539
670
|
dprint('Attempting to determine the value column...')
|
540
671
|
try:
|
541
672
|
val_name = self.get_columns('value')
|
542
|
-
except Exception
|
673
|
+
except Exception:
|
543
674
|
val_name = None
|
544
675
|
if val_name is not None:
|
545
676
|
if debug:
|
@@ -553,11 +684,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
|
|
553
684
|
return None
|
554
685
|
try:
|
555
686
|
dt_name = self.get_columns('datetime', error=False)
|
556
|
-
except Exception
|
687
|
+
except Exception:
|
557
688
|
dt_name = None
|
558
689
|
try:
|
559
690
|
id_name = self.get_columns('id', errors=False)
|
560
|
-
except Exception
|
691
|
+
except Exception:
|
561
692
|
id_name = None
|
562
693
|
|
563
694
|
if debug:
|
@@ -596,6 +727,7 @@ def parents(self) -> List[mrsm.Pipe]:
|
|
596
727
|
"""
|
597
728
|
if 'parents' not in self.parameters:
|
598
729
|
return []
|
730
|
+
|
599
731
|
from meerschaum.utils.warnings import warn
|
600
732
|
_parents_keys = self.parameters['parents']
|
601
733
|
if not isinstance(_parents_keys, list):
|
@@ -634,6 +766,7 @@ def children(self) -> List[mrsm.Pipe]:
|
|
634
766
|
"""
|
635
767
|
if 'children' not in self.parameters:
|
636
768
|
return []
|
769
|
+
|
637
770
|
from meerschaum.utils.warnings import warn
|
638
771
|
_children_keys = self.parameters['children']
|
639
772
|
if not isinstance(_children_keys, list):
|
@@ -714,7 +847,7 @@ def target(self, _target: str) -> None:
|
|
714
847
|
Override the target of the in-memory pipe.
|
715
848
|
Call `meerschaum.Pipe.edit` to persist changes.
|
716
849
|
"""
|
717
|
-
self.
|
850
|
+
self.update_parameters({'target': _target}, persist=False)
|
718
851
|
|
719
852
|
|
720
853
|
def guess_datetime(self) -> Union[str, None]:
|
@@ -755,3 +888,192 @@ def get_indices(self) -> Dict[str, str]:
|
|
755
888
|
result = {}
|
756
889
|
|
757
890
|
return result
|
891
|
+
|
892
|
+
|
893
|
+
def update_parameters(
|
894
|
+
self,
|
895
|
+
parameters_patch: Dict[str, Any],
|
896
|
+
persist: bool = True,
|
897
|
+
debug: bool = False,
|
898
|
+
) -> mrsm.SuccessTuple:
|
899
|
+
"""
|
900
|
+
Apply a patch to a pipe's `parameters` dictionary.
|
901
|
+
|
902
|
+
Parameters
|
903
|
+
----------
|
904
|
+
parameters_patch: Dict[str, Any]
|
905
|
+
The patch to be applied to `Pipe.parameters`.
|
906
|
+
|
907
|
+
persist: bool, default True
|
908
|
+
If `True`, call `Pipe.edit()` to persist the new parameters.
|
909
|
+
"""
|
910
|
+
from meerschaum.config import apply_patch_to_config
|
911
|
+
if 'parameters' not in self._attributes:
|
912
|
+
self._attributes['parameters'] = {}
|
913
|
+
|
914
|
+
if '_parameters' not in self.__dict__:
|
915
|
+
self._parameters = {}
|
916
|
+
|
917
|
+
self._attributes['parameters'] = apply_patch_to_config(
|
918
|
+
self._attributes['parameters'],
|
919
|
+
parameters_patch,
|
920
|
+
)
|
921
|
+
|
922
|
+
if self.temporary:
|
923
|
+
persist = False
|
924
|
+
|
925
|
+
if not persist:
|
926
|
+
return True, "Success"
|
927
|
+
|
928
|
+
return self.edit(debug=debug)
|
929
|
+
|
930
|
+
|
931
|
+
def get_precision(self, debug: bool = False) -> Dict[str, Union[str, int]]:
|
932
|
+
"""
|
933
|
+
Return the timestamp precision unit and interval for the `datetime` axis.
|
934
|
+
"""
|
935
|
+
from meerschaum.utils.dtypes import (
|
936
|
+
MRSM_PRECISION_UNITS_SCALARS,
|
937
|
+
MRSM_PRECISION_UNITS_ALIASES,
|
938
|
+
MRSM_PD_DTYPES,
|
939
|
+
are_dtypes_equal,
|
940
|
+
)
|
941
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
942
|
+
|
943
|
+
if self.__dict__.get('_precision', None):
|
944
|
+
if debug:
|
945
|
+
dprint(f"Returning cached precision: {self._precision}")
|
946
|
+
return self._precision
|
947
|
+
|
948
|
+
parameters = self.parameters
|
949
|
+
_precision = parameters.get('precision', {})
|
950
|
+
if isinstance(_precision, str):
|
951
|
+
_precision = {'unit': _precision}
|
952
|
+
default_precision_unit = STATIC_CONFIG['dtypes']['datetime']['default_precision_unit']
|
953
|
+
|
954
|
+
if not _precision:
|
955
|
+
|
956
|
+
dt_col = parameters.get('columns', {}).get('datetime', None)
|
957
|
+
if not dt_col and self.autotime:
|
958
|
+
dt_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
|
959
|
+
if not dt_col:
|
960
|
+
if debug:
|
961
|
+
dprint(f"No datetime axis, returning default precision '{default_precision_unit}'.")
|
962
|
+
return {'unit': default_precision_unit}
|
963
|
+
|
964
|
+
dt_typ = self.dtypes.get(dt_col, 'datetime')
|
965
|
+
if are_dtypes_equal(dt_typ, 'datetime'):
|
966
|
+
if dt_typ == 'datetime':
|
967
|
+
dt_typ = MRSM_PD_DTYPES['datetime']
|
968
|
+
if debug:
|
969
|
+
dprint(f"Datetime type is `datetime`, assuming {dt_typ} precision.")
|
970
|
+
|
971
|
+
_precision = {
|
972
|
+
'unit': (
|
973
|
+
dt_typ
|
974
|
+
.split('[', maxsplit=1)[-1]
|
975
|
+
.split(',', maxsplit=1)[0]
|
976
|
+
.split(' ', maxsplit=1)[0]
|
977
|
+
).rstrip(']')
|
978
|
+
}
|
979
|
+
|
980
|
+
if debug:
|
981
|
+
dprint(f"Extracted precision '{_precision['unit']}' from type '{dt_typ}'.")
|
982
|
+
|
983
|
+
elif are_dtypes_equal(dt_typ, 'int'):
|
984
|
+
_precision = {
|
985
|
+
'unit': (
|
986
|
+
'second'
|
987
|
+
if '32' in dt_typ
|
988
|
+
else default_precision_unit
|
989
|
+
)
|
990
|
+
}
|
991
|
+
elif are_dtypes_equal(dt_typ, 'date'):
|
992
|
+
if debug:
|
993
|
+
dprint("Datetime axis is 'date', falling back to 'day' precision.")
|
994
|
+
_precision = {'unit': 'day'}
|
995
|
+
|
996
|
+
precision_unit = _precision.get('unit', default_precision_unit)
|
997
|
+
precision_interval = _precision.get('interval', None)
|
998
|
+
true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
|
999
|
+
if true_precision_unit is None:
|
1000
|
+
if debug:
|
1001
|
+
dprint(f"No precision could be determined, falling back to '{default_precision_unit}'.")
|
1002
|
+
true_precision_unit = default_precision_unit
|
1003
|
+
|
1004
|
+
if true_precision_unit not in MRSM_PRECISION_UNITS_SCALARS:
|
1005
|
+
from meerschaum.utils.misc import items_str
|
1006
|
+
raise ValueError(
|
1007
|
+
f"Invalid precision unit '{true_precision_unit}'.\n"
|
1008
|
+
"Accepted values are "
|
1009
|
+
f"{items_str(list(MRSM_PRECISION_UNITS_SCALARS) + list(MRSM_PRECISION_UNITS_ALIASES))}."
|
1010
|
+
)
|
1011
|
+
|
1012
|
+
self._precision = {'unit': true_precision_unit}
|
1013
|
+
if precision_interval:
|
1014
|
+
self._precision['interval'] = precision_interval
|
1015
|
+
return self._precision
|
1016
|
+
|
1017
|
+
|
1018
|
+
@property
|
1019
|
+
def precision(self) -> Dict[str, Union[str, int]]:
|
1020
|
+
"""
|
1021
|
+
Return the configured or detected precision.
|
1022
|
+
"""
|
1023
|
+
return self.get_precision()
|
1024
|
+
|
1025
|
+
|
1026
|
+
@precision.setter
|
1027
|
+
def precision(self, _precision: Union[str, Dict[str, Union[str, int]]]) -> None:
|
1028
|
+
"""
|
1029
|
+
Update the `precision` parameter.
|
1030
|
+
"""
|
1031
|
+
existing_precision = self._attributes.get('parameters', {}).get('precision', None)
|
1032
|
+
if isinstance(existing_precision, str):
|
1033
|
+
existing_precision = {'unit': existing_precision}
|
1034
|
+
|
1035
|
+
true_precision = (
|
1036
|
+
_precision
|
1037
|
+
if isinstance(_precision, dict)
|
1038
|
+
else {
|
1039
|
+
'unit': _precision,
|
1040
|
+
**(
|
1041
|
+
{
|
1042
|
+
'interval': existing_precision['interval'],
|
1043
|
+
} if existing_precision else {}
|
1044
|
+
)
|
1045
|
+
}
|
1046
|
+
)
|
1047
|
+
|
1048
|
+
self.update_parameters({'precision': true_precision}, persist=False)
|
1049
|
+
_ = self.__dict__.pop('_precision', None)
|
1050
|
+
|
1051
|
+
|
1052
|
+
def _invalidate_cache(
|
1053
|
+
self,
|
1054
|
+
hard: bool = False,
|
1055
|
+
debug: bool = False,
|
1056
|
+
) -> None:
|
1057
|
+
"""
|
1058
|
+
Invalidate temporary metadata cache.
|
1059
|
+
|
1060
|
+
Parameters
|
1061
|
+
----------
|
1062
|
+
hard: bool, default False
|
1063
|
+
If `True`, clear all temporary cache.
|
1064
|
+
Otherwise only clear soft cache.
|
1065
|
+
"""
|
1066
|
+
if debug:
|
1067
|
+
dprint(f"Invalidating {'some' if not hard else 'all'} cache for {self}.")
|
1068
|
+
|
1069
|
+
self._exists = None
|
1070
|
+
self._sync_ts = None
|
1071
|
+
|
1072
|
+
if not hard:
|
1073
|
+
return
|
1074
|
+
|
1075
|
+
_ = self.__dict__.pop('_parameters', None)
|
1076
|
+
_ = self.__dict__.pop('_precision', None)
|
1077
|
+
self._columns_types_timestamp = None
|
1078
|
+
self._columns_types = None
|
1079
|
+
self._attributes_sync_time = None
|