meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +5 -2
- meerschaum/_internal/__init__.py +1 -0
- meerschaum/_internal/arguments/_parse_arguments.py +4 -4
- meerschaum/_internal/arguments/_parser.py +33 -4
- meerschaum/_internal/cli/__init__.py +6 -0
- meerschaum/_internal/cli/daemons.py +103 -0
- meerschaum/_internal/cli/entry.py +220 -0
- meerschaum/_internal/cli/workers.py +435 -0
- meerschaum/_internal/docs/index.py +48 -2
- meerschaum/_internal/entry.py +50 -14
- meerschaum/_internal/shell/Shell.py +121 -29
- meerschaum/_internal/shell/__init__.py +4 -1
- meerschaum/_internal/static.py +359 -0
- meerschaum/_internal/term/TermPageHandler.py +1 -2
- meerschaum/_internal/term/__init__.py +40 -6
- meerschaum/_internal/term/tools.py +33 -8
- meerschaum/actions/__init__.py +6 -4
- meerschaum/actions/api.py +53 -13
- meerschaum/actions/attach.py +1 -0
- meerschaum/actions/bootstrap.py +8 -8
- meerschaum/actions/delete.py +4 -2
- meerschaum/actions/edit.py +171 -25
- meerschaum/actions/login.py +8 -8
- meerschaum/actions/register.py +143 -6
- meerschaum/actions/reload.py +22 -5
- meerschaum/actions/restart.py +14 -0
- meerschaum/actions/show.py +184 -31
- meerschaum/actions/start.py +166 -17
- meerschaum/actions/stop.py +38 -2
- meerschaum/actions/sync.py +7 -2
- meerschaum/actions/tag.py +9 -8
- meerschaum/actions/verify.py +5 -8
- meerschaum/api/__init__.py +45 -15
- meerschaum/api/_events.py +46 -4
- meerschaum/api/_oauth2.py +162 -9
- meerschaum/api/_tokens.py +102 -0
- meerschaum/api/dash/__init__.py +0 -3
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/custom.py +4 -3
- meerschaum/api/dash/callbacks/dashboard.py +198 -118
- meerschaum/api/dash/callbacks/jobs.py +14 -7
- meerschaum/api/dash/callbacks/login.py +10 -1
- meerschaum/api/dash/callbacks/pipes.py +194 -14
- meerschaum/api/dash/callbacks/plugins.py +0 -1
- meerschaum/api/dash/callbacks/register.py +10 -3
- meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
- meerschaum/api/dash/callbacks/tokens.py +389 -0
- meerschaum/api/dash/components.py +36 -15
- meerschaum/api/dash/jobs.py +1 -1
- meerschaum/api/dash/keys.py +35 -93
- meerschaum/api/dash/pages/__init__.py +2 -1
- meerschaum/api/dash/pages/dashboard.py +1 -20
- meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/dash/pages/pipes.py +16 -5
- meerschaum/api/dash/pages/settings/password_reset.py +1 -1
- meerschaum/api/dash/pages/tokens.py +53 -0
- meerschaum/api/dash/pipes.py +382 -95
- meerschaum/api/dash/sessions.py +12 -0
- meerschaum/api/dash/tokens.py +603 -0
- meerschaum/api/dash/websockets.py +1 -1
- meerschaum/api/dash/webterm.py +18 -6
- meerschaum/api/models/__init__.py +23 -3
- meerschaum/api/models/_actions.py +22 -0
- meerschaum/api/models/_pipes.py +91 -7
- meerschaum/api/models/_tokens.py +81 -0
- meerschaum/api/resources/static/js/terminado.js +3 -0
- meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
- meerschaum/api/resources/templates/termpage.html +13 -0
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +3 -4
- meerschaum/api/routes/_connectors.py +3 -7
- meerschaum/api/routes/_jobs.py +26 -35
- meerschaum/api/routes/_login.py +120 -15
- meerschaum/api/routes/_misc.py +5 -10
- meerschaum/api/routes/_pipes.py +178 -143
- meerschaum/api/routes/_plugins.py +38 -28
- meerschaum/api/routes/_tokens.py +236 -0
- meerschaum/api/routes/_users.py +47 -35
- meerschaum/api/routes/_version.py +3 -3
- meerschaum/api/routes/_webterm.py +3 -3
- meerschaum/config/__init__.py +100 -30
- meerschaum/config/_default.py +132 -64
- meerschaum/config/_edit.py +38 -32
- meerschaum/config/_formatting.py +2 -0
- meerschaum/config/_patch.py +10 -8
- meerschaum/config/_paths.py +133 -13
- meerschaum/config/_read_config.py +87 -36
- meerschaum/config/_sync.py +6 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/environment.py +262 -0
- meerschaum/config/stack/__init__.py +37 -15
- meerschaum/config/static.py +18 -0
- meerschaum/connectors/_Connector.py +11 -6
- meerschaum/connectors/__init__.py +41 -22
- meerschaum/connectors/api/_APIConnector.py +34 -6
- meerschaum/connectors/api/_actions.py +2 -2
- meerschaum/connectors/api/_jobs.py +12 -1
- meerschaum/connectors/api/_login.py +33 -7
- meerschaum/connectors/api/_misc.py +2 -2
- meerschaum/connectors/api/_pipes.py +23 -32
- meerschaum/connectors/api/_plugins.py +2 -2
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/api/_tokens.py +146 -0
- meerschaum/connectors/api/_users.py +70 -58
- meerschaum/connectors/instance/_InstanceConnector.py +83 -0
- meerschaum/connectors/instance/__init__.py +10 -0
- meerschaum/connectors/instance/_pipes.py +442 -0
- meerschaum/connectors/instance/_plugins.py +159 -0
- meerschaum/connectors/instance/_tokens.py +317 -0
- meerschaum/connectors/instance/_users.py +188 -0
- meerschaum/connectors/parse.py +5 -2
- meerschaum/connectors/sql/_SQLConnector.py +22 -5
- meerschaum/connectors/sql/_cli.py +12 -11
- meerschaum/connectors/sql/_create_engine.py +12 -168
- meerschaum/connectors/sql/_fetch.py +2 -18
- meerschaum/connectors/sql/_pipes.py +295 -278
- meerschaum/connectors/sql/_plugins.py +29 -0
- meerschaum/connectors/sql/_sql.py +46 -21
- meerschaum/connectors/sql/_users.py +36 -2
- meerschaum/connectors/sql/tables/__init__.py +254 -122
- meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
- meerschaum/connectors/valkey/_pipes.py +60 -31
- meerschaum/connectors/valkey/_plugins.py +2 -26
- meerschaum/core/Pipe/__init__.py +115 -85
- meerschaum/core/Pipe/_attributes.py +425 -124
- meerschaum/core/Pipe/_bootstrap.py +54 -24
- meerschaum/core/Pipe/_cache.py +555 -0
- meerschaum/core/Pipe/_clear.py +0 -11
- meerschaum/core/Pipe/_data.py +96 -68
- meerschaum/core/Pipe/_deduplicate.py +0 -13
- meerschaum/core/Pipe/_delete.py +12 -21
- meerschaum/core/Pipe/_drop.py +11 -23
- meerschaum/core/Pipe/_dtypes.py +49 -19
- meerschaum/core/Pipe/_edit.py +14 -4
- meerschaum/core/Pipe/_fetch.py +1 -1
- meerschaum/core/Pipe/_index.py +8 -14
- meerschaum/core/Pipe/_show.py +5 -5
- meerschaum/core/Pipe/_sync.py +123 -204
- meerschaum/core/Pipe/_verify.py +4 -4
- meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
- meerschaum/core/Plugin/__init__.py +1 -1
- meerschaum/core/Token/_Token.py +220 -0
- meerschaum/core/Token/__init__.py +12 -0
- meerschaum/core/User/_User.py +35 -10
- meerschaum/core/User/__init__.py +9 -1
- meerschaum/core/__init__.py +1 -0
- meerschaum/jobs/_Executor.py +88 -4
- meerschaum/jobs/_Job.py +149 -38
- meerschaum/jobs/__init__.py +3 -2
- meerschaum/jobs/systemd.py +8 -3
- meerschaum/models/__init__.py +35 -0
- meerschaum/models/pipes.py +247 -0
- meerschaum/models/tokens.py +38 -0
- meerschaum/models/users.py +26 -0
- meerschaum/plugins/__init__.py +301 -88
- meerschaum/plugins/bootstrap.py +510 -4
- meerschaum/utils/_get_pipes.py +97 -30
- meerschaum/utils/daemon/Daemon.py +199 -43
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
- meerschaum/utils/daemon/RotatingFile.py +63 -36
- meerschaum/utils/daemon/StdinFile.py +53 -13
- meerschaum/utils/daemon/__init__.py +47 -6
- meerschaum/utils/daemon/_names.py +6 -3
- meerschaum/utils/dataframe.py +479 -81
- meerschaum/utils/debug.py +49 -19
- meerschaum/utils/dtypes/__init__.py +476 -34
- meerschaum/utils/dtypes/sql.py +369 -29
- meerschaum/utils/formatting/__init__.py +5 -2
- meerschaum/utils/formatting/_jobs.py +1 -1
- meerschaum/utils/formatting/_pipes.py +52 -50
- meerschaum/utils/formatting/_pprint.py +1 -0
- meerschaum/utils/formatting/_shell.py +44 -18
- meerschaum/utils/misc.py +268 -186
- meerschaum/utils/packages/__init__.py +25 -40
- meerschaum/utils/packages/_packages.py +42 -34
- meerschaum/utils/pipes.py +213 -0
- meerschaum/utils/process.py +2 -2
- meerschaum/utils/prompt.py +175 -144
- meerschaum/utils/schedule.py +2 -1
- meerschaum/utils/sql.py +134 -47
- meerschaum/utils/threading.py +42 -0
- meerschaum/utils/typing.py +1 -4
- meerschaum/utils/venv/_Venv.py +2 -2
- meerschaum/utils/venv/__init__.py +7 -7
- meerschaum/utils/warnings.py +19 -13
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
- meerschaum-3.0.0.dist-info/RECORD +289 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
- meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
- meerschaum/api/models/_interfaces.py +0 -15
- meerschaum/api/models/_locations.py +0 -15
- meerschaum/api/models/_metrics.py +0 -15
- meerschaum/config/_environment.py +0 -145
- meerschaum/config/static/__init__.py +0 -186
- meerschaum-2.9.5.dist-info/RECORD +0 -263
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
@@ -8,11 +8,12 @@ Fetch and manipulate Pipes' attributes
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
|
11
|
+
import uuid
|
11
12
|
from datetime import timezone
|
12
13
|
|
13
14
|
import meerschaum as mrsm
|
14
15
|
from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
|
15
|
-
from meerschaum.utils.warnings import warn
|
16
|
+
from meerschaum.utils.warnings import warn, dprint
|
16
17
|
|
17
18
|
|
18
19
|
@property
|
@@ -21,57 +22,129 @@ def attributes(self) -> Dict[str, Any]:
|
|
21
22
|
Return a dictionary of a pipe's keys and parameters.
|
22
23
|
These values are reflected directly from the pipes table of the instance.
|
23
24
|
"""
|
24
|
-
import time
|
25
25
|
from meerschaum.config import get_config
|
26
26
|
from meerschaum.config._patch import apply_patch_to_config
|
27
27
|
from meerschaum.utils.venv import Venv
|
28
28
|
from meerschaum.connectors import get_connector_plugin
|
29
|
+
from meerschaum.utils.dtypes import get_current_timestamp
|
29
30
|
|
30
31
|
timeout_seconds = get_config('pipes', 'attributes', 'local_cache_timeout_seconds')
|
31
32
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
now = time.perf_counter()
|
36
|
-
last_refresh = self.__dict__.get('_attributes_sync_time', None)
|
33
|
+
now = get_current_timestamp('ms', as_int=True) / 1000
|
34
|
+
_attributes_sync_time = self._get_cached_value('_attributes_sync_time', debug=self.debug)
|
37
35
|
timed_out = (
|
38
|
-
|
36
|
+
_attributes_sync_time is None
|
39
37
|
or
|
40
|
-
(timeout_seconds is not None and (now -
|
38
|
+
(timeout_seconds is not None and (now - _attributes_sync_time) >= timeout_seconds)
|
41
39
|
)
|
42
40
|
if not self.temporary and timed_out:
|
43
|
-
self._attributes_sync_time =
|
44
|
-
local_attributes = self.
|
41
|
+
self._cache_value('_attributes_sync_time', now, memory_only=True, debug=self.debug)
|
42
|
+
local_attributes = self._get_cached_value('attributes', debug=self.debug) or {}
|
45
43
|
with Venv(get_connector_plugin(self.instance_connector)):
|
46
44
|
instance_attributes = self.instance_connector.get_pipe_attributes(self)
|
47
|
-
|
45
|
+
|
46
|
+
self._cache_value(
|
47
|
+
'attributes',
|
48
|
+
apply_patch_to_config(instance_attributes, local_attributes),
|
49
|
+
memory_only=True,
|
50
|
+
debug=self.debug,
|
51
|
+
)
|
52
|
+
|
48
53
|
return self._attributes
|
49
54
|
|
50
55
|
|
56
|
+
def get_parameters(
|
57
|
+
self,
|
58
|
+
apply_symlinks: bool = True,
|
59
|
+
refresh: bool = False,
|
60
|
+
debug: bool = False,
|
61
|
+
_visited: 'Optional[set[mrsm.Pipe]]' = None,
|
62
|
+
) -> Dict[str, Any]:
|
63
|
+
"""
|
64
|
+
Return the `parameters` dictionary of the pipe.
|
65
|
+
|
66
|
+
Parameters
|
67
|
+
----------
|
68
|
+
apply_symlinks: bool, default True
|
69
|
+
If `True`, resolve references to parameters from other pipes.
|
70
|
+
|
71
|
+
refresh: bool, default False
|
72
|
+
If `True`, pull the latest attributes for the pipe.
|
73
|
+
|
74
|
+
Returns
|
75
|
+
-------
|
76
|
+
The pipe's parameters dictionary.
|
77
|
+
"""
|
78
|
+
from meerschaum.config._patch import apply_patch_to_config
|
79
|
+
from meerschaum.config._read_config import search_and_substitute_config
|
80
|
+
|
81
|
+
if _visited is None:
|
82
|
+
_visited = {self}
|
83
|
+
|
84
|
+
if refresh:
|
85
|
+
_ = self._invalidate_cache(hard=True)
|
86
|
+
|
87
|
+
raw_parameters = self.attributes.get('parameters', {})
|
88
|
+
ref_keys = raw_parameters.get('reference')
|
89
|
+
if not apply_symlinks:
|
90
|
+
return raw_parameters
|
91
|
+
|
92
|
+
if ref_keys:
|
93
|
+
try:
|
94
|
+
if debug:
|
95
|
+
dprint(f"Building reference pipe from keys: {ref_keys}")
|
96
|
+
ref_pipe = mrsm.Pipe(**ref_keys)
|
97
|
+
if ref_pipe in _visited:
|
98
|
+
warn(f"Circular reference detected in {self}: chain involves {ref_pipe}.")
|
99
|
+
return search_and_substitute_config(raw_parameters)
|
100
|
+
|
101
|
+
_visited.add(ref_pipe)
|
102
|
+
base_params = ref_pipe.get_parameters(_visited=_visited, debug=debug)
|
103
|
+
except Exception as e:
|
104
|
+
warn(f"Failed to resolve reference pipe for {self}: {e}")
|
105
|
+
base_params = {}
|
106
|
+
|
107
|
+
params_to_apply = {k: v for k, v in raw_parameters.items() if k != 'reference'}
|
108
|
+
parameters = apply_patch_to_config(base_params, params_to_apply)
|
109
|
+
else:
|
110
|
+
parameters = raw_parameters
|
111
|
+
|
112
|
+
from meerschaum.utils.pipes import replace_pipes_syntax
|
113
|
+
self._symlinks = {}
|
114
|
+
|
115
|
+
def recursive_replace(obj: Any, path: tuple) -> Any:
|
116
|
+
if isinstance(obj, dict):
|
117
|
+
return {k: recursive_replace(v, path + (k,)) for k, v in obj.items()}
|
118
|
+
if isinstance(obj, list):
|
119
|
+
return [recursive_replace(elem, path + (i,)) for i, elem in enumerate(obj)]
|
120
|
+
if isinstance(obj, str):
|
121
|
+
substituted_val = replace_pipes_syntax(obj)
|
122
|
+
if substituted_val != obj:
|
123
|
+
self._symlinks[path] = {
|
124
|
+
'original': obj,
|
125
|
+
'substituted': substituted_val,
|
126
|
+
}
|
127
|
+
return substituted_val
|
128
|
+
return obj
|
129
|
+
|
130
|
+
return search_and_substitute_config(recursive_replace(parameters, tuple()))
|
131
|
+
|
132
|
+
|
51
133
|
@property
|
52
134
|
def parameters(self) -> Optional[Dict[str, Any]]:
|
53
135
|
"""
|
54
136
|
Return the parameters dictionary of the pipe.
|
55
137
|
"""
|
56
|
-
|
57
|
-
self.attributes['parameters'] = {}
|
58
|
-
_parameters = self.attributes['parameters']
|
59
|
-
dt_col = _parameters.get('columns', {}).get('datetime', None)
|
60
|
-
dt_typ = _parameters.get('dtypes', {}).get(dt_col, None) if dt_col else None
|
61
|
-
if dt_col and not dt_typ:
|
62
|
-
if 'dtypes' not in _parameters:
|
63
|
-
self.attributes['parameters']['dtypes'] = {}
|
64
|
-
self.attributes['parameters']['dtypes'][dt_col] = 'datetime'
|
65
|
-
return self.attributes['parameters']
|
138
|
+
return self.get_parameters(debug=self.debug)
|
66
139
|
|
67
140
|
|
68
141
|
@parameters.setter
|
69
|
-
def parameters(self,
|
142
|
+
def parameters(self, _parameters: Dict[str, Any]) -> None:
|
70
143
|
"""
|
71
144
|
Set the parameters dictionary of the in-memory pipe.
|
72
145
|
Call `meerschaum.Pipe.edit()` to persist changes.
|
73
146
|
"""
|
74
|
-
self.
|
147
|
+
self._attributes['parameters'] = _parameters
|
75
148
|
|
76
149
|
|
77
150
|
@property
|
@@ -79,12 +152,9 @@ def columns(self) -> Union[Dict[str, str], None]:
|
|
79
152
|
"""
|
80
153
|
Return the `columns` dictionary defined in `meerschaum.Pipe.parameters`.
|
81
154
|
"""
|
82
|
-
|
83
|
-
self.parameters['columns'] = {}
|
84
|
-
cols = self.parameters['columns']
|
155
|
+
cols = self.parameters.get('columns', {})
|
85
156
|
if not isinstance(cols, dict):
|
86
|
-
|
87
|
-
self.parameters['columns'] = cols
|
157
|
+
return {}
|
88
158
|
return {col_ix: col for col_ix, col in cols.items() if col}
|
89
159
|
|
90
160
|
|
@@ -99,7 +169,7 @@ def columns(self, _columns: Union[Dict[str, str], List[str]]) -> None:
|
|
99
169
|
if not isinstance(_columns, dict):
|
100
170
|
warn(f"{self}.columns must be a dictionary, received {type(_columns)}.")
|
101
171
|
return
|
102
|
-
self.
|
172
|
+
self.update_parameters({'columns': _columns}, persist=False)
|
103
173
|
|
104
174
|
|
105
175
|
@property
|
@@ -107,19 +177,18 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
|
|
107
177
|
"""
|
108
178
|
Return the `indices` dictionary defined in `meerschaum.Pipe.parameters`.
|
109
179
|
"""
|
180
|
+
_parameters = self.get_parameters(debug=self.debug)
|
110
181
|
indices_key = (
|
111
182
|
'indexes'
|
112
|
-
if 'indexes' in
|
183
|
+
if 'indexes' in _parameters
|
113
184
|
else 'indices'
|
114
185
|
)
|
115
|
-
|
116
|
-
|
117
|
-
_indices = self.parameters[indices_key]
|
186
|
+
|
187
|
+
_indices = _parameters.get(indices_key, {})
|
118
188
|
_columns = self.columns
|
119
189
|
dt_col = _columns.get('datetime', None)
|
120
190
|
if not isinstance(_indices, dict):
|
121
191
|
_indices = {}
|
122
|
-
self.parameters[indices_key] = _indices
|
123
192
|
unique_cols = list(set((
|
124
193
|
[dt_col]
|
125
194
|
if dt_col
|
@@ -158,7 +227,7 @@ def indices(self, _indices: Union[Dict[str, Union[str, List[str]]], List[str]])
|
|
158
227
|
if 'indexes' in self.parameters
|
159
228
|
else 'indices'
|
160
229
|
)
|
161
|
-
self.
|
230
|
+
self.update_parameters({indices_key: _indices}, persist=False)
|
162
231
|
|
163
232
|
|
164
233
|
@indexes.setter
|
@@ -174,41 +243,30 @@ def tags(self) -> Union[List[str], None]:
|
|
174
243
|
"""
|
175
244
|
If defined, return the `tags` list defined in `meerschaum.Pipe.parameters`.
|
176
245
|
"""
|
177
|
-
|
178
|
-
self.parameters['tags'] = []
|
179
|
-
return self.parameters['tags']
|
246
|
+
return self.parameters.get('tags', [])
|
180
247
|
|
181
248
|
|
182
249
|
@tags.setter
|
183
|
-
def tags(self, _tags: List[str
|
250
|
+
def tags(self, _tags: List[str]) -> None:
|
184
251
|
"""
|
185
252
|
Override the tags list of the in-memory pipe.
|
186
253
|
Call `meerschaum.Pipe.edit` to persist changes.
|
187
254
|
"""
|
188
255
|
from meerschaum.utils.warnings import error
|
189
|
-
from meerschaum.
|
256
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
190
257
|
negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
|
191
258
|
for t in _tags:
|
192
259
|
if t.startswith(negation_prefix):
|
193
260
|
error(f"Tags cannot begin with '{negation_prefix}'.")
|
194
|
-
self.
|
261
|
+
self.update_parameters({'tags': _tags}, persist=False)
|
195
262
|
|
196
263
|
|
197
264
|
@property
|
198
|
-
def dtypes(self) ->
|
265
|
+
def dtypes(self) -> Dict[str, Any]:
|
199
266
|
"""
|
200
267
|
If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
|
201
268
|
"""
|
202
|
-
|
203
|
-
from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
|
204
|
-
configured_dtypes = self.parameters.get('dtypes', {})
|
205
|
-
remote_dtypes = self.infer_dtypes(persist=False)
|
206
|
-
patched_dtypes = apply_patch_to_config(remote_dtypes, configured_dtypes)
|
207
|
-
return {
|
208
|
-
col: MRSM_ALIAS_DTYPES.get(typ, typ)
|
209
|
-
for col, typ in patched_dtypes.items()
|
210
|
-
if col and typ
|
211
|
-
}
|
269
|
+
return self.get_dtypes(refresh=False, debug=self.debug)
|
212
270
|
|
213
271
|
|
214
272
|
@dtypes.setter
|
@@ -217,7 +275,61 @@ def dtypes(self, _dtypes: Dict[str, Any]) -> None:
|
|
217
275
|
Override the dtypes dictionary of the in-memory pipe.
|
218
276
|
Call `meerschaum.Pipe.edit()` to persist changes.
|
219
277
|
"""
|
220
|
-
self.
|
278
|
+
self.update_parameters({'dtypes': _dtypes}, persist=False)
|
279
|
+
self._clear_cache_key('_remote_dtypes', debug=self.debug)
|
280
|
+
self._clear_cache_key('_remote_dtypes_timestamp', debug=self.debug)
|
281
|
+
|
282
|
+
|
283
|
+
def get_dtypes(
|
284
|
+
self,
|
285
|
+
infer: bool = True,
|
286
|
+
refresh: bool = False,
|
287
|
+
debug: bool = False,
|
288
|
+
) -> Dict[str, Any]:
|
289
|
+
"""
|
290
|
+
If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
|
291
|
+
|
292
|
+
Parameters
|
293
|
+
----------
|
294
|
+
infer: bool, default True
|
295
|
+
If `True`, include the implicit existing dtypes.
|
296
|
+
Else only return the explicitly configured dtypes (e.g. `Pipe.parameters['dtypes']`).
|
297
|
+
|
298
|
+
refresh: bool, default False
|
299
|
+
If `True`, invalidate any cache and return the latest known dtypes.
|
300
|
+
|
301
|
+
Returns
|
302
|
+
-------
|
303
|
+
A dictionary mapping column names to dtypes.
|
304
|
+
"""
|
305
|
+
from meerschaum.config._patch import apply_patch_to_config
|
306
|
+
from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
|
307
|
+
parameters = self.get_parameters(refresh=refresh, debug=debug)
|
308
|
+
configured_dtypes = parameters.get('dtypes', {})
|
309
|
+
if debug:
|
310
|
+
dprint(f"Configured dtypes for {self}:")
|
311
|
+
mrsm.pprint(configured_dtypes)
|
312
|
+
|
313
|
+
remote_dtypes = (
|
314
|
+
self.infer_dtypes(persist=False, refresh=refresh, debug=debug)
|
315
|
+
if infer
|
316
|
+
else {}
|
317
|
+
)
|
318
|
+
patched_dtypes = apply_patch_to_config((remote_dtypes or {}), (configured_dtypes or {}))
|
319
|
+
|
320
|
+
dt_col = parameters.get('columns', {}).get('datetime', None)
|
321
|
+
primary_col = parameters.get('columns', {}).get('primary', None)
|
322
|
+
_dtypes = {
|
323
|
+
col: MRSM_ALIAS_DTYPES.get(typ, typ)
|
324
|
+
for col, typ in patched_dtypes.items()
|
325
|
+
if col and typ
|
326
|
+
}
|
327
|
+
if dt_col and dt_col not in configured_dtypes:
|
328
|
+
_dtypes[dt_col] = 'datetime'
|
329
|
+
if primary_col and parameters.get('autoincrement', False) and primary_col not in _dtypes:
|
330
|
+
_dtypes[primary_col] = 'int'
|
331
|
+
|
332
|
+
return _dtypes
|
221
333
|
|
222
334
|
|
223
335
|
@property
|
@@ -225,9 +337,7 @@ def upsert(self) -> bool:
|
|
225
337
|
"""
|
226
338
|
Return whether `upsert` is set for the pipe.
|
227
339
|
"""
|
228
|
-
|
229
|
-
self.parameters['upsert'] = False
|
230
|
-
return self.parameters['upsert']
|
340
|
+
return self.parameters.get('upsert', False)
|
231
341
|
|
232
342
|
|
233
343
|
@upsert.setter
|
@@ -235,7 +345,7 @@ def upsert(self, _upsert: bool) -> None:
|
|
235
345
|
"""
|
236
346
|
Set the `upsert` parameter for the pipe.
|
237
347
|
"""
|
238
|
-
self.
|
348
|
+
self.update_parameters({'upsert': _upsert}, persist=False)
|
239
349
|
|
240
350
|
|
241
351
|
@property
|
@@ -243,9 +353,7 @@ def static(self) -> bool:
|
|
243
353
|
"""
|
244
354
|
Return whether `static` is set for the pipe.
|
245
355
|
"""
|
246
|
-
|
247
|
-
self.parameters['static'] = False
|
248
|
-
return self.parameters['static']
|
356
|
+
return self.parameters.get('static', False)
|
249
357
|
|
250
358
|
|
251
359
|
@static.setter
|
@@ -253,7 +361,8 @@ def static(self, _static: bool) -> None:
|
|
253
361
|
"""
|
254
362
|
Set the `static` parameter for the pipe.
|
255
363
|
"""
|
256
|
-
self.
|
364
|
+
self.update_parameters({'static': _static}, persist=False)
|
365
|
+
self._static = _static
|
257
366
|
|
258
367
|
|
259
368
|
@property
|
@@ -261,10 +370,7 @@ def autoincrement(self) -> bool:
|
|
261
370
|
"""
|
262
371
|
Return the `autoincrement` parameter for the pipe.
|
263
372
|
"""
|
264
|
-
|
265
|
-
self.parameters['autoincrement'] = False
|
266
|
-
|
267
|
-
return self.parameters['autoincrement']
|
373
|
+
return self.parameters.get('autoincrement', False)
|
268
374
|
|
269
375
|
|
270
376
|
@autoincrement.setter
|
@@ -272,7 +378,23 @@ def autoincrement(self, _autoincrement: bool) -> None:
|
|
272
378
|
"""
|
273
379
|
Set the `autoincrement` parameter for the pipe.
|
274
380
|
"""
|
275
|
-
self.
|
381
|
+
self.update_parameters({'autoincrement': _autoincrement}, persist=False)
|
382
|
+
|
383
|
+
|
384
|
+
@property
|
385
|
+
def autotime(self) -> bool:
|
386
|
+
"""
|
387
|
+
Return the `autotime` parameter for the pipe.
|
388
|
+
"""
|
389
|
+
return self.parameters.get('autotime', False)
|
390
|
+
|
391
|
+
|
392
|
+
@autotime.setter
|
393
|
+
def autotime(self, _autotime: bool) -> None:
|
394
|
+
"""
|
395
|
+
Set the `autotime` parameter for the pipe.
|
396
|
+
"""
|
397
|
+
self.update_parameters({'autotime': _autotime}, persist=False)
|
276
398
|
|
277
399
|
|
278
400
|
@property
|
@@ -280,18 +402,23 @@ def tzinfo(self) -> Union[None, timezone]:
|
|
280
402
|
"""
|
281
403
|
Return `timezone.utc` if the pipe is timezone-aware.
|
282
404
|
"""
|
283
|
-
|
284
|
-
if not
|
285
|
-
return None
|
405
|
+
_tzinfo = self._get_cached_value('tzinfo', debug=self.debug)
|
406
|
+
if _tzinfo is not None:
|
407
|
+
return _tzinfo if _tzinfo != 'None' else None
|
286
408
|
|
287
|
-
|
288
|
-
|
289
|
-
|
409
|
+
_tzinfo = None
|
410
|
+
dt_col = self.columns.get('datetime', None)
|
411
|
+
dt_typ = str(self.dtypes.get(dt_col, 'datetime')) if dt_col else None
|
412
|
+
if self.autotime:
|
413
|
+
ts_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
|
414
|
+
ts_typ = self.dtypes.get(ts_col, 'datetime')
|
415
|
+
dt_typ = ts_typ
|
290
416
|
|
291
|
-
if dt_typ == '
|
292
|
-
|
417
|
+
if dt_typ and 'utc' in dt_typ.lower() or dt_typ == 'datetime':
|
418
|
+
_tzinfo = timezone.utc
|
293
419
|
|
294
|
-
|
420
|
+
self._cache_value('tzinfo', (_tzinfo if _tzinfo is not None else 'None'), debug=self.debug)
|
421
|
+
return _tzinfo
|
295
422
|
|
296
423
|
|
297
424
|
@property
|
@@ -299,10 +426,7 @@ def enforce(self) -> bool:
|
|
299
426
|
"""
|
300
427
|
Return the `enforce` parameter for the pipe.
|
301
428
|
"""
|
302
|
-
|
303
|
-
self.parameters['enforce'] = True
|
304
|
-
|
305
|
-
return self.parameters['enforce']
|
429
|
+
return self.parameters.get('enforce', True)
|
306
430
|
|
307
431
|
|
308
432
|
@enforce.setter
|
@@ -310,7 +434,7 @@ def enforce(self, _enforce: bool) -> None:
|
|
310
434
|
"""
|
311
435
|
Set the `enforce` parameter for the pipe.
|
312
436
|
"""
|
313
|
-
self.
|
437
|
+
self.update_parameters({'enforce': _enforce}, persist=False)
|
314
438
|
|
315
439
|
|
316
440
|
@property
|
@@ -318,10 +442,7 @@ def null_indices(self) -> bool:
|
|
318
442
|
"""
|
319
443
|
Return the `null_indices` parameter for the pipe.
|
320
444
|
"""
|
321
|
-
|
322
|
-
self.parameters['null_indices'] = True
|
323
|
-
|
324
|
-
return self.parameters['null_indices']
|
445
|
+
return self.parameters.get('null_indices', True)
|
325
446
|
|
326
447
|
|
327
448
|
@null_indices.setter
|
@@ -329,7 +450,23 @@ def null_indices(self, _null_indices: bool) -> None:
|
|
329
450
|
"""
|
330
451
|
Set the `null_indices` parameter for the pipe.
|
331
452
|
"""
|
332
|
-
self.
|
453
|
+
self.update_parameters({'null_indices': _null_indices}, persist=False)
|
454
|
+
|
455
|
+
|
456
|
+
@property
|
457
|
+
def mixed_numerics(self) -> bool:
|
458
|
+
"""
|
459
|
+
Return the `mixed_numerics` parameter for the pipe.
|
460
|
+
"""
|
461
|
+
return self.parameters.get('mixed_numerics', True)
|
462
|
+
|
463
|
+
|
464
|
+
@mixed_numerics.setter
|
465
|
+
def mixed_numerics(self, _mixed_numerics: bool) -> None:
|
466
|
+
"""
|
467
|
+
Set the `mixed_numerics` parameter for the pipe.
|
468
|
+
"""
|
469
|
+
self.update_parameters({'mixed_numerics': _mixed_numerics}, persist=False)
|
333
470
|
|
334
471
|
|
335
472
|
def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
|
@@ -357,7 +494,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
|
|
357
494
|
>>> pipe.get_columns('value', error=True)
|
358
495
|
Exception: 🛑 Missing 'value' column for Pipe('test', 'test').
|
359
496
|
"""
|
360
|
-
from meerschaum.utils.warnings import error as _error
|
497
|
+
from meerschaum.utils.warnings import error as _error
|
361
498
|
if not args:
|
362
499
|
args = tuple(self.columns.keys())
|
363
500
|
col_names = []
|
@@ -367,7 +504,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
|
|
367
504
|
col_name = self.columns[col]
|
368
505
|
if col_name is None and error:
|
369
506
|
_error(f"Please define the name of the '{col}' column for {self}.")
|
370
|
-
except Exception
|
507
|
+
except Exception:
|
371
508
|
col_name = None
|
372
509
|
if col_name is None and error:
|
373
510
|
_error(f"Missing '{col}'" + f" column for {self}.")
|
@@ -407,21 +544,22 @@ def get_columns_types(
|
|
407
544
|
}
|
408
545
|
>>>
|
409
546
|
"""
|
410
|
-
import time
|
411
547
|
from meerschaum.connectors import get_connector_plugin
|
412
|
-
from meerschaum.
|
413
|
-
from meerschaum.utils.warnings import dprint
|
548
|
+
from meerschaum.utils.dtypes import get_current_timestamp
|
414
549
|
|
415
|
-
now =
|
416
|
-
cache_seconds =
|
417
|
-
|
418
|
-
|
550
|
+
now = get_current_timestamp('ms', as_int=True) / 1000
|
551
|
+
cache_seconds = (
|
552
|
+
mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
|
553
|
+
if self.static
|
554
|
+
else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
|
555
|
+
)
|
419
556
|
if refresh:
|
420
|
-
|
421
|
-
|
422
|
-
|
557
|
+
self._clear_cache_key('_columns_types_timestamp', debug=debug)
|
558
|
+
self._clear_cache_key('_columns_types', debug=debug)
|
559
|
+
|
560
|
+
_columns_types = self._get_cached_value('_columns_types', debug=debug)
|
423
561
|
if _columns_types:
|
424
|
-
columns_types_timestamp = self.
|
562
|
+
columns_types_timestamp = self._get_cached_value('_columns_types_timestamp', debug=debug)
|
425
563
|
if columns_types_timestamp is not None:
|
426
564
|
delta = now - columns_types_timestamp
|
427
565
|
if delta < cache_seconds:
|
@@ -439,8 +577,8 @@ def get_columns_types(
|
|
439
577
|
else None
|
440
578
|
)
|
441
579
|
|
442
|
-
self.
|
443
|
-
self.
|
580
|
+
self._cache_value('_columns_types', _columns_types, debug=debug)
|
581
|
+
self._cache_value('_columns_types_timestamp', now, debug=debug)
|
444
582
|
return _columns_types or {}
|
445
583
|
|
446
584
|
|
@@ -452,23 +590,23 @@ def get_columns_indices(
|
|
452
590
|
"""
|
453
591
|
Return a dictionary mapping columns to index information.
|
454
592
|
"""
|
455
|
-
import time
|
456
593
|
from meerschaum.connectors import get_connector_plugin
|
457
|
-
from meerschaum.
|
458
|
-
from meerschaum.utils.warnings import dprint
|
594
|
+
from meerschaum.utils.dtypes import get_current_timestamp
|
459
595
|
|
460
|
-
now =
|
596
|
+
now = get_current_timestamp('ms', as_int=True) / 1000
|
461
597
|
cache_seconds = (
|
462
|
-
|
598
|
+
mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
|
463
599
|
if self.static
|
464
|
-
else
|
600
|
+
else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
|
465
601
|
)
|
466
602
|
if refresh:
|
467
|
-
|
468
|
-
|
469
|
-
|
603
|
+
self._clear_cache_key('_columns_indices_timestamp', debug=debug)
|
604
|
+
self._clear_cache_key('_columns_indices', debug=debug)
|
605
|
+
|
606
|
+
_columns_indices = self._get_cached_value('_columns_indices', debug=debug)
|
607
|
+
|
470
608
|
if _columns_indices:
|
471
|
-
columns_indices_timestamp = self.
|
609
|
+
columns_indices_timestamp = self._get_cached_value('_columns_indices_timestamp', debug=debug)
|
472
610
|
if columns_indices_timestamp is not None:
|
473
611
|
delta = now - columns_indices_timestamp
|
474
612
|
if delta < cache_seconds:
|
@@ -486,18 +624,19 @@ def get_columns_indices(
|
|
486
624
|
else None
|
487
625
|
)
|
488
626
|
|
489
|
-
self.
|
490
|
-
self.
|
627
|
+
self._cache_value('_columns_indices', _columns_indices, debug=debug)
|
628
|
+
self._cache_value('_columns_indices_timestamp', now, debug=debug)
|
491
629
|
return {k: v for k, v in _columns_indices.items() if k and v} or {}
|
492
630
|
|
493
631
|
|
494
|
-
def get_id(self, **kw: Any) -> Union[int, None]:
|
632
|
+
def get_id(self, **kw: Any) -> Union[int, str, None]:
|
495
633
|
"""
|
496
634
|
Fetch a pipe's ID from its instance connector.
|
497
|
-
If the pipe
|
635
|
+
If the pipe is not registered, return `None`.
|
498
636
|
"""
|
499
637
|
if self.temporary:
|
500
638
|
return None
|
639
|
+
|
501
640
|
from meerschaum.utils.venv import Venv
|
502
641
|
from meerschaum.connectors import get_connector_plugin
|
503
642
|
|
@@ -509,13 +648,16 @@ def get_id(self, **kw: Any) -> Union[int, None]:
|
|
509
648
|
|
510
649
|
|
511
650
|
@property
|
512
|
-
def id(self) -> Union[int, None]:
|
651
|
+
def id(self) -> Union[int, str, uuid.UUID, None]:
|
513
652
|
"""
|
514
653
|
Fetch and cache a pipe's ID.
|
515
654
|
"""
|
516
|
-
|
517
|
-
|
518
|
-
|
655
|
+
_id = self._get_cached_value('_id', debug=self.debug)
|
656
|
+
if not _id:
|
657
|
+
_id = self.get_id(debug=self.debug)
|
658
|
+
if _id is not None:
|
659
|
+
self._cache_value('_id', _id, debug=self.debug)
|
660
|
+
return _id
|
519
661
|
|
520
662
|
|
521
663
|
def get_val_column(self, debug: bool = False) -> Union[str, None]:
|
@@ -534,12 +676,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
|
|
534
676
|
-------
|
535
677
|
Either a string or `None`.
|
536
678
|
"""
|
537
|
-
from meerschaum.utils.debug import dprint
|
538
679
|
if debug:
|
539
680
|
dprint('Attempting to determine the value column...')
|
540
681
|
try:
|
541
682
|
val_name = self.get_columns('value')
|
542
|
-
except Exception
|
683
|
+
except Exception:
|
543
684
|
val_name = None
|
544
685
|
if val_name is not None:
|
545
686
|
if debug:
|
@@ -553,11 +694,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
|
|
553
694
|
return None
|
554
695
|
try:
|
555
696
|
dt_name = self.get_columns('datetime', error=False)
|
556
|
-
except Exception
|
697
|
+
except Exception:
|
557
698
|
dt_name = None
|
558
699
|
try:
|
559
700
|
id_name = self.get_columns('id', errors=False)
|
560
|
-
except Exception
|
701
|
+
except Exception:
|
561
702
|
id_name = None
|
562
703
|
|
563
704
|
if debug:
|
@@ -596,6 +737,7 @@ def parents(self) -> List[mrsm.Pipe]:
|
|
596
737
|
"""
|
597
738
|
if 'parents' not in self.parameters:
|
598
739
|
return []
|
740
|
+
|
599
741
|
from meerschaum.utils.warnings import warn
|
600
742
|
_parents_keys = self.parameters['parents']
|
601
743
|
if not isinstance(_parents_keys, list):
|
@@ -634,6 +776,7 @@ def children(self) -> List[mrsm.Pipe]:
|
|
634
776
|
"""
|
635
777
|
if 'children' not in self.parameters:
|
636
778
|
return []
|
779
|
+
|
637
780
|
from meerschaum.utils.warnings import warn
|
638
781
|
_children_keys = self.parameters['children']
|
639
782
|
if not isinstance(_children_keys, list):
|
@@ -714,7 +857,7 @@ def target(self, _target: str) -> None:
|
|
714
857
|
Override the target of the in-memory pipe.
|
715
858
|
Call `meerschaum.Pipe.edit` to persist changes.
|
716
859
|
"""
|
717
|
-
self.
|
860
|
+
self.update_parameters({'target': _target}, persist=False)
|
718
861
|
|
719
862
|
|
720
863
|
def guess_datetime(self) -> Union[str, None]:
|
@@ -755,3 +898,161 @@ def get_indices(self) -> Dict[str, str]:
|
|
755
898
|
result = {}
|
756
899
|
|
757
900
|
return result
|
901
|
+
|
902
|
+
|
903
|
+
def update_parameters(
|
904
|
+
self,
|
905
|
+
parameters_patch: Dict[str, Any],
|
906
|
+
persist: bool = True,
|
907
|
+
debug: bool = False,
|
908
|
+
) -> mrsm.SuccessTuple:
|
909
|
+
"""
|
910
|
+
Apply a patch to a pipe's `parameters` dictionary.
|
911
|
+
|
912
|
+
Parameters
|
913
|
+
----------
|
914
|
+
parameters_patch: Dict[str, Any]
|
915
|
+
The patch to be applied to `Pipe.parameters`.
|
916
|
+
|
917
|
+
persist: bool, default True
|
918
|
+
If `True`, call `Pipe.edit()` to persist the new parameters.
|
919
|
+
"""
|
920
|
+
from meerschaum.config import apply_patch_to_config
|
921
|
+
if 'parameters' not in self._attributes:
|
922
|
+
self._attributes['parameters'] = {}
|
923
|
+
|
924
|
+
self._attributes['parameters'] = apply_patch_to_config(
|
925
|
+
self._attributes['parameters'],
|
926
|
+
parameters_patch,
|
927
|
+
)
|
928
|
+
|
929
|
+
if self.temporary:
|
930
|
+
persist = False
|
931
|
+
|
932
|
+
if not persist:
|
933
|
+
return True, "Success"
|
934
|
+
|
935
|
+
return self.edit(debug=debug)
|
936
|
+
|
937
|
+
|
938
|
+
def get_precision(self, debug: bool = False) -> Dict[str, Union[str, int]]:
|
939
|
+
"""
|
940
|
+
Return the timestamp precision unit and interval for the `datetime` axis.
|
941
|
+
"""
|
942
|
+
from meerschaum.utils.dtypes import (
|
943
|
+
MRSM_PRECISION_UNITS_SCALARS,
|
944
|
+
MRSM_PRECISION_UNITS_ALIASES,
|
945
|
+
MRSM_PD_DTYPES,
|
946
|
+
are_dtypes_equal,
|
947
|
+
)
|
948
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
949
|
+
|
950
|
+
_precision = self._get_cached_value('precision', debug=debug)
|
951
|
+
if _precision:
|
952
|
+
if debug:
|
953
|
+
dprint(f"Returning cached precision: {_precision}")
|
954
|
+
return _precision
|
955
|
+
|
956
|
+
parameters = self.parameters
|
957
|
+
_precision = parameters.get('precision', {})
|
958
|
+
if isinstance(_precision, str):
|
959
|
+
_precision = {'unit': _precision}
|
960
|
+
default_precision_unit = STATIC_CONFIG['dtypes']['datetime']['default_precision_unit']
|
961
|
+
|
962
|
+
if not _precision:
|
963
|
+
|
964
|
+
dt_col = parameters.get('columns', {}).get('datetime', None)
|
965
|
+
if not dt_col and self.autotime:
|
966
|
+
dt_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
|
967
|
+
if not dt_col:
|
968
|
+
if debug:
|
969
|
+
dprint(f"No datetime axis, returning default precision '{default_precision_unit}'.")
|
970
|
+
return {'unit': default_precision_unit}
|
971
|
+
|
972
|
+
dt_typ = self.dtypes.get(dt_col, 'datetime')
|
973
|
+
if are_dtypes_equal(dt_typ, 'datetime'):
|
974
|
+
if dt_typ == 'datetime':
|
975
|
+
dt_typ = MRSM_PD_DTYPES['datetime']
|
976
|
+
if debug:
|
977
|
+
dprint(f"Datetime type is `datetime`, assuming {dt_typ} precision.")
|
978
|
+
|
979
|
+
_precision = {
|
980
|
+
'unit': (
|
981
|
+
dt_typ
|
982
|
+
.split('[', maxsplit=1)[-1]
|
983
|
+
.split(',', maxsplit=1)[0]
|
984
|
+
.split(' ', maxsplit=1)[0]
|
985
|
+
).rstrip(']')
|
986
|
+
}
|
987
|
+
|
988
|
+
if debug:
|
989
|
+
dprint(f"Extracted precision '{_precision['unit']}' from type '{dt_typ}'.")
|
990
|
+
|
991
|
+
elif are_dtypes_equal(dt_typ, 'int'):
|
992
|
+
_precision = {
|
993
|
+
'unit': (
|
994
|
+
'second'
|
995
|
+
if '32' in dt_typ
|
996
|
+
else default_precision_unit
|
997
|
+
)
|
998
|
+
}
|
999
|
+
elif are_dtypes_equal(dt_typ, 'date'):
|
1000
|
+
if debug:
|
1001
|
+
dprint("Datetime axis is 'date', falling back to 'day' precision.")
|
1002
|
+
_precision = {'unit': 'day'}
|
1003
|
+
|
1004
|
+
precision_unit = _precision.get('unit', default_precision_unit)
|
1005
|
+
precision_interval = _precision.get('interval', None)
|
1006
|
+
true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
|
1007
|
+
if true_precision_unit is None:
|
1008
|
+
if debug:
|
1009
|
+
dprint(f"No precision could be determined, falling back to '{default_precision_unit}'.")
|
1010
|
+
true_precision_unit = default_precision_unit
|
1011
|
+
|
1012
|
+
if true_precision_unit not in MRSM_PRECISION_UNITS_SCALARS:
|
1013
|
+
from meerschaum.utils.misc import items_str
|
1014
|
+
raise ValueError(
|
1015
|
+
f"Invalid precision unit '{true_precision_unit}'.\n"
|
1016
|
+
"Accepted values are "
|
1017
|
+
f"{items_str(list(MRSM_PRECISION_UNITS_SCALARS) + list(MRSM_PRECISION_UNITS_ALIASES))}."
|
1018
|
+
)
|
1019
|
+
|
1020
|
+
_precision = {'unit': true_precision_unit}
|
1021
|
+
if precision_interval:
|
1022
|
+
_precision['interval'] = precision_interval
|
1023
|
+
self._cache_value('precision', _precision, debug=debug)
|
1024
|
+
return self._precision
|
1025
|
+
|
1026
|
+
|
1027
|
+
@property
|
1028
|
+
def precision(self) -> Dict[str, Union[str, int]]:
|
1029
|
+
"""
|
1030
|
+
Return the configured or detected precision.
|
1031
|
+
"""
|
1032
|
+
return self.get_precision(debug=self.debug)
|
1033
|
+
|
1034
|
+
|
1035
|
+
@precision.setter
|
1036
|
+
def precision(self, _precision: Union[str, Dict[str, Union[str, int]]]) -> None:
|
1037
|
+
"""
|
1038
|
+
Update the `precision` parameter.
|
1039
|
+
"""
|
1040
|
+
existing_precision = self._attributes.get('parameters', {}).get('precision', None)
|
1041
|
+
if isinstance(existing_precision, str):
|
1042
|
+
existing_precision = {'unit': existing_precision}
|
1043
|
+
|
1044
|
+
true_precision = (
|
1045
|
+
_precision
|
1046
|
+
if isinstance(_precision, dict)
|
1047
|
+
else {
|
1048
|
+
'unit': _precision,
|
1049
|
+
**(
|
1050
|
+
{
|
1051
|
+
'interval': existing_precision['interval'],
|
1052
|
+
} if existing_precision else {}
|
1053
|
+
)
|
1054
|
+
}
|
1055
|
+
)
|
1056
|
+
|
1057
|
+
self.update_parameters({'precision': true_precision}, persist=False)
|
1058
|
+
self._clear_cache_key('precision', debug=self.debug)
|