meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +19 -2
  5. meerschaum/_internal/docs/index.py +49 -2
  6. meerschaum/_internal/entry.py +6 -6
  7. meerschaum/_internal/shell/Shell.py +1 -1
  8. meerschaum/_internal/static.py +356 -0
  9. meerschaum/actions/api.py +12 -2
  10. meerschaum/actions/bootstrap.py +7 -7
  11. meerschaum/actions/edit.py +142 -18
  12. meerschaum/actions/register.py +137 -6
  13. meerschaum/actions/show.py +117 -29
  14. meerschaum/actions/stop.py +4 -1
  15. meerschaum/actions/sync.py +1 -1
  16. meerschaum/actions/tag.py +9 -8
  17. meerschaum/actions/verify.py +5 -8
  18. meerschaum/api/__init__.py +11 -3
  19. meerschaum/api/_events.py +39 -2
  20. meerschaum/api/_oauth2.py +118 -8
  21. meerschaum/api/_tokens.py +102 -0
  22. meerschaum/api/dash/__init__.py +0 -3
  23. meerschaum/api/dash/callbacks/custom.py +2 -2
  24. meerschaum/api/dash/callbacks/dashboard.py +103 -19
  25. meerschaum/api/dash/callbacks/plugins.py +0 -1
  26. meerschaum/api/dash/callbacks/register.py +1 -1
  27. meerschaum/api/dash/callbacks/settings/__init__.py +1 -0
  28. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  29. meerschaum/api/dash/callbacks/settings/tokens.py +388 -0
  30. meerschaum/api/dash/components.py +30 -8
  31. meerschaum/api/dash/keys.py +19 -93
  32. meerschaum/api/dash/pages/dashboard.py +1 -20
  33. meerschaum/api/dash/pages/settings/__init__.py +1 -0
  34. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  35. meerschaum/api/dash/pages/settings/tokens.py +55 -0
  36. meerschaum/api/dash/pipes.py +94 -59
  37. meerschaum/api/dash/sessions.py +12 -0
  38. meerschaum/api/dash/tokens.py +606 -0
  39. meerschaum/api/dash/websockets.py +1 -1
  40. meerschaum/api/dash/webterm.py +4 -0
  41. meerschaum/api/models/__init__.py +23 -3
  42. meerschaum/api/models/_actions.py +22 -0
  43. meerschaum/api/models/_pipes.py +85 -7
  44. meerschaum/api/models/_tokens.py +81 -0
  45. meerschaum/api/resources/templates/termpage.html +12 -0
  46. meerschaum/api/routes/__init__.py +1 -0
  47. meerschaum/api/routes/_actions.py +3 -4
  48. meerschaum/api/routes/_connectors.py +3 -7
  49. meerschaum/api/routes/_jobs.py +14 -35
  50. meerschaum/api/routes/_login.py +49 -12
  51. meerschaum/api/routes/_misc.py +5 -10
  52. meerschaum/api/routes/_pipes.py +173 -140
  53. meerschaum/api/routes/_plugins.py +38 -28
  54. meerschaum/api/routes/_tokens.py +236 -0
  55. meerschaum/api/routes/_users.py +47 -35
  56. meerschaum/api/routes/_version.py +3 -3
  57. meerschaum/config/__init__.py +43 -20
  58. meerschaum/config/_default.py +43 -6
  59. meerschaum/config/_edit.py +28 -24
  60. meerschaum/config/_environment.py +1 -1
  61. meerschaum/config/_patch.py +6 -6
  62. meerschaum/config/_paths.py +5 -1
  63. meerschaum/config/_read_config.py +65 -34
  64. meerschaum/config/_sync.py +6 -3
  65. meerschaum/config/_version.py +1 -1
  66. meerschaum/config/stack/__init__.py +31 -11
  67. meerschaum/config/static.py +18 -0
  68. meerschaum/connectors/_Connector.py +10 -4
  69. meerschaum/connectors/__init__.py +4 -20
  70. meerschaum/connectors/api/_APIConnector.py +34 -6
  71. meerschaum/connectors/api/_actions.py +2 -2
  72. meerschaum/connectors/api/_jobs.py +1 -1
  73. meerschaum/connectors/api/_login.py +33 -7
  74. meerschaum/connectors/api/_misc.py +2 -2
  75. meerschaum/connectors/api/_pipes.py +16 -31
  76. meerschaum/connectors/api/_plugins.py +2 -2
  77. meerschaum/connectors/api/_request.py +1 -1
  78. meerschaum/connectors/api/_tokens.py +146 -0
  79. meerschaum/connectors/api/_users.py +70 -58
  80. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  81. meerschaum/connectors/instance/__init__.py +10 -0
  82. meerschaum/connectors/instance/_pipes.py +442 -0
  83. meerschaum/connectors/instance/_plugins.py +151 -0
  84. meerschaum/connectors/instance/_tokens.py +296 -0
  85. meerschaum/connectors/instance/_users.py +181 -0
  86. meerschaum/connectors/parse.py +4 -1
  87. meerschaum/connectors/sql/_SQLConnector.py +8 -5
  88. meerschaum/connectors/sql/_cli.py +12 -11
  89. meerschaum/connectors/sql/_create_engine.py +9 -168
  90. meerschaum/connectors/sql/_fetch.py +2 -18
  91. meerschaum/connectors/sql/_pipes.py +156 -190
  92. meerschaum/connectors/sql/_plugins.py +29 -0
  93. meerschaum/connectors/sql/_sql.py +46 -21
  94. meerschaum/connectors/sql/_users.py +29 -2
  95. meerschaum/connectors/sql/tables/__init__.py +1 -1
  96. meerschaum/connectors/valkey/_ValkeyConnector.py +2 -4
  97. meerschaum/connectors/valkey/_pipes.py +53 -26
  98. meerschaum/connectors/valkey/_plugins.py +2 -26
  99. meerschaum/core/Pipe/__init__.py +59 -19
  100. meerschaum/core/Pipe/_attributes.py +412 -90
  101. meerschaum/core/Pipe/_bootstrap.py +54 -24
  102. meerschaum/core/Pipe/_data.py +96 -18
  103. meerschaum/core/Pipe/_dtypes.py +48 -18
  104. meerschaum/core/Pipe/_edit.py +14 -4
  105. meerschaum/core/Pipe/_fetch.py +1 -1
  106. meerschaum/core/Pipe/_show.py +5 -5
  107. meerschaum/core/Pipe/_sync.py +118 -193
  108. meerschaum/core/Pipe/_verify.py +4 -4
  109. meerschaum/{plugins → core/Plugin}/_Plugin.py +9 -11
  110. meerschaum/core/Plugin/__init__.py +1 -1
  111. meerschaum/core/Token/_Token.py +220 -0
  112. meerschaum/core/Token/__init__.py +12 -0
  113. meerschaum/core/User/_User.py +34 -8
  114. meerschaum/core/User/__init__.py +9 -1
  115. meerschaum/core/__init__.py +1 -0
  116. meerschaum/jobs/_Job.py +3 -2
  117. meerschaum/jobs/__init__.py +3 -2
  118. meerschaum/jobs/systemd.py +1 -1
  119. meerschaum/models/__init__.py +35 -0
  120. meerschaum/models/pipes.py +247 -0
  121. meerschaum/models/tokens.py +38 -0
  122. meerschaum/models/users.py +26 -0
  123. meerschaum/plugins/__init__.py +22 -7
  124. meerschaum/plugins/bootstrap.py +2 -1
  125. meerschaum/utils/_get_pipes.py +68 -27
  126. meerschaum/utils/daemon/Daemon.py +2 -1
  127. meerschaum/utils/daemon/__init__.py +30 -2
  128. meerschaum/utils/dataframe.py +473 -81
  129. meerschaum/utils/debug.py +15 -15
  130. meerschaum/utils/dtypes/__init__.py +473 -34
  131. meerschaum/utils/dtypes/sql.py +368 -28
  132. meerschaum/utils/formatting/__init__.py +1 -1
  133. meerschaum/utils/formatting/_pipes.py +5 -4
  134. meerschaum/utils/formatting/_shell.py +11 -9
  135. meerschaum/utils/misc.py +246 -148
  136. meerschaum/utils/packages/__init__.py +10 -27
  137. meerschaum/utils/packages/_packages.py +41 -34
  138. meerschaum/utils/pipes.py +181 -0
  139. meerschaum/utils/process.py +1 -1
  140. meerschaum/utils/prompt.py +3 -1
  141. meerschaum/utils/schedule.py +2 -1
  142. meerschaum/utils/sql.py +121 -44
  143. meerschaum/utils/typing.py +1 -4
  144. meerschaum/utils/venv/_Venv.py +2 -2
  145. meerschaum/utils/venv/__init__.py +5 -7
  146. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/METADATA +92 -96
  147. meerschaum-3.0.0rc2.dist-info/RECORD +283 -0
  148. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/WHEEL +1 -1
  149. meerschaum-3.0.0rc2.dist-info/licenses/NOTICE +2 -0
  150. meerschaum/api/models/_interfaces.py +0 -15
  151. meerschaum/api/models/_locations.py +0 -15
  152. meerschaum/api/models/_metrics.py +0 -15
  153. meerschaum/config/static/__init__.py +0 -186
  154. meerschaum-2.9.5.dist-info/RECORD +0 -263
  155. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/entry_points.txt +0 -0
  156. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/licenses/LICENSE +0 -0
  157. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/top_level.txt +0 -0
  158. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0rc2.dist-info}/zip-safe +0 -0
@@ -8,11 +8,12 @@ Fetch and manipulate Pipes' attributes
8
8
 
9
9
  from __future__ import annotations
10
10
 
11
+ import uuid
11
12
  from datetime import timezone
12
13
 
13
14
  import meerschaum as mrsm
14
15
  from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
15
- from meerschaum.utils.warnings import warn
16
+ from meerschaum.utils.warnings import warn, dprint
16
17
 
17
18
 
18
19
  @property
@@ -48,21 +49,89 @@ def attributes(self) -> Dict[str, Any]:
48
49
  return self._attributes
49
50
 
50
51
 
52
+ def get_parameters(
53
+ self,
54
+ apply_symlinks: bool = True,
55
+ refresh: bool = False,
56
+ debug: bool = False,
57
+ _visited: 'Optional[set[mrsm.Pipe]]' = None,
58
+ ) -> Dict[str, Any]:
59
+ """
60
+ Return the `parameters` dictionary of the pipe.
61
+
62
+ Parameters
63
+ ----------
64
+ apply_symlinks: bool, default True
65
+ If `True`, resolve references to parameters from other pipes.
66
+
67
+ refresh: bool, default False
68
+ If `True`, pull the latest attributes for the pipe.
69
+
70
+ Returns
71
+ -------
72
+ The pipe's parameters dictionary.
73
+ """
74
+ from meerschaum.config._patch import apply_patch_to_config
75
+ from meerschaum.config._read_config import search_and_substitute_config
76
+
77
+ if _visited is None:
78
+ _visited = {self}
79
+
80
+ if refresh:
81
+ self._invalidate_cache(hard=True)
82
+
83
+ raw_parameters = self.attributes.get('parameters', {})
84
+ ref_keys = raw_parameters.get('reference')
85
+ if not apply_symlinks:
86
+ return raw_parameters
87
+
88
+ if ref_keys:
89
+ try:
90
+ if debug:
91
+ dprint(f"Building reference pipe from keys: {ref_keys}")
92
+ ref_pipe = mrsm.Pipe(**ref_keys)
93
+ if ref_pipe in _visited:
94
+ warn(f"Circular reference detected in {self}: chain involves {ref_pipe}.")
95
+ return search_and_substitute_config(raw_parameters)
96
+
97
+ _visited.add(ref_pipe)
98
+ base_params = ref_pipe.get_parameters(_visited=_visited, debug=debug)
99
+ except Exception as e:
100
+ warn(f"Failed to resolve reference pipe for {self}: {e}")
101
+ base_params = {}
102
+
103
+ params_to_apply = {k: v for k, v in raw_parameters.items() if k != 'reference'}
104
+ parameters = apply_patch_to_config(base_params, params_to_apply)
105
+ else:
106
+ parameters = raw_parameters
107
+
108
+ from meerschaum.utils.pipes import replace_pipes_syntax
109
+ self._symlinks = {}
110
+
111
+ def recursive_replace(obj: Any, path: tuple) -> Any:
112
+ if isinstance(obj, dict):
113
+ return {k: recursive_replace(v, path + (k,)) for k, v in obj.items()}
114
+ if isinstance(obj, list):
115
+ return [recursive_replace(elem, path + (i,)) for i, elem in enumerate(obj)]
116
+ if isinstance(obj, str):
117
+ substituted_val = replace_pipes_syntax(obj)
118
+ if substituted_val != obj:
119
+ self._symlinks[path] = {
120
+ 'original': obj,
121
+ 'substituted': substituted_val,
122
+ }
123
+ return substituted_val
124
+ return obj
125
+
126
+ return search_and_substitute_config(recursive_replace(parameters, tuple()))
127
+
128
+
51
129
  @property
52
130
  def parameters(self) -> Optional[Dict[str, Any]]:
53
131
  """
54
132
  Return the parameters dictionary of the pipe.
55
133
  """
56
- if 'parameters' not in self.attributes:
57
- self.attributes['parameters'] = {}
58
- _parameters = self.attributes['parameters']
59
- dt_col = _parameters.get('columns', {}).get('datetime', None)
60
- dt_typ = _parameters.get('dtypes', {}).get(dt_col, None) if dt_col else None
61
- if dt_col and not dt_typ:
62
- if 'dtypes' not in _parameters:
63
- self.attributes['parameters']['dtypes'] = {}
64
- self.attributes['parameters']['dtypes'][dt_col] = 'datetime'
65
- return self.attributes['parameters']
134
+ return self.get_parameters()
66
135
 
67
136
 
68
137
  @parameters.setter
@@ -71,7 +140,9 @@ def parameters(self, parameters: Dict[str, Any]) -> None:
71
140
  Set the parameters dictionary of the in-memory pipe.
72
141
  Call `meerschaum.Pipe.edit()` to persist changes.
73
142
  """
74
- self.attributes['parameters'] = parameters
143
+ self._attributes['parameters'] = parameters
144
+ if '_parameters' in self.__dict__:
145
+ del self.__dict__['_parameters']
75
146
 
76
147
 
77
148
  @property
@@ -79,12 +150,9 @@ def columns(self) -> Union[Dict[str, str], None]:
79
150
  """
80
151
  Return the `columns` dictionary defined in `meerschaum.Pipe.parameters`.
81
152
  """
82
- if 'columns' not in self.parameters:
83
- self.parameters['columns'] = {}
84
- cols = self.parameters['columns']
153
+ cols = self.parameters.get('columns', {})
85
154
  if not isinstance(cols, dict):
86
- cols = {}
87
- self.parameters['columns'] = cols
155
+ return {}
88
156
  return {col_ix: col for col_ix, col in cols.items() if col}
89
157
 
90
158
 
@@ -99,7 +167,7 @@ def columns(self, _columns: Union[Dict[str, str], List[str]]) -> None:
99
167
  if not isinstance(_columns, dict):
100
168
  warn(f"{self}.columns must be a dictionary, received {type(_columns)}.")
101
169
  return
102
- self.parameters['columns'] = _columns
170
+ self.update_parameters({'columns': _columns}, persist=False)
103
171
 
104
172
 
105
173
  @property
@@ -112,14 +180,12 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
112
180
  if 'indexes' in self.parameters
113
181
  else 'indices'
114
182
  )
115
- if indices_key not in self.parameters:
116
- self.parameters[indices_key] = {}
117
- _indices = self.parameters[indices_key]
183
+
184
+ _indices = self.parameters.get(indices_key, {})
118
185
  _columns = self.columns
119
186
  dt_col = _columns.get('datetime', None)
120
187
  if not isinstance(_indices, dict):
121
188
  _indices = {}
122
- self.parameters[indices_key] = _indices
123
189
  unique_cols = list(set((
124
190
  [dt_col]
125
191
  if dt_col
@@ -158,7 +224,7 @@ def indices(self, _indices: Union[Dict[str, Union[str, List[str]]], List[str]])
158
224
  if 'indexes' in self.parameters
159
225
  else 'indices'
160
226
  )
161
- self.parameters[indices_key] = _indices
227
+ self.update_parameters({indices_key: _indices}, persist=False)
162
228
 
163
229
 
164
230
  @indexes.setter
@@ -174,41 +240,30 @@ def tags(self) -> Union[List[str], None]:
174
240
  """
175
241
  If defined, return the `tags` list defined in `meerschaum.Pipe.parameters`.
176
242
  """
177
- if 'tags' not in self.parameters:
178
- self.parameters['tags'] = []
179
- return self.parameters['tags']
243
+ return self.parameters.get('tags', [])
180
244
 
181
245
 
182
246
  @tags.setter
183
- def tags(self, _tags: List[str, str]) -> None:
247
+ def tags(self, _tags: List[str]) -> None:
184
248
  """
185
249
  Override the tags list of the in-memory pipe.
186
250
  Call `meerschaum.Pipe.edit` to persist changes.
187
251
  """
188
252
  from meerschaum.utils.warnings import error
189
- from meerschaum.config.static import STATIC_CONFIG
253
+ from meerschaum._internal.static import STATIC_CONFIG
190
254
  negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
191
255
  for t in _tags:
192
256
  if t.startswith(negation_prefix):
193
257
  error(f"Tags cannot begin with '{negation_prefix}'.")
194
- self.parameters['tags'] = _tags
258
+ self.update_parameters({'tags': _tags}, persist=False)
195
259
 
196
260
 
197
261
  @property
198
- def dtypes(self) -> Union[Dict[str, Any], None]:
262
+ def dtypes(self) -> Dict[str, Any]:
199
263
  """
200
264
  If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
201
265
  """
202
- from meerschaum.config._patch import apply_patch_to_config
203
- from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
204
- configured_dtypes = self.parameters.get('dtypes', {})
205
- remote_dtypes = self.infer_dtypes(persist=False)
206
- patched_dtypes = apply_patch_to_config(remote_dtypes, configured_dtypes)
207
- return {
208
- col: MRSM_ALIAS_DTYPES.get(typ, typ)
209
- for col, typ in patched_dtypes.items()
210
- if col and typ
211
- }
266
+ return self.get_dtypes(refresh=False)
212
267
 
213
268
 
214
269
  @dtypes.setter
@@ -217,7 +272,60 @@ def dtypes(self, _dtypes: Dict[str, Any]) -> None:
217
272
  Override the dtypes dictionary of the in-memory pipe.
218
273
  Call `meerschaum.Pipe.edit()` to persist changes.
219
274
  """
220
- self.parameters['dtypes'] = _dtypes
275
+ self.update_parameters({'dtypes': _dtypes}, persist=False)
276
+ _ = self.__dict__.pop('_remote_dtypes', None)
277
+ _ = self.__dict__.pop('_remote_dtypes_timestamp', None)
278
+
279
+
280
+ def get_dtypes(
281
+ self,
282
+ infer: bool = True,
283
+ refresh: bool = False,
284
+ debug: bool = False,
285
+ ) -> Dict[str, Any]:
286
+ """
287
+ If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
288
+
289
+
290
+ Parameters
291
+ ----------
292
+ infer: bool, default True
293
+ If `True`, include the implicit existing dtypes.
294
+ Else only return the explicitly configured dtypes (e.g. `Pipe.parameters['dtypes']`).
295
+
296
+ refresh: bool, default False
297
+ If `True`, invalidate any cache and return the latest known dtypes.
298
+
299
+ Returns
300
+ -------
301
+ A dictionary mapping column names to dtypes.
302
+ """
303
+ import time
304
+ from meerschaum.config._patch import apply_patch_to_config
305
+ from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
306
+ from meerschaum._internal.static import STATIC_CONFIG
307
+ parameters = self.get_parameters(refresh=refresh, debug=debug)
308
+ configured_dtypes = parameters.get('dtypes', {})
309
+ if debug:
310
+ dprint(f"Configured dtypes for {self}:")
311
+ mrsm.pprint(configured_dtypes)
312
+
313
+ remote_dtypes = self.infer_dtypes(persist=False, refresh=refresh, debug=debug)
314
+ patched_dtypes = apply_patch_to_config((remote_dtypes or {}), (configured_dtypes or {}))
315
+
316
+ dt_col = parameters.get('columns', {}).get('datetime', None)
317
+ primary_col = parameters.get('columns', {}).get('primary', None)
318
+ _dtypes = {
319
+ col: MRSM_ALIAS_DTYPES.get(typ, typ)
320
+ for col, typ in patched_dtypes.items()
321
+ if col and typ
322
+ }
323
+ if dt_col and dt_col not in configured_dtypes:
324
+ _dtypes[dt_col] = 'datetime'
325
+ if primary_col and parameters.get('autoincrement', False) and primary_col not in _dtypes:
326
+ _dtypes[primary_col] = 'int'
327
+
328
+ return _dtypes
221
329
 
222
330
 
223
331
  @property
@@ -225,9 +333,7 @@ def upsert(self) -> bool:
225
333
  """
226
334
  Return whether `upsert` is set for the pipe.
227
335
  """
228
- if 'upsert' not in self.parameters:
229
- self.parameters['upsert'] = False
230
- return self.parameters['upsert']
336
+ return self.parameters.get('upsert', False)
231
337
 
232
338
 
233
339
  @upsert.setter
@@ -235,7 +341,7 @@ def upsert(self, _upsert: bool) -> None:
235
341
  """
236
342
  Set the `upsert` parameter for the pipe.
237
343
  """
238
- self.parameters['upsert'] = _upsert
344
+ self.update_parameters({'upsert': _upsert}, persist=False)
239
345
 
240
346
 
241
347
  @property
@@ -243,9 +349,7 @@ def static(self) -> bool:
243
349
  """
244
350
  Return whether `static` is set for the pipe.
245
351
  """
246
- if 'static' not in self.parameters:
247
- self.parameters['static'] = False
248
- return self.parameters['static']
352
+ return self.parameters.get('static', False)
249
353
 
250
354
 
251
355
  @static.setter
@@ -253,7 +357,7 @@ def static(self, _static: bool) -> None:
253
357
  """
254
358
  Set the `static` parameter for the pipe.
255
359
  """
256
- self.parameters['static'] = _static
360
+ self.update_parameters({'static': _static}, persist=False)
257
361
 
258
362
 
259
363
  @property
@@ -261,10 +365,7 @@ def autoincrement(self) -> bool:
261
365
  """
262
366
  Return the `autoincrement` parameter for the pipe.
263
367
  """
264
- if 'autoincrement' not in self.parameters:
265
- self.parameters['autoincrement'] = False
266
-
267
- return self.parameters['autoincrement']
368
+ return self.parameters.get('autoincrement', False)
268
369
 
269
370
 
270
371
  @autoincrement.setter
@@ -272,7 +373,23 @@ def autoincrement(self, _autoincrement: bool) -> None:
272
373
  """
273
374
  Set the `autoincrement` parameter for the pipe.
274
375
  """
275
- self.parameters['autoincrement'] = _autoincrement
376
+ self.update_parameters({'autoincrement': _autoincrement}, persist=False)
377
+
378
+
379
+ @property
380
+ def autotime(self) -> bool:
381
+ """
382
+ Return the `autotime` parameter for the pipe.
383
+ """
384
+ return self.parameters.get('autotime', False)
385
+
386
+
387
+ @autotime.setter
388
+ def autotime(self, _autotime: bool) -> None:
389
+ """
390
+ Set the `autotime` parameter for the pipe.
391
+ """
392
+ self.update_parameters({'autotime': _autotime}, persist=False)
276
393
 
277
394
 
278
395
  @property
@@ -280,18 +397,22 @@ def tzinfo(self) -> Union[None, timezone]:
280
397
  """
281
398
  Return `timezone.utc` if the pipe is timezone-aware.
282
399
  """
283
- dt_col = self.columns.get('datetime', None)
284
- if not dt_col:
285
- return None
400
+ if '_tzinfo' in self.__dict__:
401
+ return self.__dict__['_tzinfo']
286
402
 
287
- dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
288
- if 'utc' in dt_typ.lower() or dt_typ == 'datetime':
289
- return timezone.utc
403
+ _tzinfo = None
404
+ dt_col = self.columns.get('datetime', None)
405
+ dt_typ = str(self.dtypes.get(dt_col, 'datetime')) if dt_col else None
406
+ if self.autotime:
407
+ ts_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
408
+ ts_typ = self.dtypes.get(ts_col, 'datetime')
409
+ dt_typ = ts_typ
290
410
 
291
- if dt_typ == 'datetime64[ns]':
292
- return None
411
+ if dt_typ and 'utc' in dt_typ.lower() or dt_typ == 'datetime':
412
+ _tzinfo = timezone.utc
293
413
 
294
- return None
414
+ self._tzinfo = _tzinfo
415
+ return _tzinfo
295
416
 
296
417
 
297
418
  @property
@@ -299,10 +420,7 @@ def enforce(self) -> bool:
299
420
  """
300
421
  Return the `enforce` parameter for the pipe.
301
422
  """
302
- if 'enforce' not in self.parameters:
303
- self.parameters['enforce'] = True
304
-
305
- return self.parameters['enforce']
423
+ return self.parameters.get('enforce', True)
306
424
 
307
425
 
308
426
  @enforce.setter
@@ -310,7 +428,7 @@ def enforce(self, _enforce: bool) -> None:
310
428
  """
311
429
  Set the `enforce` parameter for the pipe.
312
430
  """
313
- self.parameters['enforce'] = _enforce
431
+ self.update_parameters({'enforce': _enforce}, persist=False)
314
432
 
315
433
 
316
434
  @property
@@ -318,10 +436,7 @@ def null_indices(self) -> bool:
318
436
  """
319
437
  Return the `null_indices` parameter for the pipe.
320
438
  """
321
- if 'null_indices' not in self.parameters:
322
- self.parameters['null_indices'] = True
323
-
324
- return self.parameters['null_indices']
439
+ return self.parameters.get('null_indices', True)
325
440
 
326
441
 
327
442
  @null_indices.setter
@@ -329,7 +444,23 @@ def null_indices(self, _null_indices: bool) -> None:
329
444
  """
330
445
  Set the `null_indices` parameter for the pipe.
331
446
  """
332
- self.parameters['null_indices'] = _null_indices
447
+ self.update_parameters({'null_indices': _null_indices}, persist=False)
448
+
449
+
450
+ @property
451
+ def mixed_numerics(self) -> bool:
452
+ """
453
+ Return the `mixed_numerics` parameter for the pipe.
454
+ """
455
+ return self.parameters.get('mixed_numerics', True)
456
+
457
+
458
+ @mixed_numerics.setter
459
+ def mixed_numerics(self, _mixed_numerics: bool) -> None:
460
+ """
461
+ Set the `mixed_numerics` parameter for the pipe.
462
+ """
463
+ self.update_parameters({'mixed_numerics': _mixed_numerics}, persist=False)
333
464
 
334
465
 
335
466
  def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
@@ -357,7 +488,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
357
488
  >>> pipe.get_columns('value', error=True)
358
489
  Exception: 🛑 Missing 'value' column for Pipe('test', 'test').
359
490
  """
360
- from meerschaum.utils.warnings import error as _error, warn
491
+ from meerschaum.utils.warnings import error as _error
361
492
  if not args:
362
493
  args = tuple(self.columns.keys())
363
494
  col_names = []
@@ -367,7 +498,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
367
498
  col_name = self.columns[col]
368
499
  if col_name is None and error:
369
500
  _error(f"Please define the name of the '{col}' column for {self}.")
370
- except Exception as e:
501
+ except Exception:
371
502
  col_name = None
372
503
  if col_name is None and error:
373
504
  _error(f"Missing '{col}'" + f" column for {self}.")
@@ -409,16 +540,18 @@ def get_columns_types(
409
540
  """
410
541
  import time
411
542
  from meerschaum.connectors import get_connector_plugin
412
- from meerschaum.config.static import STATIC_CONFIG
413
- from meerschaum.utils.warnings import dprint
543
+ from meerschaum._internal.static import STATIC_CONFIG
414
544
 
415
545
  now = time.perf_counter()
416
- cache_seconds = STATIC_CONFIG['pipes']['static_schema_cache_seconds']
417
- if not self.static:
418
- refresh = True
546
+ cache_seconds = (
547
+ mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
548
+ if self.static
549
+ else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
550
+ )
419
551
  if refresh:
420
552
  _ = self.__dict__.pop('_columns_types_timestamp', None)
421
553
  _ = self.__dict__.pop('_columns_types', None)
554
+
422
555
  _columns_types = self.__dict__.get('_columns_types', None)
423
556
  if _columns_types:
424
557
  columns_types_timestamp = self.__dict__.get('_columns_types_timestamp', None)
@@ -454,14 +587,13 @@ def get_columns_indices(
454
587
  """
455
588
  import time
456
589
  from meerschaum.connectors import get_connector_plugin
457
- from meerschaum.config.static import STATIC_CONFIG
458
- from meerschaum.utils.warnings import dprint
590
+ from meerschaum._internal.static import STATIC_CONFIG
459
591
 
460
592
  now = time.perf_counter()
461
593
  cache_seconds = (
462
- STATIC_CONFIG['pipes']['static_schema_cache_seconds']
594
+ mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
463
595
  if self.static
464
- else STATIC_CONFIG['pipes']['exists_timeout_seconds']
596
+ else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
465
597
  )
466
598
  if refresh:
467
599
  _ = self.__dict__.pop('_columns_indices_timestamp', None)
@@ -509,7 +641,7 @@ def get_id(self, **kw: Any) -> Union[int, None]:
509
641
 
510
642
 
511
643
  @property
512
- def id(self) -> Union[int, None]:
644
+ def id(self) -> Union[int, str, uuid.UUID, None]:
513
645
  """
514
646
  Fetch and cache a pipe's ID.
515
647
  """
@@ -534,12 +666,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
534
666
  -------
535
667
  Either a string or `None`.
536
668
  """
537
- from meerschaum.utils.debug import dprint
538
669
  if debug:
539
670
  dprint('Attempting to determine the value column...')
540
671
  try:
541
672
  val_name = self.get_columns('value')
542
- except Exception as e:
673
+ except Exception:
543
674
  val_name = None
544
675
  if val_name is not None:
545
676
  if debug:
@@ -553,11 +684,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
553
684
  return None
554
685
  try:
555
686
  dt_name = self.get_columns('datetime', error=False)
556
- except Exception as e:
687
+ except Exception:
557
688
  dt_name = None
558
689
  try:
559
690
  id_name = self.get_columns('id', errors=False)
560
- except Exception as e:
691
+ except Exception:
561
692
  id_name = None
562
693
 
563
694
  if debug:
@@ -596,6 +727,7 @@ def parents(self) -> List[mrsm.Pipe]:
596
727
  """
597
728
  if 'parents' not in self.parameters:
598
729
  return []
730
+
599
731
  from meerschaum.utils.warnings import warn
600
732
  _parents_keys = self.parameters['parents']
601
733
  if not isinstance(_parents_keys, list):
@@ -634,6 +766,7 @@ def children(self) -> List[mrsm.Pipe]:
634
766
  """
635
767
  if 'children' not in self.parameters:
636
768
  return []
769
+
637
770
  from meerschaum.utils.warnings import warn
638
771
  _children_keys = self.parameters['children']
639
772
  if not isinstance(_children_keys, list):
@@ -714,7 +847,7 @@ def target(self, _target: str) -> None:
714
847
  Override the target of the in-memory pipe.
715
848
  Call `meerschaum.Pipe.edit` to persist changes.
716
849
  """
717
- self.parameters['target'] = _target
850
+ self.update_parameters({'target': _target}, persist=False)
718
851
 
719
852
 
720
853
  def guess_datetime(self) -> Union[str, None]:
@@ -755,3 +888,192 @@ def get_indices(self) -> Dict[str, str]:
755
888
  result = {}
756
889
 
757
890
  return result
891
+
892
+
893
+ def update_parameters(
894
+ self,
895
+ parameters_patch: Dict[str, Any],
896
+ persist: bool = True,
897
+ debug: bool = False,
898
+ ) -> mrsm.SuccessTuple:
899
+ """
900
+ Apply a patch to a pipe's `parameters` dictionary.
901
+
902
+ Parameters
903
+ ----------
904
+ parameters_patch: Dict[str, Any]
905
+ The patch to be applied to `Pipe.parameters`.
906
+
907
+ persist: bool, default True
908
+ If `True`, call `Pipe.edit()` to persist the new parameters.
909
+ """
910
+ from meerschaum.config import apply_patch_to_config
911
+ if 'parameters' not in self._attributes:
912
+ self._attributes['parameters'] = {}
913
+
914
+ if '_parameters' not in self.__dict__:
915
+ self._parameters = {}
916
+
917
+ self._attributes['parameters'] = apply_patch_to_config(
918
+ self._attributes['parameters'],
919
+ parameters_patch,
920
+ )
921
+
922
+ if self.temporary:
923
+ persist = False
924
+
925
+ if not persist:
926
+ return True, "Success"
927
+
928
+ return self.edit(debug=debug)
929
+
930
+
931
+ def get_precision(self, debug: bool = False) -> Dict[str, Union[str, int]]:
932
+ """
933
+ Return the timestamp precision unit and interval for the `datetime` axis.
934
+ """
935
+ from meerschaum.utils.dtypes import (
936
+ MRSM_PRECISION_UNITS_SCALARS,
937
+ MRSM_PRECISION_UNITS_ALIASES,
938
+ MRSM_PD_DTYPES,
939
+ are_dtypes_equal,
940
+ )
941
+ from meerschaum._internal.static import STATIC_CONFIG
942
+
943
+ if self.__dict__.get('_precision', None):
944
+ if debug:
945
+ dprint(f"Returning cached precision: {self._precision}")
946
+ return self._precision
947
+
948
+ parameters = self.parameters
949
+ _precision = parameters.get('precision', {})
950
+ if isinstance(_precision, str):
951
+ _precision = {'unit': _precision}
952
+ default_precision_unit = STATIC_CONFIG['dtypes']['datetime']['default_precision_unit']
953
+
954
+ if not _precision:
955
+
956
+ dt_col = parameters.get('columns', {}).get('datetime', None)
957
+ if not dt_col and self.autotime:
958
+ dt_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
959
+ if not dt_col:
960
+ if debug:
961
+ dprint(f"No datetime axis, returning default precision '{default_precision_unit}'.")
962
+ return {'unit': default_precision_unit}
963
+
964
+ dt_typ = self.dtypes.get(dt_col, 'datetime')
965
+ if are_dtypes_equal(dt_typ, 'datetime'):
966
+ if dt_typ == 'datetime':
967
+ dt_typ = MRSM_PD_DTYPES['datetime']
968
+ if debug:
969
+ dprint(f"Datetime type is `datetime`, assuming {dt_typ} precision.")
970
+
971
+ _precision = {
972
+ 'unit': (
973
+ dt_typ
974
+ .split('[', maxsplit=1)[-1]
975
+ .split(',', maxsplit=1)[0]
976
+ .split(' ', maxsplit=1)[0]
977
+ ).rstrip(']')
978
+ }
979
+
980
+ if debug:
981
+ dprint(f"Extracted precision '{_precision['unit']}' from type '{dt_typ}'.")
982
+
983
+ elif are_dtypes_equal(dt_typ, 'int'):
984
+ _precision = {
985
+ 'unit': (
986
+ 'second'
987
+ if '32' in dt_typ
988
+ else default_precision_unit
989
+ )
990
+ }
991
+ elif are_dtypes_equal(dt_typ, 'date'):
992
+ if debug:
993
+ dprint("Datetime axis is 'date', falling back to 'day' precision.")
994
+ _precision = {'unit': 'day'}
995
+
996
+ precision_unit = _precision.get('unit', default_precision_unit)
997
+ precision_interval = _precision.get('interval', None)
998
+ true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
999
+ if true_precision_unit is None:
1000
+ if debug:
1001
+ dprint(f"No precision could be determined, falling back to '{default_precision_unit}'.")
1002
+ true_precision_unit = default_precision_unit
1003
+
1004
+ if true_precision_unit not in MRSM_PRECISION_UNITS_SCALARS:
1005
+ from meerschaum.utils.misc import items_str
1006
+ raise ValueError(
1007
+ f"Invalid precision unit '{true_precision_unit}'.\n"
1008
+ "Accepted values are "
1009
+ f"{items_str(list(MRSM_PRECISION_UNITS_SCALARS) + list(MRSM_PRECISION_UNITS_ALIASES))}."
1010
+ )
1011
+
1012
+ self._precision = {'unit': true_precision_unit}
1013
+ if precision_interval:
1014
+ self._precision['interval'] = precision_interval
1015
+ return self._precision
1016
+
1017
+
1018
+ @property
1019
+ def precision(self) -> Dict[str, Union[str, int]]:
1020
+ """
1021
+ Return the configured or detected precision.
1022
+ """
1023
+ return self.get_precision()
1024
+
1025
+
1026
+ @precision.setter
1027
+ def precision(self, _precision: Union[str, Dict[str, Union[str, int]]]) -> None:
1028
+ """
1029
+ Update the `precision` parameter.
1030
+ """
1031
+ existing_precision = self._attributes.get('parameters', {}).get('precision', None)
1032
+ if isinstance(existing_precision, str):
1033
+ existing_precision = {'unit': existing_precision}
1034
+
1035
+ true_precision = (
1036
+ _precision
1037
+ if isinstance(_precision, dict)
1038
+ else {
1039
+ 'unit': _precision,
1040
+ **(
1041
+ {
1042
+ 'interval': existing_precision['interval'],
1043
+ } if existing_precision else {}
1044
+ )
1045
+ }
1046
+ )
1047
+
1048
+ self.update_parameters({'precision': true_precision}, persist=False)
1049
+ _ = self.__dict__.pop('_precision', None)
1050
+
1051
+
1052
+ def _invalidate_cache(
1053
+ self,
1054
+ hard: bool = False,
1055
+ debug: bool = False,
1056
+ ) -> None:
1057
+ """
1058
+ Invalidate temporary metadata cache.
1059
+
1060
+ Parameters
1061
+ ----------
1062
+ hard: bool, default False
1063
+ If `True`, clear all temporary cache.
1064
+ Otherwise only clear soft cache.
1065
+ """
1066
+ if debug:
1067
+ dprint(f"Invalidating {'some' if not hard else 'all'} cache for {self}.")
1068
+
1069
+ self._exists = None
1070
+ self._sync_ts = None
1071
+
1072
+ if not hard:
1073
+ return
1074
+
1075
+ _ = self.__dict__.pop('_parameters', None)
1076
+ _ = self.__dict__.pop('_precision', None)
1077
+ self._columns_types_timestamp = None
1078
+ self._columns_types = None
1079
+ self._attributes_sync_time = None