meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. meerschaum/__init__.py +5 -2
  2. meerschaum/_internal/__init__.py +1 -0
  3. meerschaum/_internal/arguments/_parse_arguments.py +4 -4
  4. meerschaum/_internal/arguments/_parser.py +33 -4
  5. meerschaum/_internal/cli/__init__.py +6 -0
  6. meerschaum/_internal/cli/daemons.py +103 -0
  7. meerschaum/_internal/cli/entry.py +220 -0
  8. meerschaum/_internal/cli/workers.py +435 -0
  9. meerschaum/_internal/docs/index.py +48 -2
  10. meerschaum/_internal/entry.py +50 -14
  11. meerschaum/_internal/shell/Shell.py +121 -29
  12. meerschaum/_internal/shell/__init__.py +4 -1
  13. meerschaum/_internal/static.py +359 -0
  14. meerschaum/_internal/term/TermPageHandler.py +1 -2
  15. meerschaum/_internal/term/__init__.py +40 -6
  16. meerschaum/_internal/term/tools.py +33 -8
  17. meerschaum/actions/__init__.py +6 -4
  18. meerschaum/actions/api.py +53 -13
  19. meerschaum/actions/attach.py +1 -0
  20. meerschaum/actions/bootstrap.py +8 -8
  21. meerschaum/actions/delete.py +4 -2
  22. meerschaum/actions/edit.py +171 -25
  23. meerschaum/actions/login.py +8 -8
  24. meerschaum/actions/register.py +143 -6
  25. meerschaum/actions/reload.py +22 -5
  26. meerschaum/actions/restart.py +14 -0
  27. meerschaum/actions/show.py +184 -31
  28. meerschaum/actions/start.py +166 -17
  29. meerschaum/actions/stop.py +38 -2
  30. meerschaum/actions/sync.py +7 -2
  31. meerschaum/actions/tag.py +9 -8
  32. meerschaum/actions/verify.py +5 -8
  33. meerschaum/api/__init__.py +45 -15
  34. meerschaum/api/_events.py +46 -4
  35. meerschaum/api/_oauth2.py +162 -9
  36. meerschaum/api/_tokens.py +102 -0
  37. meerschaum/api/dash/__init__.py +0 -3
  38. meerschaum/api/dash/callbacks/__init__.py +1 -0
  39. meerschaum/api/dash/callbacks/custom.py +4 -3
  40. meerschaum/api/dash/callbacks/dashboard.py +198 -118
  41. meerschaum/api/dash/callbacks/jobs.py +14 -7
  42. meerschaum/api/dash/callbacks/login.py +10 -1
  43. meerschaum/api/dash/callbacks/pipes.py +194 -14
  44. meerschaum/api/dash/callbacks/plugins.py +0 -1
  45. meerschaum/api/dash/callbacks/register.py +10 -3
  46. meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
  47. meerschaum/api/dash/callbacks/tokens.py +389 -0
  48. meerschaum/api/dash/components.py +36 -15
  49. meerschaum/api/dash/jobs.py +1 -1
  50. meerschaum/api/dash/keys.py +35 -93
  51. meerschaum/api/dash/pages/__init__.py +2 -1
  52. meerschaum/api/dash/pages/dashboard.py +1 -20
  53. meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
  54. meerschaum/api/dash/pages/login.py +2 -2
  55. meerschaum/api/dash/pages/pipes.py +16 -5
  56. meerschaum/api/dash/pages/settings/password_reset.py +1 -1
  57. meerschaum/api/dash/pages/tokens.py +53 -0
  58. meerschaum/api/dash/pipes.py +382 -95
  59. meerschaum/api/dash/sessions.py +12 -0
  60. meerschaum/api/dash/tokens.py +603 -0
  61. meerschaum/api/dash/websockets.py +1 -1
  62. meerschaum/api/dash/webterm.py +18 -6
  63. meerschaum/api/models/__init__.py +23 -3
  64. meerschaum/api/models/_actions.py +22 -0
  65. meerschaum/api/models/_pipes.py +91 -7
  66. meerschaum/api/models/_tokens.py +81 -0
  67. meerschaum/api/resources/static/js/terminado.js +3 -0
  68. meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
  69. meerschaum/api/resources/templates/termpage.html +13 -0
  70. meerschaum/api/routes/__init__.py +1 -0
  71. meerschaum/api/routes/_actions.py +3 -4
  72. meerschaum/api/routes/_connectors.py +3 -7
  73. meerschaum/api/routes/_jobs.py +26 -35
  74. meerschaum/api/routes/_login.py +120 -15
  75. meerschaum/api/routes/_misc.py +5 -10
  76. meerschaum/api/routes/_pipes.py +178 -143
  77. meerschaum/api/routes/_plugins.py +38 -28
  78. meerschaum/api/routes/_tokens.py +236 -0
  79. meerschaum/api/routes/_users.py +47 -35
  80. meerschaum/api/routes/_version.py +3 -3
  81. meerschaum/api/routes/_webterm.py +3 -3
  82. meerschaum/config/__init__.py +100 -30
  83. meerschaum/config/_default.py +132 -64
  84. meerschaum/config/_edit.py +38 -32
  85. meerschaum/config/_formatting.py +2 -0
  86. meerschaum/config/_patch.py +10 -8
  87. meerschaum/config/_paths.py +133 -13
  88. meerschaum/config/_read_config.py +87 -36
  89. meerschaum/config/_sync.py +6 -3
  90. meerschaum/config/_version.py +1 -1
  91. meerschaum/config/environment.py +262 -0
  92. meerschaum/config/stack/__init__.py +37 -15
  93. meerschaum/config/static.py +18 -0
  94. meerschaum/connectors/_Connector.py +11 -6
  95. meerschaum/connectors/__init__.py +41 -22
  96. meerschaum/connectors/api/_APIConnector.py +34 -6
  97. meerschaum/connectors/api/_actions.py +2 -2
  98. meerschaum/connectors/api/_jobs.py +12 -1
  99. meerschaum/connectors/api/_login.py +33 -7
  100. meerschaum/connectors/api/_misc.py +2 -2
  101. meerschaum/connectors/api/_pipes.py +23 -32
  102. meerschaum/connectors/api/_plugins.py +2 -2
  103. meerschaum/connectors/api/_request.py +1 -1
  104. meerschaum/connectors/api/_tokens.py +146 -0
  105. meerschaum/connectors/api/_users.py +70 -58
  106. meerschaum/connectors/instance/_InstanceConnector.py +83 -0
  107. meerschaum/connectors/instance/__init__.py +10 -0
  108. meerschaum/connectors/instance/_pipes.py +442 -0
  109. meerschaum/connectors/instance/_plugins.py +159 -0
  110. meerschaum/connectors/instance/_tokens.py +317 -0
  111. meerschaum/connectors/instance/_users.py +188 -0
  112. meerschaum/connectors/parse.py +5 -2
  113. meerschaum/connectors/sql/_SQLConnector.py +22 -5
  114. meerschaum/connectors/sql/_cli.py +12 -11
  115. meerschaum/connectors/sql/_create_engine.py +12 -168
  116. meerschaum/connectors/sql/_fetch.py +2 -18
  117. meerschaum/connectors/sql/_pipes.py +295 -278
  118. meerschaum/connectors/sql/_plugins.py +29 -0
  119. meerschaum/connectors/sql/_sql.py +46 -21
  120. meerschaum/connectors/sql/_users.py +36 -2
  121. meerschaum/connectors/sql/tables/__init__.py +254 -122
  122. meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
  123. meerschaum/connectors/valkey/_pipes.py +60 -31
  124. meerschaum/connectors/valkey/_plugins.py +2 -26
  125. meerschaum/core/Pipe/__init__.py +115 -85
  126. meerschaum/core/Pipe/_attributes.py +425 -124
  127. meerschaum/core/Pipe/_bootstrap.py +54 -24
  128. meerschaum/core/Pipe/_cache.py +555 -0
  129. meerschaum/core/Pipe/_clear.py +0 -11
  130. meerschaum/core/Pipe/_data.py +96 -68
  131. meerschaum/core/Pipe/_deduplicate.py +0 -13
  132. meerschaum/core/Pipe/_delete.py +12 -21
  133. meerschaum/core/Pipe/_drop.py +11 -23
  134. meerschaum/core/Pipe/_dtypes.py +49 -19
  135. meerschaum/core/Pipe/_edit.py +14 -4
  136. meerschaum/core/Pipe/_fetch.py +1 -1
  137. meerschaum/core/Pipe/_index.py +8 -14
  138. meerschaum/core/Pipe/_show.py +5 -5
  139. meerschaum/core/Pipe/_sync.py +123 -204
  140. meerschaum/core/Pipe/_verify.py +4 -4
  141. meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
  142. meerschaum/core/Plugin/__init__.py +1 -1
  143. meerschaum/core/Token/_Token.py +220 -0
  144. meerschaum/core/Token/__init__.py +12 -0
  145. meerschaum/core/User/_User.py +35 -10
  146. meerschaum/core/User/__init__.py +9 -1
  147. meerschaum/core/__init__.py +1 -0
  148. meerschaum/jobs/_Executor.py +88 -4
  149. meerschaum/jobs/_Job.py +149 -38
  150. meerschaum/jobs/__init__.py +3 -2
  151. meerschaum/jobs/systemd.py +8 -3
  152. meerschaum/models/__init__.py +35 -0
  153. meerschaum/models/pipes.py +247 -0
  154. meerschaum/models/tokens.py +38 -0
  155. meerschaum/models/users.py +26 -0
  156. meerschaum/plugins/__init__.py +301 -88
  157. meerschaum/plugins/bootstrap.py +510 -4
  158. meerschaum/utils/_get_pipes.py +97 -30
  159. meerschaum/utils/daemon/Daemon.py +199 -43
  160. meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
  161. meerschaum/utils/daemon/RotatingFile.py +63 -36
  162. meerschaum/utils/daemon/StdinFile.py +53 -13
  163. meerschaum/utils/daemon/__init__.py +47 -6
  164. meerschaum/utils/daemon/_names.py +6 -3
  165. meerschaum/utils/dataframe.py +479 -81
  166. meerschaum/utils/debug.py +49 -19
  167. meerschaum/utils/dtypes/__init__.py +476 -34
  168. meerschaum/utils/dtypes/sql.py +369 -29
  169. meerschaum/utils/formatting/__init__.py +5 -2
  170. meerschaum/utils/formatting/_jobs.py +1 -1
  171. meerschaum/utils/formatting/_pipes.py +52 -50
  172. meerschaum/utils/formatting/_pprint.py +1 -0
  173. meerschaum/utils/formatting/_shell.py +44 -18
  174. meerschaum/utils/misc.py +268 -186
  175. meerschaum/utils/packages/__init__.py +25 -40
  176. meerschaum/utils/packages/_packages.py +42 -34
  177. meerschaum/utils/pipes.py +213 -0
  178. meerschaum/utils/process.py +2 -2
  179. meerschaum/utils/prompt.py +175 -144
  180. meerschaum/utils/schedule.py +2 -1
  181. meerschaum/utils/sql.py +134 -47
  182. meerschaum/utils/threading.py +42 -0
  183. meerschaum/utils/typing.py +1 -4
  184. meerschaum/utils/venv/_Venv.py +2 -2
  185. meerschaum/utils/venv/__init__.py +7 -7
  186. meerschaum/utils/warnings.py +19 -13
  187. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
  188. meerschaum-3.0.0.dist-info/RECORD +289 -0
  189. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
  190. meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
  191. meerschaum/api/models/_interfaces.py +0 -15
  192. meerschaum/api/models/_locations.py +0 -15
  193. meerschaum/api/models/_metrics.py +0 -15
  194. meerschaum/config/_environment.py +0 -145
  195. meerschaum/config/static/__init__.py +0 -186
  196. meerschaum-2.9.5.dist-info/RECORD +0 -263
  197. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
  198. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
  199. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
  200. {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
@@ -8,11 +8,12 @@ Fetch and manipulate Pipes' attributes
8
8
 
9
9
  from __future__ import annotations
10
10
 
11
+ import uuid
11
12
  from datetime import timezone
12
13
 
13
14
  import meerschaum as mrsm
14
15
  from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
15
- from meerschaum.utils.warnings import warn
16
+ from meerschaum.utils.warnings import warn, dprint
16
17
 
17
18
 
18
19
  @property
@@ -21,57 +22,129 @@ def attributes(self) -> Dict[str, Any]:
21
22
  Return a dictionary of a pipe's keys and parameters.
22
23
  These values are reflected directly from the pipes table of the instance.
23
24
  """
24
- import time
25
25
  from meerschaum.config import get_config
26
26
  from meerschaum.config._patch import apply_patch_to_config
27
27
  from meerschaum.utils.venv import Venv
28
28
  from meerschaum.connectors import get_connector_plugin
29
+ from meerschaum.utils.dtypes import get_current_timestamp
29
30
 
30
31
  timeout_seconds = get_config('pipes', 'attributes', 'local_cache_timeout_seconds')
31
32
 
32
- if '_attributes' not in self.__dict__:
33
- self._attributes = {}
34
-
35
- now = time.perf_counter()
36
- last_refresh = self.__dict__.get('_attributes_sync_time', None)
33
+ now = get_current_timestamp('ms', as_int=True) / 1000
34
+ _attributes_sync_time = self._get_cached_value('_attributes_sync_time', debug=self.debug)
37
35
  timed_out = (
38
- last_refresh is None
36
+ _attributes_sync_time is None
39
37
  or
40
- (timeout_seconds is not None and (now - last_refresh) >= timeout_seconds)
38
+ (timeout_seconds is not None and (now - _attributes_sync_time) >= timeout_seconds)
41
39
  )
42
40
  if not self.temporary and timed_out:
43
- self._attributes_sync_time = now
44
- local_attributes = self.__dict__.get('_attributes', {})
41
+ self._cache_value('_attributes_sync_time', now, memory_only=True, debug=self.debug)
42
+ local_attributes = self._get_cached_value('attributes', debug=self.debug) or {}
45
43
  with Venv(get_connector_plugin(self.instance_connector)):
46
44
  instance_attributes = self.instance_connector.get_pipe_attributes(self)
47
- self._attributes = apply_patch_to_config(instance_attributes, local_attributes)
45
+
46
+ self._cache_value(
47
+ 'attributes',
48
+ apply_patch_to_config(instance_attributes, local_attributes),
49
+ memory_only=True,
50
+ debug=self.debug,
51
+ )
52
+
48
53
  return self._attributes
49
54
 
50
55
 
56
+ def get_parameters(
57
+ self,
58
+ apply_symlinks: bool = True,
59
+ refresh: bool = False,
60
+ debug: bool = False,
61
+ _visited: 'Optional[set[mrsm.Pipe]]' = None,
62
+ ) -> Dict[str, Any]:
63
+ """
64
+ Return the `parameters` dictionary of the pipe.
65
+
66
+ Parameters
67
+ ----------
68
+ apply_symlinks: bool, default True
69
+ If `True`, resolve references to parameters from other pipes.
70
+
71
+ refresh: bool, default False
72
+ If `True`, pull the latest attributes for the pipe.
73
+
74
+ Returns
75
+ -------
76
+ The pipe's parameters dictionary.
77
+ """
78
+ from meerschaum.config._patch import apply_patch_to_config
79
+ from meerschaum.config._read_config import search_and_substitute_config
80
+
81
+ if _visited is None:
82
+ _visited = {self}
83
+
84
+ if refresh:
85
+ _ = self._invalidate_cache(hard=True)
86
+
87
+ raw_parameters = self.attributes.get('parameters', {})
88
+ ref_keys = raw_parameters.get('reference')
89
+ if not apply_symlinks:
90
+ return raw_parameters
91
+
92
+ if ref_keys:
93
+ try:
94
+ if debug:
95
+ dprint(f"Building reference pipe from keys: {ref_keys}")
96
+ ref_pipe = mrsm.Pipe(**ref_keys)
97
+ if ref_pipe in _visited:
98
+ warn(f"Circular reference detected in {self}: chain involves {ref_pipe}.")
99
+ return search_and_substitute_config(raw_parameters)
100
+
101
+ _visited.add(ref_pipe)
102
+ base_params = ref_pipe.get_parameters(_visited=_visited, debug=debug)
103
+ except Exception as e:
104
+ warn(f"Failed to resolve reference pipe for {self}: {e}")
105
+ base_params = {}
106
+
107
+ params_to_apply = {k: v for k, v in raw_parameters.items() if k != 'reference'}
108
+ parameters = apply_patch_to_config(base_params, params_to_apply)
109
+ else:
110
+ parameters = raw_parameters
111
+
112
+ from meerschaum.utils.pipes import replace_pipes_syntax
113
+ self._symlinks = {}
114
+
115
+ def recursive_replace(obj: Any, path: tuple) -> Any:
116
+ if isinstance(obj, dict):
117
+ return {k: recursive_replace(v, path + (k,)) for k, v in obj.items()}
118
+ if isinstance(obj, list):
119
+ return [recursive_replace(elem, path + (i,)) for i, elem in enumerate(obj)]
120
+ if isinstance(obj, str):
121
+ substituted_val = replace_pipes_syntax(obj)
122
+ if substituted_val != obj:
123
+ self._symlinks[path] = {
124
+ 'original': obj,
125
+ 'substituted': substituted_val,
126
+ }
127
+ return substituted_val
128
+ return obj
129
+
130
+ return search_and_substitute_config(recursive_replace(parameters, tuple()))
131
+
132
+
51
133
  @property
52
134
  def parameters(self) -> Optional[Dict[str, Any]]:
53
135
  """
54
136
  Return the parameters dictionary of the pipe.
55
137
  """
56
- if 'parameters' not in self.attributes:
57
- self.attributes['parameters'] = {}
58
- _parameters = self.attributes['parameters']
59
- dt_col = _parameters.get('columns', {}).get('datetime', None)
60
- dt_typ = _parameters.get('dtypes', {}).get(dt_col, None) if dt_col else None
61
- if dt_col and not dt_typ:
62
- if 'dtypes' not in _parameters:
63
- self.attributes['parameters']['dtypes'] = {}
64
- self.attributes['parameters']['dtypes'][dt_col] = 'datetime'
65
- return self.attributes['parameters']
138
+ return self.get_parameters(debug=self.debug)
66
139
 
67
140
 
68
141
  @parameters.setter
69
- def parameters(self, parameters: Dict[str, Any]) -> None:
142
+ def parameters(self, _parameters: Dict[str, Any]) -> None:
70
143
  """
71
144
  Set the parameters dictionary of the in-memory pipe.
72
145
  Call `meerschaum.Pipe.edit()` to persist changes.
73
146
  """
74
- self.attributes['parameters'] = parameters
147
+ self._attributes['parameters'] = _parameters
75
148
 
76
149
 
77
150
  @property
@@ -79,12 +152,9 @@ def columns(self) -> Union[Dict[str, str], None]:
79
152
  """
80
153
  Return the `columns` dictionary defined in `meerschaum.Pipe.parameters`.
81
154
  """
82
- if 'columns' not in self.parameters:
83
- self.parameters['columns'] = {}
84
- cols = self.parameters['columns']
155
+ cols = self.parameters.get('columns', {})
85
156
  if not isinstance(cols, dict):
86
- cols = {}
87
- self.parameters['columns'] = cols
157
+ return {}
88
158
  return {col_ix: col for col_ix, col in cols.items() if col}
89
159
 
90
160
 
@@ -99,7 +169,7 @@ def columns(self, _columns: Union[Dict[str, str], List[str]]) -> None:
99
169
  if not isinstance(_columns, dict):
100
170
  warn(f"{self}.columns must be a dictionary, received {type(_columns)}.")
101
171
  return
102
- self.parameters['columns'] = _columns
172
+ self.update_parameters({'columns': _columns}, persist=False)
103
173
 
104
174
 
105
175
  @property
@@ -107,19 +177,18 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
107
177
  """
108
178
  Return the `indices` dictionary defined in `meerschaum.Pipe.parameters`.
109
179
  """
180
+ _parameters = self.get_parameters(debug=self.debug)
110
181
  indices_key = (
111
182
  'indexes'
112
- if 'indexes' in self.parameters
183
+ if 'indexes' in _parameters
113
184
  else 'indices'
114
185
  )
115
- if indices_key not in self.parameters:
116
- self.parameters[indices_key] = {}
117
- _indices = self.parameters[indices_key]
186
+
187
+ _indices = _parameters.get(indices_key, {})
118
188
  _columns = self.columns
119
189
  dt_col = _columns.get('datetime', None)
120
190
  if not isinstance(_indices, dict):
121
191
  _indices = {}
122
- self.parameters[indices_key] = _indices
123
192
  unique_cols = list(set((
124
193
  [dt_col]
125
194
  if dt_col
@@ -158,7 +227,7 @@ def indices(self, _indices: Union[Dict[str, Union[str, List[str]]], List[str]])
158
227
  if 'indexes' in self.parameters
159
228
  else 'indices'
160
229
  )
161
- self.parameters[indices_key] = _indices
230
+ self.update_parameters({indices_key: _indices}, persist=False)
162
231
 
163
232
 
164
233
  @indexes.setter
@@ -174,41 +243,30 @@ def tags(self) -> Union[List[str], None]:
174
243
  """
175
244
  If defined, return the `tags` list defined in `meerschaum.Pipe.parameters`.
176
245
  """
177
- if 'tags' not in self.parameters:
178
- self.parameters['tags'] = []
179
- return self.parameters['tags']
246
+ return self.parameters.get('tags', [])
180
247
 
181
248
 
182
249
  @tags.setter
183
- def tags(self, _tags: List[str, str]) -> None:
250
+ def tags(self, _tags: List[str]) -> None:
184
251
  """
185
252
  Override the tags list of the in-memory pipe.
186
253
  Call `meerschaum.Pipe.edit` to persist changes.
187
254
  """
188
255
  from meerschaum.utils.warnings import error
189
- from meerschaum.config.static import STATIC_CONFIG
256
+ from meerschaum._internal.static import STATIC_CONFIG
190
257
  negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
191
258
  for t in _tags:
192
259
  if t.startswith(negation_prefix):
193
260
  error(f"Tags cannot begin with '{negation_prefix}'.")
194
- self.parameters['tags'] = _tags
261
+ self.update_parameters({'tags': _tags}, persist=False)
195
262
 
196
263
 
197
264
  @property
198
- def dtypes(self) -> Union[Dict[str, Any], None]:
265
+ def dtypes(self) -> Dict[str, Any]:
199
266
  """
200
267
  If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
201
268
  """
202
- from meerschaum.config._patch import apply_patch_to_config
203
- from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
204
- configured_dtypes = self.parameters.get('dtypes', {})
205
- remote_dtypes = self.infer_dtypes(persist=False)
206
- patched_dtypes = apply_patch_to_config(remote_dtypes, configured_dtypes)
207
- return {
208
- col: MRSM_ALIAS_DTYPES.get(typ, typ)
209
- for col, typ in patched_dtypes.items()
210
- if col and typ
211
- }
269
+ return self.get_dtypes(refresh=False, debug=self.debug)
212
270
 
213
271
 
214
272
  @dtypes.setter
@@ -217,7 +275,61 @@ def dtypes(self, _dtypes: Dict[str, Any]) -> None:
217
275
  Override the dtypes dictionary of the in-memory pipe.
218
276
  Call `meerschaum.Pipe.edit()` to persist changes.
219
277
  """
220
- self.parameters['dtypes'] = _dtypes
278
+ self.update_parameters({'dtypes': _dtypes}, persist=False)
279
+ self._clear_cache_key('_remote_dtypes', debug=self.debug)
280
+ self._clear_cache_key('_remote_dtypes_timestamp', debug=self.debug)
281
+
282
+
283
+ def get_dtypes(
284
+ self,
285
+ infer: bool = True,
286
+ refresh: bool = False,
287
+ debug: bool = False,
288
+ ) -> Dict[str, Any]:
289
+ """
290
+ If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
291
+
292
+ Parameters
293
+ ----------
294
+ infer: bool, default True
295
+ If `True`, include the implicit existing dtypes.
296
+ Else only return the explicitly configured dtypes (e.g. `Pipe.parameters['dtypes']`).
297
+
298
+ refresh: bool, default False
299
+ If `True`, invalidate any cache and return the latest known dtypes.
300
+
301
+ Returns
302
+ -------
303
+ A dictionary mapping column names to dtypes.
304
+ """
305
+ from meerschaum.config._patch import apply_patch_to_config
306
+ from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
307
+ parameters = self.get_parameters(refresh=refresh, debug=debug)
308
+ configured_dtypes = parameters.get('dtypes', {})
309
+ if debug:
310
+ dprint(f"Configured dtypes for {self}:")
311
+ mrsm.pprint(configured_dtypes)
312
+
313
+ remote_dtypes = (
314
+ self.infer_dtypes(persist=False, refresh=refresh, debug=debug)
315
+ if infer
316
+ else {}
317
+ )
318
+ patched_dtypes = apply_patch_to_config((remote_dtypes or {}), (configured_dtypes or {}))
319
+
320
+ dt_col = parameters.get('columns', {}).get('datetime', None)
321
+ primary_col = parameters.get('columns', {}).get('primary', None)
322
+ _dtypes = {
323
+ col: MRSM_ALIAS_DTYPES.get(typ, typ)
324
+ for col, typ in patched_dtypes.items()
325
+ if col and typ
326
+ }
327
+ if dt_col and dt_col not in configured_dtypes:
328
+ _dtypes[dt_col] = 'datetime'
329
+ if primary_col and parameters.get('autoincrement', False) and primary_col not in _dtypes:
330
+ _dtypes[primary_col] = 'int'
331
+
332
+ return _dtypes
221
333
 
222
334
 
223
335
  @property
@@ -225,9 +337,7 @@ def upsert(self) -> bool:
225
337
  """
226
338
  Return whether `upsert` is set for the pipe.
227
339
  """
228
- if 'upsert' not in self.parameters:
229
- self.parameters['upsert'] = False
230
- return self.parameters['upsert']
340
+ return self.parameters.get('upsert', False)
231
341
 
232
342
 
233
343
  @upsert.setter
@@ -235,7 +345,7 @@ def upsert(self, _upsert: bool) -> None:
235
345
  """
236
346
  Set the `upsert` parameter for the pipe.
237
347
  """
238
- self.parameters['upsert'] = _upsert
348
+ self.update_parameters({'upsert': _upsert}, persist=False)
239
349
 
240
350
 
241
351
  @property
@@ -243,9 +353,7 @@ def static(self) -> bool:
243
353
  """
244
354
  Return whether `static` is set for the pipe.
245
355
  """
246
- if 'static' not in self.parameters:
247
- self.parameters['static'] = False
248
- return self.parameters['static']
356
+ return self.parameters.get('static', False)
249
357
 
250
358
 
251
359
  @static.setter
@@ -253,7 +361,8 @@ def static(self, _static: bool) -> None:
253
361
  """
254
362
  Set the `static` parameter for the pipe.
255
363
  """
256
- self.parameters['static'] = _static
364
+ self.update_parameters({'static': _static}, persist=False)
365
+ self._static = _static
257
366
 
258
367
 
259
368
  @property
@@ -261,10 +370,7 @@ def autoincrement(self) -> bool:
261
370
  """
262
371
  Return the `autoincrement` parameter for the pipe.
263
372
  """
264
- if 'autoincrement' not in self.parameters:
265
- self.parameters['autoincrement'] = False
266
-
267
- return self.parameters['autoincrement']
373
+ return self.parameters.get('autoincrement', False)
268
374
 
269
375
 
270
376
  @autoincrement.setter
@@ -272,7 +378,23 @@ def autoincrement(self, _autoincrement: bool) -> None:
272
378
  """
273
379
  Set the `autoincrement` parameter for the pipe.
274
380
  """
275
- self.parameters['autoincrement'] = _autoincrement
381
+ self.update_parameters({'autoincrement': _autoincrement}, persist=False)
382
+
383
+
384
+ @property
385
+ def autotime(self) -> bool:
386
+ """
387
+ Return the `autotime` parameter for the pipe.
388
+ """
389
+ return self.parameters.get('autotime', False)
390
+
391
+
392
+ @autotime.setter
393
+ def autotime(self, _autotime: bool) -> None:
394
+ """
395
+ Set the `autotime` parameter for the pipe.
396
+ """
397
+ self.update_parameters({'autotime': _autotime}, persist=False)
276
398
 
277
399
 
278
400
  @property
@@ -280,18 +402,23 @@ def tzinfo(self) -> Union[None, timezone]:
280
402
  """
281
403
  Return `timezone.utc` if the pipe is timezone-aware.
282
404
  """
283
- dt_col = self.columns.get('datetime', None)
284
- if not dt_col:
285
- return None
405
+ _tzinfo = self._get_cached_value('tzinfo', debug=self.debug)
406
+ if _tzinfo is not None:
407
+ return _tzinfo if _tzinfo != 'None' else None
286
408
 
287
- dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
288
- if 'utc' in dt_typ.lower() or dt_typ == 'datetime':
289
- return timezone.utc
409
+ _tzinfo = None
410
+ dt_col = self.columns.get('datetime', None)
411
+ dt_typ = str(self.dtypes.get(dt_col, 'datetime')) if dt_col else None
412
+ if self.autotime:
413
+ ts_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
414
+ ts_typ = self.dtypes.get(ts_col, 'datetime')
415
+ dt_typ = ts_typ
290
416
 
291
- if dt_typ == 'datetime64[ns]':
292
- return None
417
+ if dt_typ and 'utc' in dt_typ.lower() or dt_typ == 'datetime':
418
+ _tzinfo = timezone.utc
293
419
 
294
- return None
420
+ self._cache_value('tzinfo', (_tzinfo if _tzinfo is not None else 'None'), debug=self.debug)
421
+ return _tzinfo
295
422
 
296
423
 
297
424
  @property
@@ -299,10 +426,7 @@ def enforce(self) -> bool:
299
426
  """
300
427
  Return the `enforce` parameter for the pipe.
301
428
  """
302
- if 'enforce' not in self.parameters:
303
- self.parameters['enforce'] = True
304
-
305
- return self.parameters['enforce']
429
+ return self.parameters.get('enforce', True)
306
430
 
307
431
 
308
432
  @enforce.setter
@@ -310,7 +434,7 @@ def enforce(self, _enforce: bool) -> None:
310
434
  """
311
435
  Set the `enforce` parameter for the pipe.
312
436
  """
313
- self.parameters['enforce'] = _enforce
437
+ self.update_parameters({'enforce': _enforce}, persist=False)
314
438
 
315
439
 
316
440
  @property
@@ -318,10 +442,7 @@ def null_indices(self) -> bool:
318
442
  """
319
443
  Return the `null_indices` parameter for the pipe.
320
444
  """
321
- if 'null_indices' not in self.parameters:
322
- self.parameters['null_indices'] = True
323
-
324
- return self.parameters['null_indices']
445
+ return self.parameters.get('null_indices', True)
325
446
 
326
447
 
327
448
  @null_indices.setter
@@ -329,7 +450,23 @@ def null_indices(self, _null_indices: bool) -> None:
329
450
  """
330
451
  Set the `null_indices` parameter for the pipe.
331
452
  """
332
- self.parameters['null_indices'] = _null_indices
453
+ self.update_parameters({'null_indices': _null_indices}, persist=False)
454
+
455
+
456
+ @property
457
+ def mixed_numerics(self) -> bool:
458
+ """
459
+ Return the `mixed_numerics` parameter for the pipe.
460
+ """
461
+ return self.parameters.get('mixed_numerics', True)
462
+
463
+
464
+ @mixed_numerics.setter
465
+ def mixed_numerics(self, _mixed_numerics: bool) -> None:
466
+ """
467
+ Set the `mixed_numerics` parameter for the pipe.
468
+ """
469
+ self.update_parameters({'mixed_numerics': _mixed_numerics}, persist=False)
333
470
 
334
471
 
335
472
  def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
@@ -357,7 +494,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
357
494
  >>> pipe.get_columns('value', error=True)
358
495
  Exception: 🛑 Missing 'value' column for Pipe('test', 'test').
359
496
  """
360
- from meerschaum.utils.warnings import error as _error, warn
497
+ from meerschaum.utils.warnings import error as _error
361
498
  if not args:
362
499
  args = tuple(self.columns.keys())
363
500
  col_names = []
@@ -367,7 +504,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
367
504
  col_name = self.columns[col]
368
505
  if col_name is None and error:
369
506
  _error(f"Please define the name of the '{col}' column for {self}.")
370
- except Exception as e:
507
+ except Exception:
371
508
  col_name = None
372
509
  if col_name is None and error:
373
510
  _error(f"Missing '{col}'" + f" column for {self}.")
@@ -407,21 +544,22 @@ def get_columns_types(
407
544
  }
408
545
  >>>
409
546
  """
410
- import time
411
547
  from meerschaum.connectors import get_connector_plugin
412
- from meerschaum.config.static import STATIC_CONFIG
413
- from meerschaum.utils.warnings import dprint
548
+ from meerschaum.utils.dtypes import get_current_timestamp
414
549
 
415
- now = time.perf_counter()
416
- cache_seconds = STATIC_CONFIG['pipes']['static_schema_cache_seconds']
417
- if not self.static:
418
- refresh = True
550
+ now = get_current_timestamp('ms', as_int=True) / 1000
551
+ cache_seconds = (
552
+ mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
553
+ if self.static
554
+ else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
555
+ )
419
556
  if refresh:
420
- _ = self.__dict__.pop('_columns_types_timestamp', None)
421
- _ = self.__dict__.pop('_columns_types', None)
422
- _columns_types = self.__dict__.get('_columns_types', None)
557
+ self._clear_cache_key('_columns_types_timestamp', debug=debug)
558
+ self._clear_cache_key('_columns_types', debug=debug)
559
+
560
+ _columns_types = self._get_cached_value('_columns_types', debug=debug)
423
561
  if _columns_types:
424
- columns_types_timestamp = self.__dict__.get('_columns_types_timestamp', None)
562
+ columns_types_timestamp = self._get_cached_value('_columns_types_timestamp', debug=debug)
425
563
  if columns_types_timestamp is not None:
426
564
  delta = now - columns_types_timestamp
427
565
  if delta < cache_seconds:
@@ -439,8 +577,8 @@ def get_columns_types(
439
577
  else None
440
578
  )
441
579
 
442
- self.__dict__['_columns_types'] = _columns_types
443
- self.__dict__['_columns_types_timestamp'] = now
580
+ self._cache_value('_columns_types', _columns_types, debug=debug)
581
+ self._cache_value('_columns_types_timestamp', now, debug=debug)
444
582
  return _columns_types or {}
445
583
 
446
584
 
@@ -452,23 +590,23 @@ def get_columns_indices(
452
590
  """
453
591
  Return a dictionary mapping columns to index information.
454
592
  """
455
- import time
456
593
  from meerschaum.connectors import get_connector_plugin
457
- from meerschaum.config.static import STATIC_CONFIG
458
- from meerschaum.utils.warnings import dprint
594
+ from meerschaum.utils.dtypes import get_current_timestamp
459
595
 
460
- now = time.perf_counter()
596
+ now = get_current_timestamp('ms', as_int=True) / 1000
461
597
  cache_seconds = (
462
- STATIC_CONFIG['pipes']['static_schema_cache_seconds']
598
+ mrsm.get_config('pipes', 'static', 'static_schema_cache_seconds')
463
599
  if self.static
464
- else STATIC_CONFIG['pipes']['exists_timeout_seconds']
600
+ else mrsm.get_config('pipes', 'dtypes', 'columns_types_cache_seconds')
465
601
  )
466
602
  if refresh:
467
- _ = self.__dict__.pop('_columns_indices_timestamp', None)
468
- _ = self.__dict__.pop('_columns_indices', None)
469
- _columns_indices = self.__dict__.get('_columns_indices', None)
603
+ self._clear_cache_key('_columns_indices_timestamp', debug=debug)
604
+ self._clear_cache_key('_columns_indices', debug=debug)
605
+
606
+ _columns_indices = self._get_cached_value('_columns_indices', debug=debug)
607
+
470
608
  if _columns_indices:
471
- columns_indices_timestamp = self.__dict__.get('_columns_indices_timestamp', None)
609
+ columns_indices_timestamp = self._get_cached_value('_columns_indices_timestamp', debug=debug)
472
610
  if columns_indices_timestamp is not None:
473
611
  delta = now - columns_indices_timestamp
474
612
  if delta < cache_seconds:
@@ -486,18 +624,19 @@ def get_columns_indices(
486
624
  else None
487
625
  )
488
626
 
489
- self.__dict__['_columns_indices'] = _columns_indices
490
- self.__dict__['_columns_indices_timestamp'] = now
627
+ self._cache_value('_columns_indices', _columns_indices, debug=debug)
628
+ self._cache_value('_columns_indices_timestamp', now, debug=debug)
491
629
  return {k: v for k, v in _columns_indices.items() if k and v} or {}
492
630
 
493
631
 
494
- def get_id(self, **kw: Any) -> Union[int, None]:
632
+ def get_id(self, **kw: Any) -> Union[int, str, None]:
495
633
  """
496
634
  Fetch a pipe's ID from its instance connector.
497
- If the pipe does not exist, return `None`.
635
+ If the pipe is not registered, return `None`.
498
636
  """
499
637
  if self.temporary:
500
638
  return None
639
+
501
640
  from meerschaum.utils.venv import Venv
502
641
  from meerschaum.connectors import get_connector_plugin
503
642
 
@@ -509,13 +648,16 @@ def get_id(self, **kw: Any) -> Union[int, None]:
509
648
 
510
649
 
511
650
  @property
512
- def id(self) -> Union[int, None]:
651
+ def id(self) -> Union[int, str, uuid.UUID, None]:
513
652
  """
514
653
  Fetch and cache a pipe's ID.
515
654
  """
516
- if not ('_id' in self.__dict__ and self._id):
517
- self._id = self.get_id()
518
- return self._id
655
+ _id = self._get_cached_value('_id', debug=self.debug)
656
+ if not _id:
657
+ _id = self.get_id(debug=self.debug)
658
+ if _id is not None:
659
+ self._cache_value('_id', _id, debug=self.debug)
660
+ return _id
519
661
 
520
662
 
521
663
  def get_val_column(self, debug: bool = False) -> Union[str, None]:
@@ -534,12 +676,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
534
676
  -------
535
677
  Either a string or `None`.
536
678
  """
537
- from meerschaum.utils.debug import dprint
538
679
  if debug:
539
680
  dprint('Attempting to determine the value column...')
540
681
  try:
541
682
  val_name = self.get_columns('value')
542
- except Exception as e:
683
+ except Exception:
543
684
  val_name = None
544
685
  if val_name is not None:
545
686
  if debug:
@@ -553,11 +694,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
553
694
  return None
554
695
  try:
555
696
  dt_name = self.get_columns('datetime', error=False)
556
- except Exception as e:
697
+ except Exception:
557
698
  dt_name = None
558
699
  try:
559
700
  id_name = self.get_columns('id', errors=False)
560
- except Exception as e:
701
+ except Exception:
561
702
  id_name = None
562
703
 
563
704
  if debug:
@@ -596,6 +737,7 @@ def parents(self) -> List[mrsm.Pipe]:
596
737
  """
597
738
  if 'parents' not in self.parameters:
598
739
  return []
740
+
599
741
  from meerschaum.utils.warnings import warn
600
742
  _parents_keys = self.parameters['parents']
601
743
  if not isinstance(_parents_keys, list):
@@ -634,6 +776,7 @@ def children(self) -> List[mrsm.Pipe]:
634
776
  """
635
777
  if 'children' not in self.parameters:
636
778
  return []
779
+
637
780
  from meerschaum.utils.warnings import warn
638
781
  _children_keys = self.parameters['children']
639
782
  if not isinstance(_children_keys, list):
@@ -714,7 +857,7 @@ def target(self, _target: str) -> None:
714
857
  Override the target of the in-memory pipe.
715
858
  Call `meerschaum.Pipe.edit` to persist changes.
716
859
  """
717
- self.parameters['target'] = _target
860
+ self.update_parameters({'target': _target}, persist=False)
718
861
 
719
862
 
720
863
  def guess_datetime(self) -> Union[str, None]:
@@ -755,3 +898,161 @@ def get_indices(self) -> Dict[str, str]:
755
898
  result = {}
756
899
 
757
900
  return result
901
+
902
+
903
+ def update_parameters(
904
+ self,
905
+ parameters_patch: Dict[str, Any],
906
+ persist: bool = True,
907
+ debug: bool = False,
908
+ ) -> mrsm.SuccessTuple:
909
+ """
910
+ Apply a patch to a pipe's `parameters` dictionary.
911
+
912
+ Parameters
913
+ ----------
914
+ parameters_patch: Dict[str, Any]
915
+ The patch to be applied to `Pipe.parameters`.
916
+
917
+ persist: bool, default True
918
+ If `True`, call `Pipe.edit()` to persist the new parameters.
919
+ """
920
+ from meerschaum.config import apply_patch_to_config
921
+ if 'parameters' not in self._attributes:
922
+ self._attributes['parameters'] = {}
923
+
924
+ self._attributes['parameters'] = apply_patch_to_config(
925
+ self._attributes['parameters'],
926
+ parameters_patch,
927
+ )
928
+
929
+ if self.temporary:
930
+ persist = False
931
+
932
+ if not persist:
933
+ return True, "Success"
934
+
935
+ return self.edit(debug=debug)
936
+
937
+
938
+ def get_precision(self, debug: bool = False) -> Dict[str, Union[str, int]]:
939
+ """
940
+ Return the timestamp precision unit and interval for the `datetime` axis.
941
+ """
942
+ from meerschaum.utils.dtypes import (
943
+ MRSM_PRECISION_UNITS_SCALARS,
944
+ MRSM_PRECISION_UNITS_ALIASES,
945
+ MRSM_PD_DTYPES,
946
+ are_dtypes_equal,
947
+ )
948
+ from meerschaum._internal.static import STATIC_CONFIG
949
+
950
+ _precision = self._get_cached_value('precision', debug=debug)
951
+ if _precision:
952
+ if debug:
953
+ dprint(f"Returning cached precision: {_precision}")
954
+ return _precision
955
+
956
+ parameters = self.parameters
957
+ _precision = parameters.get('precision', {})
958
+ if isinstance(_precision, str):
959
+ _precision = {'unit': _precision}
960
+ default_precision_unit = STATIC_CONFIG['dtypes']['datetime']['default_precision_unit']
961
+
962
+ if not _precision:
963
+
964
+ dt_col = parameters.get('columns', {}).get('datetime', None)
965
+ if not dt_col and self.autotime:
966
+ dt_col = mrsm.get_config('pipes', 'autotime', 'column_name_if_datetime_missing')
967
+ if not dt_col:
968
+ if debug:
969
+ dprint(f"No datetime axis, returning default precision '{default_precision_unit}'.")
970
+ return {'unit': default_precision_unit}
971
+
972
+ dt_typ = self.dtypes.get(dt_col, 'datetime')
973
+ if are_dtypes_equal(dt_typ, 'datetime'):
974
+ if dt_typ == 'datetime':
975
+ dt_typ = MRSM_PD_DTYPES['datetime']
976
+ if debug:
977
+ dprint(f"Datetime type is `datetime`, assuming {dt_typ} precision.")
978
+
979
+ _precision = {
980
+ 'unit': (
981
+ dt_typ
982
+ .split('[', maxsplit=1)[-1]
983
+ .split(',', maxsplit=1)[0]
984
+ .split(' ', maxsplit=1)[0]
985
+ ).rstrip(']')
986
+ }
987
+
988
+ if debug:
989
+ dprint(f"Extracted precision '{_precision['unit']}' from type '{dt_typ}'.")
990
+
991
+ elif are_dtypes_equal(dt_typ, 'int'):
992
+ _precision = {
993
+ 'unit': (
994
+ 'second'
995
+ if '32' in dt_typ
996
+ else default_precision_unit
997
+ )
998
+ }
999
+ elif are_dtypes_equal(dt_typ, 'date'):
1000
+ if debug:
1001
+ dprint("Datetime axis is 'date', falling back to 'day' precision.")
1002
+ _precision = {'unit': 'day'}
1003
+
1004
+ precision_unit = _precision.get('unit', default_precision_unit)
1005
+ precision_interval = _precision.get('interval', None)
1006
+ true_precision_unit = MRSM_PRECISION_UNITS_ALIASES.get(precision_unit, precision_unit)
1007
+ if true_precision_unit is None:
1008
+ if debug:
1009
+ dprint(f"No precision could be determined, falling back to '{default_precision_unit}'.")
1010
+ true_precision_unit = default_precision_unit
1011
+
1012
+ if true_precision_unit not in MRSM_PRECISION_UNITS_SCALARS:
1013
+ from meerschaum.utils.misc import items_str
1014
+ raise ValueError(
1015
+ f"Invalid precision unit '{true_precision_unit}'.\n"
1016
+ "Accepted values are "
1017
+ f"{items_str(list(MRSM_PRECISION_UNITS_SCALARS) + list(MRSM_PRECISION_UNITS_ALIASES))}."
1018
+ )
1019
+
1020
+ _precision = {'unit': true_precision_unit}
1021
+ if precision_interval:
1022
+ _precision['interval'] = precision_interval
1023
+ self._cache_value('precision', _precision, debug=debug)
1024
+ return self._precision
1025
+
1026
+
1027
+ @property
1028
+ def precision(self) -> Dict[str, Union[str, int]]:
1029
+ """
1030
+ Return the configured or detected precision.
1031
+ """
1032
+ return self.get_precision(debug=self.debug)
1033
+
1034
+
1035
+ @precision.setter
1036
+ def precision(self, _precision: Union[str, Dict[str, Union[str, int]]]) -> None:
1037
+ """
1038
+ Update the `precision` parameter.
1039
+ """
1040
+ existing_precision = self._attributes.get('parameters', {}).get('precision', None)
1041
+ if isinstance(existing_precision, str):
1042
+ existing_precision = {'unit': existing_precision}
1043
+
1044
+ true_precision = (
1045
+ _precision
1046
+ if isinstance(_precision, dict)
1047
+ else {
1048
+ 'unit': _precision,
1049
+ **(
1050
+ {
1051
+ 'interval': existing_precision['interval'],
1052
+ } if existing_precision else {}
1053
+ )
1054
+ }
1055
+ )
1056
+
1057
+ self.update_parameters({'precision': true_precision}, persist=False)
1058
+ self._clear_cache_key('precision', debug=self.debug)