meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +2 -5
  2. meerschaum/_internal/docs/index.py +3 -2
  3. meerschaum/_internal/entry.py +13 -7
  4. meerschaum/_internal/shell/Shell.py +38 -44
  5. meerschaum/_internal/term/TermPageHandler.py +2 -3
  6. meerschaum/_internal/term/__init__.py +13 -11
  7. meerschaum/actions/api.py +10 -7
  8. meerschaum/actions/bootstrap.py +38 -11
  9. meerschaum/actions/copy.py +3 -3
  10. meerschaum/actions/delete.py +4 -1
  11. meerschaum/actions/register.py +1 -3
  12. meerschaum/actions/stack.py +24 -19
  13. meerschaum/actions/start.py +38 -40
  14. meerschaum/actions/sync.py +53 -52
  15. meerschaum/api/__init__.py +48 -14
  16. meerschaum/api/_events.py +15 -10
  17. meerschaum/api/_oauth2.py +2 -2
  18. meerschaum/api/_websockets.py +5 -4
  19. meerschaum/api/dash/__init__.py +7 -16
  20. meerschaum/api/dash/callbacks/__init__.py +1 -0
  21. meerschaum/api/dash/callbacks/dashboard.py +52 -58
  22. meerschaum/api/dash/callbacks/jobs.py +15 -16
  23. meerschaum/api/dash/callbacks/login.py +16 -10
  24. meerschaum/api/dash/callbacks/pipes.py +41 -0
  25. meerschaum/api/dash/callbacks/plugins.py +1 -1
  26. meerschaum/api/dash/callbacks/register.py +15 -11
  27. meerschaum/api/dash/components.py +54 -59
  28. meerschaum/api/dash/jobs.py +5 -9
  29. meerschaum/api/dash/pages/__init__.py +1 -0
  30. meerschaum/api/dash/pages/pipes.py +19 -0
  31. meerschaum/api/dash/pipes.py +86 -58
  32. meerschaum/api/dash/plugins.py +6 -4
  33. meerschaum/api/dash/sessions.py +176 -0
  34. meerschaum/api/dash/users.py +3 -41
  35. meerschaum/api/dash/webterm.py +12 -17
  36. meerschaum/api/resources/static/js/terminado.js +1 -1
  37. meerschaum/api/routes/_actions.py +4 -20
  38. meerschaum/api/routes/_jobs.py +8 -7
  39. meerschaum/api/routes/_login.py +4 -4
  40. meerschaum/api/routes/_pipes.py +3 -3
  41. meerschaum/api/routes/_webterm.py +5 -6
  42. meerschaum/config/_default.py +15 -2
  43. meerschaum/config/_version.py +1 -1
  44. meerschaum/config/stack/__init__.py +64 -19
  45. meerschaum/config/static/__init__.py +4 -0
  46. meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
  47. meerschaum/connectors/__init__.py +24 -14
  48. meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
  49. meerschaum/connectors/api/__init__.py +2 -1
  50. meerschaum/connectors/parse.py +18 -16
  51. meerschaum/connectors/poll.py +30 -24
  52. meerschaum/connectors/sql/__init__.py +3 -1
  53. meerschaum/connectors/sql/_pipes.py +172 -197
  54. meerschaum/connectors/sql/_plugins.py +45 -43
  55. meerschaum/connectors/sql/_users.py +46 -38
  56. meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
  57. meerschaum/connectors/valkey/__init__.py +10 -0
  58. meerschaum/connectors/valkey/_fetch.py +75 -0
  59. meerschaum/connectors/valkey/_pipes.py +844 -0
  60. meerschaum/connectors/valkey/_plugins.py +265 -0
  61. meerschaum/connectors/valkey/_users.py +305 -0
  62. meerschaum/core/Pipe/__init__.py +3 -0
  63. meerschaum/core/Pipe/_attributes.py +1 -2
  64. meerschaum/core/Pipe/_clear.py +16 -13
  65. meerschaum/core/Pipe/_copy.py +106 -0
  66. meerschaum/core/Pipe/_data.py +165 -101
  67. meerschaum/core/Pipe/_drop.py +4 -4
  68. meerschaum/core/Pipe/_dtypes.py +14 -14
  69. meerschaum/core/Pipe/_edit.py +15 -14
  70. meerschaum/core/Pipe/_sync.py +134 -53
  71. meerschaum/core/Pipe/_verify.py +11 -11
  72. meerschaum/core/User/_User.py +14 -12
  73. meerschaum/jobs/_Job.py +1 -6
  74. meerschaum/jobs/__init__.py +7 -2
  75. meerschaum/plugins/_Plugin.py +17 -13
  76. meerschaum/utils/_get_pipes.py +14 -20
  77. meerschaum/utils/dataframe.py +291 -101
  78. meerschaum/utils/dtypes/__init__.py +31 -6
  79. meerschaum/utils/dtypes/sql.py +4 -4
  80. meerschaum/utils/formatting/_shell.py +5 -6
  81. meerschaum/utils/misc.py +3 -3
  82. meerschaum/utils/packages/__init__.py +14 -9
  83. meerschaum/utils/packages/_packages.py +2 -0
  84. meerschaum/utils/schedule.py +1 -0
  85. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
  86. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/RECORD +93 -84
  87. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
  88. meerschaum/api/dash/actions.py +0 -255
  89. /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
  90. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
  91. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
  92. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
  93. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
  94. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
@@ -0,0 +1,106 @@
1
+ #! /usr/bin/env python3
2
+ # vim:fenc=utf-8
3
+
4
+ """
5
+ Define methods for copying pipes.
6
+ """
7
+
8
+ from datetime import datetime, timedelta
9
+
10
+ import meerschaum as mrsm
11
+ from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Union
12
+
13
+
14
+ def copy_to(
15
+ self,
16
+ instance_keys: str,
17
+ sync: bool = True,
18
+ begin: Union[datetime, int, None] = None,
19
+ end: Union[datetime, int, None] = None,
20
+ params: Optional[Dict[str, Any]] = None,
21
+ chunk_interval: Union[timedelta, int, None] = None,
22
+ debug: bool = False,
23
+ **kwargs: Any
24
+ ) -> SuccessTuple:
25
+ """
26
+ Copy a pipe to another instance.
27
+
28
+ Parameters
29
+ ----------
30
+ instance_keys: str
31
+ The instance to which to copy this pipe.
32
+
33
+ sync: bool, default True
34
+ If `True`, sync the source pipe's documents
35
+
36
+ begin: Union[datetime, int, None], default None
37
+ Beginning datetime value to pass to `Pipe.get_data()`.
38
+
39
+ end: Union[datetime, int, None], default None
40
+ End datetime value to pass to `Pipe.get_data()`.
41
+
42
+ params: Optional[Dict[str, Any]], default None
43
+ Parameters filter to pass to `Pipe.get_data()`.
44
+
45
+ chunk_interval: Union[timedelta, int, None], default None
46
+ The size of chunks to retrieve from `Pipe.get_data()` for syncing.
47
+
48
+ kwargs: Any
49
+ Additional flags to pass to `Pipe.get_data()` and `Pipe.sync()`, e.g. `workers`.
50
+
51
+ Returns
52
+ -------
53
+ A SuccessTuple indicating success.
54
+ """
55
+ if str(instance_keys) == self.instance_keys:
56
+ return False, f"Cannot copy {self} to instance '{instance_keys}'."
57
+
58
+ new_pipe = mrsm.Pipe(
59
+ self.connector_keys,
60
+ self.metric_key,
61
+ self.location_key,
62
+ parameters=self.parameters.copy(),
63
+ instance=instance_keys,
64
+ )
65
+
66
+ new_pipe_is_registered = new_pipe.get_id() is not None
67
+
68
+ metadata_method = new_pipe.edit if new_pipe_is_registered else new_pipe.register
69
+ metadata_success, metadata_msg = metadata_method(debug=debug)
70
+ if not metadata_success:
71
+ return metadata_success, metadata_msg
72
+
73
+ if not self.exists(debug=debug):
74
+ return True, f"{self} does not exist; nothing to sync."
75
+
76
+ original_as_iterator = kwargs.get('as_iterator', None)
77
+ kwargs['as_iterator'] = True
78
+
79
+ chunk_generator = self.get_data(
80
+ begin=begin,
81
+ end=end,
82
+ params=params,
83
+ chunk_interval=chunk_interval,
84
+ debug=debug,
85
+ **kwargs
86
+ )
87
+
88
+ if original_as_iterator is None:
89
+ _ = kwargs.pop('as_iterator', None)
90
+ else:
91
+ kwargs['as_iterator'] = original_as_iterator
92
+
93
+ sync_success, sync_msg = new_pipe.sync(
94
+ chunk_generator,
95
+ begin=begin,
96
+ end=end,
97
+ params=params,
98
+ debug=debug,
99
+ **kwargs
100
+ )
101
+ msg = (
102
+ f"Successfully synced {new_pipe}:\n{sync_msg}"
103
+ if sync_success
104
+ else f"Failed to sync {new_pipe}:\n{sync_msg}"
105
+ )
106
+ return sync_success, msg
@@ -30,6 +30,8 @@ def get_data(
30
30
  as_chunks: bool = False,
31
31
  as_dask: bool = False,
32
32
  chunk_interval: Union[timedelta, int, None] = None,
33
+ order: Optional[str] = 'asc',
34
+ limit: Optional[int] = None,
33
35
  fresh: bool = False,
34
36
  debug: bool = False,
35
37
  **kw: Any
@@ -80,6 +82,12 @@ def get_data(
80
82
  If `chunk_interval` is a `timedelta` and the `datetime` axis an integer,
81
83
  use the number of minutes in the `timedelta`.
82
84
 
85
+ order: Optional[str], default 'asc'
86
+ If `order` is not `None`, sort the resulting dataframe by indices.
87
+
88
+ limit: Optional[int], default None
89
+ If provided, cap the dataframe to this many rows.
90
+
83
91
  fresh: bool, default True
84
92
  If `True`, skip local cache and directly query the instance connector.
85
93
  Defaults to `True`.
@@ -98,7 +106,7 @@ def get_data(
98
106
  from meerschaum.connectors import get_connector_plugin
99
107
  from meerschaum.utils.misc import iterate_chunks, items_str
100
108
  from meerschaum.utils.dtypes import to_pandas_dtype
101
- from meerschaum.utils.dataframe import add_missing_cols_to_df
109
+ from meerschaum.utils.dataframe import add_missing_cols_to_df, df_is_chunk_generator
102
110
  from meerschaum.utils.packages import attempt_import
103
111
  dd = attempt_import('dask.dataframe') if as_dask else None
104
112
  dask = attempt_import('dask') if as_dask else None
@@ -113,17 +121,48 @@ def get_data(
113
121
 
114
122
  as_iterator = as_iterator or as_chunks
115
123
 
124
+ def _sort_df(_df):
125
+ if df_is_chunk_generator(_df):
126
+ return _df
127
+ dt_col = self.columns.get('datetime', None)
128
+ indices = [] if dt_col not in _df.columns else [dt_col]
129
+ non_dt_cols = [
130
+ col
131
+ for col_ix, col in self.columns.items()
132
+ if col_ix != 'datetime' and col in _df.columns
133
+ ]
134
+ indices.extend(non_dt_cols)
135
+ if 'dask' not in _df.__module__:
136
+ _df.sort_values(
137
+ by=indices,
138
+ inplace=True,
139
+ ascending=(str(order).lower() == 'asc'),
140
+ )
141
+ _df.reset_index(drop=True, inplace=True)
142
+ else:
143
+ _df = _df.sort_values(
144
+ by=indices,
145
+ ascending=(str(order).lower() == 'asc'),
146
+ )
147
+ _df = _df.reset_index(drop=True)
148
+ if limit is not None and len(_df) > limit:
149
+ return _df.head(limit)
150
+ return _df
151
+
116
152
  if as_iterator or as_chunks:
117
- return self._get_data_as_iterator(
118
- select_columns = select_columns,
119
- omit_columns = omit_columns,
120
- begin = begin,
121
- end = end,
122
- params = params,
123
- chunk_interval = chunk_interval,
124
- fresh = fresh,
125
- debug = debug,
153
+ df = self._get_data_as_iterator(
154
+ select_columns=select_columns,
155
+ omit_columns=omit_columns,
156
+ begin=begin,
157
+ end=end,
158
+ params=params,
159
+ chunk_interval=chunk_interval,
160
+ limit=limit,
161
+ order=order,
162
+ fresh=fresh,
163
+ debug=debug,
126
164
  )
165
+ return _sort_df(df)
127
166
 
128
167
  if as_dask:
129
168
  from multiprocessing.pool import ThreadPool
@@ -131,22 +170,24 @@ def get_data(
131
170
  dask.config.set(pool=dask_pool)
132
171
  chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
133
172
  bounds = self.get_chunk_bounds(
134
- begin = begin,
135
- end = end,
136
- bounded = False,
137
- chunk_interval = chunk_interval,
138
- debug = debug,
173
+ begin=begin,
174
+ end=end,
175
+ bounded=False,
176
+ chunk_interval=chunk_interval,
177
+ debug=debug,
139
178
  )
140
179
  dask_chunks = [
141
180
  dask.delayed(self.get_data)(
142
- select_columns = select_columns,
143
- omit_columns = omit_columns,
144
- begin = chunk_begin,
145
- end = chunk_end,
146
- params = params,
147
- chunk_interval = chunk_interval,
148
- fresh = fresh,
149
- debug = debug,
181
+ select_columns=select_columns,
182
+ omit_columns=omit_columns,
183
+ begin=chunk_begin,
184
+ end=chunk_end,
185
+ params=params,
186
+ chunk_interval=chunk_interval,
187
+ order=order,
188
+ limit=limit,
189
+ fresh=fresh,
190
+ debug=debug,
150
191
  )
151
192
  for (chunk_begin, chunk_end) in bounds
152
193
  ]
@@ -154,18 +195,18 @@ def get_data(
154
195
  col: to_pandas_dtype(typ)
155
196
  for col, typ in self.dtypes.items()
156
197
  }
157
- return dd.from_delayed(dask_chunks, meta=dask_meta)
198
+ return _sort_df(dd.from_delayed(dask_chunks, meta=dask_meta))
158
199
 
159
200
  if not self.exists(debug=debug):
160
201
  return None
161
-
202
+
162
203
  if self.cache_pipe is not None:
163
204
  if not fresh:
164
205
  _sync_cache_tuple = self.cache_pipe.sync(
165
- begin = begin,
166
- end = end,
167
- params = params,
168
- debug = debug,
206
+ begin=begin,
207
+ end=end,
208
+ params=params,
209
+ debug=debug,
169
210
  **kw
170
211
  )
171
212
  if not _sync_cache_tuple[0]:
@@ -174,27 +215,31 @@ def get_data(
174
215
  else: ### Successfully synced cache.
175
216
  return self.enforce_dtypes(
176
217
  self.cache_pipe.get_data(
177
- select_columns = select_columns,
178
- omit_columns = omit_columns,
179
- begin = begin,
180
- end = end,
181
- params = params,
182
- debug = debug,
183
- fresh = True,
218
+ select_columns=select_columns,
219
+ omit_columns=omit_columns,
220
+ begin=begin,
221
+ end=end,
222
+ params=params,
223
+ order=order,
224
+ limit=limit,
225
+ debug=debug,
226
+ fresh=True,
184
227
  **kw
185
228
  ),
186
- debug = debug,
229
+ debug=debug,
187
230
  )
188
231
 
189
232
  with Venv(get_connector_plugin(self.instance_connector)):
190
233
  df = self.instance_connector.get_pipe_data(
191
- pipe = self,
192
- select_columns = select_columns,
193
- omit_columns = omit_columns,
194
- begin = begin,
195
- end = end,
196
- params = params,
197
- debug = debug,
234
+ pipe=self,
235
+ select_columns=select_columns,
236
+ omit_columns=omit_columns,
237
+ begin=begin,
238
+ end=end,
239
+ params=params,
240
+ limit=limit,
241
+ order=order,
242
+ debug=debug,
198
243
  **kw
199
244
  )
200
245
  if df is None:
@@ -226,7 +271,7 @@ def get_data(
226
271
  + "Consider adding `select_columns` and `omit_columns` support to "
227
272
  + f"'{self.instance_connector.type}' connectors to improve performance."
228
273
  ),
229
- stack = False,
274
+ stack=False,
230
275
  )
231
276
  _cols_to_select = [col for col in df.columns if col not in cols_to_omit]
232
277
  df = df[_cols_to_select]
@@ -237,25 +282,31 @@ def get_data(
237
282
  f"Specified columns {items_str(cols_to_add)} were not found on {self}. "
238
283
  + "Adding these to the DataFrame as null columns."
239
284
  ),
240
- stack = False,
285
+ stack=False,
241
286
  )
242
287
  df = add_missing_cols_to_df(df, {col: 'string' for col in cols_to_add})
243
288
 
244
- return self.enforce_dtypes(df, debug=debug)
289
+ enforced_df = self.enforce_dtypes(df, debug=debug)
290
+
291
+ if order:
292
+ return _sort_df(enforced_df)
293
+ return enforced_df
245
294
 
246
295
 
247
296
  def _get_data_as_iterator(
248
- self,
249
- select_columns: Optional[List[str]] = None,
250
- omit_columns: Optional[List[str]] = None,
251
- begin: Optional[datetime] = None,
252
- end: Optional[datetime] = None,
253
- params: Optional[Dict[str, Any]] = None,
254
- chunk_interval: Union[timedelta, int, None] = None,
255
- fresh: bool = False,
256
- debug: bool = False,
257
- **kw: Any
258
- ) -> Iterator['pd.DataFrame']:
297
+ self,
298
+ select_columns: Optional[List[str]] = None,
299
+ omit_columns: Optional[List[str]] = None,
300
+ begin: Optional[datetime] = None,
301
+ end: Optional[datetime] = None,
302
+ params: Optional[Dict[str, Any]] = None,
303
+ chunk_interval: Union[timedelta, int, None] = None,
304
+ limit: Optional[int] = None,
305
+ order: Optional[str] = 'asc',
306
+ fresh: bool = False,
307
+ debug: bool = False,
308
+ **kw: Any
309
+ ) -> Iterator['pd.DataFrame']:
259
310
  """
260
311
  Return a pipe's data as a generator.
261
312
  """
@@ -305,46 +356,51 @@ def _get_data_as_iterator(
305
356
  (min_dt + chunk_interval) > max_dt
306
357
  ):
307
358
  yield self.get_data(
308
- select_columns = select_columns,
309
- omit_columns = omit_columns,
310
- begin = begin,
311
- end = end,
312
- params = params,
313
- fresh = fresh,
314
- debug = debug,
359
+ select_columns=select_columns,
360
+ omit_columns=omit_columns,
361
+ begin=begin,
362
+ end=end,
363
+ params=params,
364
+ limit=limit,
365
+ order=order,
366
+ fresh=fresh,
367
+ debug=debug,
315
368
  )
316
369
  return
317
370
 
318
371
  chunk_bounds = self.get_chunk_bounds(
319
- begin = min_dt,
320
- end = max_dt,
321
- chunk_interval = chunk_interval,
322
- debug = debug,
372
+ begin=min_dt,
373
+ end=max_dt,
374
+ chunk_interval=chunk_interval,
375
+ debug=debug,
323
376
  )
324
377
 
325
378
  for chunk_begin, chunk_end in chunk_bounds:
326
379
  chunk = self.get_data(
327
- select_columns = select_columns,
328
- omit_columns = omit_columns,
329
- begin = chunk_begin,
330
- end = chunk_end,
331
- params = params,
332
- fresh = fresh,
333
- debug = debug,
380
+ select_columns=select_columns,
381
+ omit_columns=omit_columns,
382
+ begin=chunk_begin,
383
+ end=chunk_end,
384
+ params=params,
385
+ limit=limit,
386
+ order=order,
387
+ fresh=fresh,
388
+ debug=debug,
334
389
  )
335
390
  if len(chunk) > 0:
336
391
  yield chunk
337
392
 
338
393
 
339
394
  def get_backtrack_data(
340
- self,
341
- backtrack_minutes: Optional[int] = None,
342
- begin: Union[datetime, int, None] = None,
343
- params: Optional[Dict[str, Any]] = None,
344
- fresh: bool = False,
345
- debug: bool = False,
346
- **kw: Any
347
- ) -> Optional['pd.DataFrame']:
395
+ self,
396
+ backtrack_minutes: Optional[int] = None,
397
+ begin: Union[datetime, int, None] = None,
398
+ params: Optional[Dict[str, Any]] = None,
399
+ limit: Optional[int] = None,
400
+ fresh: bool = False,
401
+ debug: bool = False,
402
+ **kw: Any
403
+ ) -> Optional['pd.DataFrame']:
348
404
  """
349
405
  Get the most recent data from the instance connector as a Pandas DataFrame.
350
406
 
@@ -371,8 +427,10 @@ def get_backtrack_data(
371
427
 
372
428
  params: Optional[Dict[str, Any]], default None
373
429
  The standard Meerschaum `params` query dictionary.
374
-
375
-
430
+
431
+ limit: Optional[int], default None
432
+ If provided, cap the number of rows to be returned.
433
+
376
434
  fresh: bool, default False
377
435
  If `True`, Ignore local cache and pull directly from the instance connector.
378
436
  Only comes into effect if a pipe was created with `cache=True`.
@@ -409,28 +467,31 @@ def get_backtrack_data(
409
467
  else: ### Successfully synced cache.
410
468
  return self.enforce_dtypes(
411
469
  self.cache_pipe.get_backtrack_data(
412
- fresh = True,
413
- begin = begin,
414
- backtrack_minutes = backtrack_minutes,
415
- params = params,
416
- debug = deubg,
470
+ fresh=True,
471
+ begin=begin,
472
+ backtrack_minutes=backtrack_minutes,
473
+ params=params,
474
+ limit=limit,
475
+ order=kw.get('order', 'desc'),
476
+ debug=debug,
417
477
  **kw
418
478
  ),
419
- debug = debug,
479
+ debug=debug,
420
480
  )
421
481
 
422
482
  if hasattr(self.instance_connector, 'get_backtrack_data'):
423
483
  with Venv(get_connector_plugin(self.instance_connector)):
424
484
  return self.enforce_dtypes(
425
485
  self.instance_connector.get_backtrack_data(
426
- pipe = self,
427
- begin = begin,
428
- backtrack_minutes = backtrack_minutes,
429
- params = params,
430
- debug = debug,
486
+ pipe=self,
487
+ begin=begin,
488
+ backtrack_minutes=backtrack_minutes,
489
+ params=params,
490
+ limit=limit,
491
+ debug=debug,
431
492
  **kw
432
493
  ),
433
- debug = debug,
494
+ debug=debug,
434
495
  )
435
496
 
436
497
  if begin is None:
@@ -445,13 +506,16 @@ def get_backtrack_data(
445
506
  begin = begin - backtrack_interval
446
507
 
447
508
  return self.get_data(
448
- begin = begin,
449
- params = params,
450
- debug = debug,
509
+ begin=begin,
510
+ params=params,
511
+ debug=debug,
512
+ limit=limit,
513
+ order=kw.get('order', 'desc'),
451
514
  **kw
452
515
  )
453
516
 
454
517
 
518
+
455
519
  def get_rowcount(
456
520
  self,
457
521
  begin: Optional[datetime] = None,
@@ -10,10 +10,10 @@ from __future__ import annotations
10
10
  from meerschaum.utils.typing import SuccessTuple, Any
11
11
 
12
12
  def drop(
13
- self,
14
- debug: bool = False,
15
- **kw: Any
16
- ) -> SuccessTuple:
13
+ self,
14
+ debug: bool = False,
15
+ **kw: Any
16
+ ) -> SuccessTuple:
17
17
  """
18
18
  Call the Pipe's instance connector's `drop_pipe()` method.
19
19
 
@@ -16,12 +16,12 @@ if TYPE_CHECKING:
16
16
  pd = mrsm.attempt_import('pandas')
17
17
 
18
18
  def enforce_dtypes(
19
- self,
20
- df: 'pd.DataFrame',
21
- chunksize: Optional[int] = -1,
22
- safe_copy: bool = True,
23
- debug: bool = False,
24
- ) -> 'pd.DataFrame':
19
+ self,
20
+ df: 'pd.DataFrame',
21
+ chunksize: Optional[int] = -1,
22
+ safe_copy: bool = True,
23
+ debug: bool = False,
24
+ ) -> 'pd.DataFrame':
25
25
  """
26
26
  Cast the input dataframe to the pipe's registered data types.
27
27
  If the pipe does not exist and dtypes are not set, return the dataframe.
@@ -35,7 +35,7 @@ def enforce_dtypes(
35
35
  if df is None:
36
36
  if debug:
37
37
  dprint(
38
- f"Received None instead of a DataFrame.\n"
38
+ "Received None instead of a DataFrame.\n"
39
39
  + " Skipping dtype enforcement..."
40
40
  )
41
41
  return df
@@ -46,24 +46,24 @@ def enforce_dtypes(
46
46
  if isinstance(df, str):
47
47
  df = parse_df_datetimes(
48
48
  pd.read_json(StringIO(df)),
49
- ignore_cols = [
49
+ ignore_cols=[
50
50
  col
51
51
  for col, dtype in pipe_dtypes.items()
52
52
  if 'datetime' not in str(dtype)
53
53
  ],
54
- chunksize = chunksize,
55
- debug = debug,
54
+ chunksize=chunksize,
55
+ debug=debug,
56
56
  )
57
57
  else:
58
58
  df = parse_df_datetimes(
59
59
  df,
60
- ignore_cols = [
60
+ ignore_cols=[
61
61
  col
62
62
  for col, dtype in pipe_dtypes.items()
63
63
  if 'datetime' not in str(dtype)
64
64
  ],
65
- chunksize = chunksize,
66
- debug = debug,
65
+ chunksize=chunksize,
66
+ debug=debug,
67
67
  )
68
68
  except Exception as e:
69
69
  warn(f"Unable to cast incoming data as a DataFrame...:\n{e}\n\n{traceback.format_exc()}")
@@ -80,7 +80,7 @@ def enforce_dtypes(
80
80
  return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
81
81
 
82
82
 
83
- def infer_dtypes(self, persist: bool=False, debug: bool=False) -> Dict[str, Any]:
83
+ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str, Any]:
84
84
  """
85
85
  If `dtypes` is not set in `meerschaum.Pipe.parameters`,
86
86
  infer the data types from the underlying table if it exists.
@@ -18,12 +18,12 @@ def update(self, *args, **kw) -> SuccessTuple:
18
18
 
19
19
 
20
20
  def edit(
21
- self,
22
- patch: bool = False,
23
- interactive: bool = False,
24
- debug: bool = False,
25
- **kw: Any
26
- ) -> SuccessTuple:
21
+ self,
22
+ patch: bool = False,
23
+ interactive: bool = False,
24
+ debug: bool = False,
25
+ **kw: Any
26
+ ) -> SuccessTuple:
27
27
  """
28
28
  Edit a Pipe's configuration.
29
29
 
@@ -50,11 +50,12 @@ def edit(
50
50
  if not interactive:
51
51
  with Venv(get_connector_plugin(self.instance_connector)):
52
52
  return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw)
53
+
53
54
  from meerschaum.config._paths import PIPES_CACHE_RESOURCES_PATH
54
55
  from meerschaum.utils.misc import edit_file
55
56
  parameters_filename = str(self) + '.yaml'
56
57
  parameters_path = PIPES_CACHE_RESOURCES_PATH / parameters_filename
57
-
58
+
58
59
  from meerschaum.utils.yaml import yaml
59
60
 
60
61
  edit_text = f"Edit the parameters for {self}"
@@ -96,13 +97,13 @@ def edit(
96
97
 
97
98
 
98
99
  def edit_definition(
99
- self,
100
- yes: bool = False,
101
- noask: bool = False,
102
- force: bool = False,
103
- debug : bool = False,
104
- **kw : Any
105
- ) -> SuccessTuple:
100
+ self,
101
+ yes: bool = False,
102
+ noask: bool = False,
103
+ force: bool = False,
104
+ debug : bool = False,
105
+ **kw : Any
106
+ ) -> SuccessTuple:
106
107
  """
107
108
  Edit a pipe's definition file and update its configuration.
108
109
  **NOTE:** This function is interactive and should not be used in automated scripts!