meerschaum 2.0.0rc6__py3-none-any.whl → 2.0.0rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +12 -1
  2. meerschaum/_internal/arguments/_parser.py +23 -1
  3. meerschaum/actions/__init__.py +97 -48
  4. meerschaum/actions/bootstrap.py +1 -1
  5. meerschaum/actions/clear.py +1 -1
  6. meerschaum/actions/deduplicate.py +1 -1
  7. meerschaum/actions/delete.py +8 -7
  8. meerschaum/actions/drop.py +1 -10
  9. meerschaum/actions/edit.py +1 -1
  10. meerschaum/actions/install.py +1 -1
  11. meerschaum/actions/pause.py +1 -1
  12. meerschaum/actions/register.py +1 -1
  13. meerschaum/actions/setup.py +1 -1
  14. meerschaum/actions/show.py +1 -1
  15. meerschaum/actions/start.py +18 -7
  16. meerschaum/actions/stop.py +5 -4
  17. meerschaum/actions/sync.py +17 -2
  18. meerschaum/actions/uninstall.py +1 -1
  19. meerschaum/actions/upgrade.py +1 -1
  20. meerschaum/actions/verify.py +54 -3
  21. meerschaum/config/_default.py +71 -65
  22. meerschaum/config/_formatting.py +26 -0
  23. meerschaum/config/_jobs.py +28 -5
  24. meerschaum/config/_paths.py +21 -5
  25. meerschaum/config/_version.py +1 -1
  26. meerschaum/connectors/api/_fetch.py +1 -1
  27. meerschaum/connectors/api/_pipes.py +6 -11
  28. meerschaum/connectors/sql/_fetch.py +29 -11
  29. meerschaum/connectors/sql/_pipes.py +11 -4
  30. meerschaum/connectors/sql/_sql.py +1 -6
  31. meerschaum/core/Pipe/__init__.py +5 -1
  32. meerschaum/core/Pipe/_data.py +58 -9
  33. meerschaum/core/Pipe/_deduplicate.py +61 -11
  34. meerschaum/core/Pipe/_dtypes.py +2 -1
  35. meerschaum/core/Pipe/_verify.py +174 -34
  36. meerschaum/plugins/__init__.py +3 -0
  37. meerschaum/utils/daemon/Daemon.py +108 -27
  38. meerschaum/utils/daemon/__init__.py +35 -1
  39. meerschaum/utils/dataframe.py +10 -5
  40. meerschaum/utils/formatting/__init__.py +144 -1
  41. meerschaum/utils/formatting/_pipes.py +28 -5
  42. meerschaum/utils/misc.py +183 -187
  43. meerschaum/utils/packages/__init__.py +1 -1
  44. meerschaum/utils/packages/_packages.py +1 -0
  45. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/METADATA +4 -1
  46. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/RECORD +52 -52
  47. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/LICENSE +0 -0
  48. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/NOTICE +0 -0
  49. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/WHEEL +0 -0
  50. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/entry_points.txt +0 -0
  51. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/top_level.txt +0 -0
  52. {meerschaum-2.0.0rc6.dist-info → meerschaum-2.0.0rc8.dist-info}/zip-safe +0 -0
@@ -8,6 +8,8 @@ This module contains functions for parsing arguments
8
8
  """
9
9
 
10
10
  from __future__ import annotations
11
+ import json
12
+ from datetime import timedelta
11
13
  from meerschaum.utils.typing import List, Dict, Any, Optional, Callable, SuccessTuple
12
14
  from meerschaum.utils.threading import RLock
13
15
 
@@ -178,6 +180,16 @@ def parse_synonyms(
178
180
  args_dict['check_existing'] = False
179
181
  if args_dict.get('venv', None) in ('None', '[None]'):
180
182
  args_dict['venv'] = None
183
+ chunk_minutes = args_dict.get('chunk_minutes', None)
184
+ chunk_hours = args_dict.get('chunk_hours', None)
185
+ chunk_days = args_dict.get('chunk_days', None)
186
+ if isinstance(chunk_minutes, int):
187
+ args_dict['chunk_interval'] = timedelta(minutes=chunk_minutes)
188
+ elif isinstance(chunk_hours, int):
189
+ args_dict['chunk_interval'] = timedelta(hours=chunk_hours)
190
+ elif isinstance(chunk_days, int):
191
+ args_dict['chunk_interval'] = timedelta(days=chunk_days)
192
+
181
193
  return args_dict
182
194
 
183
195
 
@@ -185,7 +197,6 @@ def parse_dict_to_sysargs(
185
197
  args_dict : Dict[str, Any]
186
198
  ) -> List[str]:
187
199
  """Revert an arguments dictionary back to a command line list."""
188
- import json
189
200
  from meerschaum._internal.arguments._parser import get_arguments_triggers
190
201
  sysargs = []
191
202
  sysargs += args_dict.get('action', [])
@@ -227,7 +227,22 @@ groups['sync'].add_argument(
227
227
  )
228
228
  groups['sync'].add_argument(
229
229
  '--chunksize', type=int, help=(
230
- "Specify the chunksize for syncing and retrieving data. Defaults to 900."
230
+ "Specify the database chunksize. Defaults to 10,000."
231
+ ),
232
+ )
233
+ groups['sync'].add_argument(
234
+ '--chunk-minutes', type=int, help=(
235
+ "Specify the chunk interval in minutes for verification syncs. Defaults to 1440."
236
+ ),
237
+ )
238
+ groups['sync'].add_argument(
239
+ '--chunk-hours', type=int, help=(
240
+ "Specify the chunk interval in days for verification syncs. See `--chunk-minutes`."
241
+ ),
242
+ )
243
+ groups['sync'].add_argument(
244
+ '--chunk-days', type=int, help=(
245
+ "Specify the chunk interval in days for verification syncs. See `--chunk-minutes`."
231
246
  ),
232
247
  )
233
248
  groups['sync'].add_argument(
@@ -237,6 +252,13 @@ groups['sync'].add_argument(
237
252
  '--deduplicate', '--dedup', action="store_true",
238
253
  help="Remove duplicate rows from a pipe's table.",
239
254
  )
255
+ groups['sync'].add_argument(
256
+ '--bounded', '--bound', action="store_true",
257
+ help = (
258
+ "When verifying, do not sync outside "
259
+ + "the existing oldest and newest datetime bounds."
260
+ )
261
+ )
240
262
  groups['sync'].add_argument(
241
263
  '--skip-check-existing', '--allow-duplicates', action='store_true',
242
264
  help = (
@@ -11,51 +11,6 @@ from meerschaum.utils.typing import Callable, Any, Optional, Union, List, Dict,
11
11
  from meerschaum.utils.packages import get_modules_from_package
12
12
  _custom_actions = []
13
13
 
14
- ### build __all__ from other .py files in this package
15
- import sys
16
- modules = get_modules_from_package(
17
- sys.modules[__name__],
18
- names = False,
19
- )
20
- __all__ = ['actions', 'get_subactions', 'get_action', 'get_main_action_name', 'get_completer']
21
-
22
- ### Build the actions dictionary by importing all
23
- ### functions that do not begin with '_' from all submodules.
24
- from inspect import getmembers, isfunction
25
- actions = {}
26
- "This docstring will be replaced in __pdoc__ at the end of this file."
27
-
28
- for module in modules:
29
- ### A couple important things happening here:
30
- ### 1. Find all functions in all modules in `actions` package
31
- ### (skip functions that begin with '_')
32
- ### 2. Add them as members to the Shell class
33
- ### - Original definition : meerschaum._internal.shell.Shell
34
- ### - New definition : meerschaum._internal.Shell
35
- ### 3. Populate the actions dictionary with function names and functions
36
- ###
37
- ### UPDATE:
38
- ### Shell modifications have been deferred to get_shell in order to improve lazy loading.
39
-
40
- actions.update(
41
- dict(
42
- [
43
- ### __name__ and new function pointer
44
- (ob[0], ob[1])
45
- for ob in getmembers(module)
46
- if isfunction(ob[1])
47
- ### check that the function belongs to the module
48
- and ob[0] == module.__name__.replace('_', '').split('.')[-1]
49
- ### skip functions that start with '_'
50
- and ob[0][0] != '_'
51
- ]
52
- )
53
- )
54
-
55
- original_actions = actions.copy()
56
- from meerschaum._internal.entry import entry
57
-
58
-
59
14
  def get_subactions(
60
15
  action: Union[str, List[str]],
61
16
  _actions: Optional[Dict[str, Callable[[Any], Any]]] = None,
@@ -273,14 +228,108 @@ def choose_subaction(
273
228
  return options[parsed_choice](**kw)
274
229
 
275
230
 
276
- from meerschaum._internal.entry import get_shell
231
+ def _get_subaction_names(action: str, globs: dict = None) -> List[str]:
232
+ """Use `meerschaum.actions.get_subactions()` instead."""
233
+ if globs is None:
234
+ import importlib
235
+ module = importlib.import_module(f'meerschaum.actions.{action}')
236
+ globs = vars(module)
237
+ subactions = []
238
+ for item in globs:
239
+ if f'_{action}' in item and 'complete' not in item.lstrip('_'):
240
+ subactions.append(globs[item])
241
+ return subactions
242
+
243
+
244
+ def choices_docstring(action: str, globs : Optional[Dict[str, Any]] = None) -> str:
245
+ """
246
+ Append the an action's available options to the module docstring.
247
+ This function is to be placed at the bottom of each action module.
248
+
249
+ Parameters
250
+ ----------
251
+ action: str
252
+ The name of the action module (e.g. 'install').
253
+
254
+ globs: Optional[Dict[str, Any]], default None
255
+ An optional dictionary of global variables.
256
+
257
+ Returns
258
+ -------
259
+ The generated docstring for the module.
260
+
261
+ Examples
262
+ --------
263
+ >>> from meerschaum.utils.misc import choices_docstring as _choices_docstring
264
+ >>> install.__doc__ += _choices_docstring('install')
265
+
266
+ """
267
+ options_str = f"\n Options:\n `{action} "
268
+ subactions = _get_subaction_names(action, globs=globs)
269
+ options_str += "["
270
+ sa_names = []
271
+ for sa in subactions:
272
+ try:
273
+ sa_names.append(sa.__name__[len(f"_{action}") + 1:])
274
+ except Exception as e:
275
+ print(e)
276
+ return ""
277
+ for sa_name in sorted(sa_names):
278
+ options_str += f"{sa_name}, "
279
+ options_str = options_str[:-2] + "]`"
280
+ return options_str
281
+
282
+ ### build __all__ from other .py files in this package
283
+ import sys
284
+ modules = get_modules_from_package(
285
+ sys.modules[__name__],
286
+ names = False,
287
+ )
288
+ __all__ = ['actions', 'get_subactions', 'get_action', 'get_main_action_name', 'get_completer']
289
+
290
+ ### Build the actions dictionary by importing all
291
+ ### functions that do not begin with '_' from all submodules.
292
+ from inspect import getmembers, isfunction
293
+ actions = {}
294
+ "This docstring will be replaced in __pdoc__ at the end of this file."
295
+
296
+ for module in modules:
297
+ ### A couple important things happening here:
298
+ ### 1. Find all functions in all modules in `actions` package
299
+ ### (skip functions that begin with '_')
300
+ ### 2. Add them as members to the Shell class
301
+ ### - Original definition : meerschaum._internal.shell.Shell
302
+ ### - New definition : meerschaum._internal.Shell
303
+ ### 3. Populate the actions dictionary with function names and functions
304
+ ###
305
+ ### UPDATE:
306
+ ### Shell modifications have been deferred to get_shell in order to improve lazy loading.
307
+
308
+ actions.update(
309
+ dict(
310
+ [
311
+ ### __name__ and new function pointer
312
+ (ob[0], ob[1])
313
+ for ob in getmembers(module)
314
+ if isfunction(ob[1])
315
+ ### check that the function belongs to the module
316
+ and ob[0] == module.__name__.replace('_', '').split('.')[-1]
317
+ ### skip functions that start with '_'
318
+ and ob[0][0] != '_'
319
+ ]
320
+ )
321
+ )
322
+
323
+ original_actions = actions.copy()
324
+ from meerschaum._internal.entry import entry, get_shell
277
325
  import meerschaum.plugins
278
- # meerschaum.plugins.load_plugins()
279
326
  make_action = meerschaum.plugins.make_action
280
327
 
281
328
  ### Instruct pdoc to skip the `meerschaum.actions.plugins` subdirectory.
282
329
  __pdoc__ = {
283
- 'plugins' : False, 'arguments': True, 'shell': False,
330
+ 'plugins': False,
331
+ 'arguments': True,
332
+ 'shell': False,
284
333
  'actions': (
285
334
  "Access functions of the standard Meerschaum actions.\n\n"
286
335
  + "Visit the [actions reference page](https://meerschaum.io/reference/actions/) "
@@ -416,5 +416,5 @@ def _bootstrap_config(
416
416
  ### NOTE: This must be the final statement of the module.
417
417
  ### Any subactions added below these lines will not
418
418
  ### be added to the `help` docstring.
419
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
419
+ from meerschaum.actions import choices_docstring as _choices_docstring
420
420
  bootstrap.__doc__ += _choices_docstring('bootstrap')
@@ -141,5 +141,5 @@ def _ask_with_rowcounts(
141
141
  ### NOTE: This must be the final statement of the module.
142
142
  ### Any subactions added below these lines will not
143
143
  ### be added to the `help` docstring.
144
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
144
+ from meerschaum.actions import choices_docstring as _choices_docstring
145
145
  clear.__doc__ += _choices_docstring('clear')
@@ -38,5 +38,5 @@ def _deduplicate_pipes(**kwargs) -> SuccessTuple:
38
38
  ### NOTE: This must be the final statement of the module.
39
39
  ### Any subactions added below these lines will not
40
40
  ### be added to the `help` docstring.
41
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
41
+ from meerschaum.actions import choices_docstring as _choices_docstring
42
42
  deduplicate.__doc__ += _choices_docstring('deduplicate')
@@ -374,12 +374,12 @@ def _complete_delete_connectors(
374
374
  return get_connector_labels(*types, search_term=search_term)
375
375
 
376
376
  def _delete_jobs(
377
- action : Optional[List[str]] = None,
378
- noask : bool = False,
379
- nopretty : bool = False,
380
- force : bool = False,
381
- yes : bool = False,
382
- debug : bool = False,
377
+ action: Optional[List[str]] = None,
378
+ noask: bool = False,
379
+ nopretty: bool = False,
380
+ force: bool = False,
381
+ yes: bool = False,
382
+ debug: bool = False,
383
383
  **kw
384
384
  ) -> SuccessTuple:
385
385
  """
@@ -431,6 +431,7 @@ def _delete_jobs(
431
431
  force = force,
432
432
  noask = noask,
433
433
  debug = debug,
434
+ **kw
434
435
  )
435
436
  ### Ensure the running jobs are dead.
436
437
  if get_running_daemons(daemons):
@@ -473,5 +474,5 @@ def _delete_jobs(
473
474
  ### NOTE: This must be the final statement of the module.
474
475
  ### Any subactions added below these lines will not
475
476
  ### be added to the `help` docstring.
476
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
477
+ from meerschaum.actions import choices_docstring as _choices_docstring
477
478
  delete.__doc__ += _choices_docstring('delete')
@@ -18,7 +18,6 @@ def drop(
18
18
  from meerschaum.actions import choose_subaction
19
19
  options = {
20
20
  'pipes' : _drop_pipes,
21
- 'tables' : _drop_tables,
22
21
  }
23
22
  return choose_subaction(action, options, **kw)
24
23
 
@@ -79,17 +78,9 @@ def _drop_pipes(
79
78
  )
80
79
  return successes > 0, msg
81
80
 
82
- def _drop_tables(
83
- action: Optional[List[str]] = None,
84
- **kw: Any
85
- ) -> SuccessTuple:
86
- """
87
- Drop SQL tables. WARNING: This is very dangerous!
88
- """
89
- return False, "Not implemented"
90
81
 
91
82
  ### NOTE: This must be the final statement of the module.
92
83
  ### Any subactions added below these lines will not
93
84
  ### be added to the `help` docstring.
94
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
85
+ from meerschaum.actions import choices_docstring as _choices_docstring
95
86
  drop.__doc__ += _choices_docstring('drop')
@@ -292,5 +292,5 @@ def _edit_users(
292
292
  ### NOTE: This must be the final statement of the module.
293
293
  ### Any subactions added below these lines will not
294
294
  ### be added to the `help` docstring.
295
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
295
+ from meerschaum.actions import choices_docstring as _choices_docstring
296
296
  edit.__doc__ += _choices_docstring('edit')
@@ -236,5 +236,5 @@ def _complete_install_required(*args, **kw) -> List[str]:
236
236
  ### NOTE: This must be the final statement of the module.
237
237
  ### Any subactions added below these lines will not
238
238
  ### be added to the `help` docstring.
239
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
239
+ from meerschaum.actions import choices_docstring as _choices_docstring
240
240
  install.__doc__ += _choices_docstring('install')
@@ -124,5 +124,5 @@ def _pause_jobs(
124
124
  ### NOTE: This must be the final statement of the module.
125
125
  ### Any subactions added below these lines will not
126
126
  ### be added to the `help` docstring.
127
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
127
+ from meerschaum.actions import choices_docstring as _choices_docstring
128
128
  pause.__doc__ += _choices_docstring('pause')
@@ -336,5 +336,5 @@ def _register_users(
336
336
  ### NOTE: This must be the final statement of the module.
337
337
  ### Any subactions added below these lines will not
338
338
  ### be added to the `help` docstring.
339
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
339
+ from meerschaum.actions import choices_docstring as _choices_docstring
340
340
  register.__doc__ += _choices_docstring('register')
@@ -108,5 +108,5 @@ def _complete_setup_plugins(*args, **kw) -> List[str]:
108
108
  ### NOTE: This must be the final statement of the module.
109
109
  ### Any subactions added below these lines will not
110
110
  ### be added to the `help` docstring.
111
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
111
+ from meerschaum.actions import choices_docstring as _choices_docstring
112
112
  setup.__doc__ += _choices_docstring('setup')
@@ -733,5 +733,5 @@ def _show_environment(
733
733
  ### NOTE: This must be the final statement of the module.
734
734
  ### Any subactions added below these lines will not
735
735
  ### be added to the `help` docstring.
736
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
736
+ from meerschaum.actions import choices_docstring as _choices_docstring
737
737
  show.__doc__ += _choices_docstring('show')
@@ -128,12 +128,16 @@ def _start_jobs(
128
128
  daemon_ids = get_daemon_ids()
129
129
 
130
130
  new_job = len(list(action)) > 0
131
- _potential_jobs = {'known' : [], 'unknown' : []}
131
+ _potential_jobs = {'known': [], 'unknown': []}
132
132
 
133
133
  if action:
134
134
  for a in action:
135
- _potential_jobs[('known' if a in daemon_ids else 'unknown')].append(a)
136
-
135
+ _potential_jobs[(
136
+ 'known'
137
+ if a in daemon_ids
138
+ else 'unknown'
139
+ )].append(a)
140
+
137
141
  ### Check if the job is named after an action.
138
142
  if (
139
143
  _potential_jobs['known']
@@ -144,7 +148,7 @@ def _start_jobs(
144
148
  _potential_jobs['unknown'].insert(0, _potential_jobs['known'][0])
145
149
  del _potential_jobs['known'][0]
146
150
 
147
- ### Only spawn a new job if we don't don't find any jobs.
151
+ ### Only spawn a new job if we don't find any jobs.
148
152
  new_job = (len(_potential_jobs['known']) == 0)
149
153
  if not new_job and _potential_jobs['unknown']:
150
154
  if not kw.get('nopretty', False):
@@ -171,7 +175,7 @@ def _start_jobs(
171
175
  return False, msg
172
176
 
173
177
  ### No action provided but a --name was. Start job if possible.
174
- ### E.g. `start job --myjob`
178
+ ### E.g. `start job --name myjob`
175
179
  elif name is not None:
176
180
  new_job = False
177
181
  names = [name]
@@ -215,7 +219,10 @@ def _start_jobs(
215
219
  try:
216
220
  _daemon_sysargs = daemon.properties['target']['args'][0]
217
221
  except KeyError:
218
- return False, "Failed to get arguments for daemon '{dameon.daemon_id}'."
222
+ return (
223
+ (False, f"Failed to get arguments for daemon '{daemon.daemon_id}'."),
224
+ daemon.daemon_id
225
+ )
219
226
  _daemon_kw = parse_arguments(_daemon_sysargs)
220
227
  _daemon_kw['name'] = daemon.daemon_id
221
228
  _action_success_tuple = daemon_action(**_daemon_kw)
@@ -277,7 +284,11 @@ def _start_jobs(
277
284
 
278
285
  _successes, _failures = [], []
279
286
  for _name in names:
280
- success_tuple, __name = _run_new_job(_name) if new_job else _run_existing_job(_name)
287
+ success_tuple, __name = (
288
+ _run_new_job(_name)
289
+ if new_job
290
+ else _run_existing_job(_name)
291
+ )
281
292
  if not kw.get('nopretty', False):
282
293
  print_tuple(success_tuple)
283
294
  _successes.append(_name) if success_tuple[0] else _failures.append(_name)
@@ -50,6 +50,7 @@ def _complete_stop(
50
50
 
51
51
  def _stop_jobs(
52
52
  action: Optional[List[str]] = None,
53
+ timeout_seconds: Optional[int] = None,
53
54
  noask: bool = False,
54
55
  force: bool = False,
55
56
  yes: bool = False,
@@ -93,18 +94,18 @@ def _stop_jobs(
93
94
 
94
95
  _quit_daemons, _kill_daemons = [], []
95
96
  for d in daemons_to_stop:
96
- quit_success, quit_msg = d.quit()
97
+ quit_success, quit_msg = d.quit(timeout=timeout_seconds)
97
98
  if quit_success:
98
99
  _quit_daemons.append(d)
99
100
  continue
100
101
  else:
101
102
  warn(
102
- f"Failed to gracefully quit job '{d.daemon_id}', attempting to terminate:\n"
103
+ f"Failed to gracefully quit job '{d.daemon_id}', attempting to terminate:\n "
103
104
  + f"{quit_msg}",
104
105
  stack = False,
105
106
  )
106
107
 
107
- kill_success, kill_msg = d.kill()
108
+ kill_success, kill_msg = d.kill(timeout=timeout_seconds)
108
109
  if kill_success:
109
110
  _kill_daemons.append(d)
110
111
  continue
@@ -126,5 +127,5 @@ def _stop_jobs(
126
127
  ### NOTE: This must be the final statement of the module.
127
128
  ### Any subactions added below these lines will not
128
129
  ### be added to the `help` docstring.
129
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
130
+ from meerschaum.actions import choices_docstring as _choices_docstring
130
131
  stop.__doc__ += _choices_docstring('stop')
@@ -9,7 +9,8 @@ NOTE: `sync` required a SQL connection and is not intended for client use
9
9
  """
10
10
 
11
11
  from __future__ import annotations
12
- from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple
12
+ from datetime import timedelta
13
+ from meerschaum.utils.typing import SuccessTuple, Any, List, Optional, Tuple, Union
13
14
 
14
15
  def sync(
15
16
  action: Optional[List[str]] = None,
@@ -43,6 +44,8 @@ def _pipes_lap(
43
44
  min_seconds: int = 1,
44
45
  verify: bool = False,
45
46
  deduplicate: bool = False,
47
+ bounded: Optional[bool] = None,
48
+ chunk_interval: Union[timedelta, int, None] = None,
46
49
  mrsm_instance: Optional[str] = None,
47
50
  timeout_seconds: Optional[int] = None,
48
51
  nopretty: bool = False,
@@ -87,6 +90,8 @@ def _pipes_lap(
87
90
  'mrsm_instance': mrsm_instance,
88
91
  'verify': verify,
89
92
  'deduplicate': deduplicate,
93
+ 'bounded': bounded,
94
+ 'chunk_interval': chunk_interval,
90
95
  })
91
96
  locks = {'remaining_count': Lock(), 'results_dict': Lock(), 'pipes_threads': Lock(),}
92
97
  pipes = get_pipes(
@@ -149,6 +154,7 @@ def _pipes_lap(
149
154
  ) + msg + '\n'
150
155
  print_tuple(
151
156
  (success, msg),
157
+ calm = True,
152
158
  _progress = _progress,
153
159
  )
154
160
  _checkpoint(_progress=_progress, _task=_task)
@@ -238,6 +244,8 @@ def _sync_pipes(
238
244
  unblock: bool = False,
239
245
  verify: bool = False,
240
246
  deduplicate: bool = False,
247
+ bounded: Optional[bool] = None,
248
+ chunk_interval: Union[timedelta, int, None] = None,
241
249
  shell: bool = False,
242
250
  nopretty: bool = False,
243
251
  debug: bool = False,
@@ -280,12 +288,15 @@ def _sync_pipes(
280
288
  lap_begin = time.perf_counter()
281
289
 
282
290
  try:
291
+ results_dict = {}
283
292
  with cm:
284
293
  results_dict = _pipes_lap(
285
294
  min_seconds = min_seconds,
286
295
  _progress = _progress,
287
296
  verify = verify,
288
297
  deduplicate = deduplicate,
298
+ bounded = bounded,
299
+ chunk_interval = chunk_interval,
289
300
  unblock = unblock,
290
301
  debug = debug,
291
302
  nopretty = nopretty,
@@ -373,6 +384,8 @@ def _wrap_sync_pipe(
373
384
  workers = None,
374
385
  verify: bool = False,
375
386
  deduplicate: bool = False,
387
+ bounded: Optional[bool] = None,
388
+ chunk_interval: Union[timedelta, int, None] = None,
376
389
  **kw
377
390
  ):
378
391
  """
@@ -395,6 +408,8 @@ def _wrap_sync_pipe(
395
408
  debug = debug,
396
409
  min_seconds = min_seconds,
397
410
  workers = workers,
411
+ bounded = bounded,
412
+ chunk_interval = chunk_interval,
398
413
  **{k: v for k, v in kw.items() if k != 'blocking'}
399
414
  )
400
415
  except Exception as e:
@@ -410,5 +425,5 @@ def _wrap_sync_pipe(
410
425
  ### NOTE: This must be the final statement of the module.
411
426
  ### Any subactions added below these lines will not
412
427
  ### be added to the `help` docstring.
413
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
428
+ from meerschaum.actions import choices_docstring as _choices_docstring
414
429
  sync.__doc__ += _choices_docstring('sync')
@@ -192,5 +192,5 @@ def _uninstall_packages(
192
192
  ### NOTE: This must be the final statement of the module.
193
193
  ### Any subactions added below these lines will not
194
194
  ### be added to the `help` docstring.
195
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
195
+ from meerschaum.actions import choices_docstring as _choices_docstring
196
196
  uninstall.__doc__ += _choices_docstring('uninstall')
@@ -213,5 +213,5 @@ def _upgrade_plugins(
213
213
  ### NOTE: This must be the final statement of the module.
214
214
  ### Any subactions added below these lines will not
215
215
  ### be added to the `help` docstring.
216
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
216
+ from meerschaum.actions import choices_docstring as _choices_docstring
217
217
  upgrade.__doc__ += _choices_docstring('upgrade')
@@ -11,7 +11,7 @@ from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional
11
11
 
12
12
  def verify(
13
13
  action: Optional[List[str]] = None,
14
- **kw
14
+ **kwargs: Any
15
15
  ) -> SuccessTuple:
16
16
  """
17
17
  Verify the states of pipes, packages, and more.
@@ -21,8 +21,9 @@ def verify(
21
21
  'pipes': _verify_pipes,
22
22
  'packages': _verify_packages,
23
23
  'venvs': _verify_venvs,
24
+ 'plugins': _verify_plugins,
24
25
  }
25
- return choose_subaction(action, options, **kw)
26
+ return choose_subaction(action, options, **kwargs)
26
27
 
27
28
 
28
29
  def _verify_pipes(**kwargs) -> SuccessTuple:
@@ -92,8 +93,58 @@ def _verify_venvs(
92
93
  return True, "Success"
93
94
 
94
95
 
96
+ def _verify_plugins(
97
+ action: Optional[List[str]] = None,
98
+ **kwargs: Any
99
+ ) -> SuccessTuple:
100
+ """
101
+ Verify that all of the available plugins are able to be imported as expected.
102
+ """
103
+ from meerschaum.utils.formatting import print_options, UNICODE, print_tuple
104
+ from meerschaum.plugins import import_plugins, get_plugins_names, Plugin
105
+ from meerschaum.config import get_config
106
+ from meerschaum.utils.misc import items_str
107
+
108
+ plugins_names_to_verify = action or get_plugins_names()
109
+ if not plugins_names_to_verify:
110
+ if not action:
111
+ return True, "There are no installed plugins; nothing to do."
112
+ return False, f"Unable to verify plugins {items_str(action)}."
113
+
114
+ header = f"Verifying {len(plugins_names_to_verify)} plugins"
115
+ print_options(
116
+ plugins_names_to_verify,
117
+ header = header,
118
+ )
119
+
120
+ failed_to_import = []
121
+ for plugin_name in plugins_names_to_verify:
122
+ plugin = Plugin(plugin_name)
123
+ plugin_success = plugin.module is not None
124
+ if not plugin_success:
125
+ failed_to_import.append(plugin)
126
+ plugin_msg = (
127
+ (f"Imported plugin '{plugin}'.")
128
+ if plugin_success
129
+ else (f"Failed to import plugin '{plugin}'.")
130
+ )
131
+ print_tuple((plugin_success, plugin_msg), calm=True)
132
+
133
+ success = len(failed_to_import) == 0
134
+ message = (
135
+ f"Successfully imported {len(plugins_names_to_verify)} plugins."
136
+ if success
137
+ else (
138
+ f"Failed to import plugin"
139
+ + ('s' if len(failed_to_import) != 1 else '')
140
+ + f" {items_str(failed_to_import)}."
141
+ )
142
+ )
143
+ return success, message
144
+
145
+
95
146
  ### NOTE: This must be the final statement of the module.
96
147
  ### Any subactions added below these lines will not
97
148
  ### be added to the `help` docstring.
98
- from meerschaum.utils.misc import choices_docstring as _choices_docstring
149
+ from meerschaum.actions import choices_docstring as _choices_docstring
99
150
  verify.__doc__ += _choices_docstring('verify')