meerschaum 2.2.5.dev3__py3-none-any.whl → 2.2.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. meerschaum/__init__.py +4 -1
  2. meerschaum/__main__.py +10 -5
  3. meerschaum/_internal/arguments/_parser.py +13 -2
  4. meerschaum/_internal/docs/index.py +523 -26
  5. meerschaum/_internal/entry.py +13 -13
  6. meerschaum/_internal/shell/Shell.py +26 -22
  7. meerschaum/_internal/shell/updates.py +175 -0
  8. meerschaum/_internal/term/__init__.py +2 -2
  9. meerschaum/actions/bootstrap.py +13 -14
  10. meerschaum/actions/python.py +11 -8
  11. meerschaum/actions/register.py +149 -37
  12. meerschaum/actions/show.py +79 -71
  13. meerschaum/actions/stop.py +11 -11
  14. meerschaum/actions/sync.py +3 -3
  15. meerschaum/actions/upgrade.py +28 -36
  16. meerschaum/api/dash/callbacks/login.py +21 -13
  17. meerschaum/api/dash/pages/login.py +2 -2
  18. meerschaum/api/routes/_login.py +5 -5
  19. meerschaum/api/routes/_pipes.py +20 -20
  20. meerschaum/config/__init__.py +8 -1
  21. meerschaum/config/_formatting.py +1 -0
  22. meerschaum/config/_paths.py +24 -2
  23. meerschaum/config/_shell.py +78 -66
  24. meerschaum/config/_version.py +1 -1
  25. meerschaum/config/paths.py +21 -2
  26. meerschaum/config/static/__init__.py +2 -0
  27. meerschaum/connectors/Connector.py +7 -2
  28. meerschaum/connectors/__init__.py +7 -5
  29. meerschaum/connectors/api/APIConnector.py +7 -2
  30. meerschaum/connectors/api/_actions.py +23 -31
  31. meerschaum/connectors/api/_misc.py +1 -1
  32. meerschaum/connectors/api/_request.py +13 -9
  33. meerschaum/connectors/api/_uri.py +5 -5
  34. meerschaum/core/Pipe/__init__.py +7 -3
  35. meerschaum/core/Pipe/_data.py +23 -15
  36. meerschaum/core/Pipe/_deduplicate.py +1 -1
  37. meerschaum/core/Pipe/_dtypes.py +5 -0
  38. meerschaum/core/Pipe/_fetch.py +18 -16
  39. meerschaum/core/Pipe/_sync.py +23 -15
  40. meerschaum/plugins/_Plugin.py +6 -6
  41. meerschaum/plugins/__init__.py +1 -1
  42. meerschaum/utils/daemon/Daemon.py +88 -129
  43. meerschaum/utils/daemon/FileDescriptorInterceptor.py +14 -5
  44. meerschaum/utils/daemon/RotatingFile.py +23 -17
  45. meerschaum/utils/daemon/__init__.py +28 -21
  46. meerschaum/utils/dataframe.py +12 -4
  47. meerschaum/utils/debug.py +9 -15
  48. meerschaum/utils/formatting/__init__.py +92 -46
  49. meerschaum/utils/formatting/_jobs.py +47 -9
  50. meerschaum/utils/misc.py +117 -11
  51. meerschaum/utils/packages/__init__.py +28 -16
  52. meerschaum/utils/prompt.py +5 -0
  53. meerschaum/utils/schedule.py +21 -15
  54. meerschaum/utils/typing.py +1 -0
  55. meerschaum/utils/venv/__init__.py +5 -1
  56. meerschaum/utils/warnings.py +8 -1
  57. meerschaum/utils/yaml.py +2 -2
  58. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/METADATA +1 -1
  59. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/RECORD +65 -64
  60. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/WHEEL +1 -1
  61. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/LICENSE +0 -0
  62. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/NOTICE +0 -0
  63. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/entry_points.txt +0 -0
  64. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/top_level.txt +0 -0
  65. {meerschaum-2.2.5.dev3.dist-info → meerschaum-2.2.7.dist-info}/zip-safe +0 -0
@@ -8,11 +8,14 @@ Functions for fetching new data into the Pipe
8
8
 
9
9
  from __future__ import annotations
10
10
  from datetime import timedelta, datetime
11
+
11
12
  import meerschaum as mrsm
12
- from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
13
+ from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator, TYPE_CHECKING
13
14
  from meerschaum.config import get_config
14
15
  from meerschaum.utils.warnings import warn
15
- from meerschaum.utils.misc import filter_keywords
16
+
17
+ if TYPE_CHECKING:
18
+ pd = mrsm.attempt_import('pandas')
16
19
 
17
20
  def fetch(
18
21
  self,
@@ -55,6 +58,7 @@ def fetch(
55
58
 
56
59
  from meerschaum.connectors import custom_types, get_connector_plugin
57
60
  from meerschaum.utils.debug import dprint, _checkpoint
61
+ from meerschaum.utils.misc import filter_arguments
58
62
 
59
63
  _chunk_hook = kw.pop('chunk_hook', None)
60
64
  kw['workers'] = self.get_num_workers(kw.get('workers', None))
@@ -72,24 +76,22 @@ def fetch(
72
76
  chunk_message = '\n' + chunk_label + '\n' + chunk_message
73
77
  return chunk_success, chunk_message
74
78
 
75
-
76
79
  with mrsm.Venv(get_connector_plugin(self.connector)):
77
- df = self.connector.fetch(
80
+ _args, _kwargs = filter_arguments(
81
+ self.connector.fetch,
78
82
  self,
79
- **filter_keywords(
80
- self.connector.fetch,
81
- begin=_determine_begin(
82
- self,
83
- begin,
84
- check_existing=check_existing,
85
- debug=debug,
86
- ),
87
- end=end,
88
- chunk_hook=_chunk_hook,
83
+ begin=_determine_begin(
84
+ self,
85
+ begin,
86
+ check_existing=check_existing,
89
87
  debug=debug,
90
- **kw
91
- )
88
+ ),
89
+ end=end,
90
+ chunk_hook=_chunk_hook,
91
+ debug=debug,
92
+ **kw
92
93
  )
94
+ df = self.connector.fetch(*_args, **_kwargs)
93
95
  return df
94
96
 
95
97
 
@@ -14,7 +14,9 @@ import threading
14
14
  import multiprocessing
15
15
  import functools
16
16
  from datetime import datetime, timedelta
17
+ from typing import TYPE_CHECKING
17
18
 
19
+ import meerschaum as mrsm
18
20
  from meerschaum.utils.typing import (
19
21
  Union,
20
22
  Optional,
@@ -26,13 +28,16 @@ from meerschaum.utils.typing import (
26
28
  List,
27
29
  Iterable,
28
30
  Generator,
29
- Iterator,
30
31
  )
31
32
  from meerschaum.utils.warnings import warn, error
32
33
 
34
+ if TYPE_CHECKING:
35
+ pd = mrsm.attempt_import('pandas')
36
+
33
37
  class InferFetch:
34
38
  MRSM_INFER_FETCH: bool = True
35
39
 
40
+
36
41
  def sync(
37
42
  self,
38
43
  df: Union[
@@ -125,7 +130,7 @@ def sync(
125
130
  from meerschaum.utils.formatting import get_console
126
131
  from meerschaum.utils.venv import Venv
127
132
  from meerschaum.connectors import get_connector_plugin
128
- from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
133
+ from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords, filter_arguments
129
134
  from meerschaum.utils.pool import get_pool
130
135
  from meerschaum.config import get_config
131
136
 
@@ -189,6 +194,9 @@ def sync(
189
194
  if hasattr(df, 'MRSM_INFER_FETCH'):
190
195
  try:
191
196
  if p.connector is None:
197
+ if ':' not in p.connector_keys:
198
+ return True, f"{p} does not support fetching; nothing to do."
199
+
192
200
  msg = f"{p} does not have a valid connector."
193
201
  if p.connector_keys.startswith('plugin:'):
194
202
  msg += f"\n Perhaps {p.connector_keys} has a syntax error?"
@@ -210,28 +218,28 @@ def sync(
210
218
  ):
211
219
  with Venv(get_connector_plugin(self.instance_connector)):
212
220
  p._exists = None
213
- return self.instance_connector.sync_pipe_inplace(
221
+ _args, _kwargs = filter_arguments(
222
+ p.instance_connector.sync_pipe_inplace,
214
223
  p,
215
- **filter_keywords(
216
- p.instance_connector.sync_pipe_inplace,
217
- debug=debug,
218
- **kw
219
- )
224
+ debug=debug,
225
+ **kw
226
+ )
227
+ return self.instance_connector.sync_pipe_inplace(
228
+ *_args,
229
+ **_kwargs
220
230
  )
221
-
222
231
 
223
232
  ### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
224
233
  try:
225
234
  if getattr(p.connector, 'sync', None) is not None:
226
235
  with Venv(get_connector_plugin(p.connector), debug=debug):
227
- return_tuple = p.connector.sync(
236
+ _args, _kwargs = filter_arguments(
237
+ p.connector.sync,
228
238
  p,
229
- **filter_keywords(
230
- p.connector.sync,
231
- debug=debug,
232
- **kw
233
- )
239
+ debug=debug,
240
+ **kw
234
241
  )
242
+ return_tuple = p.connector.sync(*_args, **_kwargs)
235
243
  p._exists = None
236
244
  if not isinstance(return_tuple, tuple):
237
245
  return_tuple = (
@@ -654,17 +654,18 @@ class Plugin:
654
654
  import ast, re
655
655
  ### NOTE: This technically would break
656
656
  ### if `required` was the very first line of the file.
657
- req_start_match = re.search(r'\nrequired(\s?)=', text)
657
+ req_start_match = re.search(r'required(:\s*)?.*=', text)
658
658
  if not req_start_match:
659
659
  return []
660
660
  req_start = req_start_match.start()
661
+ equals_sign = req_start + text[req_start:].find('=')
661
662
 
662
663
  ### Dependencies may have brackets within the strings, so push back the index.
663
- first_opening_brace = req_start + 1 + text[req_start:].find('[')
664
+ first_opening_brace = equals_sign + 1 + text[equals_sign:].find('[')
664
665
  if first_opening_brace == -1:
665
666
  return []
666
667
 
667
- next_closing_brace = req_start + 1 + text[req_start:].find(']')
668
+ next_closing_brace = equals_sign + 1 + text[equals_sign:].find(']')
668
669
  if next_closing_brace == -1:
669
670
  return []
670
671
 
@@ -681,12 +682,11 @@ class Plugin:
681
682
 
682
683
  req_end = end_ix + 1
683
684
  req_text = (
684
- text[req_start:req_end]
685
- .lstrip()
686
- .replace('required', '', 1)
685
+ text[(first_opening_brace-1):req_end]
687
686
  .lstrip()
688
687
  .replace('=', '', 1)
689
688
  .lstrip()
689
+ .rstrip()
690
690
  )
691
691
  try:
692
692
  required = ast.literal_eval(req_text)
@@ -27,7 +27,7 @@ _locks = {
27
27
  'PLUGINS_INTERNAL_LOCK_PATH': RLock(),
28
28
  }
29
29
  __all__ = (
30
- "Plugin", "make_action", "api_plugin", "import_plugins",
30
+ "Plugin", "make_action", "api_plugin", "dash_plugin", "import_plugins",
31
31
  "reload_plugins", "get_plugins", "get_data_plugins", "add_plugin_argument",
32
32
  "pre_sync_hook", "post_sync_hook",
33
33
  )