meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. meerschaum/_internal/arguments/_parse_arguments.py +2 -5
  2. meerschaum/_internal/docs/index.py +3 -2
  3. meerschaum/_internal/entry.py +13 -7
  4. meerschaum/_internal/shell/Shell.py +38 -44
  5. meerschaum/_internal/term/TermPageHandler.py +2 -3
  6. meerschaum/_internal/term/__init__.py +13 -11
  7. meerschaum/actions/api.py +10 -7
  8. meerschaum/actions/bootstrap.py +38 -11
  9. meerschaum/actions/copy.py +3 -3
  10. meerschaum/actions/delete.py +4 -1
  11. meerschaum/actions/register.py +1 -3
  12. meerschaum/actions/stack.py +24 -19
  13. meerschaum/actions/start.py +38 -40
  14. meerschaum/actions/sync.py +53 -52
  15. meerschaum/api/__init__.py +48 -14
  16. meerschaum/api/_events.py +15 -10
  17. meerschaum/api/_oauth2.py +2 -2
  18. meerschaum/api/_websockets.py +5 -4
  19. meerschaum/api/dash/__init__.py +7 -16
  20. meerschaum/api/dash/callbacks/__init__.py +1 -0
  21. meerschaum/api/dash/callbacks/dashboard.py +52 -58
  22. meerschaum/api/dash/callbacks/jobs.py +15 -16
  23. meerschaum/api/dash/callbacks/login.py +16 -10
  24. meerschaum/api/dash/callbacks/pipes.py +41 -0
  25. meerschaum/api/dash/callbacks/plugins.py +1 -1
  26. meerschaum/api/dash/callbacks/register.py +15 -11
  27. meerschaum/api/dash/components.py +54 -59
  28. meerschaum/api/dash/jobs.py +5 -9
  29. meerschaum/api/dash/pages/__init__.py +1 -0
  30. meerschaum/api/dash/pages/pipes.py +19 -0
  31. meerschaum/api/dash/pipes.py +86 -58
  32. meerschaum/api/dash/plugins.py +6 -4
  33. meerschaum/api/dash/sessions.py +176 -0
  34. meerschaum/api/dash/users.py +3 -41
  35. meerschaum/api/dash/webterm.py +12 -17
  36. meerschaum/api/resources/static/js/terminado.js +1 -1
  37. meerschaum/api/routes/_actions.py +4 -20
  38. meerschaum/api/routes/_jobs.py +8 -7
  39. meerschaum/api/routes/_login.py +4 -4
  40. meerschaum/api/routes/_pipes.py +3 -3
  41. meerschaum/api/routes/_webterm.py +5 -6
  42. meerschaum/config/_default.py +15 -2
  43. meerschaum/config/_version.py +1 -1
  44. meerschaum/config/stack/__init__.py +64 -19
  45. meerschaum/config/static/__init__.py +4 -0
  46. meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
  47. meerschaum/connectors/__init__.py +24 -14
  48. meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
  49. meerschaum/connectors/api/__init__.py +2 -1
  50. meerschaum/connectors/parse.py +18 -16
  51. meerschaum/connectors/poll.py +30 -24
  52. meerschaum/connectors/sql/__init__.py +3 -1
  53. meerschaum/connectors/sql/_pipes.py +172 -197
  54. meerschaum/connectors/sql/_plugins.py +45 -43
  55. meerschaum/connectors/sql/_users.py +46 -38
  56. meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
  57. meerschaum/connectors/valkey/__init__.py +10 -0
  58. meerschaum/connectors/valkey/_fetch.py +75 -0
  59. meerschaum/connectors/valkey/_pipes.py +844 -0
  60. meerschaum/connectors/valkey/_plugins.py +265 -0
  61. meerschaum/connectors/valkey/_users.py +305 -0
  62. meerschaum/core/Pipe/__init__.py +3 -0
  63. meerschaum/core/Pipe/_attributes.py +1 -2
  64. meerschaum/core/Pipe/_clear.py +16 -13
  65. meerschaum/core/Pipe/_copy.py +106 -0
  66. meerschaum/core/Pipe/_data.py +165 -101
  67. meerschaum/core/Pipe/_drop.py +4 -4
  68. meerschaum/core/Pipe/_dtypes.py +14 -14
  69. meerschaum/core/Pipe/_edit.py +15 -14
  70. meerschaum/core/Pipe/_sync.py +134 -53
  71. meerschaum/core/Pipe/_verify.py +11 -11
  72. meerschaum/core/User/_User.py +14 -12
  73. meerschaum/jobs/_Job.py +1 -6
  74. meerschaum/jobs/__init__.py +7 -2
  75. meerschaum/plugins/_Plugin.py +17 -13
  76. meerschaum/utils/_get_pipes.py +14 -20
  77. meerschaum/utils/dataframe.py +291 -101
  78. meerschaum/utils/dtypes/__init__.py +31 -6
  79. meerschaum/utils/dtypes/sql.py +4 -4
  80. meerschaum/utils/formatting/_shell.py +5 -6
  81. meerschaum/utils/misc.py +3 -3
  82. meerschaum/utils/packages/__init__.py +14 -9
  83. meerschaum/utils/packages/_packages.py +2 -0
  84. meerschaum/utils/schedule.py +1 -0
  85. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
  86. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/RECORD +93 -84
  87. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
  88. meerschaum/api/dash/actions.py +0 -255
  89. /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
  90. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
  91. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
  92. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
  93. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
  94. {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
@@ -26,8 +26,6 @@ from meerschaum.utils.typing import (
26
26
  SuccessTuple,
27
27
  Dict,
28
28
  List,
29
- Iterable,
30
- Generator,
31
29
  )
32
30
  from meerschaum.utils.warnings import warn, error
33
31
 
@@ -266,7 +264,6 @@ def sync(
266
264
  **kw
267
265
  )
268
266
  )
269
-
270
267
  except Exception as e:
271
268
  get_console().print_exception(
272
269
  suppress=[
@@ -369,6 +366,11 @@ def sync(
369
366
 
370
367
  ### Cast to a dataframe and ensure datatypes are what we expect.
371
368
  df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
369
+
370
+ ### Capture `numeric` and `json` columns.
371
+ self._persist_new_json_columns(df, debug=debug)
372
+ self._persist_new_numeric_columns(df, debug=debug)
373
+
372
374
  if debug:
373
375
  dprint(
374
376
  "DataFrame to sync:\n"
@@ -554,14 +556,15 @@ def exists(
554
556
 
555
557
 
556
558
  def filter_existing(
557
- self,
558
- df: 'pd.DataFrame',
559
- safe_copy: bool = True,
560
- date_bound_only: bool = False,
561
- chunksize: Optional[int] = -1,
562
- debug: bool = False,
563
- **kw
564
- ) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
559
+ self,
560
+ df: 'pd.DataFrame',
561
+ safe_copy: bool = True,
562
+ date_bound_only: bool = False,
563
+ include_unchanged_columns: bool = False,
564
+ chunksize: Optional[int] = -1,
565
+ debug: bool = False,
566
+ **kw
567
+ ) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
565
568
  """
566
569
  Inspect a dataframe and filter out rows which already exist in the pipe.
567
570
 
@@ -569,7 +572,7 @@ def filter_existing(
569
572
  ----------
570
573
  df: 'pd.DataFrame'
571
574
  The dataframe to inspect and filter.
572
-
575
+
573
576
  safe_copy: bool, default True
574
577
  If `True`, create a copy before comparing and modifying the dataframes.
575
578
  Setting to `False` may mutate the DataFrames.
@@ -578,6 +581,10 @@ def filter_existing(
578
581
  date_bound_only: bool, default False
579
582
  If `True`, only use the datetime index to fetch the sample dataframe.
580
583
 
584
+ include_unchanged_columns: bool, default False
585
+ If `True`, include the backtrack columns which haven't changed in the update dataframe.
586
+ This is useful if you can't update individual keys.
587
+
581
588
  chunksize: Optional[int], default -1
582
589
  The `chunksize` used when fetching existing data.
583
590
 
@@ -605,7 +612,7 @@ def filter_existing(
605
612
  from meerschaum.config import get_config
606
613
  pd = import_pandas()
607
614
  pandas = attempt_import('pandas')
608
- if not 'dataframe' in str(type(df)).lower():
615
+ if 'dataframe' not in str(type(df)).lower():
609
616
  df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
610
617
  is_dask = 'dask' in df.__module__
611
618
  if is_dask:
@@ -615,8 +622,21 @@ def filter_existing(
615
622
  else:
616
623
  merge = pd.merge
617
624
  NA = pd.NA
625
+
626
+ def get_empty_df():
627
+ empty_df = pd.DataFrame([])
628
+ dtypes = dict(df.dtypes) if df is not None else {}
629
+ dtypes.update(self.dtypes)
630
+ pd_dtypes = {
631
+ col: to_pandas_dtype(str(typ))
632
+ for col, typ in dtypes.items()
633
+ }
634
+ return add_missing_cols_to_df(empty_df, pd_dtypes)
635
+
618
636
  if df is None:
619
- return df, df, df
637
+ empty_df = get_empty_df()
638
+ return empty_df, empty_df, empty_df
639
+
620
640
  if (df.empty if not is_dask else len(df) == 0):
621
641
  return df, df, df
622
642
 
@@ -633,7 +653,7 @@ def filter_existing(
633
653
  if min_dt_val is not None and 'datetime' in str(dt_type)
634
654
  else min_dt_val
635
655
  )
636
- except Exception as e:
656
+ except Exception:
637
657
  min_dt = None
638
658
  if not ('datetime' in str(type(min_dt))) or str(min_dt) == 'NaT':
639
659
  if 'int' not in str(type(min_dt)).lower():
@@ -643,7 +663,7 @@ def filter_existing(
643
663
  begin = (
644
664
  round_time(
645
665
  min_dt,
646
- to = 'down'
666
+ to='down'
647
667
  ) - timedelta(minutes=1)
648
668
  )
649
669
  elif dt_type and 'int' in dt_type.lower():
@@ -661,7 +681,7 @@ def filter_existing(
661
681
  if max_dt_val is not None and 'datetime' in str(dt_type)
662
682
  else max_dt_val
663
683
  )
664
- except Exception as e:
684
+ except Exception:
665
685
  import traceback
666
686
  traceback.print_exc()
667
687
  max_dt = None
@@ -674,14 +694,14 @@ def filter_existing(
674
694
  end = (
675
695
  round_time(
676
696
  max_dt,
677
- to = 'down'
697
+ to='down'
678
698
  ) + timedelta(minutes=1)
679
699
  )
680
700
  elif dt_type and 'int' in dt_type.lower():
681
701
  end = max_dt + 1
682
702
 
683
703
  if max_dt is not None and min_dt is not None and min_dt > max_dt:
684
- warn(f"Detected minimum datetime greater than maximum datetime.")
704
+ warn("Detected minimum datetime greater than maximum datetime.")
685
705
 
686
706
  if begin is not None and end is not None and begin > end:
687
707
  if isinstance(begin, datetime):
@@ -710,13 +730,18 @@ def filter_existing(
710
730
  dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
711
731
 
712
732
  backtrack_df = self.get_data(
713
- begin = begin,
714
- end = end,
715
- chunksize = chunksize,
716
- params = params,
717
- debug = debug,
733
+ begin=begin,
734
+ end=end,
735
+ chunksize=chunksize,
736
+ params=params,
737
+ debug=debug,
718
738
  **kw
719
739
  )
740
+ if backtrack_df is None:
741
+ if debug:
742
+ dprint(f"No backtrack data was found for {self}.")
743
+ return df, get_empty_df(), df
744
+
720
745
  if debug:
721
746
  dprint(f"Existing data for {self}:\n" + str(backtrack_df), **kw)
722
747
  dprint(f"Existing dtypes for {self}:\n" + str(backtrack_df.dtypes))
@@ -743,18 +768,19 @@ def filter_existing(
743
768
  filter_unseen_df(
744
769
  backtrack_df,
745
770
  df,
746
- dtypes = {
771
+ dtypes={
747
772
  col: to_pandas_dtype(typ)
748
773
  for col, typ in self_dtypes.items()
749
774
  },
750
- safe_copy = safe_copy,
751
- debug = debug
775
+ safe_copy=safe_copy,
776
+ debug=debug
752
777
  ),
753
778
  on_cols_dtypes,
754
779
  )
755
780
 
756
781
  ### Cast dicts or lists to strings so we can merge.
757
782
  serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
783
+
758
784
  def deserializer(x):
759
785
  return json.loads(x) if isinstance(x, str) else x
760
786
 
@@ -767,12 +793,12 @@ def filter_existing(
767
793
  casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
768
794
 
769
795
  joined_df = merge(
770
- delta_df.fillna(NA),
771
- backtrack_df.fillna(NA),
772
- how = 'left',
773
- on = on_cols,
774
- indicator = True,
775
- suffixes = ('', '_old'),
796
+ delta_df.infer_objects(copy=False).fillna(NA),
797
+ backtrack_df.infer_objects(copy=False).fillna(NA),
798
+ how='left',
799
+ on=on_cols,
800
+ indicator=True,
801
+ suffixes=('', '_old'),
776
802
  ) if on_cols else delta_df
777
803
  for col in casted_cols:
778
804
  if col in joined_df.columns:
@@ -782,20 +808,13 @@ def filter_existing(
782
808
 
783
809
  ### Determine which rows are completely new.
784
810
  new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None
785
- cols = list(backtrack_df.columns)
811
+ cols = list(delta_df.columns)
786
812
 
787
813
  unseen_df = (
788
- (
789
- joined_df
790
- .where(new_rows_mask)
791
- .dropna(how='all')[cols]
792
- .reset_index(drop=True)
793
- ) if not is_dask else (
794
- joined_df
795
- .where(new_rows_mask)
796
- .dropna(how='all')[cols]
797
- .reset_index(drop=True)
798
- )
814
+ joined_df
815
+ .where(new_rows_mask)
816
+ .dropna(how='all')[cols]
817
+ .reset_index(drop=True)
799
818
  ) if on_cols else delta_df
800
819
 
801
820
  ### Rows that have already been inserted but values have changed.
@@ -804,20 +823,33 @@ def filter_existing(
804
823
  .where(~new_rows_mask)
805
824
  .dropna(how='all')[cols]
806
825
  .reset_index(drop=True)
807
- ) if on_cols else None
826
+ ) if on_cols else get_empty_df()
827
+
828
+ if include_unchanged_columns and on_cols:
829
+ unchanged_backtrack_cols = [
830
+ col
831
+ for col in backtrack_df.columns
832
+ if col in on_cols or col not in update_df.columns
833
+ ]
834
+ update_df = merge(
835
+ backtrack_df[unchanged_backtrack_cols],
836
+ update_df,
837
+ how='inner',
838
+ on=on_cols,
839
+ )
808
840
 
809
841
  return unseen_df, update_df, delta_df
810
842
 
811
843
 
812
844
  @staticmethod
813
845
  def _get_chunk_label(
814
- chunk: Union[
815
- 'pd.DataFrame',
816
- List[Dict[str, Any]],
817
- Dict[str, List[Any]]
818
- ],
819
- dt_col: str,
820
- ) -> str:
846
+ chunk: Union[
847
+ 'pd.DataFrame',
848
+ List[Dict[str, Any]],
849
+ Dict[str, List[Any]]
850
+ ],
851
+ dt_col: str,
852
+ ) -> str:
821
853
  """
822
854
  Return the min - max label for the chunk.
823
855
  """
@@ -870,3 +902,52 @@ def get_num_workers(self, workers: Optional[int] = None) -> int:
870
902
  (desired_workers - current_num_connections),
871
903
  1,
872
904
  )
905
+
906
+
907
+ def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
908
+ """
909
+ Check for new numeric columns and update the parameters.
910
+ """
911
+ from meerschaum.utils.dataframe import get_numeric_cols
912
+ numeric_cols = get_numeric_cols(df)
913
+ existing_numeric_cols = [col for col, typ in self.dtypes.items() if typ == 'numeric']
914
+ new_numeric_cols = [col for col in numeric_cols if col not in existing_numeric_cols]
915
+ if not new_numeric_cols:
916
+ return True, "Success"
917
+
918
+ dtypes = self.parameters.get('dtypes', {})
919
+ dtypes.update({col: 'numeric' for col in numeric_cols})
920
+ self.parameters['dtypes'] = dtypes
921
+ if not self.temporary:
922
+ edit_success, edit_msg = self.edit(interactive=False, debug=debug)
923
+ if not edit_success:
924
+ warn(f"Unable to update NUMERIC dtypes for {self}:\n{edit_msg}")
925
+
926
+ return edit_success, edit_msg
927
+
928
+ return True, "Success"
929
+
930
+
931
+ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
932
+ """
933
+ Check for new JSON columns and update the parameters.
934
+ """
935
+ from meerschaum.utils.dataframe import get_json_cols
936
+ json_cols = get_json_cols(df)
937
+ existing_json_cols = [col for col, typ in self.dtypes.items() if typ == 'json']
938
+ new_json_cols = [col for col in json_cols if col not in existing_json_cols]
939
+ if not new_json_cols:
940
+ return True, "Success"
941
+
942
+ dtypes = self.parameters.get('dtypes', {})
943
+ dtypes.update({col: 'json' for col in json_cols})
944
+ self.parameters['dtypes'] = dtypes
945
+
946
+ if not self.temporary:
947
+ edit_success, edit_msg = self.edit(interactive=False, debug=debug)
948
+ if not edit_success:
949
+ warn(f"Unable to update JSON dtypes for {self}:\n{edit_msg}")
950
+
951
+ return edit_success, edit_msg
952
+
953
+ return True, "Success"
@@ -12,17 +12,17 @@ from meerschaum.utils.warnings import warn, info
12
12
  from meerschaum.utils.debug import dprint
13
13
 
14
14
  def verify(
15
- self,
16
- begin: Union[datetime, int, None] = None,
17
- end: Union[datetime, int, None] = None,
18
- params: Optional[Dict[str, Any]] = None,
19
- chunk_interval: Union[timedelta, int, None] = None,
20
- bounded: Optional[bool] = None,
21
- deduplicate: bool = False,
22
- workers: Optional[int] = None,
23
- debug: bool = False,
24
- **kwargs: Any
25
- ) -> SuccessTuple:
15
+ self,
16
+ begin: Union[datetime, int, None] = None,
17
+ end: Union[datetime, int, None] = None,
18
+ params: Optional[Dict[str, Any]] = None,
19
+ chunk_interval: Union[timedelta, int, None] = None,
20
+ bounded: Optional[bool] = None,
21
+ deduplicate: bool = False,
22
+ workers: Optional[int] = None,
23
+ debug: bool = False,
24
+ **kwargs: Any
25
+ ) -> SuccessTuple:
26
26
  """
27
27
  Verify the contents of the pipe by resyncing its interval.
28
28
 
@@ -11,7 +11,7 @@ import os
11
11
  import hashlib
12
12
  import hmac
13
13
  from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
14
- from meerschaum.utils.typing import Optional, Dict, Any, Tuple
14
+ from meerschaum.utils.typing import Optional, Dict, Any, Union
15
15
  from meerschaum.config.static import STATIC_CONFIG
16
16
  from meerschaum.utils.warnings import warn
17
17
 
@@ -19,10 +19,10 @@ from meerschaum.utils.warnings import warn
19
19
  __all__ = ('hash_password', 'verify_password', 'User')
20
20
 
21
21
  def hash_password(
22
- password: str,
23
- salt: Optional[bytes] = None,
24
- rounds: Optional[int] = None,
25
- ) -> str:
22
+ password: str,
23
+ salt: Optional[bytes] = None,
24
+ rounds: Optional[int] = None,
25
+ ) -> str:
26
26
  """
27
27
  Return an encoded hash string from the given password.
28
28
 
@@ -68,9 +68,9 @@ def hash_password(
68
68
 
69
69
 
70
70
  def verify_password(
71
- password: str,
72
- password_hash: str,
73
- ) -> bool:
71
+ password: str,
72
+ password_hash: str,
73
+ ) -> bool:
74
74
  """
75
75
  Return `True` if the password matches the provided hash.
76
76
 
@@ -197,26 +197,28 @@ class User:
197
197
  return self._attributes
198
198
 
199
199
  @property
200
- def instance_connector(self) -> meerschaum.connectors.Connector:
201
- """ """
200
+ def instance_connector(self) -> 'mrsm.connectors.Connector':
202
201
  from meerschaum.connectors.parse import parse_instance_keys
203
202
  if '_instance_connector' not in self.__dict__:
204
203
  self._instance_connector = parse_instance_keys(self._instance_keys)
205
204
  return self._instance_connector
206
205
 
207
206
  @property
208
- def user_id(self) -> int:
207
+ def user_id(self) -> Union[int, str, None]:
209
208
  """NOTE: This causes recursion with the API,
210
209
  so don't try to get fancy with read-only attributes.
211
210
  """
212
211
  return self._user_id
213
212
 
214
213
  @user_id.setter
215
- def user_id(self, user_id):
214
+ def user_id(self, user_id: Union[int, str, None]):
216
215
  self._user_id = user_id
217
216
 
218
217
  @property
219
218
  def password_hash(self):
219
+ """
220
+ Return the hash of the user's password.
221
+ """
220
222
  _password_hash = self.__dict__.get('_password_hash', None)
221
223
  if _password_hash is not None:
222
224
  return _password_hash
meerschaum/jobs/_Job.py CHANGED
@@ -10,10 +10,7 @@ from __future__ import annotations
10
10
 
11
11
  import shlex
12
12
  import asyncio
13
- import threading
14
- import json
15
13
  import pathlib
16
- import os
17
14
  import sys
18
15
  import traceback
19
16
  from functools import partial
@@ -206,13 +203,11 @@ class Job:
206
203
  jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
207
204
  daemon_dir = jobs_dir / daemon_id
208
205
  pid_file = daemon_dir / 'process.pid'
209
- properties_path = daemon_dir / 'properties.json'
210
- pickle_path = daemon_dir / 'pickle.pkl'
211
206
 
212
207
  if pid_file.exists():
213
208
  with open(pid_file, 'r', encoding='utf-8') as f:
214
209
  daemon_pid = int(f.read())
215
-
210
+
216
211
  if pid != daemon_pid:
217
212
  raise EnvironmentError(f"Differing PIDs: {pid=}, {daemon_pid=}")
218
213
  else:
@@ -9,9 +9,9 @@ Higher-level utilities for managing `meerschaum.utils.daemon.Daemon`.
9
9
  import pathlib
10
10
 
11
11
  import meerschaum as mrsm
12
- from meerschaum.utils.typing import Dict, Optional, List, Callable, Any, SuccessTuple
12
+ from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
13
13
 
14
- from meerschaum.jobs._Job import Job, StopMonitoringLogs
14
+ from meerschaum.jobs._Job import Job
15
15
  from meerschaum.jobs._Executor import Executor
16
16
 
17
17
  __all__ = (
@@ -403,9 +403,14 @@ def get_executor_keys_from_context() -> str:
403
403
  if _context_keys is not None:
404
404
  return _context_keys
405
405
 
406
+ from meerschaum.config import get_config
406
407
  from meerschaum.config.paths import ROOT_DIR_PATH, DEFAULT_ROOT_DIR_PATH
407
408
  from meerschaum.utils.misc import is_systemd_available
408
409
 
410
+ configured_executor = get_config('meerschaum', 'executor', warn=False)
411
+ if configured_executor is not None:
412
+ return configured_executor
413
+
409
414
  _context_keys = (
410
415
  'systemd'
411
416
  if is_systemd_available() and ROOT_DIR_PATH == DEFAULT_ROOT_DIR_PATH
@@ -7,8 +7,12 @@ Plugin metadata class
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- import os, pathlib, shutil
11
- from site import venv
10
+
11
+ import os
12
+ import pathlib
13
+ import shutil
14
+
15
+ import meerschaum as mrsm
12
16
  from meerschaum.utils.typing import (
13
17
  Dict,
14
18
  List,
@@ -40,8 +44,8 @@ class Plugin:
40
44
  attributes: Optional[Dict[str, Any]] = None,
41
45
  archive_path: Optional[pathlib.Path] = None,
42
46
  venv_path: Optional[pathlib.Path] = None,
43
- repo_connector: Optional['meerschaum.connectors.api.APIConnector'] = None,
44
- repo: Union['meerschaum.connectors.api.APIConnector', str, None] = None,
47
+ repo_connector: Optional['mrsm.connectors.api.APIConnector'] = None,
48
+ repo: Union['mrsm.connectors.api.APIConnector', str, None] = None,
45
49
  ):
46
50
  from meerschaum.config.static import STATIC_CONFIG
47
51
  sep = STATIC_CONFIG['plugins']['repo_separator']
@@ -470,9 +474,9 @@ class Plugin:
470
474
 
471
475
 
472
476
  def remove_archive(
473
- self,
474
- debug: bool = False
475
- ) -> SuccessTuple:
477
+ self,
478
+ debug: bool = False
479
+ ) -> SuccessTuple:
476
480
  """Remove a plugin's archive file."""
477
481
  if not self.archive_path.exists():
478
482
  return True, f"Archive file for plugin '{self}' does not exist."
@@ -484,9 +488,9 @@ class Plugin:
484
488
 
485
489
 
486
490
  def remove_venv(
487
- self,
488
- debug: bool = False
489
- ) -> SuccessTuple:
491
+ self,
492
+ debug: bool = False
493
+ ) -> SuccessTuple:
490
494
  """Remove a plugin's virtual environment."""
491
495
  if not self.venv_path.exists():
492
496
  return True, f"Virtual environment for plugin '{self}' does not exist."
@@ -608,9 +612,9 @@ class Plugin:
608
612
 
609
613
 
610
614
  def get_dependencies(
611
- self,
612
- debug: bool = False,
613
- ) -> List[str]:
615
+ self,
616
+ debug: bool = False,
617
+ ) -> List[str]:
614
618
  """
615
619
  If the Plugin has specified dependencies in a list called `required`, return the list.
616
620
 
@@ -16,19 +16,18 @@ from meerschaum.utils.typing import (
16
16
  __pdoc__ = {'get_pipes': True, 'fetch_pipes_keys': True}
17
17
 
18
18
  def get_pipes(
19
- connector_keys: Union[str, List[str], None] = None,
20
- metric_keys: Union[str, List[str], None] = None,
21
- location_keys: Union[str, List[str], None] = None,
22
- tags: Optional[List[str]] = None,
23
- params: Optional[Dict[str, Any]] = None,
24
- mrsm_instance: Union[str, InstanceConnector, None] = None,
25
- instance: Union[str, InstanceConnector, None] = None,
26
- as_list: bool = False,
27
- method: str = 'registered',
28
- wait: bool = False,
29
- debug: bool = False,
30
- **kw: Any
31
- ) -> Union[PipesDict, List[mrsm.Pipe]]:
19
+ connector_keys: Union[str, List[str], None] = None,
20
+ metric_keys: Union[str, List[str], None] = None,
21
+ location_keys: Union[str, List[str], None] = None,
22
+ tags: Optional[List[str]] = None,
23
+ params: Optional[Dict[str, Any]] = None,
24
+ mrsm_instance: Union[str, InstanceConnector, None] = None,
25
+ instance: Union[str, InstanceConnector, None] = None,
26
+ as_list: bool = False,
27
+ method: str = 'registered',
28
+ debug: bool = False,
29
+ **kw: Any
30
+ ) -> Union[PipesDict, List[mrsm.Pipe]]:
32
31
  """
33
32
  Return a dictionary or list of `meerschaum.Pipe` objects.
34
33
 
@@ -72,10 +71,6 @@ def get_pipes(
72
71
  If `'all'`, create pipes from predefined metrics and locations. Required `connector_keys`.
73
72
  **NOTE:** Method `'all'` is not implemented!
74
73
 
75
- wait: bool, default False
76
- Wait for a connection before getting Pipes. Should only be true for cases where the
77
- database might not be running (like the API).
78
-
79
74
  **kw: Any:
80
75
  Keyword arguments to pass to the `meerschaum.Pipe` constructor.
81
76
 
@@ -133,15 +128,14 @@ def get_pipes(
133
128
  location_keys = [location_keys]
134
129
 
135
130
  ### Get SQL or API connector (keys come from `connector.fetch_pipes_keys()`).
136
- ### If `wait`, wait until a connection is made
137
131
  if mrsm_instance is None:
138
132
  mrsm_instance = instance
139
133
  if mrsm_instance is None:
140
134
  mrsm_instance = get_config('meerschaum', 'instance', patch=True)
141
135
  if isinstance(mrsm_instance, str):
142
136
  from meerschaum.connectors.parse import parse_instance_keys
143
- connector = parse_instance_keys(keys=mrsm_instance, wait=wait, debug=debug)
144
- else: ### NOTE: mrsm_instance MUST be a SQL or API connector for this to work
137
+ connector = parse_instance_keys(keys=mrsm_instance, debug=debug)
138
+ else:
145
139
  from meerschaum.connectors import instance_types
146
140
  valid_connector = False
147
141
  if hasattr(mrsm_instance, 'type'):