meerschaum 2.2.7__py3-none-any.whl → 2.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. meerschaum/__init__.py +6 -1
  2. meerschaum/__main__.py +0 -5
  3. meerschaum/_internal/arguments/__init__.py +1 -1
  4. meerschaum/_internal/arguments/_parse_arguments.py +72 -6
  5. meerschaum/_internal/arguments/_parser.py +45 -15
  6. meerschaum/_internal/docs/index.py +265 -8
  7. meerschaum/_internal/entry.py +154 -24
  8. meerschaum/_internal/shell/Shell.py +264 -77
  9. meerschaum/actions/__init__.py +29 -17
  10. meerschaum/actions/api.py +12 -12
  11. meerschaum/actions/attach.py +113 -0
  12. meerschaum/actions/copy.py +68 -41
  13. meerschaum/actions/delete.py +112 -50
  14. meerschaum/actions/edit.py +3 -3
  15. meerschaum/actions/install.py +40 -32
  16. meerschaum/actions/pause.py +44 -27
  17. meerschaum/actions/restart.py +107 -0
  18. meerschaum/actions/show.py +130 -159
  19. meerschaum/actions/start.py +161 -100
  20. meerschaum/actions/stop.py +78 -42
  21. meerschaum/api/_events.py +25 -1
  22. meerschaum/api/_oauth2.py +2 -0
  23. meerschaum/api/_websockets.py +2 -2
  24. meerschaum/api/dash/callbacks/jobs.py +36 -44
  25. meerschaum/api/dash/jobs.py +89 -78
  26. meerschaum/api/routes/__init__.py +1 -0
  27. meerschaum/api/routes/_actions.py +148 -17
  28. meerschaum/api/routes/_jobs.py +407 -0
  29. meerschaum/api/routes/_pipes.py +5 -5
  30. meerschaum/config/_default.py +1 -0
  31. meerschaum/config/_jobs.py +1 -1
  32. meerschaum/config/_paths.py +7 -0
  33. meerschaum/config/_shell.py +8 -3
  34. meerschaum/config/_version.py +1 -1
  35. meerschaum/config/static/__init__.py +17 -0
  36. meerschaum/connectors/Connector.py +13 -7
  37. meerschaum/connectors/__init__.py +28 -15
  38. meerschaum/connectors/api/APIConnector.py +27 -1
  39. meerschaum/connectors/api/_actions.py +71 -6
  40. meerschaum/connectors/api/_jobs.py +368 -0
  41. meerschaum/connectors/api/_pipes.py +85 -84
  42. meerschaum/connectors/parse.py +27 -15
  43. meerschaum/core/Pipe/_bootstrap.py +16 -8
  44. meerschaum/jobs/_Executor.py +69 -0
  45. meerschaum/jobs/_Job.py +899 -0
  46. meerschaum/jobs/__init__.py +396 -0
  47. meerschaum/jobs/systemd.py +694 -0
  48. meerschaum/plugins/__init__.py +97 -12
  49. meerschaum/utils/daemon/Daemon.py +276 -30
  50. meerschaum/utils/daemon/FileDescriptorInterceptor.py +5 -5
  51. meerschaum/utils/daemon/RotatingFile.py +14 -7
  52. meerschaum/utils/daemon/StdinFile.py +121 -0
  53. meerschaum/utils/daemon/__init__.py +15 -7
  54. meerschaum/utils/daemon/_names.py +15 -13
  55. meerschaum/utils/formatting/__init__.py +2 -1
  56. meerschaum/utils/formatting/_jobs.py +115 -62
  57. meerschaum/utils/formatting/_shell.py +6 -0
  58. meerschaum/utils/misc.py +41 -22
  59. meerschaum/utils/packages/_packages.py +9 -6
  60. meerschaum/utils/process.py +9 -9
  61. meerschaum/utils/prompt.py +16 -8
  62. meerschaum/utils/venv/__init__.py +2 -2
  63. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
  64. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/RECORD +70 -61
  65. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
  66. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
  67. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
  68. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
  69. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
  70. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
@@ -11,14 +11,15 @@ import time
11
11
  import json
12
12
  from io import StringIO
13
13
  from datetime import datetime
14
+
14
15
  import meerschaum as mrsm
15
16
  from meerschaum.utils.debug import dprint
16
17
  from meerschaum.utils.warnings import warn, error
17
18
  from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
18
19
 
19
20
  def pipe_r_url(
20
- pipe: mrsm.Pipe
21
- ) -> str:
21
+ pipe: mrsm.Pipe
22
+ ) -> str:
22
23
  """Return a relative URL path from a Pipe's keys."""
23
24
  from meerschaum.config.static import STATIC_CONFIG
24
25
  location_key = pipe.location_key
@@ -30,10 +31,10 @@ def pipe_r_url(
30
31
  )
31
32
 
32
33
  def register_pipe(
33
- self,
34
- pipe: mrsm.Pipe,
35
- debug: bool = False
36
- ) -> SuccessTuple:
34
+ self,
35
+ pipe: mrsm.Pipe,
36
+ debug: bool = False
37
+ ) -> SuccessTuple:
37
38
  """Submit a POST to the API to register a new Pipe object.
38
39
  Returns a tuple of (success_bool, response_dict).
39
40
  """
@@ -59,11 +60,11 @@ def register_pipe(
59
60
 
60
61
 
61
62
  def edit_pipe(
62
- self,
63
- pipe: mrsm.Pipe,
64
- patch: bool = False,
65
- debug: bool = False,
66
- ) -> SuccessTuple:
63
+ self,
64
+ pipe: mrsm.Pipe,
65
+ patch: bool = False,
66
+ debug: bool = False,
67
+ ) -> SuccessTuple:
67
68
  """Submit a PATCH to the API to edit an existing Pipe object.
68
69
  Returns a tuple of (success_bool, response_dict).
69
70
  """
@@ -89,14 +90,14 @@ def edit_pipe(
89
90
 
90
91
 
91
92
  def fetch_pipes_keys(
92
- self,
93
- connector_keys: Optional[List[str]] = None,
94
- metric_keys: Optional[List[str]] = None,
95
- location_keys: Optional[List[str]] = None,
96
- tags: Optional[List[str]] = None,
97
- params: Optional[Dict[str, Any]] = None,
98
- debug: bool = False
99
- ) -> Union[List[Tuple[str, str, Union[str, None]]]]:
93
+ self,
94
+ connector_keys: Optional[List[str]] = None,
95
+ metric_keys: Optional[List[str]] = None,
96
+ location_keys: Optional[List[str]] = None,
97
+ tags: Optional[List[str]] = None,
98
+ params: Optional[Dict[str, Any]] = None,
99
+ debug: bool = False
100
+ ) -> Union[List[Tuple[str, str, Union[str, None]]]]:
100
101
  """
101
102
  Fetch registered Pipes' keys from the API.
102
103
 
@@ -158,13 +159,13 @@ def fetch_pipes_keys(
158
159
 
159
160
 
160
161
  def sync_pipe(
161
- self,
162
- pipe: mrsm.Pipe,
163
- df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
164
- chunksize: Optional[int] = -1,
165
- debug: bool = False,
166
- **kw: Any
167
- ) -> SuccessTuple:
162
+ self,
163
+ pipe: mrsm.Pipe,
164
+ df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
165
+ chunksize: Optional[int] = -1,
166
+ debug: bool = False,
167
+ **kw: Any
168
+ ) -> SuccessTuple:
168
169
  """Sync a DataFrame into a Pipe."""
169
170
  from decimal import Decimal
170
171
  from meerschaum.utils.debug import dprint
@@ -303,10 +304,10 @@ def sync_pipe(
303
304
 
304
305
 
305
306
  def delete_pipe(
306
- self,
307
- pipe: Optional[meerschaum.Pipe] = None,
308
- debug: bool = None,
309
- ) -> SuccessTuple:
307
+ self,
308
+ pipe: Optional[meerschaum.Pipe] = None,
309
+ debug: bool = None,
310
+ ) -> SuccessTuple:
310
311
  """Delete a Pipe and drop its table."""
311
312
  if pipe is None:
312
313
  error(f"Pipe cannot be None.")
@@ -327,17 +328,17 @@ def delete_pipe(
327
328
 
328
329
 
329
330
  def get_pipe_data(
330
- self,
331
- pipe: meerschaum.Pipe,
332
- select_columns: Optional[List[str]] = None,
333
- omit_columns: Optional[List[str]] = None,
334
- begin: Union[str, datetime, int, None] = None,
335
- end: Union[str, datetime, int, None] = None,
336
- params: Optional[Dict[str, Any]] = None,
337
- as_chunks: bool = False,
338
- debug: bool = False,
339
- **kw: Any
340
- ) -> Union[pandas.DataFrame, None]:
331
+ self,
332
+ pipe: meerschaum.Pipe,
333
+ select_columns: Optional[List[str]] = None,
334
+ omit_columns: Optional[List[str]] = None,
335
+ begin: Union[str, datetime, int, None] = None,
336
+ end: Union[str, datetime, int, None] = None,
337
+ params: Optional[Dict[str, Any]] = None,
338
+ as_chunks: bool = False,
339
+ debug: bool = False,
340
+ **kw: Any
341
+ ) -> Union[pandas.DataFrame, None]:
341
342
  """Fetch data from the API."""
342
343
  r_url = pipe_r_url(pipe)
343
344
  chunks_list = []
@@ -389,10 +390,10 @@ def get_pipe_data(
389
390
 
390
391
 
391
392
  def get_pipe_id(
392
- self,
393
- pipe: meerschuam.Pipe,
394
- debug: bool = False,
395
- ) -> int:
393
+ self,
394
+ pipe: meerschuam.Pipe,
395
+ debug: bool = False,
396
+ ) -> int:
396
397
  """Get a Pipe's ID from the API."""
397
398
  from meerschaum.utils.misc import is_int
398
399
  r_url = pipe_r_url(pipe)
@@ -411,10 +412,10 @@ def get_pipe_id(
411
412
 
412
413
 
413
414
  def get_pipe_attributes(
414
- self,
415
- pipe: meerschaum.Pipe,
416
- debug: bool = False,
417
- ) -> Dict[str, Any]:
415
+ self,
416
+ pipe: meerschaum.Pipe,
417
+ debug: bool = False,
418
+ ) -> Dict[str, Any]:
418
419
  """Get a Pipe's attributes from the API
419
420
 
420
421
  Parameters
@@ -437,12 +438,12 @@ def get_pipe_attributes(
437
438
 
438
439
 
439
440
  def get_sync_time(
440
- self,
441
- pipe: 'meerschaum.Pipe',
442
- params: Optional[Dict[str, Any]] = None,
443
- newest: bool = True,
444
- debug: bool = False,
445
- ) -> Union[datetime, int, None]:
441
+ self,
442
+ pipe: 'meerschaum.Pipe',
443
+ params: Optional[Dict[str, Any]] = None,
444
+ newest: bool = True,
445
+ debug: bool = False,
446
+ ) -> Union[datetime, int, None]:
446
447
  """Get a Pipe's most recent datetime value from the API.
447
448
 
448
449
  Parameters
@@ -492,10 +493,10 @@ def get_sync_time(
492
493
 
493
494
 
494
495
  def pipe_exists(
495
- self,
496
- pipe: 'meerschaum.Pipe',
497
- debug: bool = False
498
- ) -> bool:
496
+ self,
497
+ pipe: mrsm.Pipe,
498
+ debug: bool = False
499
+ ) -> bool:
499
500
  """Check the API to see if a Pipe exists.
500
501
 
501
502
  Parameters
@@ -523,9 +524,9 @@ def pipe_exists(
523
524
 
524
525
 
525
526
  def create_metadata(
526
- self,
527
- debug: bool = False
528
- ) -> bool:
527
+ self,
528
+ debug: bool = False
529
+ ) -> bool:
529
530
  """Create metadata tables.
530
531
 
531
532
  Returns
@@ -547,14 +548,14 @@ def create_metadata(
547
548
 
548
549
 
549
550
  def get_pipe_rowcount(
550
- self,
551
- pipe: 'meerschaum.Pipe',
552
- begin: Optional[datetime] = None,
553
- end: Optional[datetime] = None,
554
- params: Optional[Dict[str, Any]] = None,
555
- remote: bool = False,
556
- debug: bool = False,
557
- ) -> int:
551
+ self,
552
+ pipe: mrsm.Pipe,
553
+ begin: Optional[datetime] = None,
554
+ end: Optional[datetime] = None,
555
+ params: Optional[Dict[str, Any]] = None,
556
+ remote: bool = False,
557
+ debug: bool = False,
558
+ ) -> int:
558
559
  """Get a pipe's row count from the API.
559
560
 
560
561
  Parameters
@@ -600,10 +601,10 @@ def get_pipe_rowcount(
600
601
 
601
602
 
602
603
  def drop_pipe(
603
- self,
604
- pipe: meerschaum.Pipe,
605
- debug: bool = False
606
- ) -> SuccessTuple:
604
+ self,
605
+ pipe: mrsm.Pipe,
606
+ debug: bool = False
607
+ ) -> SuccessTuple:
607
608
  """
608
609
  Drop a pipe's table but maintain its registration.
609
610
 
@@ -644,11 +645,11 @@ def drop_pipe(
644
645
 
645
646
 
646
647
  def clear_pipe(
647
- self,
648
- pipe: meerschaum.Pipe,
649
- debug: bool = False,
650
- **kw
651
- ) -> SuccessTuple:
648
+ self,
649
+ pipe: mrsm.Pipe,
650
+ debug: bool = False,
651
+ **kw
652
+ ) -> SuccessTuple:
652
653
  """
653
654
  Delete rows in a pipe's table.
654
655
 
@@ -666,7 +667,7 @@ def clear_pipe(
666
667
  kw.pop('location_keys', None)
667
668
  kw.pop('action', None)
668
669
  kw.pop('force', None)
669
- return self.do_action(
670
+ return self.do_action_legacy(
670
671
  ['clear', 'pipes'],
671
672
  connector_keys = pipe.connector_keys,
672
673
  metric_keys = pipe.metric_key,
@@ -678,10 +679,10 @@ def clear_pipe(
678
679
 
679
680
 
680
681
  def get_pipe_columns_types(
681
- self,
682
- pipe: meerschaum.Pipe,
683
- debug: bool = False,
684
- ) -> Union[Dict[str, str], None]:
682
+ self,
683
+ pipe: mrsm.Pipe,
684
+ debug: bool = False,
685
+ ) -> Union[Dict[str, str], None]:
685
686
  """
686
687
  Fetch the columns and types of the pipe's table.
687
688
 
@@ -10,11 +10,11 @@ from __future__ import annotations
10
10
  from meerschaum.utils.typing import Mapping, Any, SuccessTuple, Union, Optional, Dict, Tuple
11
11
 
12
12
  def parse_connector_keys(
13
- keys: str,
14
- construct: bool = True,
15
- as_tuple: bool = False,
16
- **kw: Any
17
- ) -> (
13
+ keys: str,
14
+ construct: bool = True,
15
+ as_tuple: bool = False,
16
+ **kw: Any
17
+ ) -> (
18
18
  Union[
19
19
  meerschaum.connectors.Connector,
20
20
  Dict[str, Any],
@@ -87,19 +87,17 @@ def parse_connector_keys(
87
87
 
88
88
 
89
89
  def parse_instance_keys(
90
- keys: Optional[str],
91
- construct: bool = True,
92
- as_tuple: bool = False,
93
- **kw
94
- ):
90
+ keys: Optional[str],
91
+ construct: bool = True,
92
+ as_tuple: bool = False,
93
+ **kw
94
+ ):
95
95
  """
96
96
  Parse the Meerschaum instance value into a Connector object.
97
97
  """
98
98
  from meerschaum.utils.warnings import warn
99
99
  from meerschaum.config import get_config
100
100
 
101
- ### TODO Check for valid types? Not sure how to do that if construct = False.
102
-
103
101
  if keys is None:
104
102
  keys = get_config('meerschaum', 'instance')
105
103
  keys = str(keys)
@@ -119,10 +117,24 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
119
117
  return parse_connector_keys(keys, **kw)
120
118
 
121
119
 
120
+ def parse_executor_keys(keys: Optional[str] = None, **kw):
121
+ """Parse the executor keys into an APIConnector or string."""
122
+ from meerschaum.jobs import get_executor_keys_from_context
123
+ if keys is None:
124
+ keys = get_executor_keys_from_context()
125
+
126
+ if keys is None or keys == 'local':
127
+ return 'local'
128
+
129
+ keys = str(keys)
130
+ return parse_connector_keys(keys, **kw)
131
+
132
+
122
133
  def is_valid_connector_keys(
123
- keys: str
124
- ) -> bool:
125
- """Verify a connector_keys string references a valid connector.
134
+ keys: str
135
+ ) -> bool:
136
+ """
137
+ Verify a connector_keys string references a valid connector.
126
138
  """
127
139
  try:
128
140
  success = parse_connector_keys(keys, construct=False) is not None
@@ -88,21 +88,29 @@ def bootstrap(
88
88
 
89
89
  try:
90
90
  if yes_no(
91
- f"Would you like to edit the definition for {self}?", yes=yes, noask=noask
91
+ f"Would you like to edit the definition for {self}?",
92
+ yes=yes,
93
+ noask=noask,
94
+ default='n',
92
95
  ):
93
96
  edit_tuple = self.edit_definition(debug=debug)
94
97
  if not edit_tuple[0]:
95
98
  return edit_tuple
96
99
 
97
- if yes_no(f"Would you like to try syncing {self} now?", yes=yes, noask=noask):
100
+ if yes_no(
101
+ f"Would you like to try syncing {self} now?",
102
+ yes=yes,
103
+ noask=noask,
104
+ default='n',
105
+ ):
98
106
  sync_tuple = actions['sync'](
99
107
  ['pipes'],
100
- connector_keys = [self.connector_keys],
101
- metric_keys = [self.metric_key],
102
- location_keys = [self.location_key],
103
- mrsm_instance = str(self.instance_connector),
104
- debug = debug,
105
- shell = shell,
108
+ connector_keys=[self.connector_keys],
109
+ metric_keys=[self.metric_key],
110
+ location_keys=[self.location_key],
111
+ mrsm_instance=str(self.instance_connector),
112
+ debug=debug,
113
+ shell=shell,
106
114
  )
107
115
  if not sync_tuple[0]:
108
116
  return sync_tuple
@@ -0,0 +1,69 @@
1
+ #! /usr/bin/env python3
2
+ # vim:fenc=utf-8
3
+
4
+ """
5
+ Define the base class for a Job executor.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from abc import abstractmethod
11
+
12
+ from meerschaum.connectors import Connector
13
+ from meerschaum.utils.typing import List, Dict, SuccessTuple, TYPE_CHECKING
14
+
15
+ if TYPE_CHECKING:
16
+ from meerschaum.jobs import Job
17
+
18
+ class Executor(Connector):
19
+ """
20
+ Define the methods for managing jobs.
21
+ """
22
+
23
+ @abstractmethod
24
+ def get_job_exists(self, name: str, debug: bool = False) -> bool:
25
+ """
26
+ Return whether a job exists.
27
+ """
28
+
29
+ @abstractmethod
30
+ def get_jobs(self) -> Dict[str, Job]:
31
+ """
32
+ Return a dictionary of names -> Jobs.
33
+ """
34
+
35
+ @abstractmethod
36
+ def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
37
+ """
38
+ Create a new job.
39
+ """
40
+
41
+ @abstractmethod
42
+ def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
43
+ """
44
+ Start a job.
45
+ """
46
+
47
+ @abstractmethod
48
+ def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
49
+ """
50
+ Stop a job.
51
+ """
52
+
53
+ @abstractmethod
54
+ def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
55
+ """
56
+ Pause a job.
57
+ """
58
+
59
+ @abstractmethod
60
+ def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
61
+ """
62
+ Delete a job.
63
+ """
64
+
65
+ @abstractmethod
66
+ def get_logs(self, name: str, debug: bool = False) -> str:
67
+ """
68
+ Return a job's log output.
69
+ """