meerschaum 2.2.7__py3-none-any.whl → 2.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +6 -1
- meerschaum/__main__.py +0 -5
- meerschaum/_internal/arguments/__init__.py +1 -1
- meerschaum/_internal/arguments/_parse_arguments.py +72 -6
- meerschaum/_internal/arguments/_parser.py +45 -15
- meerschaum/_internal/docs/index.py +265 -8
- meerschaum/_internal/entry.py +154 -24
- meerschaum/_internal/shell/Shell.py +264 -77
- meerschaum/actions/__init__.py +29 -17
- meerschaum/actions/api.py +12 -12
- meerschaum/actions/attach.py +113 -0
- meerschaum/actions/copy.py +68 -41
- meerschaum/actions/delete.py +112 -50
- meerschaum/actions/edit.py +3 -3
- meerschaum/actions/install.py +40 -32
- meerschaum/actions/pause.py +44 -27
- meerschaum/actions/restart.py +107 -0
- meerschaum/actions/show.py +130 -159
- meerschaum/actions/start.py +161 -100
- meerschaum/actions/stop.py +78 -42
- meerschaum/api/_events.py +25 -1
- meerschaum/api/_oauth2.py +2 -0
- meerschaum/api/_websockets.py +2 -2
- meerschaum/api/dash/callbacks/jobs.py +36 -44
- meerschaum/api/dash/jobs.py +89 -78
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +148 -17
- meerschaum/api/routes/_jobs.py +407 -0
- meerschaum/api/routes/_pipes.py +5 -5
- meerschaum/config/_default.py +1 -0
- meerschaum/config/_jobs.py +1 -1
- meerschaum/config/_paths.py +7 -0
- meerschaum/config/_shell.py +8 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +17 -0
- meerschaum/connectors/Connector.py +13 -7
- meerschaum/connectors/__init__.py +28 -15
- meerschaum/connectors/api/APIConnector.py +27 -1
- meerschaum/connectors/api/_actions.py +71 -6
- meerschaum/connectors/api/_jobs.py +368 -0
- meerschaum/connectors/api/_pipes.py +85 -84
- meerschaum/connectors/parse.py +27 -15
- meerschaum/core/Pipe/_bootstrap.py +16 -8
- meerschaum/jobs/_Executor.py +69 -0
- meerschaum/jobs/_Job.py +899 -0
- meerschaum/jobs/__init__.py +396 -0
- meerschaum/jobs/systemd.py +694 -0
- meerschaum/plugins/__init__.py +97 -12
- meerschaum/utils/daemon/Daemon.py +276 -30
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +5 -5
- meerschaum/utils/daemon/RotatingFile.py +14 -7
- meerschaum/utils/daemon/StdinFile.py +121 -0
- meerschaum/utils/daemon/__init__.py +15 -7
- meerschaum/utils/daemon/_names.py +15 -13
- meerschaum/utils/formatting/__init__.py +2 -1
- meerschaum/utils/formatting/_jobs.py +115 -62
- meerschaum/utils/formatting/_shell.py +6 -0
- meerschaum/utils/misc.py +41 -22
- meerschaum/utils/packages/_packages.py +9 -6
- meerschaum/utils/process.py +9 -9
- meerschaum/utils/prompt.py +16 -8
- meerschaum/utils/venv/__init__.py +2 -2
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/RECORD +70 -61
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
@@ -11,14 +11,15 @@ import time
|
|
11
11
|
import json
|
12
12
|
from io import StringIO
|
13
13
|
from datetime import datetime
|
14
|
+
|
14
15
|
import meerschaum as mrsm
|
15
16
|
from meerschaum.utils.debug import dprint
|
16
17
|
from meerschaum.utils.warnings import warn, error
|
17
18
|
from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
|
18
19
|
|
19
20
|
def pipe_r_url(
|
20
|
-
|
21
|
-
|
21
|
+
pipe: mrsm.Pipe
|
22
|
+
) -> str:
|
22
23
|
"""Return a relative URL path from a Pipe's keys."""
|
23
24
|
from meerschaum.config.static import STATIC_CONFIG
|
24
25
|
location_key = pipe.location_key
|
@@ -30,10 +31,10 @@ def pipe_r_url(
|
|
30
31
|
)
|
31
32
|
|
32
33
|
def register_pipe(
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
self,
|
35
|
+
pipe: mrsm.Pipe,
|
36
|
+
debug: bool = False
|
37
|
+
) -> SuccessTuple:
|
37
38
|
"""Submit a POST to the API to register a new Pipe object.
|
38
39
|
Returns a tuple of (success_bool, response_dict).
|
39
40
|
"""
|
@@ -59,11 +60,11 @@ def register_pipe(
|
|
59
60
|
|
60
61
|
|
61
62
|
def edit_pipe(
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
63
|
+
self,
|
64
|
+
pipe: mrsm.Pipe,
|
65
|
+
patch: bool = False,
|
66
|
+
debug: bool = False,
|
67
|
+
) -> SuccessTuple:
|
67
68
|
"""Submit a PATCH to the API to edit an existing Pipe object.
|
68
69
|
Returns a tuple of (success_bool, response_dict).
|
69
70
|
"""
|
@@ -89,14 +90,14 @@ def edit_pipe(
|
|
89
90
|
|
90
91
|
|
91
92
|
def fetch_pipes_keys(
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
93
|
+
self,
|
94
|
+
connector_keys: Optional[List[str]] = None,
|
95
|
+
metric_keys: Optional[List[str]] = None,
|
96
|
+
location_keys: Optional[List[str]] = None,
|
97
|
+
tags: Optional[List[str]] = None,
|
98
|
+
params: Optional[Dict[str, Any]] = None,
|
99
|
+
debug: bool = False
|
100
|
+
) -> Union[List[Tuple[str, str, Union[str, None]]]]:
|
100
101
|
"""
|
101
102
|
Fetch registered Pipes' keys from the API.
|
102
103
|
|
@@ -158,13 +159,13 @@ def fetch_pipes_keys(
|
|
158
159
|
|
159
160
|
|
160
161
|
def sync_pipe(
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
162
|
+
self,
|
163
|
+
pipe: mrsm.Pipe,
|
164
|
+
df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
|
165
|
+
chunksize: Optional[int] = -1,
|
166
|
+
debug: bool = False,
|
167
|
+
**kw: Any
|
168
|
+
) -> SuccessTuple:
|
168
169
|
"""Sync a DataFrame into a Pipe."""
|
169
170
|
from decimal import Decimal
|
170
171
|
from meerschaum.utils.debug import dprint
|
@@ -303,10 +304,10 @@ def sync_pipe(
|
|
303
304
|
|
304
305
|
|
305
306
|
def delete_pipe(
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
307
|
+
self,
|
308
|
+
pipe: Optional[meerschaum.Pipe] = None,
|
309
|
+
debug: bool = None,
|
310
|
+
) -> SuccessTuple:
|
310
311
|
"""Delete a Pipe and drop its table."""
|
311
312
|
if pipe is None:
|
312
313
|
error(f"Pipe cannot be None.")
|
@@ -327,17 +328,17 @@ def delete_pipe(
|
|
327
328
|
|
328
329
|
|
329
330
|
def get_pipe_data(
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
331
|
+
self,
|
332
|
+
pipe: meerschaum.Pipe,
|
333
|
+
select_columns: Optional[List[str]] = None,
|
334
|
+
omit_columns: Optional[List[str]] = None,
|
335
|
+
begin: Union[str, datetime, int, None] = None,
|
336
|
+
end: Union[str, datetime, int, None] = None,
|
337
|
+
params: Optional[Dict[str, Any]] = None,
|
338
|
+
as_chunks: bool = False,
|
339
|
+
debug: bool = False,
|
340
|
+
**kw: Any
|
341
|
+
) -> Union[pandas.DataFrame, None]:
|
341
342
|
"""Fetch data from the API."""
|
342
343
|
r_url = pipe_r_url(pipe)
|
343
344
|
chunks_list = []
|
@@ -389,10 +390,10 @@ def get_pipe_data(
|
|
389
390
|
|
390
391
|
|
391
392
|
def get_pipe_id(
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
393
|
+
self,
|
394
|
+
pipe: meerschuam.Pipe,
|
395
|
+
debug: bool = False,
|
396
|
+
) -> int:
|
396
397
|
"""Get a Pipe's ID from the API."""
|
397
398
|
from meerschaum.utils.misc import is_int
|
398
399
|
r_url = pipe_r_url(pipe)
|
@@ -411,10 +412,10 @@ def get_pipe_id(
|
|
411
412
|
|
412
413
|
|
413
414
|
def get_pipe_attributes(
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
415
|
+
self,
|
416
|
+
pipe: meerschaum.Pipe,
|
417
|
+
debug: bool = False,
|
418
|
+
) -> Dict[str, Any]:
|
418
419
|
"""Get a Pipe's attributes from the API
|
419
420
|
|
420
421
|
Parameters
|
@@ -437,12 +438,12 @@ def get_pipe_attributes(
|
|
437
438
|
|
438
439
|
|
439
440
|
def get_sync_time(
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
441
|
+
self,
|
442
|
+
pipe: 'meerschaum.Pipe',
|
443
|
+
params: Optional[Dict[str, Any]] = None,
|
444
|
+
newest: bool = True,
|
445
|
+
debug: bool = False,
|
446
|
+
) -> Union[datetime, int, None]:
|
446
447
|
"""Get a Pipe's most recent datetime value from the API.
|
447
448
|
|
448
449
|
Parameters
|
@@ -492,10 +493,10 @@ def get_sync_time(
|
|
492
493
|
|
493
494
|
|
494
495
|
def pipe_exists(
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
496
|
+
self,
|
497
|
+
pipe: mrsm.Pipe,
|
498
|
+
debug: bool = False
|
499
|
+
) -> bool:
|
499
500
|
"""Check the API to see if a Pipe exists.
|
500
501
|
|
501
502
|
Parameters
|
@@ -523,9 +524,9 @@ def pipe_exists(
|
|
523
524
|
|
524
525
|
|
525
526
|
def create_metadata(
|
526
|
-
|
527
|
-
|
528
|
-
|
527
|
+
self,
|
528
|
+
debug: bool = False
|
529
|
+
) -> bool:
|
529
530
|
"""Create metadata tables.
|
530
531
|
|
531
532
|
Returns
|
@@ -547,14 +548,14 @@ def create_metadata(
|
|
547
548
|
|
548
549
|
|
549
550
|
def get_pipe_rowcount(
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
551
|
+
self,
|
552
|
+
pipe: mrsm.Pipe,
|
553
|
+
begin: Optional[datetime] = None,
|
554
|
+
end: Optional[datetime] = None,
|
555
|
+
params: Optional[Dict[str, Any]] = None,
|
556
|
+
remote: bool = False,
|
557
|
+
debug: bool = False,
|
558
|
+
) -> int:
|
558
559
|
"""Get a pipe's row count from the API.
|
559
560
|
|
560
561
|
Parameters
|
@@ -600,10 +601,10 @@ def get_pipe_rowcount(
|
|
600
601
|
|
601
602
|
|
602
603
|
def drop_pipe(
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
604
|
+
self,
|
605
|
+
pipe: mrsm.Pipe,
|
606
|
+
debug: bool = False
|
607
|
+
) -> SuccessTuple:
|
607
608
|
"""
|
608
609
|
Drop a pipe's table but maintain its registration.
|
609
610
|
|
@@ -644,11 +645,11 @@ def drop_pipe(
|
|
644
645
|
|
645
646
|
|
646
647
|
def clear_pipe(
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
648
|
+
self,
|
649
|
+
pipe: mrsm.Pipe,
|
650
|
+
debug: bool = False,
|
651
|
+
**kw
|
652
|
+
) -> SuccessTuple:
|
652
653
|
"""
|
653
654
|
Delete rows in a pipe's table.
|
654
655
|
|
@@ -666,7 +667,7 @@ def clear_pipe(
|
|
666
667
|
kw.pop('location_keys', None)
|
667
668
|
kw.pop('action', None)
|
668
669
|
kw.pop('force', None)
|
669
|
-
return self.
|
670
|
+
return self.do_action_legacy(
|
670
671
|
['clear', 'pipes'],
|
671
672
|
connector_keys = pipe.connector_keys,
|
672
673
|
metric_keys = pipe.metric_key,
|
@@ -678,10 +679,10 @@ def clear_pipe(
|
|
678
679
|
|
679
680
|
|
680
681
|
def get_pipe_columns_types(
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
682
|
+
self,
|
683
|
+
pipe: mrsm.Pipe,
|
684
|
+
debug: bool = False,
|
685
|
+
) -> Union[Dict[str, str], None]:
|
685
686
|
"""
|
686
687
|
Fetch the columns and types of the pipe's table.
|
687
688
|
|
meerschaum/connectors/parse.py
CHANGED
@@ -10,11 +10,11 @@ from __future__ import annotations
|
|
10
10
|
from meerschaum.utils.typing import Mapping, Any, SuccessTuple, Union, Optional, Dict, Tuple
|
11
11
|
|
12
12
|
def parse_connector_keys(
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
13
|
+
keys: str,
|
14
|
+
construct: bool = True,
|
15
|
+
as_tuple: bool = False,
|
16
|
+
**kw: Any
|
17
|
+
) -> (
|
18
18
|
Union[
|
19
19
|
meerschaum.connectors.Connector,
|
20
20
|
Dict[str, Any],
|
@@ -87,19 +87,17 @@ def parse_connector_keys(
|
|
87
87
|
|
88
88
|
|
89
89
|
def parse_instance_keys(
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
90
|
+
keys: Optional[str],
|
91
|
+
construct: bool = True,
|
92
|
+
as_tuple: bool = False,
|
93
|
+
**kw
|
94
|
+
):
|
95
95
|
"""
|
96
96
|
Parse the Meerschaum instance value into a Connector object.
|
97
97
|
"""
|
98
98
|
from meerschaum.utils.warnings import warn
|
99
99
|
from meerschaum.config import get_config
|
100
100
|
|
101
|
-
### TODO Check for valid types? Not sure how to do that if construct = False.
|
102
|
-
|
103
101
|
if keys is None:
|
104
102
|
keys = get_config('meerschaum', 'instance')
|
105
103
|
keys = str(keys)
|
@@ -119,10 +117,24 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
|
|
119
117
|
return parse_connector_keys(keys, **kw)
|
120
118
|
|
121
119
|
|
120
|
+
def parse_executor_keys(keys: Optional[str] = None, **kw):
|
121
|
+
"""Parse the executor keys into an APIConnector or string."""
|
122
|
+
from meerschaum.jobs import get_executor_keys_from_context
|
123
|
+
if keys is None:
|
124
|
+
keys = get_executor_keys_from_context()
|
125
|
+
|
126
|
+
if keys is None or keys == 'local':
|
127
|
+
return 'local'
|
128
|
+
|
129
|
+
keys = str(keys)
|
130
|
+
return parse_connector_keys(keys, **kw)
|
131
|
+
|
132
|
+
|
122
133
|
def is_valid_connector_keys(
|
123
|
-
|
124
|
-
|
125
|
-
"""
|
134
|
+
keys: str
|
135
|
+
) -> bool:
|
136
|
+
"""
|
137
|
+
Verify a connector_keys string references a valid connector.
|
126
138
|
"""
|
127
139
|
try:
|
128
140
|
success = parse_connector_keys(keys, construct=False) is not None
|
@@ -88,21 +88,29 @@ def bootstrap(
|
|
88
88
|
|
89
89
|
try:
|
90
90
|
if yes_no(
|
91
|
-
f"Would you like to edit the definition for {self}?",
|
91
|
+
f"Would you like to edit the definition for {self}?",
|
92
|
+
yes=yes,
|
93
|
+
noask=noask,
|
94
|
+
default='n',
|
92
95
|
):
|
93
96
|
edit_tuple = self.edit_definition(debug=debug)
|
94
97
|
if not edit_tuple[0]:
|
95
98
|
return edit_tuple
|
96
99
|
|
97
|
-
if yes_no(
|
100
|
+
if yes_no(
|
101
|
+
f"Would you like to try syncing {self} now?",
|
102
|
+
yes=yes,
|
103
|
+
noask=noask,
|
104
|
+
default='n',
|
105
|
+
):
|
98
106
|
sync_tuple = actions['sync'](
|
99
107
|
['pipes'],
|
100
|
-
connector_keys
|
101
|
-
metric_keys
|
102
|
-
location_keys
|
103
|
-
mrsm_instance
|
104
|
-
debug
|
105
|
-
shell
|
108
|
+
connector_keys=[self.connector_keys],
|
109
|
+
metric_keys=[self.metric_key],
|
110
|
+
location_keys=[self.location_key],
|
111
|
+
mrsm_instance=str(self.instance_connector),
|
112
|
+
debug=debug,
|
113
|
+
shell=shell,
|
106
114
|
)
|
107
115
|
if not sync_tuple[0]:
|
108
116
|
return sync_tuple
|
@@ -0,0 +1,69 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# vim:fenc=utf-8
|
3
|
+
|
4
|
+
"""
|
5
|
+
Define the base class for a Job executor.
|
6
|
+
"""
|
7
|
+
|
8
|
+
from __future__ import annotations
|
9
|
+
|
10
|
+
from abc import abstractmethod
|
11
|
+
|
12
|
+
from meerschaum.connectors import Connector
|
13
|
+
from meerschaum.utils.typing import List, Dict, SuccessTuple, TYPE_CHECKING
|
14
|
+
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
from meerschaum.jobs import Job
|
17
|
+
|
18
|
+
class Executor(Connector):
|
19
|
+
"""
|
20
|
+
Define the methods for managing jobs.
|
21
|
+
"""
|
22
|
+
|
23
|
+
@abstractmethod
|
24
|
+
def get_job_exists(self, name: str, debug: bool = False) -> bool:
|
25
|
+
"""
|
26
|
+
Return whether a job exists.
|
27
|
+
"""
|
28
|
+
|
29
|
+
@abstractmethod
|
30
|
+
def get_jobs(self) -> Dict[str, Job]:
|
31
|
+
"""
|
32
|
+
Return a dictionary of names -> Jobs.
|
33
|
+
"""
|
34
|
+
|
35
|
+
@abstractmethod
|
36
|
+
def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
|
37
|
+
"""
|
38
|
+
Create a new job.
|
39
|
+
"""
|
40
|
+
|
41
|
+
@abstractmethod
|
42
|
+
def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
43
|
+
"""
|
44
|
+
Start a job.
|
45
|
+
"""
|
46
|
+
|
47
|
+
@abstractmethod
|
48
|
+
def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
49
|
+
"""
|
50
|
+
Stop a job.
|
51
|
+
"""
|
52
|
+
|
53
|
+
@abstractmethod
|
54
|
+
def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
55
|
+
"""
|
56
|
+
Pause a job.
|
57
|
+
"""
|
58
|
+
|
59
|
+
@abstractmethod
|
60
|
+
def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
|
61
|
+
"""
|
62
|
+
Delete a job.
|
63
|
+
"""
|
64
|
+
|
65
|
+
@abstractmethod
|
66
|
+
def get_logs(self, name: str, debug: bool = False) -> str:
|
67
|
+
"""
|
68
|
+
Return a job's log output.
|
69
|
+
"""
|