meerschaum 2.3.0rc2__py3-none-any.whl → 2.3.0rc5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/__init__.py +1 -1
- meerschaum/_internal/arguments/_parse_arguments.py +17 -12
- meerschaum/_internal/arguments/_parser.py +3 -6
- meerschaum/_internal/entry.py +42 -4
- meerschaum/_internal/shell/Shell.py +84 -72
- meerschaum/actions/delete.py +4 -0
- meerschaum/actions/show.py +5 -5
- meerschaum/actions/start.py +71 -1
- meerschaum/api/dash/callbacks/jobs.py +36 -44
- meerschaum/api/dash/jobs.py +24 -15
- meerschaum/api/routes/_actions.py +54 -6
- meerschaum/api/routes/_jobs.py +19 -1
- meerschaum/config/_jobs.py +1 -1
- meerschaum/config/_paths.py +1 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +2 -0
- meerschaum/connectors/api/APIConnector.py +7 -1
- meerschaum/connectors/api/_actions.py +77 -1
- meerschaum/connectors/api/_jobs.py +13 -2
- meerschaum/connectors/api/_pipes.py +85 -84
- meerschaum/jobs/_Job.py +53 -12
- meerschaum/jobs/_SystemdExecutor.py +111 -30
- meerschaum/jobs/__init__.py +9 -2
- meerschaum/utils/daemon/Daemon.py +14 -0
- meerschaum/utils/daemon/StdinFile.py +1 -0
- meerschaum/utils/daemon/_names.py +15 -13
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/METADATA +1 -1
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/RECORD +34 -34
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/WHEEL +0 -0
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.0rc2.dist-info → meerschaum-2.3.0rc5.dist-info}/zip-safe +0 -0
@@ -11,14 +11,15 @@ import time
|
|
11
11
|
import json
|
12
12
|
from io import StringIO
|
13
13
|
from datetime import datetime
|
14
|
+
|
14
15
|
import meerschaum as mrsm
|
15
16
|
from meerschaum.utils.debug import dprint
|
16
17
|
from meerschaum.utils.warnings import warn, error
|
17
18
|
from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
|
18
19
|
|
19
20
|
def pipe_r_url(
|
20
|
-
|
21
|
-
|
21
|
+
pipe: mrsm.Pipe
|
22
|
+
) -> str:
|
22
23
|
"""Return a relative URL path from a Pipe's keys."""
|
23
24
|
from meerschaum.config.static import STATIC_CONFIG
|
24
25
|
location_key = pipe.location_key
|
@@ -30,10 +31,10 @@ def pipe_r_url(
|
|
30
31
|
)
|
31
32
|
|
32
33
|
def register_pipe(
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
self,
|
35
|
+
pipe: mrsm.Pipe,
|
36
|
+
debug: bool = False
|
37
|
+
) -> SuccessTuple:
|
37
38
|
"""Submit a POST to the API to register a new Pipe object.
|
38
39
|
Returns a tuple of (success_bool, response_dict).
|
39
40
|
"""
|
@@ -59,11 +60,11 @@ def register_pipe(
|
|
59
60
|
|
60
61
|
|
61
62
|
def edit_pipe(
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
63
|
+
self,
|
64
|
+
pipe: mrsm.Pipe,
|
65
|
+
patch: bool = False,
|
66
|
+
debug: bool = False,
|
67
|
+
) -> SuccessTuple:
|
67
68
|
"""Submit a PATCH to the API to edit an existing Pipe object.
|
68
69
|
Returns a tuple of (success_bool, response_dict).
|
69
70
|
"""
|
@@ -89,14 +90,14 @@ def edit_pipe(
|
|
89
90
|
|
90
91
|
|
91
92
|
def fetch_pipes_keys(
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
93
|
+
self,
|
94
|
+
connector_keys: Optional[List[str]] = None,
|
95
|
+
metric_keys: Optional[List[str]] = None,
|
96
|
+
location_keys: Optional[List[str]] = None,
|
97
|
+
tags: Optional[List[str]] = None,
|
98
|
+
params: Optional[Dict[str, Any]] = None,
|
99
|
+
debug: bool = False
|
100
|
+
) -> Union[List[Tuple[str, str, Union[str, None]]]]:
|
100
101
|
"""
|
101
102
|
Fetch registered Pipes' keys from the API.
|
102
103
|
|
@@ -158,13 +159,13 @@ def fetch_pipes_keys(
|
|
158
159
|
|
159
160
|
|
160
161
|
def sync_pipe(
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
162
|
+
self,
|
163
|
+
pipe: mrsm.Pipe,
|
164
|
+
df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
|
165
|
+
chunksize: Optional[int] = -1,
|
166
|
+
debug: bool = False,
|
167
|
+
**kw: Any
|
168
|
+
) -> SuccessTuple:
|
168
169
|
"""Sync a DataFrame into a Pipe."""
|
169
170
|
from decimal import Decimal
|
170
171
|
from meerschaum.utils.debug import dprint
|
@@ -303,10 +304,10 @@ def sync_pipe(
|
|
303
304
|
|
304
305
|
|
305
306
|
def delete_pipe(
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
307
|
+
self,
|
308
|
+
pipe: Optional[meerschaum.Pipe] = None,
|
309
|
+
debug: bool = None,
|
310
|
+
) -> SuccessTuple:
|
310
311
|
"""Delete a Pipe and drop its table."""
|
311
312
|
if pipe is None:
|
312
313
|
error(f"Pipe cannot be None.")
|
@@ -327,17 +328,17 @@ def delete_pipe(
|
|
327
328
|
|
328
329
|
|
329
330
|
def get_pipe_data(
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
331
|
+
self,
|
332
|
+
pipe: meerschaum.Pipe,
|
333
|
+
select_columns: Optional[List[str]] = None,
|
334
|
+
omit_columns: Optional[List[str]] = None,
|
335
|
+
begin: Union[str, datetime, int, None] = None,
|
336
|
+
end: Union[str, datetime, int, None] = None,
|
337
|
+
params: Optional[Dict[str, Any]] = None,
|
338
|
+
as_chunks: bool = False,
|
339
|
+
debug: bool = False,
|
340
|
+
**kw: Any
|
341
|
+
) -> Union[pandas.DataFrame, None]:
|
341
342
|
"""Fetch data from the API."""
|
342
343
|
r_url = pipe_r_url(pipe)
|
343
344
|
chunks_list = []
|
@@ -389,10 +390,10 @@ def get_pipe_data(
|
|
389
390
|
|
390
391
|
|
391
392
|
def get_pipe_id(
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
393
|
+
self,
|
394
|
+
pipe: meerschuam.Pipe,
|
395
|
+
debug: bool = False,
|
396
|
+
) -> int:
|
396
397
|
"""Get a Pipe's ID from the API."""
|
397
398
|
from meerschaum.utils.misc import is_int
|
398
399
|
r_url = pipe_r_url(pipe)
|
@@ -411,10 +412,10 @@ def get_pipe_id(
|
|
411
412
|
|
412
413
|
|
413
414
|
def get_pipe_attributes(
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
415
|
+
self,
|
416
|
+
pipe: meerschaum.Pipe,
|
417
|
+
debug: bool = False,
|
418
|
+
) -> Dict[str, Any]:
|
418
419
|
"""Get a Pipe's attributes from the API
|
419
420
|
|
420
421
|
Parameters
|
@@ -437,12 +438,12 @@ def get_pipe_attributes(
|
|
437
438
|
|
438
439
|
|
439
440
|
def get_sync_time(
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
441
|
+
self,
|
442
|
+
pipe: 'meerschaum.Pipe',
|
443
|
+
params: Optional[Dict[str, Any]] = None,
|
444
|
+
newest: bool = True,
|
445
|
+
debug: bool = False,
|
446
|
+
) -> Union[datetime, int, None]:
|
446
447
|
"""Get a Pipe's most recent datetime value from the API.
|
447
448
|
|
448
449
|
Parameters
|
@@ -492,10 +493,10 @@ def get_sync_time(
|
|
492
493
|
|
493
494
|
|
494
495
|
def pipe_exists(
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
496
|
+
self,
|
497
|
+
pipe: mrsm.Pipe,
|
498
|
+
debug: bool = False
|
499
|
+
) -> bool:
|
499
500
|
"""Check the API to see if a Pipe exists.
|
500
501
|
|
501
502
|
Parameters
|
@@ -523,9 +524,9 @@ def pipe_exists(
|
|
523
524
|
|
524
525
|
|
525
526
|
def create_metadata(
|
526
|
-
|
527
|
-
|
528
|
-
|
527
|
+
self,
|
528
|
+
debug: bool = False
|
529
|
+
) -> bool:
|
529
530
|
"""Create metadata tables.
|
530
531
|
|
531
532
|
Returns
|
@@ -547,14 +548,14 @@ def create_metadata(
|
|
547
548
|
|
548
549
|
|
549
550
|
def get_pipe_rowcount(
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
551
|
+
self,
|
552
|
+
pipe: mrsm.Pipe,
|
553
|
+
begin: Optional[datetime] = None,
|
554
|
+
end: Optional[datetime] = None,
|
555
|
+
params: Optional[Dict[str, Any]] = None,
|
556
|
+
remote: bool = False,
|
557
|
+
debug: bool = False,
|
558
|
+
) -> int:
|
558
559
|
"""Get a pipe's row count from the API.
|
559
560
|
|
560
561
|
Parameters
|
@@ -600,10 +601,10 @@ def get_pipe_rowcount(
|
|
600
601
|
|
601
602
|
|
602
603
|
def drop_pipe(
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
604
|
+
self,
|
605
|
+
pipe: mrsm.Pipe,
|
606
|
+
debug: bool = False
|
607
|
+
) -> SuccessTuple:
|
607
608
|
"""
|
608
609
|
Drop a pipe's table but maintain its registration.
|
609
610
|
|
@@ -644,11 +645,11 @@ def drop_pipe(
|
|
644
645
|
|
645
646
|
|
646
647
|
def clear_pipe(
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
648
|
+
self,
|
649
|
+
pipe: mrsm.Pipe,
|
650
|
+
debug: bool = False,
|
651
|
+
**kw
|
652
|
+
) -> SuccessTuple:
|
652
653
|
"""
|
653
654
|
Delete rows in a pipe's table.
|
654
655
|
|
@@ -666,7 +667,7 @@ def clear_pipe(
|
|
666
667
|
kw.pop('location_keys', None)
|
667
668
|
kw.pop('action', None)
|
668
669
|
kw.pop('force', None)
|
669
|
-
return self.
|
670
|
+
return self.do_action_legacy(
|
670
671
|
['clear', 'pipes'],
|
671
672
|
connector_keys = pipe.connector_keys,
|
672
673
|
metric_keys = pipe.metric_key,
|
@@ -678,10 +679,10 @@ def clear_pipe(
|
|
678
679
|
|
679
680
|
|
680
681
|
def get_pipe_columns_types(
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
682
|
+
self,
|
683
|
+
pipe: mrsm.Pipe,
|
684
|
+
debug: bool = False,
|
685
|
+
) -> Union[Dict[str, str], None]:
|
685
686
|
"""
|
686
687
|
Fetch the columns and types of the pipe's table.
|
687
688
|
|
meerschaum/jobs/_Job.py
CHANGED
@@ -62,8 +62,9 @@ class Job:
|
|
62
62
|
_properties: Optional[Dict[str, Any]] = None,
|
63
63
|
_rotating_log = None,
|
64
64
|
_stdin_file = None,
|
65
|
-
_status_hook = None,
|
66
|
-
_result_hook = None,
|
65
|
+
_status_hook: Optional[Callable[[], str]] = None,
|
66
|
+
_result_hook: Optional[Callable[[], SuccessTuple]] = None,
|
67
|
+
_externally_managed: bool = False,
|
67
68
|
):
|
68
69
|
"""
|
69
70
|
Create a new job to manage a `meerschaum.utils.daemon.Daemon`.
|
@@ -133,7 +134,10 @@ class Job:
|
|
133
134
|
if _result_hook is not None:
|
134
135
|
self._result_hook = _result_hook
|
135
136
|
|
137
|
+
self._externally_managed = _externally_managed
|
136
138
|
self._properties_patch = _properties or {}
|
139
|
+
if _externally_managed:
|
140
|
+
self._properties_patch.update({'externally_managed': _externally_managed})
|
137
141
|
|
138
142
|
daemon_sysargs = (
|
139
143
|
self._daemon.properties.get('target', {}).get('args', [None])[0]
|
@@ -154,9 +158,6 @@ class Job:
|
|
154
158
|
self._properties_patch.update({'restart': True})
|
155
159
|
break
|
156
160
|
|
157
|
-
if '--systemd' in self._sysargs:
|
158
|
-
self._properties_patch.update({'systemd': True})
|
159
|
-
|
160
161
|
@staticmethod
|
161
162
|
def from_pid(pid: int, executor_keys: Optional[str] = None) -> Job:
|
162
163
|
"""
|
@@ -740,7 +741,7 @@ class Job:
|
|
740
741
|
The datetime when the job began running.
|
741
742
|
"""
|
742
743
|
if self.executor is not None:
|
743
|
-
began_str = self.executor.get_job_began(name)
|
744
|
+
began_str = self.executor.get_job_began(self.name)
|
744
745
|
if began_str is None:
|
745
746
|
return None
|
746
747
|
return (
|
@@ -762,6 +763,8 @@ class Job:
|
|
762
763
|
"""
|
763
764
|
if self.executor is not None:
|
764
765
|
ended_str = self.executor.get_job_ended(self.name)
|
766
|
+
if ended_str is None:
|
767
|
+
return None
|
765
768
|
return (
|
766
769
|
datetime.fromisoformat(ended_str)
|
767
770
|
.astimezone(timezone.utc)
|
@@ -779,6 +782,16 @@ class Job:
|
|
779
782
|
"""
|
780
783
|
The datetime when the job was suspended while running.
|
781
784
|
"""
|
785
|
+
if self.executor is not None:
|
786
|
+
paused_str = self.executor.get_job_paused(self.name)
|
787
|
+
if paused_str is None:
|
788
|
+
return None
|
789
|
+
return (
|
790
|
+
datetime.fromisoformat(paused_str)
|
791
|
+
.astimezone(timezone.utc)
|
792
|
+
.replace(tzinfo=None)
|
793
|
+
)
|
794
|
+
|
782
795
|
paused_str = self.daemon.properties.get('process', {}).get('paused', None)
|
783
796
|
if paused_str is None:
|
784
797
|
return None
|
@@ -796,11 +809,8 @@ class Job:
|
|
796
809
|
if not self.daemon.stop_path.exists():
|
797
810
|
return None
|
798
811
|
|
799
|
-
|
800
|
-
|
801
|
-
stop_data = json.load(f)
|
802
|
-
except Exception as e:
|
803
|
-
warn(f"Failed to read stop file for {self}:\n{e}")
|
812
|
+
stop_data = self.daemon._read_stop_file()
|
813
|
+
if not stop_data:
|
804
814
|
return None
|
805
815
|
|
806
816
|
stop_time_str = stop_data.get('stop_time', None)
|
@@ -815,7 +825,11 @@ class Job:
|
|
815
825
|
"""
|
816
826
|
Return a bool indicating whether this job should be displayed.
|
817
827
|
"""
|
818
|
-
return
|
828
|
+
return (
|
829
|
+
self.name.startswith('_')
|
830
|
+
or self.name.startswith('.')
|
831
|
+
or self._is_externally_managed
|
832
|
+
)
|
819
833
|
|
820
834
|
def check_restart(self) -> SuccessTuple:
|
821
835
|
"""
|
@@ -841,6 +855,33 @@ class Job:
|
|
841
855
|
"""
|
842
856
|
return shlex.join(self.sysargs).replace(' + ', '\n+ ')
|
843
857
|
|
858
|
+
@property
|
859
|
+
def _externally_managed_file(self) -> pathlib.Path:
|
860
|
+
"""
|
861
|
+
Return the path to the externally managed file.
|
862
|
+
"""
|
863
|
+
return self.daemon.path / '.externally-managed'
|
864
|
+
|
865
|
+
def _set_externally_managed(self):
|
866
|
+
"""
|
867
|
+
Set this job as externally managed.
|
868
|
+
"""
|
869
|
+
self._externally_managed = True
|
870
|
+
try:
|
871
|
+
self._externally_managed_file.parent.mkdir(exist_ok=True, parents=True)
|
872
|
+
self._externally_managed_file.touch()
|
873
|
+
except Exception as e:
|
874
|
+
warn(e)
|
875
|
+
|
876
|
+
@property
|
877
|
+
def _is_externally_managed(self) -> bool:
|
878
|
+
"""
|
879
|
+
Return whether this job is externally managed.
|
880
|
+
"""
|
881
|
+
return self.executor_keys in (None, 'local') and (
|
882
|
+
self._externally_managed or self._externally_managed_file.exists()
|
883
|
+
)
|
884
|
+
|
844
885
|
def __str__(self) -> str:
|
845
886
|
sysargs = self.sysargs
|
846
887
|
sysargs_str = shlex.join(sysargs) if sysargs else ''
|
@@ -13,12 +13,14 @@ import asyncio
|
|
13
13
|
import json
|
14
14
|
import time
|
15
15
|
import traceback
|
16
|
+
import shutil
|
16
17
|
from datetime import datetime, timezone
|
17
18
|
from functools import partial
|
18
19
|
|
19
20
|
import meerschaum as mrsm
|
20
21
|
from meerschaum.jobs import Job, Executor, make_executor
|
21
22
|
from meerschaum.utils.typing import Dict, Any, List, SuccessTuple, Union, Optional, Callable
|
23
|
+
from meerschaum.config import get_config
|
22
24
|
from meerschaum.config.static import STATIC_CONFIG
|
23
25
|
from meerschaum.utils.warnings import warn, dprint
|
24
26
|
from meerschaum._internal.arguments._parse_arguments import parse_arguments
|
@@ -36,10 +38,10 @@ class SystemdExecutor(Executor):
|
|
36
38
|
"""
|
37
39
|
Return a list of existing jobs, including hidden ones.
|
38
40
|
"""
|
39
|
-
from meerschaum.config.paths import
|
41
|
+
from meerschaum.config.paths import SYSTEMD_USER_RESOURCES_PATH
|
40
42
|
return [
|
41
43
|
service_name[len('mrsm-'):(-1 * len('.service'))]
|
42
|
-
for service_name in os.listdir(
|
44
|
+
for service_name in os.listdir(SYSTEMD_USER_RESOURCES_PATH)
|
43
45
|
if service_name.startswith('mrsm-') and service_name.endswith('.service')
|
44
46
|
]
|
45
47
|
|
@@ -72,6 +74,13 @@ class SystemdExecutor(Executor):
|
|
72
74
|
"""
|
73
75
|
return f"mrsm-{name.replace(' ', '-')}.service"
|
74
76
|
|
77
|
+
def get_service_job_path(self, name: str, debug: bool = False) -> pathlib.Path:
|
78
|
+
"""
|
79
|
+
Return the path for the job's files under the root directory.
|
80
|
+
"""
|
81
|
+
from meerschaum.config.paths import SYSTEMD_JOBS_RESOURCES_PATH
|
82
|
+
return SYSTEMD_JOBS_RESOURCES_PATH / name
|
83
|
+
|
75
84
|
def get_service_symlink_file_path(self, name: str, debug: bool = False) -> pathlib.Path:
|
76
85
|
"""
|
77
86
|
Return the path to where to create the service symlink.
|
@@ -83,8 +92,10 @@ class SystemdExecutor(Executor):
|
|
83
92
|
"""
|
84
93
|
Return the path to a Job's service file.
|
85
94
|
"""
|
86
|
-
|
87
|
-
|
95
|
+
return (
|
96
|
+
self.get_service_job_path(name, debug=debug)
|
97
|
+
/ self.get_service_name(name, debug=debug)
|
98
|
+
)
|
88
99
|
|
89
100
|
def get_service_logs_path(self, name: str, debug: bool = False) -> pathlib.Path:
|
90
101
|
"""
|
@@ -93,29 +104,22 @@ class SystemdExecutor(Executor):
|
|
93
104
|
from meerschaum.config.paths import SYSTEMD_LOGS_RESOURCES_PATH
|
94
105
|
return SYSTEMD_LOGS_RESOURCES_PATH / (self.get_service_name(name, debug=debug) + '.log')
|
95
106
|
|
96
|
-
def get_service_socket_path(self, name: str, debug: bool = False) -> pathlib.Path:
|
97
|
-
"""
|
98
|
-
Return the path to the unit file for the socket (not the socket itself).
|
99
|
-
"""
|
100
|
-
from meerschaum.config.paths import SYSTEMD_USER_RESOURCES_PATH
|
101
|
-
return SYSTEMD_USER_RESOURCES_PATH / (
|
102
|
-
self.get_service_name(name, debug=debug).replace('.service', '.socket')
|
103
|
-
)
|
104
|
-
|
105
107
|
def get_socket_path(self, name: str, debug: bool = False) -> pathlib.Path:
|
106
108
|
"""
|
107
109
|
Return the path to the FIFO file.
|
108
110
|
"""
|
109
|
-
|
110
|
-
|
111
|
+
return (
|
112
|
+
self.get_service_job_path(name, debug=debug)
|
113
|
+
/ (self.get_service_name(name, debug=debug) + '.stdin')
|
114
|
+
)
|
111
115
|
|
112
116
|
def get_result_path(self, name: str, debug: bool = False) -> pathlib.Path:
|
113
117
|
"""
|
114
118
|
Return the path to the result file.
|
115
119
|
"""
|
116
|
-
|
117
|
-
|
118
|
-
self.get_service_name(name, debug=debug) + '.result.json'
|
120
|
+
return (
|
121
|
+
self.get_service_job_path(name, debug=debug)
|
122
|
+
/ (self.get_service_name(name, debug=debug) + '.result.json')
|
119
123
|
)
|
120
124
|
|
121
125
|
def get_service_file_text(self, name: str, sysargs: List[str], debug: bool = False) -> str:
|
@@ -141,7 +145,8 @@ class SystemdExecutor(Executor):
|
|
141
145
|
STATIC_CONFIG['environment']['systemd_log_path']: service_logs_path.as_posix(),
|
142
146
|
STATIC_CONFIG['environment']['systemd_result_path']: result_path.as_posix(),
|
143
147
|
STATIC_CONFIG['environment']['systemd_stdin_path']: socket_path.as_posix(),
|
144
|
-
|
148
|
+
'LINES': get_config('jobs', 'terminal', 'lines'),
|
149
|
+
'COLUMNS': get_config('jobs', 'terminal', 'columns'),
|
145
150
|
})
|
146
151
|
environment_lines = [
|
147
152
|
f"Environment={key}={val}"
|
@@ -189,17 +194,19 @@ class SystemdExecutor(Executor):
|
|
189
194
|
"""
|
190
195
|
Return the hidden "sister" job to store a job's parameters.
|
191
196
|
"""
|
192
|
-
|
193
|
-
|
194
|
-
return Job(
|
195
|
-
hidden_name,
|
197
|
+
job = Job(
|
198
|
+
name,
|
196
199
|
sysargs,
|
197
200
|
executor_keys='local',
|
198
201
|
_rotating_log=self.get_job_rotating_file(name, debug=debug),
|
199
202
|
_stdin_file=self.get_job_stdin_file(name, debug=debug),
|
200
203
|
_status_hook=partial(self.get_job_status, name),
|
201
204
|
_result_hook=partial(self.get_job_result, name),
|
205
|
+
_externally_managed=True,
|
202
206
|
)
|
207
|
+
job._set_externally_managed()
|
208
|
+
return job
|
209
|
+
|
203
210
|
|
204
211
|
def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
|
205
212
|
"""
|
@@ -259,7 +266,10 @@ class SystemdExecutor(Executor):
|
|
259
266
|
return None
|
260
267
|
|
261
268
|
psutil = mrsm.attempt_import('psutil')
|
262
|
-
|
269
|
+
try:
|
270
|
+
return psutil.Process(pid)
|
271
|
+
except Exception:
|
272
|
+
return None
|
263
273
|
|
264
274
|
def get_job_status(self, name: str, debug: bool = False) -> str:
|
265
275
|
"""
|
@@ -271,13 +281,19 @@ class SystemdExecutor(Executor):
|
|
271
281
|
debug=debug,
|
272
282
|
)
|
273
283
|
|
284
|
+
if output == 'activating':
|
285
|
+
return 'running'
|
286
|
+
|
274
287
|
if output == 'active':
|
275
288
|
process = self.get_job_process(name, debug=debug)
|
276
289
|
if process is None:
|
277
290
|
return 'stopped'
|
278
291
|
|
279
|
-
|
280
|
-
|
292
|
+
try:
|
293
|
+
if process.status() == 'stopped':
|
294
|
+
return 'paused'
|
295
|
+
except Exception:
|
296
|
+
return 'stopped'
|
281
297
|
|
282
298
|
return 'running'
|
283
299
|
|
@@ -310,7 +326,7 @@ class SystemdExecutor(Executor):
|
|
310
326
|
|
311
327
|
return None
|
312
328
|
|
313
|
-
def get_job_began(self, name: str, debug: bool = False) -> Union[
|
329
|
+
def get_job_began(self, name: str, debug: bool = False) -> Union[str, None]:
|
314
330
|
"""
|
315
331
|
Return when a job began running.
|
316
332
|
"""
|
@@ -326,10 +342,62 @@ class SystemdExecutor(Executor):
|
|
326
342
|
if not output.startswith('ActiveEnterTimestamp'):
|
327
343
|
return None
|
328
344
|
|
345
|
+
dt_str = output.split('=')[-1]
|
346
|
+
if not dt_str:
|
347
|
+
return None
|
348
|
+
|
329
349
|
dateutil_parser = mrsm.attempt_import('dateutil.parser')
|
330
|
-
|
350
|
+
try:
|
351
|
+
dt = dateutil_parser.parse(dt_str)
|
352
|
+
except Exception as e:
|
353
|
+
warn(f"Cannot parse '{output}' as a datetime:\n{e}")
|
354
|
+
return None
|
355
|
+
|
331
356
|
return dt.astimezone(timezone.utc).isoformat()
|
332
357
|
|
358
|
+
def get_job_ended(self, name: str, debug: bool = False) -> Union[str, None]:
|
359
|
+
"""
|
360
|
+
Return when a job began running.
|
361
|
+
"""
|
362
|
+
output = self.run_command(
|
363
|
+
[
|
364
|
+
'show',
|
365
|
+
self.get_service_name(name, debug=debug),
|
366
|
+
'--property=InactiveEnterTimestamp'
|
367
|
+
],
|
368
|
+
as_output=True,
|
369
|
+
debug=debug,
|
370
|
+
)
|
371
|
+
if not output.startswith('InactiveEnterTimestamp'):
|
372
|
+
return None
|
373
|
+
|
374
|
+
dt_str = output.split('=')[-1]
|
375
|
+
if not dt_str:
|
376
|
+
return None
|
377
|
+
|
378
|
+
dateutil_parser = mrsm.attempt_import('dateutil.parser')
|
379
|
+
|
380
|
+
try:
|
381
|
+
dt = dateutil_parser.parse(dt_str)
|
382
|
+
except Exception as e:
|
383
|
+
warn(f"Cannot parse '{output}' as a datetime:\n{e}")
|
384
|
+
return None
|
385
|
+
return dt.astimezone(timezone.utc).isoformat()
|
386
|
+
|
387
|
+
def get_job_paused(self, name: str, debug: bool = False) -> Union[str, None]:
|
388
|
+
"""
|
389
|
+
Return when a job was paused.
|
390
|
+
"""
|
391
|
+
job = self.get_hidden_job(name, debug=debug)
|
392
|
+
if self.get_job_status(name, debug=debug) != 'paused':
|
393
|
+
return None
|
394
|
+
|
395
|
+
stop_time = job.stop_time
|
396
|
+
if stop_time is None:
|
397
|
+
return None
|
398
|
+
|
399
|
+
return stop_time.isoformat()
|
400
|
+
|
333
401
|
def get_job_result(self, name: str, debug: bool = False) -> SuccessTuple:
|
334
402
|
"""
|
335
403
|
Return the job's result SuccessTuple.
|
@@ -426,6 +494,7 @@ class SystemdExecutor(Executor):
|
|
426
494
|
|
427
495
|
if name not in self._stdin_files:
|
428
496
|
socket_path = self.get_socket_path(name, debug=debug)
|
497
|
+
socket_path.parent.mkdir(parents=True, exist_ok=True)
|
429
498
|
self._stdin_files[name] = StdinFile(socket_path)
|
430
499
|
|
431
500
|
return self._stdin_files[name]
|
@@ -441,6 +510,9 @@ class SystemdExecutor(Executor):
|
|
441
510
|
socket_stdin = self.get_job_stdin_file(name, debug=debug)
|
442
511
|
_ = socket_stdin.file_handler
|
443
512
|
|
513
|
+
### Init the `externally_managed file`.
|
514
|
+
_ = self.get_hidden_job(name, debug=debug)
|
515
|
+
|
444
516
|
with open(service_file_path, 'w+', encoding='utf-8') as f:
|
445
517
|
f.write(self.get_service_file_text(name, sysargs, debug=debug))
|
446
518
|
|
@@ -495,12 +567,13 @@ class SystemdExecutor(Executor):
|
|
495
567
|
debug=debug,
|
496
568
|
)
|
497
569
|
|
570
|
+
check_timeout_interval = get_config('jobs', 'check_timeout_interval_seconds')
|
498
571
|
loop_start = time.perf_counter()
|
499
|
-
while (time.perf_counter() - loop_start) <
|
572
|
+
while (time.perf_counter() - loop_start) < get_config('jobs', 'timeout_seconds'):
|
500
573
|
if self.get_job_status(name, debug=debug) == 'stopped':
|
501
574
|
return True, 'Success'
|
502
575
|
|
503
|
-
time.sleep(
|
576
|
+
time.sleep(check_timeout_interval)
|
504
577
|
|
505
578
|
return self.run_command(
|
506
579
|
['stop', self.get_service_name(name, debug=debug)],
|
@@ -530,6 +603,14 @@ class SystemdExecutor(Executor):
|
|
530
603
|
debug=debug,
|
531
604
|
)
|
532
605
|
|
606
|
+
service_job_path = self.get_service_job_path(name, debug=debug)
|
607
|
+
try:
|
608
|
+
if service_job_path.exists():
|
609
|
+
shutil.rmtree(service_job_path)
|
610
|
+
except Exception as e:
|
611
|
+
warn(e)
|
612
|
+
return False, str(e)
|
613
|
+
|
533
614
|
service_logs_path = self.get_service_logs_path(name, debug=debug)
|
534
615
|
logs_paths = [
|
535
616
|
(SYSTEMD_LOGS_RESOURCES_PATH / name)
|