libpipe 0.6__tar.gz → 0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {libpipe-0.6 → libpipe-0.7}/PKG-INFO +9 -4
- {libpipe-0.6 → libpipe-0.7}/libpipe/attrmap.py +9 -16
- {libpipe-0.6 → libpipe-0.7}/libpipe/futils.py +11 -9
- {libpipe-0.6 → libpipe-0.7}/libpipe/settings.py +4 -6
- {libpipe-0.6 → libpipe-0.7}/libpipe/tools/nodetool.py +14 -10
- {libpipe-0.6 → libpipe-0.7}/libpipe/worker.py +69 -61
- libpipe-0.7/pyproject.toml +61 -0
- libpipe-0.6/pyproject.toml +0 -34
- {libpipe-0.6 → libpipe-0.7}/LICENSE +0 -0
- {libpipe-0.6 → libpipe-0.7}/README.md +0 -0
- {libpipe-0.6 → libpipe-0.7}/libpipe/__init__.py +0 -0
- {libpipe-0.6 → libpipe-0.7}/libpipe/msutils.py +0 -0
- {libpipe-0.6 → libpipe-0.7}/libpipe/tools/__init__.py +0 -0
- {libpipe-0.6 → libpipe-0.7}/libpipe/tools/libnodes_bash.sh +0 -0
|
@@ -1,23 +1,28 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: libpipe
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7
|
|
4
4
|
Summary: Pipeline orchestration and node management utilities
|
|
5
|
-
License: GPL-3.0-or-later
|
|
5
|
+
License-Expression: GPL-3.0-or-later
|
|
6
6
|
License-File: LICENSE
|
|
7
7
|
Author: Florent Mertens
|
|
8
8
|
Author-email: florent.mertens@gmail.com
|
|
9
|
-
Requires-Python: >=3.10
|
|
10
|
-
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
9
|
+
Requires-Python: >=3.10,<4.0
|
|
11
10
|
Classifier: Programming Language :: Python :: 3
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Provides-Extra: dev
|
|
17
17
|
Requires-Dist: asyncssh (>=2.0)
|
|
18
18
|
Requires-Dist: click (>=8.0)
|
|
19
19
|
Requires-Dist: progressbar2 (>=3.40)
|
|
20
|
+
Requires-Dist: pytest (>=8.0) ; extra == "dev"
|
|
21
|
+
Requires-Dist: pytest-cov (>=5.0) ; extra == "dev"
|
|
22
|
+
Requires-Dist: ruff (==0.16.1) ; extra == "dev"
|
|
20
23
|
Requires-Dist: toml (>=0.10)
|
|
24
|
+
Project-URL: Homepage, https://gitlab.com/flomertens/libpipe
|
|
25
|
+
Project-URL: Issues, https://gitlab.com/flomertens/libpipe/-/issues
|
|
21
26
|
Project-URL: Repository, https://gitlab.com/flomertens/libpipe
|
|
22
27
|
Description-Content-Type: text/markdown
|
|
23
28
|
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
# Some parts inspired by the AttrDict package written by Brendan Curran-Johnson (MIT Licence)
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import collections
|
|
4
4
|
import copy
|
|
5
5
|
import functools
|
|
6
|
-
|
|
6
|
+
import re
|
|
7
7
|
from collections.abc import Mapping
|
|
8
|
-
import collections
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
def merge(left, right):
|
|
@@ -130,7 +129,7 @@ class AttrMap(Mapping):
|
|
|
130
129
|
# sequence type seems like more trouble than it is worth.
|
|
131
130
|
# If people want full serialization, they can pickle, and in
|
|
132
131
|
# 99% of cases, sequence_type won't change anyway
|
|
133
|
-
return "AttrMap({
|
|
132
|
+
return f"AttrMap({self._mapping!r})"
|
|
134
133
|
|
|
135
134
|
@classmethod
|
|
136
135
|
def _valid_name(cls, key):
|
|
@@ -158,9 +157,7 @@ class AttrMap(Mapping):
|
|
|
158
157
|
|
|
159
158
|
if key not in self or not self._valid_name(key):
|
|
160
159
|
raise AttributeError(
|
|
161
|
-
"'{
|
|
162
|
-
cls=self.__class__.__name__, name=key
|
|
163
|
-
)
|
|
160
|
+
f"'{self.__class__.__name__}' instance has no attribute '{key}'"
|
|
164
161
|
)
|
|
165
162
|
|
|
166
163
|
return self._build(self[key])
|
|
@@ -251,9 +248,7 @@ class AttrMap(Mapping):
|
|
|
251
248
|
Mapping.__setattr__(self, key, value)
|
|
252
249
|
else:
|
|
253
250
|
raise TypeError(
|
|
254
|
-
"'{
|
|
255
|
-
cls=self.__class__.__name__
|
|
256
|
-
)
|
|
251
|
+
f"'{self.__class__.__name__}' does not allow attribute creation."
|
|
257
252
|
)
|
|
258
253
|
|
|
259
254
|
def __delattr__(self, key, force=False): # pragma: no cover
|
|
@@ -267,9 +262,7 @@ class AttrMap(Mapping):
|
|
|
267
262
|
Mapping.__delattr__(self, key, force=force)
|
|
268
263
|
else:
|
|
269
264
|
raise TypeError(
|
|
270
|
-
"'{
|
|
271
|
-
cls=self.__class__.__name__
|
|
272
|
-
)
|
|
265
|
+
f"'{self.__class__.__name__}' does not allow attribute deletion."
|
|
273
266
|
)
|
|
274
267
|
|
|
275
268
|
def set_data(self, mapping):
|
|
@@ -301,7 +294,7 @@ class AttrMap(Mapping):
|
|
|
301
294
|
|
|
302
295
|
def get(self, key, default=None):
|
|
303
296
|
try:
|
|
304
|
-
return functools.reduce(lambda x, y: x[y], [self
|
|
297
|
+
return functools.reduce(lambda x, y: x[y], [self, *key.split('.')])
|
|
305
298
|
except KeyError:
|
|
306
299
|
return None
|
|
307
300
|
|
|
@@ -356,8 +349,8 @@ class OrderedSet(collections.abc.MutableSet):
|
|
|
356
349
|
|
|
357
350
|
def __repr__(self):
|
|
358
351
|
if not self:
|
|
359
|
-
return '
|
|
360
|
-
return '
|
|
352
|
+
return f'{self.__class__.__name__}()'
|
|
353
|
+
return f'{self.__class__.__name__}({list(self)!r})'
|
|
361
354
|
|
|
362
355
|
def __eq__(self, other):
|
|
363
356
|
if isinstance(other, OrderedSet):
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
#
|
|
3
3
|
# Author: F. Mertens
|
|
4
4
|
|
|
5
|
-
import re
|
|
6
5
|
import os
|
|
7
|
-
import
|
|
6
|
+
import re
|
|
8
7
|
import shutil
|
|
8
|
+
import time
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def file_not_found_retry(fct, n_try, *args, **kargs):
|
|
@@ -28,7 +28,7 @@ def reglob(path, exp, invert=False):
|
|
|
28
28
|
else:
|
|
29
29
|
res = [f for f in os.listdir(path) if not m.search(f)]
|
|
30
30
|
|
|
31
|
-
return
|
|
31
|
+
return (f"{path}/{x}" for x in res)
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
def rm_if_exist(directory, verbose=False):
|
|
@@ -54,21 +54,23 @@ def is_file(f):
|
|
|
54
54
|
def zip_copy(dir_ins, dir_outs, filename, filename_out=None, verbose=True, n_retry=1):
|
|
55
55
|
if not filename_out:
|
|
56
56
|
filename_out = filename
|
|
57
|
-
for dir_in, dir_out in zip(dir_ins, dir_outs):
|
|
57
|
+
for dir_in, dir_out in zip(dir_ins, dir_outs, strict=False):
|
|
58
58
|
if verbose:
|
|
59
59
|
print(f'Copy {dir_in}/{filename} -> {dir_out}/{filename_out}')
|
|
60
|
-
file_not_found_retry(shutil.copyfile, n_retry, f'{dir_in}/{filename}',
|
|
60
|
+
file_not_found_retry(shutil.copyfile, n_retry, f'{dir_in}/{filename}',
|
|
61
|
+
f'{dir_out}/{filename_out}')
|
|
61
62
|
|
|
62
63
|
|
|
63
64
|
def zip_copytree(dir_ins, dir_outs, dirname, verbose=True, n_retry=1):
|
|
64
|
-
for dir_in, dir_out in zip(dir_ins, dir_outs):
|
|
65
|
+
for dir_in, dir_out in zip(dir_ins, dir_outs, strict=False):
|
|
65
66
|
if verbose:
|
|
66
67
|
print(f'Copy {dir_in}/{dirname} -> {dir_out}/{dirname}')
|
|
67
|
-
file_not_found_retry(shutil.copytree, n_retry, f'{dir_in}/{dirname}',
|
|
68
|
+
file_not_found_retry(shutil.copytree, n_retry, f'{dir_in}/{dirname}',
|
|
69
|
+
f'{dir_out}/{dirname}')
|
|
68
70
|
|
|
69
71
|
|
|
70
72
|
def zip_rename(dir_ins, dir_outs, filename_or_dirname=None, verbose=True, n_retry=1):
|
|
71
|
-
for f_in, f_out in zip(dir_ins, dir_outs):
|
|
73
|
+
for f_in, f_out in zip(dir_ins, dir_outs, strict=False):
|
|
72
74
|
if filename_or_dirname is not None:
|
|
73
75
|
f_in = f'{f_in}/{filename_or_dirname}'
|
|
74
76
|
f_out = f'{f_out}/{filename_or_dirname}'
|
|
@@ -78,7 +80,7 @@ def zip_rename(dir_ins, dir_outs, filename_or_dirname=None, verbose=True, n_retr
|
|
|
78
80
|
|
|
79
81
|
|
|
80
82
|
def zip_rename_reg(dir_ins, dir_outs, regex, verbose=True, invert=False, n_retry=1):
|
|
81
|
-
for f_in, f_out in zip(dir_ins, dir_outs):
|
|
83
|
+
for f_in, f_out in zip(dir_ins, dir_outs, strict=False):
|
|
82
84
|
for f_in2 in reglob(f_in, regex, invert=invert):
|
|
83
85
|
f_out2 = f'{f_out}/{os.path.basename(f_in2)}'
|
|
84
86
|
if verbose:
|
|
@@ -71,11 +71,8 @@ class BaseSettings(attrmap.AttrMap):
|
|
|
71
71
|
else:
|
|
72
72
|
for k, v in c_file.items():
|
|
73
73
|
if isinstance(v, dict):
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
else:
|
|
77
|
-
t = k
|
|
78
|
-
if t not in c_default.keys():
|
|
74
|
+
t = v.get('type', k)
|
|
75
|
+
if t not in c_default:
|
|
79
76
|
print(f'Warning: no default for section {k} of type {t}')
|
|
80
77
|
continue
|
|
81
78
|
t_default = c_default[t]
|
|
@@ -120,7 +117,8 @@ class BaseSettings(attrmap.AttrMap):
|
|
|
120
117
|
with open(file, mode='w') as f:
|
|
121
118
|
toml.dump(dict(m), f)
|
|
122
119
|
|
|
123
|
-
def duplicate(self, out_file, copy_parset_files=False, strip_parents_keys=False,
|
|
120
|
+
def duplicate(self, out_file, copy_parset_files=False, strip_parents_keys=False,
|
|
121
|
+
**config_modifiers):
|
|
124
122
|
s_out = self.copy()
|
|
125
123
|
s_out.set_file(out_file)
|
|
126
124
|
s_out['default_settings'] = ''
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
|
|
3
|
-
import os
|
|
4
3
|
import json
|
|
4
|
+
import os
|
|
5
5
|
|
|
6
6
|
import click
|
|
7
7
|
|
|
8
|
-
from libpipe import worker
|
|
9
|
-
from libpipe import __version__
|
|
10
|
-
|
|
8
|
+
from libpipe import __version__, worker
|
|
11
9
|
|
|
12
10
|
t_file = click.Path(exists=True, dir_okay=False)
|
|
13
11
|
|
|
@@ -21,7 +19,8 @@ def main():
|
|
|
21
19
|
@main.command('run_all')
|
|
22
20
|
@click.argument('cmd_list', type=t_file)
|
|
23
21
|
@click.option('--nodes_string', '-n', help='Range of nodes to use', type=str, default='localhost')
|
|
24
|
-
@click.option('--max_concurrent', '-m',
|
|
22
|
+
@click.option('--max_concurrent', '-m', type=int, default=5,
|
|
23
|
+
help='Maximum concurrent tasks on a node')
|
|
25
24
|
@click.option('--env_file', '-e', help='Environment source file', type=t_file)
|
|
26
25
|
@click.option('--dry_run', '-d', help='Dry run, only print commands', is_flag=True)
|
|
27
26
|
@click.option('--log_file', help='Log output to log_file', type=str, default=None)
|
|
@@ -31,7 +30,10 @@ def run_all(cmd_list, nodes_string, max_concurrent, env_file, dry_run, log_file)
|
|
|
31
30
|
env_file=env_file, dry_run=dry_run, debug=dry_run)
|
|
32
31
|
|
|
33
32
|
i_log_file = log_file
|
|
34
|
-
|
|
33
|
+
with open(cmd_list) as f:
|
|
34
|
+
cmds = f.readlines()
|
|
35
|
+
|
|
36
|
+
for i, cmd in enumerate(cmds):
|
|
35
37
|
if log_file is not None:
|
|
36
38
|
i_log_file = log_file + f'.{i + 1}'
|
|
37
39
|
pool.add(cmd.strip(), output_file=i_log_file)
|
|
@@ -64,7 +66,7 @@ def run_on_all_nodes(cmd, nodes_string, env_file, dry_run):
|
|
|
64
66
|
@click.option('--log_file', '-l', help='Log output to log_file', type=str, default=None)
|
|
65
67
|
def init_job(name, cmd, target_node, log_file):
|
|
66
68
|
''' Add a job to a job list file NAME. If job file does not exist, create one. '''
|
|
67
|
-
if os.path.exists(name):
|
|
69
|
+
if os.path.exists(name):
|
|
68
70
|
with open(name) as json_file:
|
|
69
71
|
jobs = json.load(json_file)
|
|
70
72
|
else:
|
|
@@ -78,9 +80,11 @@ def init_job(name, cmd, target_node, log_file):
|
|
|
78
80
|
|
|
79
81
|
@main.command('run_joblist')
|
|
80
82
|
@click.argument('name', type=t_file)
|
|
81
|
-
@click.option('--nodes_string', '-n',
|
|
83
|
+
@click.option('--nodes_string', '-n',
|
|
84
|
+
help='Range of nodes to use. By default use the target nodes in the joblist',
|
|
82
85
|
type=str, default='joblist')
|
|
83
|
-
@click.option('--max_concurrent', '-m',
|
|
86
|
+
@click.option('--max_concurrent', '-m', type=int, default=5,
|
|
87
|
+
help='Maximum concurrent tasks on a node')
|
|
84
88
|
@click.option('--env_file', '-e', help='Environment source file', type=t_file)
|
|
85
89
|
@click.option('--dry_run', '-d', help='Dry run, only print commands', is_flag=True)
|
|
86
90
|
@click.option('--worker_name', '-w', help='Name of the worker', type=str, default='Run joblist')
|
|
@@ -90,7 +94,7 @@ def run_joblist(name, nodes_string, max_concurrent, env_file, dry_run, worker_na
|
|
|
90
94
|
jobs = json.load(json_file)
|
|
91
95
|
|
|
92
96
|
if nodes_string == 'joblist':
|
|
93
|
-
all_nodes =
|
|
97
|
+
all_nodes = {job['target_node'] for job in jobs['jobs'] if job['target_node'] is not None}
|
|
94
98
|
if len(all_nodes) == 0:
|
|
95
99
|
all_nodes = ['localhost']
|
|
96
100
|
else:
|
|
@@ -2,24 +2,24 @@
|
|
|
2
2
|
#
|
|
3
3
|
# Authors: F.Mertens
|
|
4
4
|
|
|
5
|
+
import asyncio
|
|
6
|
+
import collections
|
|
7
|
+
import contextlib
|
|
8
|
+
import datetime
|
|
9
|
+
import getpass
|
|
10
|
+
import itertools
|
|
5
11
|
import os
|
|
6
12
|
import re
|
|
7
|
-
import sys
|
|
8
|
-
import time
|
|
9
13
|
import shlex
|
|
10
|
-
import socket
|
|
11
14
|
import signal
|
|
12
|
-
import
|
|
13
|
-
import
|
|
14
|
-
import
|
|
15
|
-
import itertools
|
|
16
|
-
import collections
|
|
15
|
+
import socket
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
17
18
|
from enum import Enum
|
|
18
19
|
|
|
19
20
|
import asyncssh
|
|
20
21
|
import progressbar
|
|
21
|
-
from click import
|
|
22
|
-
|
|
22
|
+
from click import secho, style
|
|
23
23
|
|
|
24
24
|
localhost_shortname = socket.gethostname().split('.', 1)[0]
|
|
25
25
|
|
|
@@ -48,7 +48,7 @@ def expend_num_ranges(s):
|
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
def get_hosts(host_string):
|
|
51
|
-
hosts = list(
|
|
51
|
+
hosts = list({host for k in host_string.split(',') for host in expend_num_ranges(k)})
|
|
52
52
|
return [localhost_shortname if h == 'localhost' else h for h in hosts]
|
|
53
53
|
|
|
54
54
|
|
|
@@ -74,14 +74,14 @@ class WorkerStatus(Enum):
|
|
|
74
74
|
CLOSED = 4
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
class Process
|
|
77
|
+
class Process:
|
|
78
78
|
|
|
79
79
|
def __init__(self, client, process):
|
|
80
80
|
self.client = client
|
|
81
81
|
self.process = process
|
|
82
82
|
|
|
83
83
|
|
|
84
|
-
class Task
|
|
84
|
+
class Task:
|
|
85
85
|
"""Represent a task executed by the worker pool
|
|
86
86
|
|
|
87
87
|
Attributes:
|
|
@@ -115,46 +115,50 @@ class Task(object):
|
|
|
115
115
|
try:
|
|
116
116
|
self._done_callback()
|
|
117
117
|
except Exception as exc:
|
|
118
|
-
err_msg = f'Error executing return callback of task {self.name}: {
|
|
118
|
+
err_msg = f'Error executing return callback of task {self.name}: {exc!s}\n'
|
|
119
119
|
if self._task_done_callback is not None:
|
|
120
120
|
try:
|
|
121
121
|
self._task_done_callback(self)
|
|
122
122
|
except Exception as exc:
|
|
123
|
-
err_msg = f'Error executing return callback of task {self.name}: {
|
|
123
|
+
err_msg = f'Error executing return callback of task {self.name}: {exc!s}\n'
|
|
124
124
|
|
|
125
125
|
return err_msg
|
|
126
126
|
|
|
127
127
|
def init_log(self):
|
|
128
128
|
if self.output_file is not None:
|
|
129
|
-
self.fd = open(self.output_file, 'w')
|
|
129
|
+
self.fd = open(self.output_file, 'w') # noqa: SIM115 - closed in close_log()
|
|
130
130
|
self.fd.write(f'# Logging starting at {datetime.datetime.now()}\n')
|
|
131
131
|
self.fd.write(f'# Input command: {self.command}\n')
|
|
132
132
|
|
|
133
133
|
def set_process(self, process):
|
|
134
134
|
self.process = process
|
|
135
135
|
if hasattr(self.process.process, 'pid'):
|
|
136
|
-
self.log(f'Starting on host {self.process.client.host} with PID
|
|
137
|
-
self.process.client.host)
|
|
136
|
+
self.log(f'Starting on host {self.process.client.host} with PID '
|
|
137
|
+
f'{self.process.process.pid}\n\n', self.process.client.host)
|
|
138
138
|
if self.pid_filename is not None:
|
|
139
139
|
try:
|
|
140
140
|
with open(self.pid_filename, 'w') as fd:
|
|
141
141
|
fd.write(f'{self.process.client.host} {self.process.process.pid}')
|
|
142
|
-
self.log(f'Starting on host {self.process.client.host} with PID
|
|
143
|
-
self.process.client.host)
|
|
142
|
+
self.log(f'Starting on host {self.process.client.host} with PID '
|
|
143
|
+
f'{self.process.process.pid}\n\n', self.process.client.host)
|
|
144
144
|
except Exception:
|
|
145
|
-
self.log(f'Error creating PID file {self.pid_filename}',
|
|
145
|
+
self.log(f'Error creating PID file {self.pid_filename}',
|
|
146
|
+
self.process.client.host, err=True)
|
|
146
147
|
|
|
147
148
|
async def terminate(self):
|
|
148
149
|
if self.process is not None and hasattr(self.process.process, 'pid'):
|
|
149
150
|
if self.process.client.host == localhost_shortname:
|
|
150
|
-
self.log(f'Killing local process PID {self.process.process.pid}\n',
|
|
151
|
+
self.log(f'Killing local process PID {self.process.process.pid}\n',
|
|
152
|
+
'local', err=True)
|
|
151
153
|
try:
|
|
152
154
|
os.killpg(os.getpgid(self.process.process.pid), signal.SIGTERM)
|
|
153
155
|
except ProcessLookupError:
|
|
154
156
|
print(f'Process {self.process.process.pid} not found')
|
|
155
157
|
else:
|
|
156
|
-
self.log(f'Killing remote process PID {self.process.process.pid}\n',
|
|
157
|
-
|
|
158
|
+
self.log(f'Killing remote process PID {self.process.process.pid}\n',
|
|
159
|
+
self.process.client.host, err=True)
|
|
160
|
+
kill_remote(self.process.client.host, self.process.client.user,
|
|
161
|
+
self.process.process.pid)
|
|
158
162
|
|
|
159
163
|
def log(self, line, host, err=False):
|
|
160
164
|
if isinstance(line, bytes):
|
|
@@ -178,12 +182,13 @@ class Task(object):
|
|
|
178
182
|
self.log(line, host, err=err)
|
|
179
183
|
|
|
180
184
|
def close_log(self):
|
|
181
|
-
if self.pid_filename and os.path.exists(self.pid_filename)
|
|
182
|
-
|
|
183
|
-
|
|
185
|
+
if (self.pid_filename and os.path.exists(self.pid_filename)
|
|
186
|
+
and (not self.keep_pid_file_on_error or self.returncode == 0)):
|
|
187
|
+
os.remove(self.pid_filename)
|
|
184
188
|
if self.fd is not None:
|
|
185
189
|
try:
|
|
186
|
-
self.fd.write(f'\n\n# Logging stopped at {datetime.datetime.now()}
|
|
190
|
+
self.fd.write(f'\n\n# Logging stopped at {datetime.datetime.now()} '
|
|
191
|
+
f'with code: {self.returncode}\n')
|
|
187
192
|
self.fd.close()
|
|
188
193
|
except Exception:
|
|
189
194
|
print(f'Error closing log file for task {self.name}. Ignoring.')
|
|
@@ -225,7 +230,7 @@ class SSHClient(asyncssh.SSHClient):
|
|
|
225
230
|
|
|
226
231
|
def connection_made(self, conn):
|
|
227
232
|
self.connected = True
|
|
228
|
-
print('Connection made to
|
|
233
|
+
print('Connection made to {}.'.format(conn.get_extra_info('peername')[0]))
|
|
229
234
|
|
|
230
235
|
def connection_lost(self, exc):
|
|
231
236
|
if exc:
|
|
@@ -234,7 +239,7 @@ class SSHClient(asyncssh.SSHClient):
|
|
|
234
239
|
self.connected = False
|
|
235
240
|
|
|
236
241
|
|
|
237
|
-
class Client
|
|
242
|
+
class Client:
|
|
238
243
|
|
|
239
244
|
def __init__(self, host, user=None, force_sync=True, password=None, connect_timeout=15):
|
|
240
245
|
self.host = host
|
|
@@ -295,7 +300,7 @@ class Client(object):
|
|
|
295
300
|
|
|
296
301
|
async with self.creating_session:
|
|
297
302
|
process = await self.conn.create_process('echo $$;' + task.command)
|
|
298
|
-
|
|
303
|
+
|
|
299
304
|
try:
|
|
300
305
|
process.pid = int(await process.stdout.readline())
|
|
301
306
|
except ValueError:
|
|
@@ -325,7 +330,7 @@ class Client(object):
|
|
|
325
330
|
return self.client is not None and self.client.connected
|
|
326
331
|
|
|
327
332
|
|
|
328
|
-
class LocalClient
|
|
333
|
+
class LocalClient:
|
|
329
334
|
|
|
330
335
|
def __init__(self, host='local', force_sync=True):
|
|
331
336
|
self.host = host
|
|
@@ -336,8 +341,9 @@ class LocalClient(object):
|
|
|
336
341
|
self.started.set()
|
|
337
342
|
|
|
338
343
|
async def execute(self, task):
|
|
339
|
-
process = await asyncio.create_subprocess_shell(
|
|
340
|
-
|
|
344
|
+
process = await asyncio.create_subprocess_shell(
|
|
345
|
+
task.command, stdout=asyncio.subprocess.PIPE,
|
|
346
|
+
stderr=asyncio.subprocess.PIPE, preexec_fn=os.setpgrp)
|
|
341
347
|
task.set_process(Process(self, process))
|
|
342
348
|
|
|
343
349
|
await asyncio.gather(
|
|
@@ -358,7 +364,7 @@ class LocalClient(object):
|
|
|
358
364
|
pass
|
|
359
365
|
|
|
360
366
|
|
|
361
|
-
class Worker
|
|
367
|
+
class Worker:
|
|
362
368
|
|
|
363
369
|
def __init__(self, client, name):
|
|
364
370
|
self.client = client
|
|
@@ -376,13 +382,11 @@ class Worker(object):
|
|
|
376
382
|
try:
|
|
377
383
|
await self.client.start()
|
|
378
384
|
except Exception as exc:
|
|
379
|
-
err_msg = f'Error starting client: {
|
|
385
|
+
err_msg = f'Error starting client: {exc!s}\n'
|
|
380
386
|
# transient failure: allow task retry to pick a worker again
|
|
381
387
|
self.status = WorkerStatus.IDLE
|
|
382
|
-
|
|
388
|
+
with contextlib.suppress(Exception):
|
|
383
389
|
self.client.close() # ensure clean slate
|
|
384
|
-
except Exception:
|
|
385
|
-
pass
|
|
386
390
|
return False, err_msg
|
|
387
391
|
|
|
388
392
|
await self.client.started.wait()
|
|
@@ -390,18 +394,16 @@ class Worker(object):
|
|
|
390
394
|
try:
|
|
391
395
|
await self.client.execute(task)
|
|
392
396
|
except ConnectionError as exc:
|
|
393
|
-
err_msg = f'Error connecting client: {
|
|
397
|
+
err_msg = f'Error connecting client: {exc!s}\n'
|
|
394
398
|
self.status = WorkerStatus.IDLE
|
|
395
|
-
|
|
399
|
+
with contextlib.suppress(Exception):
|
|
396
400
|
self.client.close()
|
|
397
|
-
except Exception:
|
|
398
|
-
pass
|
|
399
401
|
return False, err_msg
|
|
400
402
|
except asyncio.CancelledError:
|
|
401
403
|
err_msg = f'Task {self.name} has been canceled\n'
|
|
402
404
|
return True, err_msg
|
|
403
405
|
except Exception as exc:
|
|
404
|
-
err_msg = f'Error executing task {self.name}: {
|
|
406
|
+
err_msg = f'Error executing task {self.name}: {exc!s}\n'
|
|
405
407
|
self.status = WorkerStatus.IDLE
|
|
406
408
|
return False, err_msg
|
|
407
409
|
|
|
@@ -452,7 +454,7 @@ class WorkerPoolReport:
|
|
|
452
454
|
return task.run_on_host or "?"
|
|
453
455
|
|
|
454
456
|
|
|
455
|
-
class WorkerPool
|
|
457
|
+
class WorkerPool:
|
|
456
458
|
|
|
457
459
|
def __init__(self, hosts, name='Worker', max_tasks_per_worker=4,
|
|
458
460
|
env_source_file=None, user=None, max_time=None, force_sync=False,
|
|
@@ -463,11 +465,13 @@ class WorkerPool(object):
|
|
|
463
465
|
Args:
|
|
464
466
|
name (str): Name of the worker pool.
|
|
465
467
|
hosts (list): List of host names, reachable via ssh (if not local host).
|
|
466
|
-
max_tasks_per_worker (int, optional): Maximum number of tasks to execute
|
|
468
|
+
max_tasks_per_worker (int, optional): Maximum number of tasks to execute
|
|
469
|
+
concurrently on an host.
|
|
467
470
|
env_source_file (str, optional): Name of a file to source before executing a task.
|
|
468
471
|
user (str, optional): User to connect
|
|
469
472
|
connect_timeout (int, optional): Per-attempt SSH connection timeout, in seconds.
|
|
470
|
-
host_cooldown_s (int, optional): Time a just-failed host is avoided while
|
|
473
|
+
host_cooldown_s (int, optional): Time a just-failed host is avoided while
|
|
474
|
+
a healthy one exists.
|
|
471
475
|
"""
|
|
472
476
|
if asyncio.get_event_loop().is_closed():
|
|
473
477
|
asyncio.set_event_loop(asyncio.new_event_loop())
|
|
@@ -534,9 +538,10 @@ class WorkerPool(object):
|
|
|
534
538
|
|
|
535
539
|
await asyncio.sleep(0.1)
|
|
536
540
|
|
|
537
|
-
def add(self, command, name=None, output_file=None, task_done_callback=None, done_callback=None,
|
|
541
|
+
def add(self, command, name=None, output_file=None, task_done_callback=None, done_callback=None,
|
|
538
542
|
run_on_host=None, n_try=3, pid_filename=None, keep_pid_file_on_error=False):
|
|
539
|
-
"""Add a command to execute by a worker in the pool. Optionally output result
|
|
543
|
+
"""Add a command to execute by a worker in the pool. Optionally output result
|
|
544
|
+
in output_file.
|
|
540
545
|
|
|
541
546
|
Args:
|
|
542
547
|
command (str): Command to execute.
|
|
@@ -544,13 +549,11 @@ class WorkerPool(object):
|
|
|
544
549
|
output_file (str, optional): Optional filename to log output into.
|
|
545
550
|
"""
|
|
546
551
|
if run_on_host is not None and run_on_host not in self.hosts:
|
|
547
|
-
print(f'Requested run_on_host "{run_on_host}" is not in the hosts list.
|
|
552
|
+
print(f'Requested run_on_host "{run_on_host}" is not in the hosts list. '
|
|
553
|
+
'Releasing constraint ...')
|
|
548
554
|
run_on_host = None
|
|
549
555
|
|
|
550
|
-
if self.env_file
|
|
551
|
-
script = f". {shlex.quote(self.env_file)}; {command}"
|
|
552
|
-
else:
|
|
553
|
-
script = command
|
|
556
|
+
script = f". {shlex.quote(self.env_file)}; {command}" if self.env_file else command
|
|
554
557
|
|
|
555
558
|
command = f"bash -lc {shlex.quote(script)}"
|
|
556
559
|
|
|
@@ -563,8 +566,9 @@ class WorkerPool(object):
|
|
|
563
566
|
return
|
|
564
567
|
|
|
565
568
|
self.tasks.append(Task(name, command, output_file=output_file, done_callback=done_callback,
|
|
566
|
-
task_done_callback=task_done_callback, run_on_host=run_on_host,
|
|
567
|
-
pid_filename=pid_filename,
|
|
569
|
+
task_done_callback=task_done_callback, run_on_host=run_on_host,
|
|
570
|
+
n_try=n_try, pid_filename=pid_filename,
|
|
571
|
+
keep_pid_file_on_error=keep_pid_file_on_error))
|
|
568
572
|
|
|
569
573
|
def get_all_tasks(self):
|
|
570
574
|
tasks = collections.defaultdict(list)
|
|
@@ -589,8 +593,10 @@ class WorkerPool(object):
|
|
|
589
593
|
if running_time > (self.max_time + update_interval):
|
|
590
594
|
task = worker.running_task
|
|
591
595
|
task.timed_out = True
|
|
592
|
-
task.log(f'Task timeout: waited for {running_time:.2f} s',
|
|
593
|
-
|
|
596
|
+
task.log(f'Task timeout: waited for {running_time:.2f} s',
|
|
597
|
+
worker.client.host, err=True)
|
|
598
|
+
task.log(f'Command was: {task.command}',
|
|
599
|
+
worker.client.host, err=True)
|
|
594
600
|
await task.terminate()
|
|
595
601
|
|
|
596
602
|
except asyncio.CancelledError:
|
|
@@ -632,7 +638,8 @@ class WorkerPool(object):
|
|
|
632
638
|
|
|
633
639
|
loop = asyncio.get_event_loop()
|
|
634
640
|
|
|
635
|
-
loop.add_signal_handler(signal.SIGTERM,
|
|
641
|
+
loop.add_signal_handler(signal.SIGTERM,
|
|
642
|
+
lambda: asyncio.ensure_future(self.shutdown(exit_code=1)))
|
|
636
643
|
|
|
637
644
|
try:
|
|
638
645
|
loop.run_until_complete(self.process_tasks())
|
|
@@ -640,7 +647,7 @@ class WorkerPool(object):
|
|
|
640
647
|
loop.run_until_complete(self.terminate_all())
|
|
641
648
|
raise
|
|
642
649
|
except asyncio.CancelledError:
|
|
643
|
-
secho(
|
|
650
|
+
secho('Error: one (or more) task(s) have been canceled', fg='red')
|
|
644
651
|
finally:
|
|
645
652
|
for worker in self.workers:
|
|
646
653
|
worker.close()
|
|
@@ -668,7 +675,8 @@ class WorkerPool(object):
|
|
|
668
675
|
if report.failed:
|
|
669
676
|
print("\nFailed tasks:")
|
|
670
677
|
for t in report.failed:
|
|
671
|
-
print(f" - {t.name} | status={t.status.name} |
|
|
678
|
+
print(f" - {t.name} | status={t.status.name} | "
|
|
679
|
+
f"host={report.host_of(t)} | rc={t.returncode}")
|
|
672
680
|
|
|
673
681
|
print("==========================\n")
|
|
674
682
|
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["poetry-core>=2.0"]
|
|
3
|
+
build-backend = "poetry.core.masonry.api"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "libpipe"
|
|
7
|
+
version = "0.7"
|
|
8
|
+
description = "Pipeline orchestration and node management utilities"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "GPL-3.0-or-later"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Florent Mertens", email = "florent.mertens@gmail.com" }
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.10,<4.0"
|
|
15
|
+
|
|
16
|
+
dependencies = [
|
|
17
|
+
"asyncssh>=2.0",
|
|
18
|
+
"progressbar2>=3.40",
|
|
19
|
+
"toml>=0.10",
|
|
20
|
+
"click>=8.0",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
dev = [
|
|
25
|
+
"pytest>=8.0",
|
|
26
|
+
"pytest-cov>=5.0",
|
|
27
|
+
"ruff==0.16.1",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://gitlab.com/flomertens/libpipe"
|
|
32
|
+
Repository = "https://gitlab.com/flomertens/libpipe"
|
|
33
|
+
Issues = "https://gitlab.com/flomertens/libpipe/-/issues"
|
|
34
|
+
|
|
35
|
+
[project.scripts]
|
|
36
|
+
nodetool = "libpipe.tools.nodetool:main"
|
|
37
|
+
|
|
38
|
+
[tool.ruff]
|
|
39
|
+
line-length = 100
|
|
40
|
+
target-version = "py310"
|
|
41
|
+
|
|
42
|
+
[tool.ruff.lint]
|
|
43
|
+
select = [
|
|
44
|
+
"E",
|
|
45
|
+
"W",
|
|
46
|
+
"F",
|
|
47
|
+
"I",
|
|
48
|
+
"B",
|
|
49
|
+
"UP",
|
|
50
|
+
"C4",
|
|
51
|
+
"SIM",
|
|
52
|
+
"PIE",
|
|
53
|
+
"NPY",
|
|
54
|
+
"RUF",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[tool.ruff.lint.per-file-ignores]
|
|
58
|
+
"**/__init__.py" = ["F401"]
|
|
59
|
+
|
|
60
|
+
[tool.pytest.ini_options]
|
|
61
|
+
testpaths = ["tests"]
|
libpipe-0.6/pyproject.toml
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["poetry-core>=1.9.0"]
|
|
3
|
-
build-backend = "poetry.core.masonry.api"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "libpipe"
|
|
7
|
-
version = "0.6"
|
|
8
|
-
description = "Pipeline orchestration and node management utilities"
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
license = { text = "GPL-3.0-or-later" }
|
|
11
|
-
authors = [
|
|
12
|
-
{ name = "Florent Mertens", email = "florent.mertens@gmail.com" }
|
|
13
|
-
]
|
|
14
|
-
requires-python = ">=3.10"
|
|
15
|
-
|
|
16
|
-
dependencies = [
|
|
17
|
-
"asyncssh>=2.0",
|
|
18
|
-
"progressbar2>=3.40",
|
|
19
|
-
"toml>=0.10",
|
|
20
|
-
"click>=8.0",
|
|
21
|
-
]
|
|
22
|
-
|
|
23
|
-
[project.urls]
|
|
24
|
-
Repository = "https://gitlab.com/flomertens/libpipe"
|
|
25
|
-
|
|
26
|
-
[project.scripts]
|
|
27
|
-
nodetool = "libpipe.tools.nodetool:main"
|
|
28
|
-
|
|
29
|
-
[tool.poetry.group.dev.dependencies]
|
|
30
|
-
pytest = "^8.0"
|
|
31
|
-
pytest-cov = "^5.0"
|
|
32
|
-
|
|
33
|
-
[tool.pytest.ini_options]
|
|
34
|
-
testpaths = ["tests"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|