libpipe 0.6__tar.gz → 0.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,23 +1,28 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: libpipe
3
- Version: 0.6
3
+ Version: 0.7
4
4
  Summary: Pipeline orchestration and node management utilities
5
- License: GPL-3.0-or-later
5
+ License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE
7
7
  Author: Florent Mertens
8
8
  Author-email: florent.mertens@gmail.com
9
- Requires-Python: >=3.10
10
- Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
9
+ Requires-Python: >=3.10,<4.0
11
10
  Classifier: Programming Language :: Python :: 3
12
11
  Classifier: Programming Language :: Python :: 3.10
13
12
  Classifier: Programming Language :: Python :: 3.11
14
13
  Classifier: Programming Language :: Python :: 3.12
15
14
  Classifier: Programming Language :: Python :: 3.13
16
15
  Classifier: Programming Language :: Python :: 3.14
16
+ Provides-Extra: dev
17
17
  Requires-Dist: asyncssh (>=2.0)
18
18
  Requires-Dist: click (>=8.0)
19
19
  Requires-Dist: progressbar2 (>=3.40)
20
+ Requires-Dist: pytest (>=8.0) ; extra == "dev"
21
+ Requires-Dist: pytest-cov (>=5.0) ; extra == "dev"
22
+ Requires-Dist: ruff (==0.16.1) ; extra == "dev"
20
23
  Requires-Dist: toml (>=0.10)
24
+ Project-URL: Homepage, https://gitlab.com/flomertens/libpipe
25
+ Project-URL: Issues, https://gitlab.com/flomertens/libpipe/-/issues
21
26
  Project-URL: Repository, https://gitlab.com/flomertens/libpipe
22
27
  Description-Content-Type: text/markdown
23
28
 
@@ -1,11 +1,10 @@
1
1
  # Some parts inspired by the AttrDict package written by Brendan Curran-Johnson (MIT Licence)
2
2
 
3
- import re
3
+ import collections
4
4
  import copy
5
5
  import functools
6
-
6
+ import re
7
7
  from collections.abc import Mapping
8
- import collections
9
8
 
10
9
 
11
10
  def merge(left, right):
@@ -130,7 +129,7 @@ class AttrMap(Mapping):
130
129
  # sequence type seems like more trouble than it is worth.
131
130
  # If people want full serialization, they can pickle, and in
132
131
  # 99% of cases, sequence_type won't change anyway
133
- return "AttrMap({mapping})".format(mapping=repr(self._mapping))
132
+ return f"AttrMap({self._mapping!r})"
134
133
 
135
134
  @classmethod
136
135
  def _valid_name(cls, key):
@@ -158,9 +157,7 @@ class AttrMap(Mapping):
158
157
 
159
158
  if key not in self or not self._valid_name(key):
160
159
  raise AttributeError(
161
- "'{cls}' instance has no attribute '{name}'".format(
162
- cls=self.__class__.__name__, name=key
163
- )
160
+ f"'{self.__class__.__name__}' instance has no attribute '{key}'"
164
161
  )
165
162
 
166
163
  return self._build(self[key])
@@ -251,9 +248,7 @@ class AttrMap(Mapping):
251
248
  Mapping.__setattr__(self, key, value)
252
249
  else:
253
250
  raise TypeError(
254
- "'{cls}' does not allow attribute creation.".format(
255
- cls=self.__class__.__name__
256
- )
251
+ f"'{self.__class__.__name__}' does not allow attribute creation."
257
252
  )
258
253
 
259
254
  def __delattr__(self, key, force=False): # pragma: no cover
@@ -267,9 +262,7 @@ class AttrMap(Mapping):
267
262
  Mapping.__delattr__(self, key, force=force)
268
263
  else:
269
264
  raise TypeError(
270
- "'{cls}' does not allow attribute deletion.".format(
271
- cls=self.__class__.__name__
272
- )
265
+ f"'{self.__class__.__name__}' does not allow attribute deletion."
273
266
  )
274
267
 
275
268
  def set_data(self, mapping):
@@ -301,7 +294,7 @@ class AttrMap(Mapping):
301
294
 
302
295
  def get(self, key, default=None):
303
296
  try:
304
- return functools.reduce(lambda x, y: x[y], [self] + key.split('.'))
297
+ return functools.reduce(lambda x, y: x[y], [self, *key.split('.')])
305
298
  except KeyError:
306
299
  return None
307
300
 
@@ -356,8 +349,8 @@ class OrderedSet(collections.abc.MutableSet):
356
349
 
357
350
  def __repr__(self):
358
351
  if not self:
359
- return '%s()' % (self.__class__.__name__,)
360
- return '%s(%r)' % (self.__class__.__name__, list(self))
352
+ return f'{self.__class__.__name__}()'
353
+ return f'{self.__class__.__name__}({list(self)!r})'
361
354
 
362
355
  def __eq__(self, other):
363
356
  if isinstance(other, OrderedSet):
@@ -2,10 +2,10 @@
2
2
  #
3
3
  # Author: F. Mertens
4
4
 
5
- import re
6
5
  import os
7
- import time
6
+ import re
8
7
  import shutil
8
+ import time
9
9
 
10
10
 
11
11
  def file_not_found_retry(fct, n_try, *args, **kargs):
@@ -28,7 +28,7 @@ def reglob(path, exp, invert=False):
28
28
  else:
29
29
  res = [f for f in os.listdir(path) if not m.search(f)]
30
30
 
31
- return map(lambda x: "%s/%s" % (path, x, ), res)
31
+ return (f"{path}/{x}" for x in res)
32
32
 
33
33
 
34
34
  def rm_if_exist(directory, verbose=False):
@@ -54,21 +54,23 @@ def is_file(f):
54
54
  def zip_copy(dir_ins, dir_outs, filename, filename_out=None, verbose=True, n_retry=1):
55
55
  if not filename_out:
56
56
  filename_out = filename
57
- for dir_in, dir_out in zip(dir_ins, dir_outs):
57
+ for dir_in, dir_out in zip(dir_ins, dir_outs, strict=False):
58
58
  if verbose:
59
59
  print(f'Copy {dir_in}/{filename} -> {dir_out}/{filename_out}')
60
- file_not_found_retry(shutil.copyfile, n_retry, f'{dir_in}/{filename}', f'{dir_out}/{filename_out}')
60
+ file_not_found_retry(shutil.copyfile, n_retry, f'{dir_in}/{filename}',
61
+ f'{dir_out}/{filename_out}')
61
62
 
62
63
 
63
64
  def zip_copytree(dir_ins, dir_outs, dirname, verbose=True, n_retry=1):
64
- for dir_in, dir_out in zip(dir_ins, dir_outs):
65
+ for dir_in, dir_out in zip(dir_ins, dir_outs, strict=False):
65
66
  if verbose:
66
67
  print(f'Copy {dir_in}/{dirname} -> {dir_out}/{dirname}')
67
- file_not_found_retry(shutil.copytree, n_retry, f'{dir_in}/{dirname}', f'{dir_out}/{dirname}')
68
+ file_not_found_retry(shutil.copytree, n_retry, f'{dir_in}/{dirname}',
69
+ f'{dir_out}/{dirname}')
68
70
 
69
71
 
70
72
  def zip_rename(dir_ins, dir_outs, filename_or_dirname=None, verbose=True, n_retry=1):
71
- for f_in, f_out in zip(dir_ins, dir_outs):
73
+ for f_in, f_out in zip(dir_ins, dir_outs, strict=False):
72
74
  if filename_or_dirname is not None:
73
75
  f_in = f'{f_in}/{filename_or_dirname}'
74
76
  f_out = f'{f_out}/{filename_or_dirname}'
@@ -78,7 +80,7 @@ def zip_rename(dir_ins, dir_outs, filename_or_dirname=None, verbose=True, n_retr
78
80
 
79
81
 
80
82
  def zip_rename_reg(dir_ins, dir_outs, regex, verbose=True, invert=False, n_retry=1):
81
- for f_in, f_out in zip(dir_ins, dir_outs):
83
+ for f_in, f_out in zip(dir_ins, dir_outs, strict=False):
82
84
  for f_in2 in reglob(f_in, regex, invert=invert):
83
85
  f_out2 = f'{f_out}/{os.path.basename(f_in2)}'
84
86
  if verbose:
@@ -71,11 +71,8 @@ class BaseSettings(attrmap.AttrMap):
71
71
  else:
72
72
  for k, v in c_file.items():
73
73
  if isinstance(v, dict):
74
- if 'type' in v.keys():
75
- t = v['type']
76
- else:
77
- t = k
78
- if t not in c_default.keys():
74
+ t = v.get('type', k)
75
+ if t not in c_default:
79
76
  print(f'Warning: no default for section {k} of type {t}')
80
77
  continue
81
78
  t_default = c_default[t]
@@ -120,7 +117,8 @@ class BaseSettings(attrmap.AttrMap):
120
117
  with open(file, mode='w') as f:
121
118
  toml.dump(dict(m), f)
122
119
 
123
- def duplicate(self, out_file, copy_parset_files=False, strip_parents_keys=False, **config_modifiers):
120
+ def duplicate(self, out_file, copy_parset_files=False, strip_parents_keys=False,
121
+ **config_modifiers):
124
122
  s_out = self.copy()
125
123
  s_out.set_file(out_file)
126
124
  s_out['default_settings'] = ''
@@ -1,13 +1,11 @@
1
1
  #!/usr/bin/env python
2
2
 
3
- import os
4
3
  import json
4
+ import os
5
5
 
6
6
  import click
7
7
 
8
- from libpipe import worker
9
- from libpipe import __version__
10
-
8
+ from libpipe import __version__, worker
11
9
 
12
10
  t_file = click.Path(exists=True, dir_okay=False)
13
11
 
@@ -21,7 +19,8 @@ def main():
21
19
  @main.command('run_all')
22
20
  @click.argument('cmd_list', type=t_file)
23
21
  @click.option('--nodes_string', '-n', help='Range of nodes to use', type=str, default='localhost')
24
- @click.option('--max_concurrent', '-m', help='Maximum concurrent tasks on a node', type=int, default=5)
22
+ @click.option('--max_concurrent', '-m', type=int, default=5,
23
+ help='Maximum concurrent tasks on a node')
25
24
  @click.option('--env_file', '-e', help='Environment source file', type=t_file)
26
25
  @click.option('--dry_run', '-d', help='Dry run, only print commands', is_flag=True)
27
26
  @click.option('--log_file', help='Log output to log_file', type=str, default=None)
@@ -31,7 +30,10 @@ def run_all(cmd_list, nodes_string, max_concurrent, env_file, dry_run, log_file)
31
30
  env_file=env_file, dry_run=dry_run, debug=dry_run)
32
31
 
33
32
  i_log_file = log_file
34
- for i, cmd in enumerate(open(cmd_list).readlines()):
33
+ with open(cmd_list) as f:
34
+ cmds = f.readlines()
35
+
36
+ for i, cmd in enumerate(cmds):
35
37
  if log_file is not None:
36
38
  i_log_file = log_file + f'.{i + 1}'
37
39
  pool.add(cmd.strip(), output_file=i_log_file)
@@ -64,7 +66,7 @@ def run_on_all_nodes(cmd, nodes_string, env_file, dry_run):
64
66
  @click.option('--log_file', '-l', help='Log output to log_file', type=str, default=None)
65
67
  def init_job(name, cmd, target_node, log_file):
66
68
  ''' Add a job to a job list file NAME. If job file does not exist, create one. '''
67
- if os.path.exists(name):
69
+ if os.path.exists(name):
68
70
  with open(name) as json_file:
69
71
  jobs = json.load(json_file)
70
72
  else:
@@ -78,9 +80,11 @@ def init_job(name, cmd, target_node, log_file):
78
80
 
79
81
  @main.command('run_joblist')
80
82
  @click.argument('name', type=t_file)
81
- @click.option('--nodes_string', '-n', help='Range of nodes to use. By default use the target nodes in the joblist',
83
+ @click.option('--nodes_string', '-n',
84
+ help='Range of nodes to use. By default use the target nodes in the joblist',
82
85
  type=str, default='joblist')
83
- @click.option('--max_concurrent', '-m', help='Maximum concurrent tasks on a node', type=int, default=5)
86
+ @click.option('--max_concurrent', '-m', type=int, default=5,
87
+ help='Maximum concurrent tasks on a node')
84
88
  @click.option('--env_file', '-e', help='Environment source file', type=t_file)
85
89
  @click.option('--dry_run', '-d', help='Dry run, only print commands', is_flag=True)
86
90
  @click.option('--worker_name', '-w', help='Name of the worker', type=str, default='Run joblist')
@@ -90,7 +94,7 @@ def run_joblist(name, nodes_string, max_concurrent, env_file, dry_run, worker_na
90
94
  jobs = json.load(json_file)
91
95
 
92
96
  if nodes_string == 'joblist':
93
- all_nodes = set([job['target_node'] for job in jobs['jobs'] if job['target_node'] is not None])
97
+ all_nodes = {job['target_node'] for job in jobs['jobs'] if job['target_node'] is not None}
94
98
  if len(all_nodes) == 0:
95
99
  all_nodes = ['localhost']
96
100
  else:
@@ -2,24 +2,24 @@
2
2
  #
3
3
  # Authors: F.Mertens
4
4
 
5
+ import asyncio
6
+ import collections
7
+ import contextlib
8
+ import datetime
9
+ import getpass
10
+ import itertools
5
11
  import os
6
12
  import re
7
- import sys
8
- import time
9
13
  import shlex
10
- import socket
11
14
  import signal
12
- import getpass
13
- import asyncio
14
- import datetime
15
- import itertools
16
- import collections
15
+ import socket
16
+ import sys
17
+ import time
17
18
  from enum import Enum
18
19
 
19
20
  import asyncssh
20
21
  import progressbar
21
- from click import style, secho
22
-
22
+ from click import secho, style
23
23
 
24
24
  localhost_shortname = socket.gethostname().split('.', 1)[0]
25
25
 
@@ -48,7 +48,7 @@ def expend_num_ranges(s):
48
48
 
49
49
 
50
50
  def get_hosts(host_string):
51
- hosts = list(set(host for k in host_string.split(',') for host in expend_num_ranges(k)))
51
+ hosts = list({host for k in host_string.split(',') for host in expend_num_ranges(k)})
52
52
  return [localhost_shortname if h == 'localhost' else h for h in hosts]
53
53
 
54
54
 
@@ -74,14 +74,14 @@ class WorkerStatus(Enum):
74
74
  CLOSED = 4
75
75
 
76
76
 
77
- class Process(object):
77
+ class Process:
78
78
 
79
79
  def __init__(self, client, process):
80
80
  self.client = client
81
81
  self.process = process
82
82
 
83
83
 
84
- class Task(object):
84
+ class Task:
85
85
  """Represent a task executed by the worker pool
86
86
 
87
87
  Attributes:
@@ -115,46 +115,50 @@ class Task(object):
115
115
  try:
116
116
  self._done_callback()
117
117
  except Exception as exc:
118
- err_msg = f'Error executing return callback of task {self.name}: {str(exc)}\n'
118
+ err_msg = f'Error executing return callback of task {self.name}: {exc!s}\n'
119
119
  if self._task_done_callback is not None:
120
120
  try:
121
121
  self._task_done_callback(self)
122
122
  except Exception as exc:
123
- err_msg = f'Error executing return callback of task {self.name}: {str(exc)}\n'
123
+ err_msg = f'Error executing return callback of task {self.name}: {exc!s}\n'
124
124
 
125
125
  return err_msg
126
126
 
127
127
  def init_log(self):
128
128
  if self.output_file is not None:
129
- self.fd = open(self.output_file, 'w')
129
+ self.fd = open(self.output_file, 'w') # noqa: SIM115 - closed in close_log()
130
130
  self.fd.write(f'# Logging starting at {datetime.datetime.now()}\n')
131
131
  self.fd.write(f'# Input command: {self.command}\n')
132
132
 
133
133
  def set_process(self, process):
134
134
  self.process = process
135
135
  if hasattr(self.process.process, 'pid'):
136
- self.log(f'Starting on host {self.process.client.host} with PID {self.process.process.pid}\n\n',
137
- self.process.client.host)
136
+ self.log(f'Starting on host {self.process.client.host} with PID '
137
+ f'{self.process.process.pid}\n\n', self.process.client.host)
138
138
  if self.pid_filename is not None:
139
139
  try:
140
140
  with open(self.pid_filename, 'w') as fd:
141
141
  fd.write(f'{self.process.client.host} {self.process.process.pid}')
142
- self.log(f'Starting on host {self.process.client.host} with PID {self.process.process.pid}\n\n',
143
- self.process.client.host)
142
+ self.log(f'Starting on host {self.process.client.host} with PID '
143
+ f'{self.process.process.pid}\n\n', self.process.client.host)
144
144
  except Exception:
145
- self.log(f'Error creating PID file {self.pid_filename}', self.process.client.host, err=True)
145
+ self.log(f'Error creating PID file {self.pid_filename}',
146
+ self.process.client.host, err=True)
146
147
 
147
148
  async def terminate(self):
148
149
  if self.process is not None and hasattr(self.process.process, 'pid'):
149
150
  if self.process.client.host == localhost_shortname:
150
- self.log(f'Killing local process PID {self.process.process.pid}\n', 'local', err=True)
151
+ self.log(f'Killing local process PID {self.process.process.pid}\n',
152
+ 'local', err=True)
151
153
  try:
152
154
  os.killpg(os.getpgid(self.process.process.pid), signal.SIGTERM)
153
155
  except ProcessLookupError:
154
156
  print(f'Process {self.process.process.pid} not found')
155
157
  else:
156
- self.log(f'Killing remote process PID {self.process.process.pid}\n', self.process.client.host, err=True)
157
- kill_remote(self.process.client.host, self.process.client.user, self.process.process.pid)
158
+ self.log(f'Killing remote process PID {self.process.process.pid}\n',
159
+ self.process.client.host, err=True)
160
+ kill_remote(self.process.client.host, self.process.client.user,
161
+ self.process.process.pid)
158
162
 
159
163
  def log(self, line, host, err=False):
160
164
  if isinstance(line, bytes):
@@ -178,12 +182,13 @@ class Task(object):
178
182
  self.log(line, host, err=err)
179
183
 
180
184
  def close_log(self):
181
- if self.pid_filename and os.path.exists(self.pid_filename):
182
- if not self.keep_pid_file_on_error or self.returncode == 0:
183
- os.remove(self.pid_filename)
185
+ if (self.pid_filename and os.path.exists(self.pid_filename)
186
+ and (not self.keep_pid_file_on_error or self.returncode == 0)):
187
+ os.remove(self.pid_filename)
184
188
  if self.fd is not None:
185
189
  try:
186
- self.fd.write(f'\n\n# Logging stopped at {datetime.datetime.now()} with code: {self.returncode}\n')
190
+ self.fd.write(f'\n\n# Logging stopped at {datetime.datetime.now()} '
191
+ f'with code: {self.returncode}\n')
187
192
  self.fd.close()
188
193
  except Exception:
189
194
  print(f'Error closing log file for task {self.name}. Ignoring.')
@@ -225,7 +230,7 @@ class SSHClient(asyncssh.SSHClient):
225
230
 
226
231
  def connection_made(self, conn):
227
232
  self.connected = True
228
- print('Connection made to %s.' % conn.get_extra_info('peername')[0])
233
+ print('Connection made to {}.'.format(conn.get_extra_info('peername')[0]))
229
234
 
230
235
  def connection_lost(self, exc):
231
236
  if exc:
@@ -234,7 +239,7 @@ class SSHClient(asyncssh.SSHClient):
234
239
  self.connected = False
235
240
 
236
241
 
237
- class Client(object):
242
+ class Client:
238
243
 
239
244
  def __init__(self, host, user=None, force_sync=True, password=None, connect_timeout=15):
240
245
  self.host = host
@@ -295,7 +300,7 @@ class Client(object):
295
300
 
296
301
  async with self.creating_session:
297
302
  process = await self.conn.create_process('echo $$;' + task.command)
298
-
303
+
299
304
  try:
300
305
  process.pid = int(await process.stdout.readline())
301
306
  except ValueError:
@@ -325,7 +330,7 @@ class Client(object):
325
330
  return self.client is not None and self.client.connected
326
331
 
327
332
 
328
- class LocalClient(object):
333
+ class LocalClient:
329
334
 
330
335
  def __init__(self, host='local', force_sync=True):
331
336
  self.host = host
@@ -336,8 +341,9 @@ class LocalClient(object):
336
341
  self.started.set()
337
342
 
338
343
  async def execute(self, task):
339
- process = await asyncio.create_subprocess_shell(task.command, stdout=asyncio.subprocess.PIPE,
340
- stderr=asyncio.subprocess.PIPE, preexec_fn=os.setpgrp)
344
+ process = await asyncio.create_subprocess_shell(
345
+ task.command, stdout=asyncio.subprocess.PIPE,
346
+ stderr=asyncio.subprocess.PIPE, preexec_fn=os.setpgrp)
341
347
  task.set_process(Process(self, process))
342
348
 
343
349
  await asyncio.gather(
@@ -358,7 +364,7 @@ class LocalClient(object):
358
364
  pass
359
365
 
360
366
 
361
- class Worker(object):
367
+ class Worker:
362
368
 
363
369
  def __init__(self, client, name):
364
370
  self.client = client
@@ -376,13 +382,11 @@ class Worker(object):
376
382
  try:
377
383
  await self.client.start()
378
384
  except Exception as exc:
379
- err_msg = f'Error starting client: {str(exc)}\n'
385
+ err_msg = f'Error starting client: {exc!s}\n'
380
386
  # transient failure: allow task retry to pick a worker again
381
387
  self.status = WorkerStatus.IDLE
382
- try:
388
+ with contextlib.suppress(Exception):
383
389
  self.client.close() # ensure clean slate
384
- except Exception:
385
- pass
386
390
  return False, err_msg
387
391
 
388
392
  await self.client.started.wait()
@@ -390,18 +394,16 @@ class Worker(object):
390
394
  try:
391
395
  await self.client.execute(task)
392
396
  except ConnectionError as exc:
393
- err_msg = f'Error connecting client: {str(exc)}\n'
397
+ err_msg = f'Error connecting client: {exc!s}\n'
394
398
  self.status = WorkerStatus.IDLE
395
- try:
399
+ with contextlib.suppress(Exception):
396
400
  self.client.close()
397
- except Exception:
398
- pass
399
401
  return False, err_msg
400
402
  except asyncio.CancelledError:
401
403
  err_msg = f'Task {self.name} has been canceled\n'
402
404
  return True, err_msg
403
405
  except Exception as exc:
404
- err_msg = f'Error executing task {self.name}: {str(exc)}\n'
406
+ err_msg = f'Error executing task {self.name}: {exc!s}\n'
405
407
  self.status = WorkerStatus.IDLE
406
408
  return False, err_msg
407
409
 
@@ -452,7 +454,7 @@ class WorkerPoolReport:
452
454
  return task.run_on_host or "?"
453
455
 
454
456
 
455
- class WorkerPool(object):
457
+ class WorkerPool:
456
458
 
457
459
  def __init__(self, hosts, name='Worker', max_tasks_per_worker=4,
458
460
  env_source_file=None, user=None, max_time=None, force_sync=False,
@@ -463,11 +465,13 @@ class WorkerPool(object):
463
465
  Args:
464
466
  name (str): Name of the worker pool.
465
467
  hosts (list): List of host names, reachable via ssh (if not local host).
466
- max_tasks_per_worker (int, optional): Maximum number of tasks to execute concurrently on an host.
468
+ max_tasks_per_worker (int, optional): Maximum number of tasks to execute
469
+ concurrently on an host.
467
470
  env_source_file (str, optional): Name of a file to source before executing a task.
468
471
  user (str, optional): User to connect
469
472
  connect_timeout (int, optional): Per-attempt SSH connection timeout, in seconds.
470
- host_cooldown_s (int, optional): Time a just-failed host is avoided while a healthy one exists.
473
+ host_cooldown_s (int, optional): Time a just-failed host is avoided while
474
+ a healthy one exists.
471
475
  """
472
476
  if asyncio.get_event_loop().is_closed():
473
477
  asyncio.set_event_loop(asyncio.new_event_loop())
@@ -534,9 +538,10 @@ class WorkerPool(object):
534
538
 
535
539
  await asyncio.sleep(0.1)
536
540
 
537
- def add(self, command, name=None, output_file=None, task_done_callback=None, done_callback=None,
541
+ def add(self, command, name=None, output_file=None, task_done_callback=None, done_callback=None,
538
542
  run_on_host=None, n_try=3, pid_filename=None, keep_pid_file_on_error=False):
539
- """Add a command to execute by a worker in the pool. Optionally output result in output_file.
543
+ """Add a command to execute by a worker in the pool. Optionally output result
544
+ in output_file.
540
545
 
541
546
  Args:
542
547
  command (str): Command to execute.
@@ -544,13 +549,11 @@ class WorkerPool(object):
544
549
  output_file (str, optional): Optional filename to log output into.
545
550
  """
546
551
  if run_on_host is not None and run_on_host not in self.hosts:
547
- print(f'Requested run_on_host "{run_on_host}" is not in the hosts list. Releasing constraint ...')
552
+ print(f'Requested run_on_host "{run_on_host}" is not in the hosts list. '
553
+ 'Releasing constraint ...')
548
554
  run_on_host = None
549
555
 
550
- if self.env_file:
551
- script = f". {shlex.quote(self.env_file)}; {command}"
552
- else:
553
- script = command
556
+ script = f". {shlex.quote(self.env_file)}; {command}" if self.env_file else command
554
557
 
555
558
  command = f"bash -lc {shlex.quote(script)}"
556
559
 
@@ -563,8 +566,9 @@ class WorkerPool(object):
563
566
  return
564
567
 
565
568
  self.tasks.append(Task(name, command, output_file=output_file, done_callback=done_callback,
566
- task_done_callback=task_done_callback, run_on_host=run_on_host, n_try=n_try,
567
- pid_filename=pid_filename, keep_pid_file_on_error=keep_pid_file_on_error))
569
+ task_done_callback=task_done_callback, run_on_host=run_on_host,
570
+ n_try=n_try, pid_filename=pid_filename,
571
+ keep_pid_file_on_error=keep_pid_file_on_error))
568
572
 
569
573
  def get_all_tasks(self):
570
574
  tasks = collections.defaultdict(list)
@@ -589,8 +593,10 @@ class WorkerPool(object):
589
593
  if running_time > (self.max_time + update_interval):
590
594
  task = worker.running_task
591
595
  task.timed_out = True
592
- task.log(f'Task timeout: waited for {running_time:.2f} s', worker.client.host, err=True)
593
- task.log(f'Command was: {task.command}', worker.client.host, err=True)
596
+ task.log(f'Task timeout: waited for {running_time:.2f} s',
597
+ worker.client.host, err=True)
598
+ task.log(f'Command was: {task.command}',
599
+ worker.client.host, err=True)
594
600
  await task.terminate()
595
601
 
596
602
  except asyncio.CancelledError:
@@ -632,7 +638,8 @@ class WorkerPool(object):
632
638
 
633
639
  loop = asyncio.get_event_loop()
634
640
 
635
- loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.ensure_future(self.shutdown(exit_code=1)))
641
+ loop.add_signal_handler(signal.SIGTERM,
642
+ lambda: asyncio.ensure_future(self.shutdown(exit_code=1)))
636
643
 
637
644
  try:
638
645
  loop.run_until_complete(self.process_tasks())
@@ -640,7 +647,7 @@ class WorkerPool(object):
640
647
  loop.run_until_complete(self.terminate_all())
641
648
  raise
642
649
  except asyncio.CancelledError:
643
- secho(f'Error: one (or more) task(s) have been canceled', fg='red')
650
+ secho('Error: one (or more) task(s) have been canceled', fg='red')
644
651
  finally:
645
652
  for worker in self.workers:
646
653
  worker.close()
@@ -668,7 +675,8 @@ class WorkerPool(object):
668
675
  if report.failed:
669
676
  print("\nFailed tasks:")
670
677
  for t in report.failed:
671
- print(f" - {t.name} | status={t.status.name} | host={report.host_of(t)} | rc={t.returncode}")
678
+ print(f" - {t.name} | status={t.status.name} | "
679
+ f"host={report.host_of(t)} | rc={t.returncode}")
672
680
 
673
681
  print("==========================\n")
674
682
 
@@ -0,0 +1,61 @@
1
+ [build-system]
2
+ requires = ["poetry-core>=2.0"]
3
+ build-backend = "poetry.core.masonry.api"
4
+
5
+ [project]
6
+ name = "libpipe"
7
+ version = "0.7"
8
+ description = "Pipeline orchestration and node management utilities"
9
+ readme = "README.md"
10
+ license = "GPL-3.0-or-later"
11
+ authors = [
12
+ { name = "Florent Mertens", email = "florent.mertens@gmail.com" }
13
+ ]
14
+ requires-python = ">=3.10,<4.0"
15
+
16
+ dependencies = [
17
+ "asyncssh>=2.0",
18
+ "progressbar2>=3.40",
19
+ "toml>=0.10",
20
+ "click>=8.0",
21
+ ]
22
+
23
+ [project.optional-dependencies]
24
+ dev = [
25
+ "pytest>=8.0",
26
+ "pytest-cov>=5.0",
27
+ "ruff==0.16.1",
28
+ ]
29
+
30
+ [project.urls]
31
+ Homepage = "https://gitlab.com/flomertens/libpipe"
32
+ Repository = "https://gitlab.com/flomertens/libpipe"
33
+ Issues = "https://gitlab.com/flomertens/libpipe/-/issues"
34
+
35
+ [project.scripts]
36
+ nodetool = "libpipe.tools.nodetool:main"
37
+
38
+ [tool.ruff]
39
+ line-length = 100
40
+ target-version = "py310"
41
+
42
+ [tool.ruff.lint]
43
+ select = [
44
+ "E",
45
+ "W",
46
+ "F",
47
+ "I",
48
+ "B",
49
+ "UP",
50
+ "C4",
51
+ "SIM",
52
+ "PIE",
53
+ "NPY",
54
+ "RUF",
55
+ ]
56
+
57
+ [tool.ruff.lint.per-file-ignores]
58
+ "**/__init__.py" = ["F401"]
59
+
60
+ [tool.pytest.ini_options]
61
+ testpaths = ["tests"]
@@ -1,34 +0,0 @@
1
- [build-system]
2
- requires = ["poetry-core>=1.9.0"]
3
- build-backend = "poetry.core.masonry.api"
4
-
5
- [project]
6
- name = "libpipe"
7
- version = "0.6"
8
- description = "Pipeline orchestration and node management utilities"
9
- readme = "README.md"
10
- license = { text = "GPL-3.0-or-later" }
11
- authors = [
12
- { name = "Florent Mertens", email = "florent.mertens@gmail.com" }
13
- ]
14
- requires-python = ">=3.10"
15
-
16
- dependencies = [
17
- "asyncssh>=2.0",
18
- "progressbar2>=3.40",
19
- "toml>=0.10",
20
- "click>=8.0",
21
- ]
22
-
23
- [project.urls]
24
- Repository = "https://gitlab.com/flomertens/libpipe"
25
-
26
- [project.scripts]
27
- nodetool = "libpipe.tools.nodetool:main"
28
-
29
- [tool.poetry.group.dev.dependencies]
30
- pytest = "^8.0"
31
- pytest-cov = "^5.0"
32
-
33
- [tool.pytest.ini_options]
34
- testpaths = ["tests"]
File without changes
File without changes
File without changes
File without changes
File without changes