libpipe 0.6__tar.gz → 0.8__tar.gz

libpipe-0.8/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.3
+Name: libpipe
+Version: 0.8
+Summary: Pipeline orchestration and node management utilities
+License: GPL-3.0-or-later
+Author: Florent Mertens
+Author-email: florent.mertens@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: dev
+Requires-Dist: asyncssh (>=2.0)
+Requires-Dist: click (>=8.0)
+Requires-Dist: progressbar2 (>=3.40)
+Requires-Dist: pytest (>=8.0) ; extra == "dev"
+Requires-Dist: pytest-cov (>=5.0) ; extra == "dev"
+Requires-Dist: ruff (==0.16.1) ; extra == "dev"
+Requires-Dist: toml (>=0.10)
+Project-URL: Homepage, https://gitlab.com/flomertens/libpipe
+Project-URL: Issues, https://gitlab.com/flomertens/libpipe/-/issues
+Project-URL: Repository, https://gitlab.com/flomertens/libpipe
+Description-Content-Type: text/markdown
+
+Libpipe
+=======
+
+Common pipeline framework and library
+
+Installation
+------------
+
+libpipe can be installed via pip:
+
+    $ pip install libpipe
+
+and requires Python 3.10 or higher.
+
+Worker pools on a Slurm cluster
+-------------------------------
+
+A `WorkerPool` runs its tasks locally, on other hosts over SSH, or through Slurm with the
+host `slurm`: each task is then started with `srun`, as a job of its own when the pool runs
+on a login node, as a job step inside an allocation (`salloc`, `sbatch`).
+
+    pool = worker.WorkerPool(['slurm'], name='Imaging', max_tasks_per_worker=8,
+                             slurm_options='-p regular --mem=200G --time=24:00:00')
+
+`max_tasks_per_worker` is the number of jobs in flight. The srun options (partition,
+account, resources) come from `slurm_options`, or from the `LIBPIPE_SLURM_OPTIONS`
+environment variable; `max_time` becomes the job time limit when the options set none.
+Host constraints (`run_on_host`) do not apply: Slurm places the tasks, so the data must be
+on a file system shared with the compute nodes. Give the memory (`--mem`) in the options:
+inside an allocation, a step without it can claim all the memory of its node.
+

libpipe-0.8/README.md

@@ -0,0 +1,30 @@
+Libpipe
+=======
+
+Common pipeline framework and library
+
+Installation
+------------
+
+libpipe can be installed via pip:
+
+    $ pip install libpipe
+
+and requires Python 3.10 or higher.
+
+Worker pools on a Slurm cluster
+-------------------------------
+
+A `WorkerPool` runs its tasks locally, on other hosts over SSH, or through Slurm with the
+host `slurm`: each task is then started with `srun`, as a job of its own when the pool runs
+on a login node, as a job step inside an allocation (`salloc`, `sbatch`).
+
+    pool = worker.WorkerPool(['slurm'], name='Imaging', max_tasks_per_worker=8,
+                             slurm_options='-p regular --mem=200G --time=24:00:00')
+
+`max_tasks_per_worker` is the number of jobs in flight. The srun options (partition,
+account, resources) come from `slurm_options`, or from the `LIBPIPE_SLURM_OPTIONS`
+environment variable; `max_time` becomes the job time limit when the options set none.
+Host constraints (`run_on_host`) do not apply: Slurm places the tasks, so the data must be
+on a file system shared with the compute nodes. Give the memory (`--mem`) in the options:
+inside an allocation, a step without it can claim all the memory of its node.

{libpipe-0.6 → libpipe-0.8}/libpipe/attrmap.py

@@ -1,11 +1,10 @@
 # Some parts inspired by the AttrDict package written by Brendan Curran-Johnson (MIT Licence)
 
-import re
+import collections
 import copy
 import functools
-
+import re
 from collections.abc import Mapping
-import collections
 
 
 def merge(left, right):
@@ -130,7 +129,7 @@ class AttrMap(Mapping):
         # sequence type seems like more trouble than it is worth.
         # If people want full serialization, they can pickle, and in
         # 99% of cases, sequence_type won't change anyway
-        return "AttrMap({mapping})".format(mapping=repr(self._mapping))
+        return f"AttrMap({self._mapping!r})"
 
     @classmethod
     def _valid_name(cls, key):
@@ -158,9 +157,7 @@ class AttrMap(Mapping):
 
         if key not in self or not self._valid_name(key):
             raise AttributeError(
-                "'{cls}' instance has no attribute '{name}'".format(
-                    cls=self.__class__.__name__, name=key
-                )
+                f"'{self.__class__.__name__}' instance has no attribute '{key}'"
             )
 
         return self._build(self[key])
@@ -251,9 +248,7 @@ class AttrMap(Mapping):
             Mapping.__setattr__(self, key, value)
         else:
             raise TypeError(
-                "'{cls}' does not allow attribute creation.".format(
-                    cls=self.__class__.__name__
-                )
+                f"'{self.__class__.__name__}' does not allow attribute creation."
             )
 
     def __delattr__(self, key, force=False):  # pragma: no cover
@@ -267,9 +262,7 @@ class AttrMap(Mapping):
             Mapping.__delattr__(self, key, force=force)
         else:
             raise TypeError(
-                "'{cls}' does not allow attribute deletion.".format(
-                    cls=self.__class__.__name__
-                )
+                f"'{self.__class__.__name__}' does not allow attribute deletion."
             )
 
     def set_data(self, mapping):
@@ -301,7 +294,7 @@ class AttrMap(Mapping):
 
     def get(self, key, default=None):
         try:
-            return functools.reduce(lambda x, y: x[y], [self] + key.split('.'))
+            return functools.reduce(lambda x, y: x[y], [self, *key.split('.')])
         except KeyError:
             return None
 
@@ -356,8 +349,8 @@ class OrderedSet(collections.abc.MutableSet):
 
     def __repr__(self):
         if not self:
-            return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, list(self))
+            return f'{self.__class__.__name__}()'
+        return f'{self.__class__.__name__}({list(self)!r})'
 
     def __eq__(self, other):
         if isinstance(other, OrderedSet):

{libpipe-0.6 → libpipe-0.8}/libpipe/futils.py

@@ -2,10 +2,10 @@
 #
 # Author: F. Mertens
 
-import re
 import os
-import time
+import re
 import shutil
+import time
 
 
 def file_not_found_retry(fct, n_try, *args, **kargs):
@@ -28,7 +28,7 @@ def reglob(path, exp, invert=False):
     else:
         res = [f for f in os.listdir(path) if not m.search(f)]
 
-    return map(lambda x: "%s/%s" % (path, x, ), res)
+    return (f"{path}/{x}" for x in res)
 
 
 def rm_if_exist(directory, verbose=False):
@@ -54,21 +54,23 @@ def is_file(f):
 def zip_copy(dir_ins, dir_outs, filename, filename_out=None, verbose=True, n_retry=1):
     if not filename_out:
         filename_out = filename
-    for dir_in, dir_out in zip(dir_ins, dir_outs):
+    for dir_in, dir_out in zip(dir_ins, dir_outs, strict=False):
         if verbose:
             print(f'Copy {dir_in}/{filename} -> {dir_out}/{filename_out}')
-        file_not_found_retry(shutil.copyfile, n_retry, f'{dir_in}/{filename}', f'{dir_out}/{filename_out}')
+        file_not_found_retry(shutil.copyfile, n_retry, f'{dir_in}/{filename}',
+                             f'{dir_out}/{filename_out}')
 
 
 def zip_copytree(dir_ins, dir_outs, dirname, verbose=True, n_retry=1):
-    for dir_in, dir_out in zip(dir_ins, dir_outs):
+    for dir_in, dir_out in zip(dir_ins, dir_outs, strict=False):
         if verbose:
             print(f'Copy {dir_in}/{dirname} -> {dir_out}/{dirname}')
-        file_not_found_retry(shutil.copytree, n_retry, f'{dir_in}/{dirname}', f'{dir_out}/{dirname}')
+        file_not_found_retry(shutil.copytree, n_retry, f'{dir_in}/{dirname}',
+                             f'{dir_out}/{dirname}')
 
 
 def zip_rename(dir_ins, dir_outs, filename_or_dirname=None, verbose=True, n_retry=1):
-    for f_in, f_out in zip(dir_ins, dir_outs):
+    for f_in, f_out in zip(dir_ins, dir_outs, strict=False):
         if filename_or_dirname is not None:
             f_in = f'{f_in}/{filename_or_dirname}'
             f_out = f'{f_out}/{filename_or_dirname}'
@@ -78,7 +80,7 @@ def zip_rename(dir_ins, dir_outs, filename_or_dirname=None, verbose=True, n_retr
 
 
 def zip_rename_reg(dir_ins, dir_outs, regex, verbose=True, invert=False, n_retry=1):
-    for f_in, f_out in zip(dir_ins, dir_outs):
+    for f_in, f_out in zip(dir_ins, dir_outs, strict=False):
         for f_in2 in reglob(f_in, regex, invert=invert):
             f_out2 = f'{f_out}/{os.path.basename(f_in2)}'
             if verbose:

{libpipe-0.6 → libpipe-0.8}/libpipe/settings.py

@@ -71,11 +71,8 @@ class BaseSettings(attrmap.AttrMap):
         else:
             for k, v in c_file.items():
                 if isinstance(v, dict):
-                    if 'type' in v.keys():
-                        t = v['type']
-                    else:
-                        t = k
-                    if t not in c_default.keys():
+                    t = v.get('type', k)
+                    if t not in c_default:
                         print(f'Warning: no default for section {k} of type {t}')
                         continue
                     t_default = c_default[t]
@@ -120,7 +117,8 @@ class BaseSettings(attrmap.AttrMap):
             with open(file, mode='w') as f:
                 toml.dump(dict(m), f)
 
-    def duplicate(self, out_file, copy_parset_files=False, strip_parents_keys=False, **config_modifiers):
+    def duplicate(self, out_file, copy_parset_files=False, strip_parents_keys=False,
+                  **config_modifiers):
         s_out = self.copy()
         s_out.set_file(out_file)
         s_out['default_settings'] = ''

{libpipe-0.6 → libpipe-0.8}/libpipe/tools/nodetool.py

@@ -1,13 +1,11 @@
 #!/usr/bin/env python
 
-import os
 import json
+import os
 
 import click
 
-from libpipe import worker
-from libpipe import __version__
-
+from libpipe import __version__, worker
 
 t_file = click.Path(exists=True, dir_okay=False)
 
@@ -21,7 +19,8 @@ def main():
 @main.command('run_all')
 @click.argument('cmd_list', type=t_file)
 @click.option('--nodes_string', '-n', help='Range of nodes to use', type=str, default='localhost')
-@click.option('--max_concurrent', '-m', help='Maximum concurrent tasks on a node', type=int, default=5)
+@click.option('--max_concurrent', '-m', type=int, default=5,
+              help='Maximum concurrent tasks on a node')
 @click.option('--env_file', '-e', help='Environment source file', type=t_file)
 @click.option('--dry_run', '-d', help='Dry run, only print commands', is_flag=True)
 @click.option('--log_file', help='Log output to log_file', type=str, default=None)
@@ -31,7 +30,10 @@ def run_all(cmd_list, nodes_string, max_concurrent, env_file, dry_run, log_file)
                                   env_file=env_file, dry_run=dry_run, debug=dry_run)
 
     i_log_file = log_file
-    for i, cmd in enumerate(open(cmd_list).readlines()):
+    with open(cmd_list) as f:
+        cmds = f.readlines()
+
+    for i, cmd in enumerate(cmds):
         if log_file is not None:
             i_log_file = log_file + f'.{i + 1}'
         pool.add(cmd.strip(), output_file=i_log_file)
@@ -64,7 +66,7 @@ def run_on_all_nodes(cmd, nodes_string, env_file, dry_run):
 @click.option('--log_file', '-l', help='Log output to log_file', type=str, default=None)
 def init_job(name, cmd, target_node, log_file):
     ''' Add a job to a job list file NAME. If job file does not exist, create one. '''
-    if os.path.exists(name):        
+    if os.path.exists(name):
         with open(name) as json_file:
             jobs = json.load(json_file)
     else:
@@ -78,9 +80,11 @@ def init_job(name, cmd, target_node, log_file):
 
 @main.command('run_joblist')
 @click.argument('name', type=t_file)
-@click.option('--nodes_string', '-n', help='Range of nodes to use. By default use the target nodes in the joblist',
+@click.option('--nodes_string', '-n',
+              help='Range of nodes to use. By default use the target nodes in the joblist',
               type=str, default='joblist')
-@click.option('--max_concurrent', '-m', help='Maximum concurrent tasks on a node', type=int, default=5)
+@click.option('--max_concurrent', '-m', type=int, default=5,
+              help='Maximum concurrent tasks on a node')
 @click.option('--env_file', '-e', help='Environment source file', type=t_file)
 @click.option('--dry_run', '-d', help='Dry run, only print commands', is_flag=True)
 @click.option('--worker_name', '-w', help='Name of the worker', type=str, default='Run joblist')
@@ -90,7 +94,7 @@ def run_joblist(name, nodes_string, max_concurrent, env_file, dry_run, worker_na
         jobs = json.load(json_file)
 
     if nodes_string == 'joblist':
-        all_nodes = set([job['target_node'] for job in jobs['jobs'] if job['target_node'] is not None])
+        all_nodes = {job['target_node'] for job in jobs['jobs'] if job['target_node'] is not None}
         if len(all_nodes) == 0:
             all_nodes = ['localhost']
     else: