libpipe 0.5__tar.gz → 0.6__tar.gz

{libpipe-0.5 → libpipe-0.6}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: libpipe
-Version: 0.5
+Version: 0.6
 Summary: Pipeline orchestration and node management utilities
 License: GPL-3.0-or-later
+License-File: LICENSE
 Author: Florent Mertens
 Author-email: florent.mertens@gmail.com
 Requires-Python: >=3.10
@@ -12,10 +13,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: asyncssh (>=2.0)
 Requires-Dist: click (>=8.0)
 Requires-Dist: progressbar2 (>=3.40)
 Requires-Dist: toml (>=0.10)
+Project-URL: Repository, https://gitlab.com/flomertens/libpipe
 Description-Content-Type: text/markdown
 
 Libpipe

{libpipe-0.5 → libpipe-0.6}/libpipe/worker.py

@@ -6,6 +6,7 @@ import os
 import re
 import sys
 import time
+import shlex
 import socket
 import signal
 import getpass
@@ -70,8 +71,7 @@ class WorkerStatus(Enum):
     IDLE = 1
     COMMITED = 2
     RUNNING = 3
-    ERROR = 4
-    CLOSED = 5
+    CLOSED = 4
 
 
 class Process(object):
@@ -146,7 +146,7 @@ class Task(object):
 
     async def terminate(self):
         if self.process is not None and hasattr(self.process.process, 'pid'):
-            if self.process.client.host is localhost_shortname:
+            if self.process.client.host == localhost_shortname:
                 self.log(f'Killing local process PID {self.process.process.pid}\n', 'local', err=True)
                 try:
                     os.killpg(os.getpgid(self.process.process.pid), signal.SIGTERM)
@@ -169,12 +169,13 @@ class Task(object):
     async def log_stream(self, stream, host, err=False):
         while True:
             try:
-                async for line in stream:
-                    self.log(line, host, err=err)
+                line = await stream.readline()
             except ValueError:
                 self.log('(line too long)', host, err=True)
                 continue
-            break
+            if not line:
+                break
+            self.log(line, host, err=err)
 
     def close_log(self):
         if self.pid_filename and os.path.exists(self.pid_filename):
@@ -198,6 +199,11 @@ class Task(object):
 
             ret_status, err_msg = await worker.run(self)
 
+            if not ret_status:
+                # mark host temporarily bad so next retry likely uses a different host
+                host = worker.client.host
+                client_pool.bad_hosts_until[host] = time.time() + client_pool.host_cooldown_s
+
             if err_msg:
                 self.log(err_msg, worker.client.host, err=True)
 
@@ -230,10 +236,11 @@ class SSHClient(asyncssh.SSHClient):
 
 class Client(object):
 
-    def __init__(self, host, user=None, force_sync=True, password=None):
+    def __init__(self, host, user=None, force_sync=True, password=None, connect_timeout=15):
         self.host = host
         self.user = user
         self.password = password
+        self.connect_timeout = connect_timeout
         self.conn = None
         self.client = None
         self.starting = False
@@ -241,18 +248,45 @@ class Client(object):
         self.closing = False
         self.force_sync = force_sync
         self.creating_session = asyncio.Lock()
+        self.start_lock = asyncio.Lock()
 
     async def start(self):
-        if not self.starting:
+        # Only one concurrent connection attempt per Client
+        async with self.start_lock:
+            # If already connected, nothing to do
+            if self.connected():
+                self.started.set()
+                return
+
+            # Allow callers to wait on "started" for this attempt
+            if self.started.is_set():
+                self.started.clear()
+
             self.starting = True
             print(f'Starting client {self.host} ...')
+
             try:
-                self.conn, self.client = await asyncssh.create_connection(SSHClient, self.host, username=self.user,
-                                                                          password=self.password, agent_forwarding=True)
+                self.conn, self.client = await asyncssh.create_connection(
+                    SSHClient,
+                    self.host,
+                    username=self.user,
+                    password=self.password,
+                    agent_forwarding=True,
+                    connect_timeout=self.connect_timeout
+                )
                 self.conn.set_keepalive(120, 5)
+
             except Exception:
+                # IMPORTANT: reset state so a later retry actually retries
+                self.conn = None
+                self.client = None
+                self.starting = False
+                self.started.set()
                 raise
-            finally:
+
+            else:
+                # Connected successfully
+                self.starting = False
                 self.started.set()
 
     async def execute(self, task):
@@ -269,8 +303,10 @@ class Client(object):
 
         task.set_process(Process(self, process))
 
-        await task.log_stream(process.stdout, self.host)
-        await task.log_stream(process.stderr, self.host, err=True)
+        await asyncio.gather(
+            task.log_stream(process.stdout, self.host),
+            task.log_stream(process.stderr, self.host, err=True),
+        )
 
         process.channel.close()
         task.returncode = process.returncode
@@ -304,9 +340,10 @@ class LocalClient(object):
                                                         stderr=asyncio.subprocess.PIPE, preexec_fn=os.setpgrp)
         task.set_process(Process(self, process))
 
-        await task.log_stream(process.stdout, self.host)
-        await task.log_stream(process.stderr, self.host, err=True)
-
+        await asyncio.gather(
+            task.log_stream(process.stdout, self.host),
+            task.log_stream(process.stderr, self.host, err=True),
+        )
         task.returncode = await process.wait()
 
         if self.force_sync:
@@ -340,7 +377,12 @@ class Worker(object):
             await self.client.start()
         except Exception as exc:
             err_msg = f'Error starting client: {str(exc)}\n'
-            self.status = WorkerStatus.ERROR
+            # transient failure: allow task retry to pick a worker again
+            self.status = WorkerStatus.IDLE
+            try:
+                self.client.close()  # ensure clean slate
+            except Exception:
+                pass
             return False, err_msg
 
         await self.client.started.wait()
@@ -349,7 +391,11 @@ class Worker(object):
             await self.client.execute(task)
         except ConnectionError as exc:
             err_msg = f'Error connecting client: {str(exc)}\n'
-            self.status = WorkerStatus.ERROR
+            self.status = WorkerStatus.IDLE
+            try:
+                self.client.close()
+            except Exception:
+                pass
             return False, err_msg
         except asyncio.CancelledError:
             err_msg = f'Task {self.name} has been canceled\n'
@@ -360,6 +406,12 @@ class Worker(object):
             return False, err_msg
 
         err_msg = task.done_callback()
+        # err_msg = ''
+        # try:
+        #     loop = asyncio.get_running_loop()
+        #     err_msg = await loop.run_in_executor(None, task.done_callback)
+        # except Exception as exc:
+        #     err_msg = f'Error executing callback of task {self.name}: {exc}\n'
 
         self.status = WorkerStatus.IDLE
 
@@ -370,11 +422,42 @@ class Worker(object):
         self.client.close()
 
 
+class WorkerPoolReport:
+
+    def __init__(self, all_tasks):
+        self.all_tasks = list(all_tasks)
+        self.n_total = len(self.all_tasks)
+
+        # Buckets by status
+        self.success = [t for t in self.all_tasks if t.status == TaskStatus.SUCCESS]
+        self.failed = [t for t in self.all_tasks if t.status != TaskStatus.SUCCESS]
+
+        # Within SUCCESS, split by return code
+        self.success_rc0 = [t for t in self.success if t.returncode == 0]
+        self.success_nonzero = [t for t in self.success if t.returncode not in (None, 0)]
+        self.success_unknown_rc = [t for t in self.success if t.returncode is None]
+
+        # Counts
+        self.n_success = len(self.success)
+        self.n_failed = len(self.failed)
+        self.n_success_rc0 = len(self.success_rc0)
+        self.n_success_nonzero = len(self.success_nonzero)
+        self.n_success_unknown_rc = len(self.success_unknown_rc)
+
+        self.ok = (self.n_failed == 0 and self.n_success_nonzero == 0)
+
+    def host_of(self, task):
+        if task.process and task.process.client:
+            return getattr(task.process.client, "host", "?")
+        return task.run_on_host or "?"
+
+
 class WorkerPool(object):
 
     def __init__(self, hosts, name='Worker', max_tasks_per_worker=4,
                  env_source_file=None, user=None, max_time=None, force_sync=False,
-                 debug=False, dry_run=False, password=None):
+                 debug=False, dry_run=False, password=None, connect_timeout=15,
+                 host_cooldown_s=10):
         """Initiate a worker pool.
 
         Args:
@@ -383,6 +466,8 @@ class WorkerPool(object):
             max_tasks_per_worker (int, optional): Maximum number of tasks to execute concurrently on an host.
             env_source_file (str, optional): Name of a file to source before executing a task.
             user (str, optional): User to connect
+            connect_timeout (int, optional): Per-attempt SSH connection timeout, in seconds.
+            host_cooldown_s (int, optional): Time a just-failed host is avoided while a healthy one exists.
         """
         if asyncio.get_event_loop().is_closed():
             asyncio.set_event_loop(asyncio.new_event_loop())
@@ -399,13 +484,17 @@ class WorkerPool(object):
         self.dry_run = dry_run
         self.hosts = hosts
         self.futures = None
-        self.active_hosts = hosts
+        self.last_report = None
+        # host -> epoch time until which a just-failed host is avoided (steering only)
+        self.host_cooldown_s = host_cooldown_s
+        self.bad_hosts_until = {}
 
         for host in self.hosts:
             if host == localhost_shortname or host == 'localhost':
                 client = LocalClient(localhost_shortname, force_sync=force_sync)
             else:
-                client = Client(host, user, force_sync=force_sync, password=password)
+                client = Client(host, user, force_sync=force_sync, password=password,
+                                connect_timeout=connect_timeout)
 
             for i in range(max_tasks_per_worker):
                 worker = Worker(client, f'{client.host}:{i}')
@@ -423,15 +512,26 @@ class WorkerPool(object):
 
     async def get_worker(self, host=None):
         while True:
+            now = time.time()
+
+            # hosts not currently in cooldown
+            good_hosts = {h for h in self.hosts if self.bad_hosts_until.get(h, 0) <= now}
+
             for worker in self.workers:
-                if host is not None and worker.client.host != host:
-                    if host not in self.active_hosts:
-                        print(f'Requested run_on_host "{host}" not available anymore. Releasing constraint ...')
-                        host = None
+                w_host = worker.client.host
+
+                # respect run_on_host if requested
+                if host is not None and w_host != host:
+                    continue
+
+                # if we have at least one good host, avoid bad ones
+                if host is None and good_hosts and w_host not in good_hosts:
                     continue
+
                 if worker.status is WorkerStatus.IDLE:
                     worker.status = WorkerStatus.COMMITED
                     return worker
+
             await asyncio.sleep(0.1)
 
     def add(self, command, name=None, output_file=None, task_done_callback=None, done_callback=None, 
@@ -448,9 +548,11 @@ class WorkerPool(object):
             run_on_host = None
 
         if self.env_file:
-            command = f"bash -c '. {self.env_file}; {command}'"
+            script = f". {shlex.quote(self.env_file)}; {command}"
         else:
-            command = f"bash -c '{command}'"
+            script = command
+
+        command = f"bash -lc {shlex.quote(script)}"
 
         if name is None:
             name = f'T{len(self.tasks)}'
@@ -480,26 +582,16 @@ class WorkerPool(object):
                 n_todo = len(tasks[TaskStatus.WAITING]) + len(tasks[TaskStatus.RUNNING])
                 self.pbar.update(n_done)
 
-                self.active_hosts = set(w.client.host for w in self.workers if w.status != WorkerStatus.ERROR)
-                n_active_workers = 0
-
-                for worker in self.workers:
-                    n_active_workers += worker.status != WorkerStatus.ERROR
-
-                    if worker.status == WorkerStatus.RUNNING and self.max_time is not None:
-                        running_time = time.time() - worker.execute_start_time
-                        if running_time > (self.max_time + update_interval):
-                            task = worker.running_task
-                            task.timed_out = True
-                            task.log(f'Task timeout: waited for {running_time:.2f} s', worker.client.host, err=True)
-                            task.log(f'Command was: {task.command}', worker.client.host, err=True)
-                            await task.terminate()
-
-                if n_active_workers == 0:
-                    secho("No active workers, canceling all remaining tasks ...", fg='red')
-                    if self.futures is not None:
-                        for future in self.futures[::-1]:
-                            future.cancel()
+                if self.max_time is not None:
+                    for worker in self.workers:
+                        if worker.status == WorkerStatus.RUNNING:
+                            running_time = time.time() - worker.execute_start_time
+                            if running_time > (self.max_time + update_interval):
+                                task = worker.running_task
+                                task.timed_out = True
+                                task.log(f'Task timeout: waited for {running_time:.2f} s', worker.client.host, err=True)
+                                task.log(f'Command was: {task.command}', worker.client.host, err=True)
+                                await task.terminate()
 
         except asyncio.CancelledError:
             pass
@@ -530,7 +622,10 @@ class WorkerPool(object):
 
         await asyncio.gather(*self.futures, return_exceptions=False)
 
-    def execute(self):
+    def get_report(self):
+        return self.last_report
+
+    def execute(self, print_summary=False):
         if not self.tasks:
             self.pbar.finish()
             return [], []
@@ -556,5 +651,26 @@ class WorkerPool(object):
         self.pbar.update(n_success)
         self.pbar.finish(dirty=n_success != len(self.tasks))
 
+        self.last_report = WorkerPoolReport(self.tasks)
+        report = self.last_report
+
+        if print_summary:
+            print("\n=== WorkerPool summary ===")
+            print(f"Successful tasks (rc=0):        {report.n_success_rc0} / {report.n_total}")
+            print(f"Completed with non-zero rc:     {report.n_success_nonzero} / {report.n_total}")
+            print(f"Failed tasks (exception/kill):  {report.n_failed} / {report.n_total}")
+
+            if report.success_nonzero:
+                print("\nTasks completed with non-zero return code:")
+                for t in report.success_nonzero:
+                    print(f" - {t.name} | host={report.host_of(t)} | rc={t.returncode}")
+
+            if report.failed:
+                print("\nFailed tasks:")
+                for t in report.failed:
+                    print(f" - {t.name} | status={t.status.name} | host={report.host_of(t)} | rc={t.returncode}")
+
+            print("==========================\n")
+
         return tasks[TaskStatus.SUCCESS], tasks[TaskStatus.ERROR] + tasks[TaskStatus.CANCELED] \
             + tasks[TaskStatus.WAITING] + tasks[TaskStatus.RUNNING]

{libpipe-0.5 → libpipe-0.6}/pyproject.toml

@@ -4,14 +4,13 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "libpipe"
-version = "0.5"
+version = "0.6"
 description = "Pipeline orchestration and node management utilities"
 readme = "README.md"
 license = { text = "GPL-3.0-or-later" }
 authors = [
     { name = "Florent Mertens", email = "florent.mertens@gmail.com" }
 ]
-repository = "https://gitlab.com/flomertens/libpipe"
 requires-python = ">=3.10"
 
 dependencies = [
@@ -21,9 +20,15 @@ dependencies = [
     "click>=8.0",
 ]
 
+[project.urls]
+Repository = "https://gitlab.com/flomertens/libpipe"
+
 [project.scripts]
 nodetool = "libpipe.tools.nodetool:main"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0"
 pytest-cov = "^5.0"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

libpipe-0.5/libpipe/tests/settings_test.toml

@@ -1,73 +0,0 @@
-data_dir = '.'
-default_settings = ''
-debug = false
-longitude = 6.57
-
-[worker]
-nodes = 'localhost'
-max_concurrent = 4
-env_file = ''
-
-[merge_ms]
-apply_aoflagger = true
-data_col = 'CORRECTED_DATA'
-obs_id_out_suffix = 'flagged'
-blmin = 0
-blmax = 1000
-flag_baselines = ''
-avg_timestep = 1
-time_start_index = 0
-time_end_index = 0
-
-[ssins]
-apply_ssins = true
-n_time_avg = [1, 4, 8]
-nsigmas.1 = [8, 5, 5]
-nsigmas.4 = [8, 5, 5]
-nsigmas.8 = [8, 5, 5]
-percentage_freq_full_flag = 0.4
-percentage_time_full_flag = 0.4
-time_freq_threshold = 0.6
-baseline_threshold = 0.75
-snapshot_threshold = 0.5
-
-[image]
-name = 'default'
-umin = 50
-umax = 250
-scale = '0.5amin'
-size = '512 512'
-data_col = 'CORRECTED_DATA'
-channels_out = 'all'
-time_start_index = 0
-time_end_index = 0
-lst_bins = []
-stokes = 'IV'
-split_even_odd = true
-weight = 'natural'
-
-[vis_cube]
-fov = 4
-umin = 50
-umax = 500
-win_fct = '(tukey,0.2)'
-
-[vis_to_sph]
-obs_id_out_suffix = 'sph'
-pre_flag = ''
-
-[combine]
-obs_id_out_prefix = '001'
-scale_with_noise = false
-weights_mode = 'full'
-inhomogeneous = false
-pre_flag = ''
-
-[power_spectra]
-eor_bin_list = ''
-ps_config = ''
-flagger = ''
-
-[gpr]
-config_i = ''
-config_v = ''

libpipe-0.5/libpipe/tests/test_settings.py

@@ -1,305 +0,0 @@
-import os
-import pathlib
-from unittest import mock
-
-from libpipe import settings
-
-
-class Settings(settings.BaseSettings):
-
-    DEFAULT_SETTINGS = pathlib.Path(__file__).absolute().parent / 'settings_test.toml'
-
-
-def test_settings(tmp_path):
-    c = Settings(None, {'vis': {'umin': 20, 'umax': 40, 'list': ['a', 'b']}, 'data_path': 'some_path'})
-
-    assert c.vis.umin == 20
-    assert c.vis.umax == 40
-    assert c.data_path == 'some_path'
-    assert c['vis'] == {'umin': 20, 'umax': 40, 'list': ['a', 'b']}
-    assert c.vis.list == ['a', 'b']
-    assert c.vis.list[1] == 'b'
-
-    c.image = {'lmin': 100, 'data_col': 'DATA'}
-    c.obs_ids = [1, 2, 3, 4]
-
-    assert c.image.lmin == 100
-    assert c.image.data_col == 'DATA'
-    assert c.obs_ids == [1, 2, 3, 4]
-    assert c.obs_ids[0] == 1
-
-    s = '''
-    db_path = 'some_other_path'
-
-    [vis]
-    list = ['c', 'd']
-
-    [worker]
-    nodes = 'node1[0-1][1-5]'
-    concurrent = 2
-    '''
-
-    filename = tmp_path / 'test.toml'
-    with open(str(filename), 'w') as f:
-        f.write(s)
-
-    c2 = Settings.load(str(filename))
-
-    assert c2.vis.list == ['c', 'd']
-    assert c2.worker.nodes == 'node1[0-1][1-5]'
-
-    c3 = c2.copy()
-    c3.worker.concurrent = 3
-    c3.new = 'a'
-
-    assert c2.worker.concurrent == 2
-    assert c3.worker.concurrent == 3
-    assert hasattr(c3, 'new')
-    assert not hasattr(c2, 'new')
-
-    c4 = c + c2
-
-    assert c4.vis.umin == 20
-    assert c4.vis.umax == 40
-    assert c4.vis.list == ['c', 'd']
-    assert c4.worker.nodes == 'node1[0-1][1-5]'
-    assert c4.worker.concurrent == 2
-
-    c5 = c + c2 + c3
-
-    assert c5.worker.concurrent == 3
-    assert c5.new == 'a'
-
-    c = Settings.load_from_string(s)
-
-    assert c.vis.list == ['c', 'd']
-    assert c.worker.nodes == 'node1[0-1][1-5]'
-
-    c = Settings.load_from_string(['db_path="a"', 'vis.c=2'])
-
-    assert c.db_path == 'a'
-    assert c.vis.c == 2
-
-
-def test_default_settings(tmp_path):
-    s = '''
-    db_path = 'some_path'
-    default_settings = ''
-
-    [vis_cube]
-    umax = 200
-    win_fct = 'hann'
-
-    [worker]
-    nodes = 'node1[0-1][1-5]'
-    concurrent = 2
-    '''
-
-    filename = tmp_path / 'config.toml'
-    with open(str(filename), 'w') as f:
-        f.write(s)
-
-    c = Settings.load_with_defaults(filename)
-
-    assert c.db_path == 'some_path'
-    assert c.default_settings == ''
-    assert c.vis_cube.umax == 200
-    assert c.vis_cube.win_fct == 'hann'
-    assert c.worker.nodes == 'node1[0-1][1-5]'
-    assert c.worker.concurrent == 2
-    assert c.data_dir == '.'
-    assert c.vis_cube.umin == 50
-
-    s = f'''
-    db_path = 'some_other_path'
-    default_settings = '{str(filename)}'
-
-    [vis_cube]
-    umax = 400
-
-    [image]
-    umax = 5000
-
-    [worker]
-    concurrent = 2
-    '''
-
-    filename2 = tmp_path / 'config2.toml'
-    with open(str(filename2), 'w') as f:
-        f.write(s)
-
-    with mock.patch.object(Settings, 'DEFAULT_SETTINGS', filename):
-        c = Settings.load_with_defaults(filename2)
-
-        assert c.db_path == 'some_other_path'
-        assert os.path.abspath(c.default_settings) == os.path.abspath(filename)
-        assert c.vis_cube.umax == 400
-        assert c.vis_cube.win_fct == 'hann'
-        assert c.worker.nodes == 'node1[0-1][1-5]'
-        assert c.worker.concurrent == 2
-        assert c.image.umax == 5000
-
-        c.save(strip_parents_keys=True)
-
-        c = Settings.load_with_defaults(filename2)
-
-        assert c.db_path == 'some_other_path'
-        assert os.path.abspath(c.default_settings) == os.path.abspath(filename)
-        assert c.vis_cube.umax == 400
-        assert c.vis_cube.win_fct == 'hann'
-        assert c.worker.nodes == 'node1[0-1][1-5]'
-        assert c.worker.concurrent == 2
-        assert c.image.umax == 5000
-
-        c = Settings.load(filename2)
-
-        print(open(filename2).readlines())
-        assert c.db_path == 'some_other_path'
-        assert os.path.abspath(c.default_settings) == os.path.abspath(filename)
-        assert c.image.umax == 5000
-        assert c.vis_cube.umax == 400
-        assert not hasattr(c, 'worker')
-        assert not hasattr(c.image, 'win_fct')
-
-
-def test_duplicate(tmp_path):
-    s = '''
-    db_path = 'some_path'
-    default_settings = ''
-
-    [worker]
-    nodes = 'node1[0-1][1-5]'
-    concurrent = 2
-    some_parset = 'config/some_file.parset'
-    '''
-    os.chdir(tmp_path)
-
-    filename = tmp_path / 'config.toml'
-    with open(str(filename), 'w') as f:
-        f.write(s)
-
-    os.makedirs(tmp_path / 'config')
-
-    (tmp_path / 'config' / 'some_file.parset').touch()
-
-    s = Settings.load_with_defaults(filename)
-
-    new_file = tmp_path / 'new_config.toml'
-
-    ns = s.duplicate(new_file)
-
-    assert ns.db_path == 'some_path'
-    assert ns.worker.some_parset == 'config/some_file.parset'
-    assert not (tmp_path / 'new_dir' / 'config' / 'some_file.parset').is_file()
-
-    new_file = tmp_path / 'new_dir' / 'new_config.toml'
-
-    ns = s.duplicate(new_file, copy_parset_files=False)
-
-    assert ns.db_path == 'some_path'
-    assert ns.worker.some_parset == str(tmp_path / 'config' / 'some_file.parset')
-
-    new_file = tmp_path / 'new_dir' / 'new_config.toml'
-
-    ns = s.duplicate(new_file, copy_parset_files=True)
-
-    assert ns.db_path == 'some_path'
-    assert ns.worker.some_parset == 'config/some_file.parset'
-    assert (tmp_path / 'new_dir' / 'config' / 'some_file.parset').is_file()
-
-    ns = s.duplicate(new_file, update_relative_path=True, **{'db_path': 'added_path', 'worker': {'concurrent': 4}})
-
-    assert ns.db_path == 'added_path'
-    assert ns.worker.concurrent == 4
-
-
-def test_get_path(tmp_path):
-    c = Settings(None, {'data_dir': 'a'})
-    assert c.get_path('data_dir') == str(pathlib.Path.cwd() / 'a')
-
-    c = Settings('/some/file.toml', {'data_dir': 'a'})
-    assert c.get_path('data_dir') == '/some/a'
-
-    c = Settings(None, {'data_dir': '/a'})
-    assert c.get_path('data_dir') == '/a'
-
-    c = Settings('/some/file.toml', {'data_dir': '/a'})
-    assert c.get_path('data_dir') == '/a'
-
-    c = Settings.load_from_string('a.b.data_dir = "/some_dir"')
-    assert c.get_path('a.b.data_dir') == '/some_dir'
-
-    c = Settings.load_from_string('a.b.data_dir = "some_dir"')
-    assert c.get_path('a.b.data_dir') == str(pathlib.Path.cwd() / 'some_dir')
-
-    c = Settings.load_from_string('a.b.c.data_dir = "/some_dir"')
-    assert c.a.b.get_path('c.data_dir') == '/some_dir'
-
-
-def test_default_type(tmp_path):
-    s = '''
-    [type1]
-    a = 200
-    b = 'h'
-
-    [type2]
-    c = [1, 2, 3]
-    d.k = 2
-
-    [type3]
-    pa.b = 3
-    pa.c = '2'
-    v = [1]
-
-    [type4]
-    a = 0
-
-    [type5]
-    u = 0
-    '''
-
-    filename = tmp_path / 'default.toml'
-    with open(str(filename), 'w') as f:
-        f.write(s)
-
-    s = '''
-    [action1]
-    type = 'type1'
-    a = 100
-
-    [action2]
-    type = 'type1'
-    c = 200
-
-    [action3]
-    type = 'type2'
-    c = [2, 3, 1, 2]
-    d = 4
-
-    [type3]
-    pa.a = 1
-    pa.c = '5'
-
-    [type5]
-
-    '''
-
-    filename2 = tmp_path / 'config.toml'
-    with open(str(filename2), 'w') as f:
-        f.write(s)
-
-    with mock.patch.object(Settings, 'DEFAULT_SETTINGS', filename):
-        c = Settings.load_with_defaults(filename2, with_type_keyword=True)
-
-    assert c.action1.a == 100
-    assert c.action1.b == 'h'
-    assert c.action2.a == 200
-    assert c.action2.c == 200
-    assert c.action3.d == 4
-    assert c.action3.c == [2, 3, 1, 2]
-    assert c.type3.pa.a == 1
-    assert c.type3.pa.b == 3
-    assert c.type3.pa.c == '5'
-    assert c.type3.v == [1]
-    assert 'type4' not in c
-    assert 'type5' in c

libpipe-0.5/libpipe/tests/test_worker.py

@@ -1,274 +0,0 @@
-import os
-import string
-import random
-import functools
-
-from unittest import mock
-
-from libpipe import worker
-
-
-def test_worker(tmp_path):
-    w = worker.WorkerPool([worker.localhost_shortname, 'test_node'], 'Test', max_tasks_per_worker=5)
-
-    assert len(w.workers) == 10
-    assert isinstance(w.workers[0].client, worker.LocalClient)
-    assert isinstance(w.workers[6].client, worker.Client)
-
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test')
-
-    for i in range(20):
-        w.add('touch %s/t%d' % (str(tmp_path), i))
-
-    success, errors = w.execute()
-
-    assert len(success) == 20
-    assert len(errors) == 0
-
-    for i in range(20):
-        assert os.path.isfile('%s/t%d' % (str(tmp_path), i))
-
-
-def test_worker_log(tmp_path):
-    w = worker.WorkerPool(['wrong_server', worker.localhost_shortname], 'Test')
-
-    for i in range(20):
-        w.add(f'echo Task {i}', output_file=str(tmp_path / f't{i}.log'))
-        print(str(tmp_path / f't{i}.log'))
-
-    success, errors = w.execute()
-
-    assert len(success) == 20
-    assert len(errors) == 0
-
-    for i, task in enumerate(success):
-        assert os.path.isfile(tmp_path / f't{i}.log')
-        with open(tmp_path / f't{i}.log') as f:
-            s = f.readlines()
-        assert len(s) == 10 or len(s) == 8
-        assert s[0].startswith('# Logging starting at ')
-        assert s[1] == f"# Input command: bash -c 'echo Task {i}'\n"
-        if len(s) == 8:
-            assert s[2] == f'Starting on host {task.process.client.host} with PID {task.process.process.pid}\n'
-            assert s[4] == f'Task {i}\n'
-        if len(s) == 10:
-            assert s[2].startswith('Error')
-            assert s[3] == f'Will retry 1 / 3 ...\n'
-            assert s[4] == f'Starting on host {task.process.client.host} with PID {task.process.process.pid}\n'
-            assert s[6] == f'Task {i}\n'
-        assert s[-1].startswith('# Logging stopped at ')
-
-
-def test_pidfile(tmp_path):
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test')
-
-    for i in range(20):
-        w.add(f'echo Task {i}; exit {i}', pid_filename=f't{i}.pid', keep_pid_file_on_error=True)
-
-    success, errors = w.execute()
-
-    assert len(success) == 20
-    assert len(errors) == 0
-
-    for task in success:
-        assert os.path.exists(task.pid_filename) != (task.returncode == 0)
-        if task.returncode > 0:
-            with open(task.pid_filename) as fd:
-                s = fd.read()
-            assert s == f'{task.process.client.host} {task.process.process.pid}'
-
-
-def test_worker_wrong_server(tmp_path):
-    w = worker.WorkerPool(['wrong_server'] * 6, 'Test')
-
-    for i in range(20):
-        w.add(f'echo Task {i}')
-
-    success, errors = w.execute()
-
-    assert len(success) == 0
-    assert len(errors) == 20
-
-
-def test_worker_env_file(tmp_path):
-    with open(tmp_path / 'env', 'w') as f:
-        f.write('export WORKER_TEST=test_env')
-
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test', env_source_file=str(tmp_path / 'env'))
-
-    os.listdir(tmp_path)
-
-    for i in range(20):
-        w.add('echo env:${WORKER_TEST}; touch %s/t_${WORKER_TEST}_%d' % (str(tmp_path), i))
-
-    success, errors = w.execute()
-
-    assert len(success) == 20
-    assert len(errors) == 0
-
-    for i in range(20):
-        assert os.path.isfile('%s/t_test_env_%d' % (str(tmp_path), i))
-
-
-def test_worker_retcode():
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test')
-
-    w.add('set -e; cat non_existing_file', name=f't0')
-
-    success, errors = w.execute()
-
-    assert len(success) == 1
-    assert len(errors) == 0
-    assert success[0].returncode == 1
-
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test')
-
-    for i in range(10):
-        w.add(f'echo Task {i}; exit {i}', name=f't{i}')
-
-    success, errors = w.execute()
-
-    print([k.process.client for k in success])
-
-    assert len(success) == 10
-    assert len(errors) == 0
-    assert [f't{k.returncode}' for k in success] == [k.name for k in success]
-
-
-def test_timeout():
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test', max_time=2)
-
-    w.add('echo 0')
-
-    success, errors = w.execute()
-
-    assert len(success) == 1
-    assert len(errors) == 0
-
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test', max_time=1)
-
-    w.add('sleep 100')
-
-    success, errors = w.execute()
-
-    # Currently falling. Need to be fixed
-    assert len(success) == 0
-    assert len(errors) == 1
-
-
-def test_done_callback():
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test', max_time=2)
-
-    done = False
-
-    def callback():
-        nonlocal done
-        done = True
-
-    w.add('echo 0', done_callback=callback)
-
-    success, errors = w.execute()
-
-    assert len(success) == 1
-    assert len(errors) == 0
-    assert done
-
-
-def test_task_done_callback():
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test', max_time=2)
-
-    done = 0
-
-    def callback(task):
-        print(task)
-        nonlocal done
-        assert isinstance(task, worker.Task)
-        if task.name == 'Error':
-            assert task.returncode == 1
-        else:    
-            assert task.returncode == 0
-        done += 1
-
-    def callback_with_extra(name, task):
-        assert name == 'test'
-        callback(task)
-
-    w.add('echo 0', task_done_callback=callback, name='Success1')
-    w.add('echo 0', task_done_callback=callback, name='Success2')
-    w.add('echo 3; cat non_existing_file', task_done_callback=functools.partial(callback_with_extra, 'test'), name='Error')
-    w.add('echo 4;', task_done_callback=functools.partial(callback_with_extra, 'test_err'), name='CBK Error')
-
-    success, errors = w.execute()
-
-    assert len(success) == 4
-    assert len(errors) == 0
-    assert done == 3
-
-
-class FakeSSHClient(worker.LocalClient):
-
-    def __init__(self, host, user=None, force_sync=True, password=None):
-        worker.LocalClient.__init__(self, force_sync=force_sync)
-        self.host = host
-
-    async def execute(self, task):
-        await worker.LocalClient.execute(self, task)
-        task.remote_host = self.host
-
-
-def test_run_on_host():
-    with mock.patch.object(worker, 'Client', FakeSSHClient):
-        n_hosts = 10
-        hosts = [f'host{k}' for k in range(n_hosts)]
-        w = worker.WorkerPool(hosts, 'Test', max_time=2)
-
-        for i in range(20 * n_hosts):
-            w.add('echo 0', run_on_host=random.choice(hosts))
-        w.add('echo 0', run_on_host='wrong_host')
-        task_last = w.tasks[-1]
-
-        success, errors = w.execute()
-
-        assert len(errors) == 0
-
-        for task in success:
-            if task == task_last:
-                assert task.run_on_host == None
-            else:
-                assert task.remote_host == task.run_on_host
-
-
-def test_line_too_long(tmp_path):
-    w = worker.WorkerPool([worker.localhost_shortname], 'Test')
-
-    N = 100000
-    s = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(N))
-
-    log_file = str(tmp_path / f'text.log')
-
-    w.add(f'echo Start; echo Other task; echo Done')
-    w.add(f'echo Start; echo {s}; echo Done', output_file=log_file)
-    w.add(f'echo Start; echo Other task; echo Done')
-
-    success, errors = w.execute()
-
-    has_long_line = False
-    has_done = False
-
-    with open(log_file) as f:
-        for line in f.readlines():
-            if '(line too long)' in line:
-                has_long_line = True
-            if 'Done' in line:
-                has_done = True
-
-    assert has_long_line
-    assert has_done
-    assert len(errors) == 0
-    assert len(success) == 3
-
-
-def test_client():
-    client = worker.Client('localhost')
-
-    task = worker.Task('Check size', 'du -hs *')