hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/scheduler.py

@@ -1,323 +0,0 @@
-"""`hpcflow.scheduler.py`"""
-
-from datetime import datetime
-from subprocess import run, PIPE
-from pprint import pprint
-
-from hpcflow.config import Config as CONFIG
-from hpcflow._version import __version__
-
-
-class Scheduler(object):
-
-    options = None
-    output_dir = None
-    error_dir = None
-
-    def __repr__(self):
-        out = ('{}('
-               'options={!r}, '
-               'output_dir={!r}, '
-               'error_dir={!r}'
-               ')').format(
-            self.__class__.__name__,
-            self.options,
-            self.output_dir,
-            self.error_dir,
-        )
-        return out
-
-    def __init__(self, options=None, output_dir=None, error_dir=None):
-
-        self.output_dir = output_dir or CONFIG.get('default_output_dir')
-        self.error_dir = error_dir or CONFIG.get('default_error_dir')
-        self.options = options
-
-
-class SunGridEngine(Scheduler):
-
-    _NAME = 'sge'
-    SHEBANG = '#!/bin/bash --login'
-
-    STATS_DELIM = '==============================================================\n'
-
-    # Options that determine how to set the output/error directories:
-    STDOUT_OPT = 'o'
-    STDERR_OPT = 'e'
-    STDOUT_OPT_FMT = '{}/'
-    STDERR_OPT_FMT = '{}/'
-
-    # Required options to ensure the job scripts work with hpcflow:
-    REQ_OPT = ['cwd']
-    REQ_PARAMETRISED_OPT = {}
-
-    ALLOWED_USER_OPTS = [
-        'pe',       # Parallel environment
-        'l',        # Resource request
-        'tc',       # Max running tasks
-        'P',        # Project name (e.g. to which account jobs are accounted against)
-    ]
-
-    NUM_CORES_VAR = '$NSLOTS'
-
-    def __init__(self, options=None, output_dir=None, error_dir=None):
-
-        for i in options:
-            if i not in SunGridEngine.ALLOWED_USER_OPTS:
-                msg = ('Option "{}" is not allowed for scheduler "{}". Allowed options '
-                       'are: {}.')
-                raise ValueError(
-                    msg.format(i, SunGridEngine._NAME, SunGridEngine.ALLOWED_USER_OPTS))
-
-        super().__init__(options=options, output_dir=output_dir, error_dir=error_dir)
-
-    def get_formatted_options(self, max_num_tasks, task_step_size, user_opt=True,
-                              name=None):
-
-        opts = ['#$ -{}'.format(i) for i in SunGridEngine.REQ_OPT]
-        opts.append('#$ -{} {}'.format(
-            SunGridEngine.STDOUT_OPT,
-            SunGridEngine.STDOUT_OPT_FMT.format(self.output_dir)),
-        )
-        opts.append('#$ -{} {}'.format(
-            SunGridEngine.STDERR_OPT,
-            SunGridEngine.STDERR_OPT_FMT.format(self.error_dir)),
-        )
-        opts += ['#$ -{} {}'.format(i, j)
-                 for i, j in SunGridEngine.REQ_PARAMETRISED_OPT.items()]
-
-        if name:
-            opts += [f'#$ -N {name}']
-
-        if user_opt:
-            opts += ['#$ -{} {}'.format(k, v).strip()
-                     for k, v in sorted(self.options.items())]
-
-        opts += ['', '#$ -t 1-{}:{}'.format(max_num_tasks, task_step_size)]
-
-        return opts
-
-    def write_jobscript(self, dir_path, workflow_directory, command_group_order,
-                        max_num_tasks, task_step_size, environment, archive,
-                        alternate_scratch_dir, command_group_submission_id, name):
-        """Write the jobscript.
-
-        Parameters
-        ----------
-        archive : bool
-
-        """
-
-        js_ext = CONFIG.get('jobscript_ext')
-        js_name = 'js_{}'.format(command_group_order)
-        js_fn = js_name + js_ext
-        js_path = dir_path.joinpath(js_fn)
-
-        cmd_name = 'cmd_{}'.format(command_group_order)
-        cmd_fn = cmd_name + js_ext
-
-        submit_dir_relative = dir_path.relative_to(workflow_directory).as_posix()
-
-        wk_dirs_path = ('${{ITER_DIR}}/working_dirs_{}{}').format(
-            command_group_order, CONFIG.get('working_dirs_file_ext'))
-
-        dt_stamp = datetime.now().strftime(r'%Y.%m.%d at %H:%M:%S')
-        about_msg = ['# --- jobscript generated by `hpcflow` (version: {}) '
-                     'on {} ---'.format(__version__, dt_stamp)]
-
-        define_dirs_A = [
-            'ROOT_DIR=`pwd`',
-            'SUBMIT_DIR=$ROOT_DIR/{}'.format(submit_dir_relative),
-            'ITER_DIR=$SUBMIT_DIR/iter_$ITER_IDX',
-            'LOG_PATH=$ITER_DIR/log_{}.$SGE_TASK_ID'.format(command_group_order),
-            'TASK_IDX=$((($SGE_TASK_ID - 1)/{}))'.format(task_step_size),
-        ]
-
-        write_cmd_exec = [(
-            f'hpcflow write-runtime-files --directory $ROOT_DIR '
-            f'--config-dir {CONFIG.get("config_dir")} '
-            f'{command_group_submission_id} $TASK_IDX $ITER_IDX > $LOG_PATH 2>&1'
-        )]
-
-        define_dirs_B = [
-            'INPUTS_DIR_REL=`sed -n "${{SGE_TASK_ID}}p" {}`'.format(wk_dirs_path),
-            'INPUTS_DIR=$ROOT_DIR/$INPUTS_DIR_REL',
-        ]
-
-        if alternate_scratch_dir:
-            alt_scratch_exc_path = '$ITER_DIR/{}_{}_$TASK_IDX{}'.format(
-                CONFIG.get('alt_scratch_exc_file'),
-                command_group_order,
-                CONFIG.get('alt_scratch_exc_file_ext'),
-            )
-            define_dirs_B.append('ALT_SCRATCH_EXC=' + alt_scratch_exc_path)
-            in_dir_scratch = 'INPUTS_DIR_SCRATCH={}/$INPUTS_DIR_REL'.format(
-                alternate_scratch_dir)
-            copy_to_alt = [
-                ('rsync -avviz --exclude-from="${ALT_SCRATCH_EXC}" '
-                 '$INPUTS_DIR/ $INPUTS_DIR_SCRATCH >> $LOG_PATH 2>&1'),
-                '',
-            ]
-            move_from_alt = [
-                '',
-                ('rsync -avviz $INPUTS_DIR_SCRATCH/ $INPUTS_DIR --remove-source-files'
-                 ' >> $LOG_PATH 2>&1'),
-                '',
-            ]
-        else:
-            in_dir_scratch = 'INPUTS_DIR_SCRATCH=$INPUTS_DIR'
-            copy_to_alt = []
-            move_from_alt = []
-
-        define_dirs_B.append(in_dir_scratch)
-
-        log_stuff = [
-            r'printf "Jobscript variables:\n" >> $LOG_PATH 2>&1',
-            r'printf "ITER_IDX:\t ${ITER_IDX}\n" >> $LOG_PATH 2>&1',
-            r'printf "ROOT_DIR:\t ${ROOT_DIR}\n" >> $LOG_PATH 2>&1',
-            r'printf "SUBMIT_DIR:\t ${SUBMIT_DIR}\n" >> $LOG_PATH 2>&1',
-            r'printf "ITER_DIR:\t ${ITER_DIR}\n" >> $LOG_PATH 2>&1',
-            r'printf "LOG_PATH:\t ${LOG_PATH}\n" >> $LOG_PATH 2>&1',
-            r'printf "SGE_TASK_ID:\t ${SGE_TASK_ID}\n" >> $LOG_PATH 2>&1',
-            r'printf "TASK_IDX:\t ${TASK_IDX}\n" >> $LOG_PATH 2>&1',
-            r'printf "INPUTS_DIR_REL:\t ${INPUTS_DIR_REL}\n" >> $LOG_PATH 2>&1',
-            r'printf "INPUTS_DIR:\t ${INPUTS_DIR}\n" >> $LOG_PATH 2>&1',
-            r'printf "INPUTS_DIR_SCRATCH:\t ${INPUTS_DIR_SCRATCH}\n" >> $LOG_PATH 2>&1',
-        ]
-
-        if alternate_scratch_dir:
-            log_stuff.append(
-                r'printf "ALT_SCRATCH_EXC:\t ${ALT_SCRATCH_EXC}\n" >> $LOG_PATH 2>&1',
-            )
-
-        log_stuff.append(r'printf "\n" >> $LOG_PATH 2>&1')
-
-        if environment:
-            loads = [''] + environment + ['']
-        else:
-            loads = []
-
-        set_task_args = (f'--directory $ROOT_DIR '
-                         f'--config-dir {CONFIG.get("config_dir")} '
-                         f'{command_group_submission_id} '
-                         f'$TASK_IDX $ITER_IDX >> $LOG_PATH 2>&1')
-        cmd_exec = [
-            f'hpcflow set-task-start {set_task_args}',
-            f'',
-            f'cd $INPUTS_DIR_SCRATCH',
-            f'. $SUBMIT_DIR/{cmd_fn}',
-            f'',
-            f'hpcflow set-task-end {set_task_args}',
-        ]
-
-        arch_lns = []
-        if archive:
-            arch_lns = [
-                (f'hpcflow archive --directory $ROOT_DIR '
-                 f'--config-dir {CONFIG.get("config_dir")} '
-                 f'{command_group_submission_id} '
-                 f'$TASK_IDX $ITER_IDX >> $LOG_PATH 2>&1'),
-                ''
-            ]
-
-        js_lines = ([SunGridEngine.SHEBANG, ''] +
-                    about_msg + [''] +
-                    self.get_formatted_options(max_num_tasks, task_step_size, name=name) +
-                    [''] +
-                    define_dirs_A + [''] +
-                    write_cmd_exec + [''] +
-                    define_dirs_B + [''] +
-                    log_stuff + [''] +
-                    loads + [''] +
-                    copy_to_alt +
-                    cmd_exec +
-                    move_from_alt +
-                    arch_lns)
-
-        # Write jobscript:
-        with js_path.open('w') as handle:
-            handle.write('\n'.join(js_lines))
-
-        return js_path
-
-    def write_stats_jobscript(self, dir_path, workflow_directory, command_group_order,
-                              max_num_tasks, task_step_size, command_group_submission_id,
-                              name):
-
-        js_ext = CONFIG.get('jobscript_ext')
-        js_name = 'st_{}'.format(command_group_order)
-        js_fn = js_name + js_ext
-        js_path = dir_path.joinpath(js_fn)
-
-        dt_stamp = datetime.now().strftime(r'%Y.%m.%d at %H:%M:%S')
-        about_msg = ['# --- jobscript generated by `hpcflow` (version: {}) '
-                     'on {} ---'.format(__version__, dt_stamp)]
-
-        submit_dir_relative = dir_path.relative_to(workflow_directory).as_posix()
-
-        define_dirs = [
-            'ROOT_DIR=`pwd`',
-            'SUBMIT_DIR=$ROOT_DIR/{}'.format(submit_dir_relative),
-            'ITER_DIR=$SUBMIT_DIR/iter_$ITER_IDX',
-            'LOG_PATH=$ITER_DIR/log_{}.$SGE_TASK_ID'.format(command_group_order),
-            'TASK_IDX=$((($SGE_TASK_ID - 1)/{}))'.format(task_step_size),
-        ]
-
-        log_stuff = [
-            r'printf "Jobscript variables:\n" >> $LOG_PATH 2>&1',
-            r'printf "ITER_IDX:\t ${ITER_IDX}\n" >> $LOG_PATH 2>&1',
-            r'printf "ROOT_DIR:\t ${ROOT_DIR}\n" >> $LOG_PATH 2>&1',
-            r'printf "SUBMIT_DIR:\t ${SUBMIT_DIR}\n" >> $LOG_PATH 2>&1',
-            r'printf "ITER_DIR:\t ${ITER_DIR}\n" >> $LOG_PATH 2>&1',
-            r'printf "LOG_PATH:\t ${LOG_PATH}\n" >> $LOG_PATH 2>&1',
-            r'printf "SGE_TASK_ID:\t ${SGE_TASK_ID}\n" >> $LOG_PATH 2>&1',
-            r'printf "TASK_IDX:\t ${TASK_IDX}\n" >> $LOG_PATH 2>&1',
-        ]
-
-        cmd_exec = [(
-            f'hpcflow get-scheduler-stats --directory $ROOT_DIR '
-            f'--config-dir {CONFIG.get("config_dir")} '
-            f'{command_group_submission_id} $TASK_IDX $ITER_IDX >> $LOG_PATH 2>&1'
-        )]
-
-        opt = self.get_formatted_options(max_num_tasks, task_step_size, user_opt=False,
-                                         name=name)
-        opt.append('#$ -l short')  # Temp (should be a profile option)
-
-        js_lines = ([SunGridEngine.SHEBANG, ''] +
-                    about_msg + [''] +
-                    opt + [''] +
-                    define_dirs + [''] +
-                    log_stuff + [''] +
-                    cmd_exec)
-
-        # Write jobscript:
-        with js_path.open('w') as handle:
-            handle.write('\n'.join(js_lines))
-
-        return js_path
-
-    def get_scheduler_stats(self, scheduler_job_id, task_id):
-
-        cmd = ['/opt/site/sge/bin/lx-amd64/qacct', '-j', str(scheduler_job_id)]
-        proc = run(cmd, stdout=PIPE, stderr=PIPE)
-        out = proc.stdout.decode().strip()
-        _ = proc.stderr.decode().strip()
-
-        info = {}
-        qacct = out.split(SunGridEngine.STATS_DELIM)
-        for i in qacct[1:]:
-            keep = False
-            for ln in i.splitlines():
-                key, val = ln.strip().split(None, 1)
-                val = val.strip()
-                info.update({key: val})
-                if key == 'taskid':
-                    if val == 'undefined' or int(val) == task_id:
-                        keep = True
-            if keep:
-                break
-            else:
-                info = {}
-
-        return info

hpcflow/utils.py

@@ -1,103 +0,0 @@
-"""`hpcflow.utils.py`
-
-Utility functions that are not particularly specific to `hpcflow`.
-
-"""
-
-import random
-
-
-def coerce_same_length(all_lists):
-    """
-    TODO: add docstring and examples
-
-    """
-
-    all_len = [len(i) for i in all_lists]
-    uniq_lens = set(all_len)
-    num_uniq_lens = len(uniq_lens)
-
-    if num_uniq_lens == 1:
-        out = all_lists
-
-    elif num_uniq_lens == 2:
-
-        if min(uniq_lens) != 1:
-            raise ValueError('bad!')
-
-        max_len = max(uniq_lens)
-        out = []
-        for i in all_lists:
-            if len(i) != max_len:
-                i = i * max_len
-            out.append(i)
-
-    else:
-        raise ValueError('bad!')
-
-    return out
-
-
-def zeropad(num, largest):
-    """Return a zero-padded string of a number, given the largest number.
-
-    TODO: want to support floating-point numbers as well? Or rename function
-    accordingly.
-
-    Parameters
-    ----------
-    num : int
-        The number to be formatted with zeros padding on the left.
-    largest : int
-        The number that determines the number of zeros to pad with.
-
-    Returns
-    -------
-    padded : str
-        The original number, `num`, formatted as a string with zeros added
-        on the left.
-
-    """
-
-    num_digits = len('{:.0f}'.format(largest))
-    padded = '{0:0{width}}'.format(num, width=num_digits)
-
-    return padded
-
-
-def datetime_to_dict(dt):
-    return {
-        'year': dt.year,
-        'month': dt.month,
-        'day': dt.day,
-        'hour': dt.hour,
-        'minute': dt.minute,
-        'second': dt.second,
-        'microsecond': dt.microsecond,
-    }
-
-
-def timedelta_to_dict(td):
-    return {
-        'days': td.days,
-        'seconds': td.seconds,
-        'microseconds': td.microseconds,
-    }
-
-
-def format_time_delta(time_delta):
-
-    days, days_rem = divmod(time_delta.total_seconds(), 3600 * 24)
-    hours, hours_rem = divmod(days_rem, 3600)
-    mins, seconds = divmod(hours_rem, 60)
-
-    time_diff_fmt = '{:02.0f}:{:02.0f}:{:02.0f}'.format(hours, mins, round(seconds))
-    if days > 0:
-        days_str = 'day' if days == 1 else 'days'
-        time_diff_fmt = '{} {}, '.format(int(days), days_str) + time_diff_fmt
-
-    return time_diff_fmt
-
-
-def get_random_hex(n=10):
-    return ''.join([random.choice('0123456789abcdef') for i in range(n)])

hpcflow/validation.py

@@ -1,167 +0,0 @@
-"""`hpcflow.validation.py`
-
-These validation functions are invoked during initialisation of the `Workflow`
-and `CommandGroup` instances. They are extracted out as functions here to allow
-re-use by the `visualise` module, whose code may be invoked without necessarily
-constructing `Workflow` and `CommandGroup` instances.
-
-"""
-
-from hpcflow.config import Config as CONFIG
-from hpcflow.nesting import NestingType
-
-
-def validate_workflow(workflow_dict):
-    """Validate the `nesting`, `order` and `num_tasks` for each in a list of
-    workflow command groups."""
-
-    return workflow_dict
-
-
-def validate_job_profile(job_profile):
-    """Validate the keys of a job profile.
-
-    Parameters
-    ----------
-    job_profile : dict
-        Dictionary representing a job profile.
-
-    Returns
-    -------
-    job_profile : dict
-        Dictionary representing a job profile.
-
-    """
-
-    # Validate profile keys:
-    keys = job_profile.keys()
-    missing_keys = list(set(CONFIG.get('profile_keys_required')) - set(keys))
-    bad_keys = list(set(keys) - set(CONFIG.get('profile_keys_allowed')))
-
-    # Check required keys exist:
-    if len(missing_keys) > 0:
-        msg = ('Job profile is missing required key(s): {}')
-        raise ValueError(msg.format(missing_keys))
-
-    # Check for unknown keys:
-    if len(bad_keys) > 0:
-        msg = ('Job profile contains unknown key(s): {}')
-        raise ValueError(msg.format(bad_keys))
-
-    return job_profile
-
-
-def validate_job_profile_list(job_profiles):
-    """Validate a list of job profiles and sort by `profile_order`.
-
-    Parameters
-    ----------
-    job_profiles : list of dict
-        List of dictionaries representing job profiles.
-
-    Returns
-    -------
-    job_profiles : list of dict
-        List of dictionaries representing job profiles, sorted by
-        `profile_order`.
-
-    """
-
-    # Some validation:
-    profile_orders = []
-    profile_names = []
-    for i in job_profiles:
-
-        # Check all profiles have a name:
-        if i.get('profile_name') is None:
-            msg = ('`profile_name` must be set for each job profile.')
-            raise ValueError(msg)
-        else:
-            profile_names.append(i['profile_name'])
-
-        if len(job_profiles) > 1:
-            # Check all profiles have an order:
-            if i.get('profile_order') is None:
-                msg = ('`profile_order` must be set for each profile if there '
-                       'are multiple job profiles, but was not specified for '
-                       'the job profile "{}"')
-                raise ValueError(msg.format(i['profile_name']))
-            else:
-                profile_orders.append(i['profile_order'])
-
-    if len(job_profiles) > 1:
-        # Check distinct profile names:
-        if len(set(profile_names)) < len(job_profiles):
-            msg = ('Each job profile must have a distinct `profile_name` if '
-                   'there are multiple job profiles, but job profile names '
-                   'are: {}')
-            raise ValueError(msg.format(profile_names))
-
-        # Check distinct profile orders:
-        if len(set(profile_orders)) < len(job_profiles):
-            msg = ('Each job profile must have a distinct `profile_order` if '
-                   'there are multiple job profiles, but job profile orders '
-                   'are: {}')
-            raise ValueError(msg.format(profile_orders))
-
-        # Sort by profile order:
-        job_profiles = sorted(job_profiles, key=lambda x: x['profile_order'])
-
-    return job_profiles
-
-
-def validate_task_multiplicity(cmd_groups, common_err_msg='',
-                               max_num_channels=1):
-    """Validate an ordered list of command groups that represent (part of) a
-    Workflow.
-
-    Parameters
-    ----------
-    cmd_groups : list of dict
-        List of dicts representing command groups.
-    common_err_msg : str, optional
-        Optional error message to prepend to any exception raised in here. By
-        default, empty.
-    max_num_channels : int, optional
-        Restrict the allowed number of channels to a given number.
-
-    Returns
-    -------
-    cmd_groups : list of dict
-        List of dicts representing command groups.    
-
-    """
-
-    if not common_err_msg.endswith(' '):
-        common_err_msg += ' '
-
-    exec_orders_all = [i.get('exec_order') for i in cmd_groups]
-    exec_orders = [i for i in exec_orders_all if i is not None]
-
-    if exec_orders:
-        # Check `exec_order` starts from zero and increases by 0 or 1:
-        uniq_exec_orders = list(set(sorted(exec_orders)))
-        if uniq_exec_orders != list(range(len(uniq_exec_orders))):
-            msg = common_err_msg + ('If specified, `exec_order` must start at '
-                                    'zero and increase by zero or one across '
-                                    'command groups.')
-            raise ValueError(msg)
-
-        for i in uniq_exec_orders:
-            # Get cmd_group indices that have this `exec_order`:
-            exec_order_idx = [idx for idx, j in enumerate(exec_orders_all)
-                              if j == i]
-
-            if len(exec_order_idx) > 1:
-
-                if len(exec_order_idx) > max_num_channels:
-                    # Temporarily restrict number of channels to 1:
-                    msg = ('Multiple channels are not yet implemented.')
-                    raise NotImplementedError(msg)
-
-    else:
-        # Set execution orders for command groups:
-        for i in range(len(cmd_groups)):
-            cmd_groups[i]['exec_order'] = i
-
-    return cmd_groups