sierra-research 1.3.6__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/hpc/adhoc/plugin.py

@@ -1,125 +0,0 @@
-# Copyright 2020 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-"""HPC plugin for running experiments with an ad-hoc set of compute nodes.
-
-E.g., whatever computers you happen to have laying around in the lab.
-
-"""
-
-# Core packages
-import os
-import typing as tp
-import argparse
-import shutil
-import pathlib
-
-# 3rd party packages
-import implements
-
-# Project packages
-from sierra.core import types, utils
-from sierra.core.experiment import bindings
-
-
-@implements.implements(bindings.IParsedCmdlineConfigurer)
-class ParsedCmdlineConfigurer():
-    """Configure SIERRA for ad-hoc HPC.
-
-    May use the following environment variables:
-
-    - ``SIERRA_NODEFILE`` - If this is not defined ``--nodefile`` must be
-      passed.
-
-    """
-
-    def __init__(self, exec_env: str) -> None:
-        pass
-
-    def __call__(self, args: argparse.Namespace) -> None:
-        if args.nodefile is None:
-            assert 'SIERRA_NODEFILE' in os.environ,\
-                ("Non-hpc.adhoc environment detected: --nodefile not "
-                 "passed and 'SIERRA_NODEFILE' not found")
-            args.nodefile = os.environ['SIERRA_NODEFILE']
-
-        assert utils.path_exists(args.nodefile), \
-            f"SIERRA_NODEFILE '{args.nodefile}' does not exist"
-
-        assert not args.platform_vc,\
-            "Platform visual capture not supported on Adhoc"
-
-
-@implements.implements(bindings.IExpShellCmdsGenerator)
-class ExpShellCmdsGenerator():
-    """Generate the cmd to invoke GNU Parallel in the ad-hoc HPC environment.
-    """
-
-    def __init__(self,
-                 cmdopts: types.Cmdopts,
-                 exp_num: int) -> None:
-        self.cmdopts = cmdopts
-
-    def pre_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def post_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def exec_exp_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
-        jobid = os.getpid()
-
-        # Even if we are passed --nodelist, we still make our own copy of it, so
-        # that the user can safely modify it (if they want to) after running
-        # stage 1.
-        nodelist = pathlib.Path(exec_opts['exp_input_root'],
-                                f"{jobid}-nodelist.txt")
-
-        resume = ''
-        # This can't be --resume, because then GNU parallel looks at the results
-        # directory, and if there is stuff in it, (apparently) assumes that the
-        # job finished...
-        if exec_opts['exec_resume']:
-            resume = '--resume-failed'
-
-        # Make sure there are no duplicate nodes
-        unique_nodes = types.ShellCmdSpec(
-            cmd='sort -u {0} > {1}'.format(exec_opts["nodefile"], nodelist),
-            shell=True,
-            wait=True)
-        # Make sure GNU parallel uses the right shell, because it seems to
-        # defaults to /bin/sh since all cmds are run in a python shell which
-        # does not have $SHELL set.
-        use_bash = types.ShellCmdSpec(
-            cmd='export PARALLEL_SHELL={0}'.format(shutil.which('bash')),
-            shell=True,
-            wait=True,
-            env=True)
-
-        # GNU parallel cmd
-        parallel = 'parallel {2} ' \
-            '--jobs {1} ' \
-            '--results {4} ' \
-            '--joblog {3} ' \
-            '--sshloginfile {0} ' \
-            '--workdir {4} < "{5}"'
-
-        log = pathlib.Path(exec_opts['scratch_dir'], "parallel.log")
-        parallel = parallel.format(nodelist,
-                                   exec_opts['n_jobs'],
-                                   resume,
-                                   log,
-                                   exec_opts['scratch_dir'],
-                                   exec_opts['cmdfile_stem_path'] + exec_opts['cmdfile_ext'])
-
-        parallel_spec = types.ShellCmdSpec(cmd=parallel,
-                                           shell=True,
-                                           wait=True)
-
-        return [unique_nodes, use_bash, parallel_spec]
-
-
-__api__ = [
-    'ParsedCmdlineConfigurer',
-    'ExpShellCmdsGenerator',
-]

sierra/plugins/hpc/local/plugin.py

@@ -1,81 +0,0 @@
-# Copyright 2020 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-"""HPC plugin for running SIERRA locally.
-
-Not necessarily HPC, but it fits well enough under that semantic umbrella.
-
-"""
-
-# Core packages
-import typing as tp
-import shutil
-import pathlib
-
-# 3rd party packages
-import implements
-
-# Project packages
-from sierra.core import types
-from sierra.core.experiment import bindings
-
-
-@implements.implements(bindings.IExpShellCmdsGenerator)
-class ExpShellCmdsGenerator():
-    """
-    Generate the command to invoke GNU parallel for local HPC.
-    """
-
-    def __init__(self,
-                 cmdopts: types.Cmdopts,
-                 exp_num: int) -> None:
-        self.cmdopts = cmdopts
-
-    def pre_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def post_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def exec_exp_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
-        resume = ''
-
-        # This can't be --resume, because then GNU parallel looks at the results
-        # directory, and if there is stuff in it, (apparently) assumes that the
-        # job finished...
-        if exec_opts['exec_resume']:
-            resume = '--resume-failed'
-
-        # Make sure GNU parallel uses the right shell, because it seems to
-        # defaults to /bin/sh since all cmds are run in a python shell which
-        # does not have $SHELL set.
-        shell = shutil.which('bash')
-        use_bash = types.ShellCmdSpec(cmd=f'export PARALLEL_SHELL={shell}',
-                                      shell=True,
-                                      wait=True,
-                                      env=True)
-
-        parallel = 'parallel {1} ' \
-            '--jobs {2} ' \
-            '--results {0} '\
-            '--joblog {3} '\
-            '--no-notice < "{4}"'
-
-        log = pathlib.Path(exec_opts['scratch_dir'], "parallel.log")
-        parallel = parallel.format(exec_opts['scratch_dir'],
-                                   resume,
-                                   exec_opts['n_jobs'],
-                                   log,
-                                   exec_opts['cmdfile_stem_path'] +
-                                   exec_opts['cmdfile_ext'])
-
-        parallel_spec = types.ShellCmdSpec(cmd=parallel,
-                                           shell=True,
-                                           wait=True)
-
-        return [use_bash, parallel_spec]
-
-
-__api__ = [
-    'ExpShellCmdsGenerator'
-]

sierra/plugins/hpc/pbs/__init__.py

@@ -1,9 +0,0 @@
-# Copyright 2021 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-
-# Core packages
-
-# 3rd party packages
-
-# Project packages

sierra/plugins/hpc/pbs/plugin.py

@@ -1,126 +0,0 @@
-# Copyright 2020 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-"""
-HPC plugin for running SIERRA on HPC clusters using the TORQUE-PBS scheduler.
-"""
-
-# Core packages
-import os
-import typing as tp
-import argparse
-import shutil
-import pathlib
-
-# 3rd party packages
-import implements
-
-# Project packages
-from sierra.core import types
-from sierra.core.experiment import bindings
-
-
-@implements.implements(bindings.IParsedCmdlineConfigurer)
-class ParsedCmdlineConfigurer():
-    """Configure SIERRA for PBS HPC.
-
-    Uses the following environment variables (if any of them are not defined an
-    assertion will be triggered):
-
-    - ``PBS_NUM_PPN``
-    - ``PBS_NODEFILE``
-    - ``PBS_JOBID``
-
-    """
-
-    def __init__(self, exec_env: str) -> None:
-        pass
-
-    def __call__(self, args: argparse.Namespace) -> None:
-        keys = ['PBS_NUM_PPN',
-                'PBS_NODEFILE',
-                'PBS_JOBID'
-                ]
-
-        for k in keys:
-            assert k in os.environ,\
-                f"Non-PBS environment detected: '{k}' not found"
-
-        assert args.exec_jobs_per_node is not None, \
-            "--exec-jobs-per-node is required (can't be computed from PBS)"
-
-        assert not args.platform_vc,\
-            "Platform visual capture not supported on PBS"
-
-
-@implements.implements(bindings.IExpShellCmdsGenerator)
-class ExpShellCmdsGenerator():
-    """Generate the cmd to invoke GNU Parallel on PBS HPC.
-
-    """
-
-    def __init__(self,
-                 cmdopts: types.Cmdopts,
-                 exp_num: int) -> None:
-        self.cmdopts = cmdopts
-
-    def pre_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def post_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def exec_exp_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
-        resume = ''
-        jobid = os.environ['PBS_JOBID']
-        nodelist = pathlib.Path(exec_opts['exp_input_root'],
-                                f"{jobid}-nodelist.txt")
-
-        resume = ''
-        # This can't be --resume, because then GNU parallel looks at the results
-        # directory, and if there is stuff in it, (apparently) assumes that the
-        # job finished...
-        if exec_opts['exec_resume']:
-            resume = '--resume-failed'
-
-        unique_nodes = types.ShellCmdSpec(cmd=f'sort -u $PBS_NODEFILE > {nodelist}',
-                                          shell=True,
-                                          wait=True)
-
-        # Make sure GNU parallel uses the right shell, because it seems to
-        # defaults to /bin/sh since all cmds are run in a python shell which
-        # does not have $SHELL set.
-        shell = shutil.which('bash')
-        use_bash = types.ShellCmdSpec(cmd=f'export PARALLEL_SHELL={shell}',
-                                      shell=True,
-                                      wait=True,
-                                      env=True)
-
-        parallel = 'parallel {2} ' \
-            '--jobs {1} ' \
-            '--results {4} ' \
-            '--joblog {3} ' \
-            '--sshloginfile {0} ' \
-            '--workdir {4} < "{5}"'
-
-        log = pathlib.Path(exec_opts['scratch_dir'], "parallel.log")
-        parallel = parallel.format(nodelist,
-                                   exec_opts['n_jobs'],
-                                   resume,
-                                   log,
-                                   exec_opts['scratch_dir'],
-                                   exec_opts['cmdfile_stem_path'] +
-                                   exec_opts['cmdfile_ext'])
-
-        parallel_spec = types.ShellCmdSpec(cmd=parallel,
-                                           shell=True,
-                                           wait=True)
-
-        return [unique_nodes, use_bash, parallel_spec]
-
-
-__api__ = [
-    'ParsedCmdlineConfigurer',
-
-
-]

sierra/plugins/hpc/slurm/__init__.py

@@ -1,9 +0,0 @@
-# Copyright 2021 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-
-# Core packages
-
-# 3rd party packages
-
-# Project packages

sierra/plugins/hpc/slurm/plugin.py

@@ -1,130 +0,0 @@
-# Copyright 2020 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-"""
-HPC plugin for running SIERRA on HPC clusters using the SLURM scheduler.
-"""
-
-# Core packages
-import typing as tp
-import argparse
-import shutil
-import pathlib
-import os
-
-# 3rd party packages
-import implements
-
-# Project packages
-from sierra.core import types
-from sierra.core.experiment import bindings
-
-
-@implements.implements(bindings.IParsedCmdlineConfigurer)
-class ParsedCmdlineConfigurer():
-    """Configure SIERRA for SLURM HPC.
-
-    Uses the following environment variables (if any of them are not defined an
-    assertion will be triggered):
-
-    - ``SLURM_CPUS_PER_TASK``
-    - ``SLURM_TASKS_PER_NODE``
-    - ``SLURM_JOB_NODELIST``
-    - ``SLURM_JOB_ID``
-
-    """
-
-    def __init__(self, exec_env: str) -> None:
-        pass
-
-    def __call__(self, args: argparse.Namespace) -> None:
-        keys = ['SLURM_CPUS_PER_TASK',
-                'SLURM_TASKS_PER_NODE',
-                'SLURM_JOB_NODELIST',
-                'SLURM_JOB_ID']
-
-        for k in keys:
-            assert k in os.environ,\
-                f"Non-SLURM environment detected: '{k}' not found"
-
-        assert not args.platform_vc,\
-            "Platform visual capture not supported on SLURM"
-
-        # SLURM_TASKS_PER_NODE can be set to things like '1(x32),3', indicating
-        # that not all nodes will run the same # of tasks. SIERRA expects all
-        # nodes to have the same # tasks allocated to each (i.e., a homogeneous
-        # allocation), so we check for this.
-        assert "," not in os.environ['SLURM_TASKS_PER_NODE'], \
-            "SLURM_TASKS_PER_NODE not homogeneous"
-
-
-@implements.implements(bindings.IExpShellCmdsGenerator)
-class ExpShellCmdsGenerator():
-    """Generate the cmd to correctly invoke GNU Parallel on SLURM HPC.
-
-    """
-
-    def __init__(self,
-                 cmdopts: types.Cmdopts,
-                 exp_num: int) -> None:
-        self.cmdopts = cmdopts
-
-    def pre_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def post_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
-        return []
-
-    def exec_exp_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
-        jobid = os.environ['SLURM_JOB_ID']
-        nodelist = pathlib.Path(exec_opts['exp_input_root'],
-                                f"{jobid}-nodelist.txt")
-
-        resume = ''
-        # This can't be --resume, because then GNU parallel looks at the results
-        # directory, and if there is stuff in it, (apparently) assumes that the
-        # job finished...
-        if exec_opts['exec_resume']:
-            resume = '--resume-failed'
-
-        unique_nodes = types.ShellCmdSpec(
-            cmd=f'scontrol show hostnames $SLURM_JOB_NODELIST > {nodelist}',
-            shell=True,
-            wait=True)
-
-        # Make sure GNU parallel uses the right shell, because it seems to
-        # defaults to /bin/sh since all cmds are run in a python shell which
-        # does not have $SHELL set.
-        shell = shutil.which('bash')
-        use_bash = types.ShellCmdSpec(cmd=f'export PARALLEL_SHELL={shell}',
-                                      shell=True,
-                                      wait=True)
-
-        parallel = 'parallel {2} ' \
-            '--jobs {1} ' \
-            '--results {4} ' \
-            '--joblog {3} ' \
-            '--sshloginfile {0} ' \
-            '--workdir {4} < "{5}"'
-
-        log = pathlib.Path(exec_opts['scratch_dir'], "parallel.log")
-        parallel = parallel.format(nodelist,
-                                   exec_opts['n_jobs'],
-                                   resume,
-                                   log,
-                                   exec_opts['scratch_dir'],
-                                   exec_opts['cmdfile_stem_path'] +
-                                   exec_opts['cmdfile_ext'])
-        parallel_spec = types.ShellCmdSpec(cmd=parallel,
-                                           shell=True,
-                                           wait=True)
-
-        return [unique_nodes, use_bash, parallel_spec]
-
-
-__api__ = [
-    'ParsedCmdlineConfigurer',
-    'ExpShellCmdsGenerator'
-
-
-]

sierra/plugins/platform/__init__.py

@@ -1,9 +0,0 @@
-# Copyright 2021 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-
-# Core packages
-
-# 3rd party packages
-
-# Project packages

sierra/plugins/platform/argos/__init__.py

@@ -1,9 +0,0 @@
-# Copyright 2021 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-
-# Core packages
-
-# 3rd party packages
-
-# Project packages