sierra-research 1.3.6__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/engine/ros1robot/variables/population_size.py

@@ -0,0 +1,146 @@
+# Copyright 2020 John Harwell, All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+"""Classes for the population size batch criteria.
+
+See :ref:`plugins/engine/ros1robot/bc/population-size` for usage
+documentation.
+
+"""
+
+# Core packages
+import typing as tp
+import logging
+import pathlib
+
+# 3rd party packages
+import implements
+
+# Project packages
+from sierra.core.variables import batch_criteria as bc
+from sierra.core import types
+from sierra.core.variables import population_size
+from sierra.core.experiment import definition
+from sierra.core.graphs import bcbridge
+
+
+@implements.implements(bcbridge.IGraphable)
+@implements.implements(bc.IQueryableBatchCriteria)
+class PopulationSize(population_size.PopulationSize):
+    """A univariate range of system sizes used to define batch experiments.
+
+    This class is a base class which should (almost) never be used on its
+    own. Instead, the ``factory()`` function should be used to dynamically
+    create derived classes expressing the user's desired size distribution.
+
+    Note: Usage of this class assumes homogeneous systems.
+
+    Attributes:
+        size_list: List of integer system sizes defining the range of the
+                   variable for the batch experiment.
+
+    """
+
+    def __init__(
+        self,
+        cli_arg: str,
+        main_config: types.YAMLDict,
+        batch_input_root: pathlib.Path,
+        robot: str,
+        sizes: tp.List[int],
+    ) -> None:
+        population_size.PopulationSize.__init__(
+            self, cli_arg, main_config, batch_input_root
+        )
+        self.sizes = sizes
+        self.robot = robot
+        self.logger = logging.getLogger(__name__)
+        self.element_adds = []  # type: tp.List[definition.ElementAddList]
+
+    def gen_element_addlist(self) -> tp.List[definition.ElementAddList]:
+        """
+        Generate XML modifications to set system sizes.
+        """
+        if not self.element_adds:
+            robot_config = self.main_config["ros"]["robots"][self.robot]
+            prefix = robot_config["prefix"]
+
+            for s in self.sizes:
+                per_robot = definition.ElementAddList()
+                per_robot.append(definition.ElementAdd(".", "master", {}, True))
+                per_robot.append(
+                    definition.ElementAdd("./master", "group", {"ns": "sierra"}, False)
+                )
+                per_robot.append(
+                    definition.ElementAdd(
+                        "./master/group/[@ns='sierra']",
+                        "param",
+                        {"name": "experiment/n_agents", "value": str(s)},
+                        False,
+                    )
+                )
+
+                for i in range(0, s):
+
+                    # Note that we don't try to do any of the robot bringup
+                    # here--we can't know the exact node/package names without
+                    # using a lot of (brittle) config.
+                    ns = f"{prefix}{i}"
+                    per_robot.append(
+                        definition.ElementAdd("./robot", "group", {"ns": ns}, True)
+                    )
+
+                    per_robot.append(
+                        definition.ElementAdd(
+                            f"./robot/group/[@ns='{ns}']",
+                            "param",
+                            {"name": "tf_prefix", "value": ns},
+                            True,
+                        )
+                    )
+
+                self.element_adds.append(per_robot)
+
+        return self.element_adds
+
+    def n_agents(self, exp_num: int) -> int:
+        return self.sizes[exp_num]
+
+    def graph_info(
+        self,
+        cmdopts: types.Cmdopts,
+        batch_output_root: tp.Optional[pathlib.Path] = None,
+        exp_names: tp.Optional[tp.List[str]] = None,
+    ) -> bcbridge.GraphInfo:
+        info = bcbridge.GraphInfo(
+            cmdopts,
+            batch_output_root,
+            exp_names if exp_names else self.gen_exp_names(),
+        )
+
+        info.xlabel = super().graph_xlabel(info.cmdopts)
+        info.xticklabels = super().graph_xticklabels(
+            info.cmdopts, info.batch_output_root, info.exp_names
+        )
+        info.xticks = super().graph_xticks(
+            info.cmdopts, info.batch_output_root, info.exp_names
+        )
+        return info
+
+
+def factory(
+    cli_arg: str,
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    batch_input_root: pathlib.Path,
+    **kwargs,
+) -> PopulationSize:
+    """Create a :class:`PopulationSize` derived class from the cmdline definition."""
+    max_sizes = population_size.parse(cli_arg)
+
+    return PopulationSize(
+        cli_arg, main_config, batch_input_root, cmdopts["robot"], max_sizes
+    )
+
+
+__all__ = ["PopulationSize"]

sierra/plugins/execenv/__init__.py

@@ -0,0 +1,11 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages

sierra/plugins/execenv/hpc/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Container module for plugins related to execution environments (HPC flavor).
+
+Driven by ``--execenv``.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+from . import cmdline
+
+
+__all__ = ["cmdline"]

sierra/plugins/execenv/hpc/adhoc/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Container module for the adhoc execution environment.
+
+See :ref:`plugins/execenv/hpc/adhoc`.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/execenv/hpc/adhoc/cmdline.py

@@ -0,0 +1,30 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Command line definitions for the :ref:`plugins/execenv/hpc/PBS`."""
+
+# Core packages
+import typing as tp
+import argparse
+
+# 3rd party packages
+
+# Project packages
+from sierra.plugins.execenv import hpc
+from sierra.core import types
+from sierra.plugins import PluginCmdline
+
+
+def build(
+    parents: tp.List[argparse.ArgumentParser], stages: tp.List[int]
+) -> PluginCmdline:
+    """
+    Get a cmdline parser supporting the ``hpc.adhoc`` execution environment.
+    """
+    return hpc.cmdline.HPCCmdline(parents, stages)
+
+
+def to_cmdopts(args: argparse.Namespace) -> types.Cmdopts:
+    return hpc.cmdline.to_cmdopts(args)

sierra/plugins/execenv/hpc/adhoc/plugin.py

@@ -0,0 +1,131 @@
+# Copyright 2020 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""HPC plugin for running experiments with an ad-hoc set of compute nodes.
+
+E.g., whatever computers you happen to have laying around in the lab.
+
+"""
+
+# Core packages
+import os
+import typing as tp
+import argparse
+import shutil
+import pathlib
+
+# 3rd party packages
+import implements
+
+# Project packages
+from sierra.core import types, utils
+from sierra.core.experiment import bindings
+
+
+def cmdline_postparse_configure(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    Configure SIERRA for ad-hoc HPC.
+
+    May use the following environment variables:
+
+    - :envvar:`SIERRA_NODEFILE` - If this is not defined ``--nodefile`` must be
+      passed.
+
+    """
+
+    if args.nodefile is None:
+        assert "SIERRA_NODEFILE" in os.environ, (
+            "Non-hpc.adhoc environment detected: --nodefile not "
+            "passed and 'SIERRA_NODEFILE' not found"
+        )
+        args.nodefile = os.environ["SIERRA_NODEFILE"]
+
+    assert utils.path_exists(
+        args.nodefile
+    ), f"SIERRA_NODEFILE '{args.nodefile}' does not exist"
+
+    assert not args.engine_vc, "Engine visual capture not supported on Adhoc"
+
+    return args
+
+
+@implements.implements(bindings.IExpShellCmdsGenerator)
+class ExpShellCmdsGenerator:
+    """Generate the cmd to invoke GNU Parallel in the ad-hoc HPC environment."""
+
+    def __init__(self, cmdopts: types.Cmdopts, exp_num: int) -> None:
+        self.cmdopts = cmdopts
+
+    def pre_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        shell = shutil.which("bash")
+
+        return [
+            # Since parallel doesn't export any envvars to child processes by
+            # default, we add some common ones.
+            types.ShellCmdSpec(
+                cmd='export PARALLEL="${PARALLEL} --env LD_LIBRARY_PATH --env PYTHONPATH"',
+                shell=True,
+                wait=True,
+                env=True,
+            ),
+            # Make sure GNU parallel uses the right shell, because it seems to
+            # defaults to /bin/sh since all cmds are run in a python shell which
+            # does not have $SHELL set.
+            types.ShellCmdSpec(
+                cmd=f"export PARALLEL_SHELL={shell}", shell=True, wait=True, env=True
+            ),
+        ]
+
+    def post_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        return []
+
+    def exec_exp_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
+        jobid = os.getpid()
+
+        # Even if we are passed --nodelist, we still make our own copy of it, so
+        # that the user can safely modify it (if they want to) after running
+        # stage 1.
+        nodelist = pathlib.Path(exec_opts["exp_input_root"], f"{jobid}-nodelist.txt")
+
+        resume = ""
+        # This can't be --resume, because then GNU parallel looks at the results
+        # directory, and if there is stuff in it, (apparently) assumes that the
+        # job finished...
+        if exec_opts["exec_resume"]:
+            resume = "--resume-failed"
+
+        # Make sure there are no duplicate nodes
+        unique_nodes = types.ShellCmdSpec(
+            cmd="sort -u {0} > {1}".format(exec_opts["nodefile"], nodelist),
+            shell=True,
+            wait=True,
+        )
+        # GNU parallel cmd
+        parallel = (
+            "env && parallel {2} "
+            "--jobs {1} "
+            "--results {4} "
+            "--joblog {3} "
+            "--sshloginfile {0} "
+            '--workdir {4} < "{5}"'
+        )
+
+        log = pathlib.Path(exec_opts["exp_scratch_root"], "parallel.log")
+        parallel = parallel.format(
+            nodelist,
+            exec_opts["n_jobs"],
+            resume,
+            log,
+            exec_opts["exp_scratch_root"],
+            exec_opts["cmdfile_stem_path"] + exec_opts["cmdfile_ext"],
+        )
+
+        parallel_spec = types.ShellCmdSpec(cmd=parallel, shell=True, wait=True)
+
+        return [unique_nodes, parallel_spec]
+
+
+__all__ = [
+    "cmdline_postparse_configure",
+    "ExpShellCmdsGenerator",
+]

sierra/plugins/execenv/hpc/cmdline.py

@@ -0,0 +1,137 @@
+# Copyright 2020 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Common cmdline classes for the various HPC plugins.
+"""
+
+# Core packages
+import argparse
+import typing as tp
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import types
+from sierra.plugins import PluginCmdline
+
+
+class HPCCmdline(PluginCmdline):
+    def __init__(
+        self, parents: tp.List[argparse.ArgumentParser], stages: tp.List[int]
+    ) -> None:
+        super().__init__(parents, stages)
+
+    def init_stage2(self) -> None:
+        """Add HPC cmdline options.
+
+        Options may be interpreted differently between :term:`Engines
+        <Engine>`, or ignored, depending. These include:
+
+        - ``--exec-jobs-per-node``
+
+        - ``--exec-no-devnull``
+
+        - ``--exec-resume``
+
+        - ``--exec-strict``
+
+        """
+        self.stage2.add_argument(
+            "--exec-jobs-per-node",
+            help="""
+
+                              Specify the maximum number of parallel jobs to run
+                              per allocated node. By default this is computed
+                              from the selected HPC environment for maximum
+                              throughput given the desired ``--n-runs`` and CPUs
+                              per allocated node. However, for some environments
+                              being able to override the computed default can be
+                              useful.
+
+                              """
+            + self.stage_usage_doc([2]),
+            type=int,
+            default=None,
+        )
+
+        self.stage2.add_argument(
+            "--exec-devnull",
+            help="""
+
+                              Redirect ALL output from simulations to
+                              /dev/null. Useful for engine where you can't
+                              disable all INFO messages at compile time, and
+                              don't want to have to grep through lots of
+                              redundant stdout files to see if there were any
+                              errors.
+
+                              """
+            + self.stage_usage_doc([1, 2]),
+            action="store_true",
+            dest="exec_devnull",
+            default=True,
+        )
+
+        self.stage2.add_argument(
+            "--exec-no-devnull",
+            help="""
+
+                              Don't redirect ALL output from simulations to
+                              /dev/null. Useful for engines where you can't
+                              disable all INFO messages at compile time, and
+                              don't want to have to grep through lots of
+                              redundant stdout files to see if there were any
+                              errors.
+
+                              """
+            + self.stage_usage_doc([1, 2]),
+            action="store_false",
+            dest="exec_devnull",
+        )
+
+        self.stage2.add_argument(
+            "--exec-resume",
+            help="""
+                 Resume a batch experiment that was killed/stopped/etc last time
+                 SIERRA was run.
+                 """
+            + self.stage_usage_doc([2]),
+            action="store_true",
+            default=False,
+        )
+
+        self.stage2.add_argument(
+            "--exec-strict",
+            help="""
+                 If passed, then if any experimental commands fail during stage
+                 2 SIERRA will exit, rather than try to keep going and execute
+                 the rest of the experiments.
+
+                 Useful for:
+
+                     - "Correctness by construction" experiments, where you know
+                       if SIERRA doesn't crash and it makes it to the end of
+                       your batch experiment then none of the individual
+                       experiments crashed.
+
+                     - CI pipelines.
+                 """,
+            action="store_true",
+        )
+
+
+def to_cmdopts(args: argparse.Namespace) -> types.Cmdopts:
+    """Update cmdopts dictionary with the HPC-specific cmdline options."""
+    return {
+        # Multistage
+        "exec_devnull": args.exec_devnull,
+        "exec_jobs_per_node": args.exec_jobs_per_node,
+        "exec_resume": args.exec_resume,
+        "exec_strict": args.exec_strict,
+    }
+
+
+__all__ = [
+    "HPCCmdline",
+]

sierra/plugins/execenv/hpc/local/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Container module for the local execution environment.
+
+See :ref:`plugins/execenv/hpc/local`.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/execenv/hpc/local/cmdline.py

@@ -0,0 +1,31 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""
+Command line definitions for the :ref:`plugins/execenv/hpc/local`.
+"""
+# Core packages
+import typing as tp
+import argparse
+
+# 3rd party packages
+
+# Project packages
+from sierra.plugins.execenv import hpc
+from sierra.core import types
+from sierra.plugins import PluginCmdline
+
+
+def build(
+    parents: tp.List[argparse.ArgumentParser], stages: tp.List[int]
+) -> PluginCmdline:
+    """
+    Get a cmdline parser supporting the ``hpc.local`` execution environment.
+    """
+    return hpc.cmdline.HPCCmdline(parents, stages)
+
+
+def to_cmdopts(args: argparse.Namespace) -> types.Cmdopts:
+    return hpc.cmdline.to_cmdopts(args)

sierra/plugins/execenv/hpc/local/plugin.py

@@ -0,0 +1,145 @@
+# Copyright 2020 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""HPC plugin for running SIERRA locally.
+
+Not necessarily HPC, but it fits well enough under that semantic umbrella.
+
+"""
+
+# Core packages
+import typing as tp
+import shutil
+import pathlib
+
+# 3rd party packages
+import implements
+
+# Project packages
+from sierra.core import types
+from sierra.core.experiment import bindings
+
+
+@implements.implements(bindings.IExpShellCmdsGenerator)
+class ExpShellCmdsGenerator:
+    """
+    Generate the commands for local HPC (experiment-level parallelism).
+    """
+
+    def __init__(self, cmdopts: types.Cmdopts, exp_num: int) -> None:
+        self.cmdopts = cmdopts
+
+    def pre_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        shell = shutil.which("bash")
+
+        return [
+            # Since parallel doesn't export any envvars to child processes by
+            # default, we add some common ones.
+            types.ShellCmdSpec(
+                cmd='export PARALLEL="${PARALLEL} --env LD_LIBRARY_PATH --env PYTHONPATH"',
+                shell=True,
+                wait=True,
+                env=True,
+            ),
+            # Make sure GNU parallel uses the right shell, because it seems to
+            # defaults to /bin/sh since all cmds are run in a python shell which
+            # does not have $SHELL set.
+            types.ShellCmdSpec(
+                cmd=f"export PARALLEL_SHELL={shell}", shell=True, wait=True, env=True
+            ),
+        ]
+
+    def post_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        return []
+
+    def exec_exp_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
+        resume = ""
+
+        # This can't be --resume, because then GNU parallel looks at the results
+        # directory, and if there is stuff in it, (apparently) assumes that the
+        # job finished...
+        if exec_opts["exec_resume"]:
+            resume = "--resume-failed"
+
+        parallel = (
+            "parallel {1} "
+            "--jobs {2} "
+            "--results {0} "
+            "--joblog {3} "
+            '--no-notice < "{4}"'
+        )
+
+        log = pathlib.Path(exec_opts["exp_scratch_root"], "parallel.log")
+        parallel = parallel.format(
+            exec_opts["exp_scratch_root"],
+            resume,
+            exec_opts["n_jobs"],
+            log,
+            exec_opts["cmdfile_stem_path"] + exec_opts["cmdfile_ext"],
+        )
+
+        return [types.ShellCmdSpec(cmd=parallel, shell=True, wait=True)]
+
+
+@implements.implements(bindings.IBatchShellCmdsGenerator)
+class BatchShellCmdsGenerator:
+    """
+    Generate the commands for local HPC (batch-level parallelism).
+    """
+
+    def __init__(self, cmdopts: types.Cmdopts) -> None:
+        self.cmdopts = cmdopts
+
+    def pre_batch_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        shell = shutil.which("bash")
+
+        return [
+            # Since parallel doesn't export any envvars to child processes by
+            # default, we add some common ones.
+            types.ShellCmdSpec(
+                cmd='export PARALLEL="${PARALLEL} --env LD_LIBRARY_PATH"',
+                shell=True,
+                wait=True,
+                env=True,
+            ),
+            # Make sure GNU parallel uses the right shell, because it seems to
+            # defaults to /bin/sh since all cmds are run in a python shell which
+            # does not have $SHELL set.
+            types.ShellCmdSpec(
+                cmd=f"export PARALLEL_SHELL={shell}", shell=True, wait=True, env=True
+            ),
+        ]
+
+    def post_batch_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        return []
+
+    def exec_batch_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
+        resume = ""
+
+        # This can't be --resume, because then GNU parallel looks at the results
+        # directory, and if there is stuff in it, (apparently) assumes that the
+        # job finished...
+        if exec_opts["exec_resume"]:
+            resume = "--resume-failed"
+
+        parallel = (
+            "env && parallel {1} "
+            "--jobs {2} "
+            "--results {0} "
+            "--joblog {3} "
+            '--no-notice < "{4}"'
+        )
+
+        log = pathlib.Path(exec_opts["batch_scratch_root"], "parallel.log")
+        parallel = parallel.format(
+            exec_opts["batch_scratch_root"],
+            resume,
+            exec_opts["n_jobs"],
+            log,
+            exec_opts["cmdfile_stem_path"] + exec_opts["cmdfile_ext"],
+        )
+
+        return [types.ShellCmdSpec(cmd=parallel, shell=True, wait=True)]
+
+
+__all__ = ["ExpShellCmdsGenerator", "BatchShellCmdsGenerator"]