sierra-research 1.3.6__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/core/pipeline/stage1/pipeline_stage1.py

@@ -2,20 +2,19 @@
 #
 #  SPDX-License-Identifier: MIT
 
-"""Stage 1 of the experimental pipeline: generating experimental inputs.
-
-"""
+"""Stage 1 of the experimental pipeline: generating experimental inputs."""
 
 # Core packges
 import logging
+import time
+import datetime
 
 # 3rd party packages
 
 # Project packages
-from sierra.core.generators.exp_generators import BatchExpDefGenerator
-from sierra.core.generators.exp_creator import BatchExpCreator
+from sierra.core.generators.experiment import BatchExpDefGenerator, BatchExpCreator
 import sierra.core.variables.batch_criteria as bc
-from sierra.core import types
+from sierra.core import types, batchroot
 
 
 class PipelineStage1:
@@ -28,15 +27,25 @@ class PipelineStage1:
 
     """
 
-    def __init__(self,
-                 cmdopts: types.Cmdopts,
-                 controller: str,
-                 criteria: bc.IConcreteBatchCriteria) -> None:
-        self.generator = BatchExpDefGenerator(controller_name=controller,
-                                              scenario_basename=cmdopts['scenario'],
-                                              criteria=criteria,
-                                              cmdopts=cmdopts)
-        self.creator = BatchExpCreator(criteria=criteria, cmdopts=cmdopts)
+    def __init__(
+        self,
+        cmdopts: types.Cmdopts,
+        pathset: batchroot.PathSet,
+        controller: str,
+        criteria: bc.XVarBatchCriteria,
+    ) -> None:
+        self.generator = BatchExpDefGenerator(
+            controller_name=controller,
+            scenario_basename=cmdopts["scenario"],
+            criteria=criteria,
+            pathset=pathset,
+            cmdopts=cmdopts,
+        )
+
+        self.creator = BatchExpCreator(
+            criteria=criteria, cmdopts=cmdopts, pathset=pathset
+        )
+        self.pathset = pathset
 
         self.cmdopts = cmdopts
         self.criteria = criteria
@@ -55,12 +64,12 @@ class PipelineStage1:
 
         #. Generate changes to be applied to to the newly written per-experiment
            "template" XML file for each experiment which are non-:term:`Batch
-           Criteria` related (e.g., :term:`Platform` changes). These changes
+           Criteria` related (e.g., :term:`Engine` changes). These changes
            apply to all experiment runs in an experiment, but may differ across
-           experimental runs (e.g., # cores used for a simulator platform).
+           experimental runs (e.g., # cores used for a simulator engine).
 
         #. Generate per-experimental run changes for each experimental run in
-           the experiment, according to platform and :term:`Project`
+           the experiment, according to engine and :term:`Project`
            configuration.
 
         #. Write the input files for all experimental runs in all experiments to
@@ -68,17 +77,22 @@ class PipelineStage1:
 
         """
 
-        self.logger.info("Generating input files for batch experiment in %s...",
-                         self.cmdopts['batch_root'])
-        self.creator.create(self.generator)
-
-        n_exp_in_batch = len(self.criteria.gen_attr_changelist()) + \
-            len(self.criteria.gen_tag_addlist())
-        self.logger.info("%d input files generated in %d experiments.",
-                         self.cmdopts['n_runs'] * n_exp_in_batch,
-                         n_exp_in_batch)
+        self.logger.info("Generating batch experiment in %s...", self.pathset.root)
 
-
-__api__ = [
-    'PipelineStage1'
-]
+        start = time.time()
+        self.creator.create(self.generator)
+        elapsed = int(time.time() - start)
+        sec = datetime.timedelta(seconds=elapsed)
+        n_exp_in_batch = len(self.criteria.gen_attr_changelist()) + len(
+            self.criteria.gen_element_addlist()
+        )
+        self.logger.info(
+            "Generation complete in %s: %d experiments, %d runs per experiment, %d runs total",
+            str(sec),
+            n_exp_in_batch,
+            self.cmdopts["n_runs"],
+            self.cmdopts["n_runs"] * n_exp_in_batch,
+        )
+
+
+__all__ = ["PipelineStage1"]

sierra/core/pipeline/stage2/__init__.py

@@ -0,0 +1,10 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""Container module for stage 2 of the pipeline."""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages

sierra/core/pipeline/stage2/pipeline_stage2.py

@@ -2,9 +2,7 @@
 #
 #  SPDX-License-Identifier: MIT
 
-"""Stage 2 of the experimental pipeline: running experiments.
-
-"""
+"""Stage 2 of the experimental pipeline: running experiments."""
 
 # Core packages
 import time
@@ -15,8 +13,8 @@ import logging
 
 # Project packages
 from sierra.core.variables import batch_criteria as bc
-from sierra.core.pipeline.stage2.exp_runner import BatchExpRunner
-from sierra.core import types
+from sierra.core.pipeline.stage2.runner import BatchExpRunner
+from sierra.core import types, batchroot
 
 
 class PipelineStage2:
@@ -30,18 +28,17 @@ class PipelineStage2:
 
     """
 
-    def __init__(self, cmdopts: types.Cmdopts) -> None:
+    def __init__(self, cmdopts: types.Cmdopts, pathset: batchroot.PathSet) -> None:
         self.logger = logging.getLogger(__name__)
         self.cmdopts = cmdopts
+        self.pathset = pathset
 
-    def run(self, criteria: bc.BatchCriteria) -> None:
+    def run(self, criteria: bc.XVarBatchCriteria) -> None:
         start = time.time()
-        BatchExpRunner(self.cmdopts, criteria)()
+        BatchExpRunner(self.cmdopts, self.pathset, criteria)()
         elapsed = int(time.time() - start)
         sec = datetime.timedelta(seconds=elapsed)
         self.logger.info("Execution complete in %s", str(sec))
 
 
-__api__ = [
-    'PipelineStage2'
-]
+__all__ = ["PipelineStage2"]

sierra/core/pipeline/stage2/runner.py

@@ -0,0 +1,401 @@
+# Copyright 2018 London Lowmanstone, John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""Classes for executing experiments via the specified ``--execenv``."""
+
+# Core packages
+import os
+import subprocess
+import time
+import sys
+import datetime
+import logging
+import pathlib
+import typing as tp  # noqa: F401
+
+# 3rd party packages
+
+# Project packages
+from sierra.core.variables import batch_criteria as bc
+from sierra.core import types, config, engine, utils, batchroot, execenv
+import sierra.core.plugin as pm
+
+
+class ExpShell:
+    """Launch a shell which persists across experimental runs.
+
+    Having a persistent shell is necessary so that running pre- and post-run
+    shell commands have an effect on the actual commands to execute the run. If
+    you set an environment variable before the simulator launches (for example),
+    and then the shell containing that change exits, and the simulator launches
+    in a new shell, then the configuration has no effect. Thus, a persistent
+    shell.
+
+    """
+
+    def __init__(self, exec_strict: bool) -> None:
+        self.env = os.environ.copy()
+        self.logger = logging.getLogger(__name__)
+        self.procs = []  # type: tp.List[subprocess.Popen]
+        self.exec_strict = exec_strict
+
+    def run_from_spec(self, spec: types.ShellCmdSpec) -> bool:
+        self.logger.trace("Cmd: %s", spec.cmd)  # type: ignore
+
+        # We use a special marker at the end of the cmd's output to know when
+        # the environment dump starts.
+        if spec.env:
+            spec.cmd += " && echo ~~~~ENV_START~~~~ && env"
+
+        proc = subprocess.Popen(
+            spec.cmd,
+            shell=spec.shell,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=self.env,
+        )
+
+        if not spec.wait:
+            self.procs.append(proc)
+            return True
+
+        # We use communicate(), not wait() to avoid issues with IO buffers
+        # becoming full (e.g., you get deadlocks with wait() regularly).
+        stdout_raw, stderr_raw = proc.communicate()
+
+        # Update the environment for all commands
+        if spec.env:
+            self._update_env(stdout_raw)
+
+        # Only show output if the process failed (i.e., did not return 0)
+        if proc.returncode != 0:
+            self.logger.error("Cmd '%s' failed!", spec.cmd)
+            stdout_str = stdout_raw.decode("utf-8")
+            stderr_str = stderr_raw.decode("utf-8")
+
+            if spec.env:
+                stdout_str = stdout_str.split("~~~~ENV_START~~~~", maxsplit=1)[0]
+                stderr_str = stderr_str.split("~~~~ENV_START~~~~", maxsplit=1)[0]
+
+            self.logger.error(
+                "Cmd stdout (last 10 lines): %s",
+                "\n + " "\n".join(stdout_str.split("\n")[-10:]),
+            )
+            self.logger.error(
+                "Cmd stderr (last 10 lines): %s",
+                "\n" + "\n".join(stderr_str.split("\n")[-10:]),
+            )
+            if self.exec_strict:
+                raise RuntimeError(
+                    ("Command failed and strict checking was " "requested")
+                )
+
+            return False
+        else:
+            return True
+
+    def _update_env(self, stdout) -> None:
+        record = False
+        for e in stdout.decode("utf-8").split("\n"):
+            if record:
+                candidate = e.strip().split("=")
+                if len(candidate) != 2:
+                    continue
+
+                key = candidate[0]
+                value = candidate[1]
+
+                if key not in self.env or self.env[key] != value:
+                    self.logger.debug(
+                        "Update experiment environment: %s=%s", key, value
+                    )
+                    self.env[key] = value
+            elif e.strip() == "~~~~ENV_START~~~~":
+                record = True
+
+
+class BatchExpRunner:
+    """Runs each :term:`Experiment` in :term:`Batch Experiment` in sequence.
+
+    Attributes:
+        batch_exp_root: Absolute path to the root directory for the batch
+                        experiment inputs (i.e. experiment directories are
+                        placed in here).
+
+        batch_stat_root: Absolute path to the root directory for statistics
+                         which are computed in stage {3,4} (i.e. experiment
+                         directories are placed in here).
+
+        batch_stat_exec_root: Absolute path to the root directory for statistics
+                              which are generated as experiments run during
+                              stage 2 (e.g., how long each experiment took).
+
+        cmdopts: Dictionary of parsed cmdline options.
+
+        criteria: Batch criteria for the experiment.
+
+        exec_exp_range: The subset of experiments in the batch to run (can be
+                        None to run all experiments in the batch).
+
+    """
+
+    def __init__(
+        self,
+        cmdopts: types.Cmdopts,
+        pathset: batchroot.PathSet,
+        criteria: bc.XVarBatchCriteria,
+    ) -> None:
+        self.cmdopts = cmdopts
+        self.criteria = criteria
+        self.pathset = pathset
+        self.exec_exp_range = self.cmdopts["exp_range"]
+
+        self.logger = logging.getLogger(__name__)
+
+        utils.dir_create_checked(self.pathset.stat_exec_root, exist_ok=True)
+        utils.dir_create_checked(self.pathset.scratch_root, exist_ok=True)
+
+    def __call__(self) -> None:
+        """
+        Execute experiments in the batch according to configuration.
+
+        """
+        self.logger.info(
+            "Engine=%s, execenv=%s",
+            self.cmdopts["engine"],
+            self.cmdopts["execenv"],
+        )
+
+        module = pm.pipeline.get_plugin_module(self.cmdopts["engine"])
+
+        # Output some useful information before running
+        if hasattr(module, "pre_exp_diagnostics"):
+            module.pre_exp_diagnostics(self.cmdopts, self.pathset, self.logger)
+
+        exp_all = [self.pathset.input_root / d for d in self.criteria.gen_exp_names()]
+
+        exp_to_run = utils.exp_range_calc(
+            self.cmdopts["exp_range"],
+            self.pathset.input_root,
+            self.criteria.gen_exp_names(),
+        )
+
+        # Verify environment is OK before running anything
+        self.logger.debug("Checking --engine execution environment")
+        engine.execenv_check(self.cmdopts)
+
+        self.logger.debug("Checking --execenv execution environment")
+        execenv.execenv_check(self.cmdopts)
+
+        # Calculate path for to file for logging execution times
+        now = datetime.datetime.now()
+        exec_times_fpath = self.pathset.stat_exec_root / now.strftime("%Y-%m-%e-%H:%M")
+
+        # Start a new process for the experiment shell so pre-run commands have
+        # an effect (if they set environment variables, etc.).
+        shell = ExpShell(self.cmdopts["exec_strict"])
+
+        if self.cmdopts["exec_parallelism_paradigm"] is not None:
+            self.logger.warning(
+                "Overriding engine=%s parallelism paradigm with %s",
+                self.cmdopts["engine"],
+                self.cmdopts["exec_parallelism_paradigm"],
+            )
+            parallelism_paradigm = self.cmdopts["exec_parallelism_paradigm"]
+        else:
+            configurer = engine.ExpConfigurer(self.cmdopts)
+            parallelism_paradigm = configurer.parallelism_paradigm()
+
+        if parallelism_paradigm == "per-batch":
+            ParallelRunner(
+                self.pathset, self.cmdopts, exec_times_fpath, exp_all, shell
+            )(exp_to_run)
+
+        else:
+            # Run the experiment!
+            for exp in exp_to_run:
+                exp_num = exp_all.index(exp)
+
+                # Run cmds for engine-specific things to setup the experiment
+                # (e.g., start daemons) if needed.
+                engine_generator = engine.ExpShellCmdsGenerator(self.cmdopts, exp_num)
+                execenv_generator = execenv.ExpShellCmdsGenerator(self.cmdopts, exp_num)
+
+                for spec in execenv_generator.pre_exp_cmds():
+                    shell.run_from_spec(spec)
+
+                for spec in engine_generator.pre_exp_cmds():
+                    shell.run_from_spec(spec)
+
+                runner = SequentialRunner(
+                    self.pathset,
+                    self.cmdopts,
+                    exec_times_fpath,
+                    execenv_generator,
+                    shell,
+                )
+                runner(exp.name, exp_num)
+
+                # Run cmds to cleanup {execenv, engine}-specific things now that
+                # the experiment is done (if needed).
+                for spec in execenv_generator.post_exp_cmds():
+                    shell.run_from_spec(spec)
+
+                for spec in engine_generator.post_exp_cmds():
+                    shell.run_from_spec(spec)
+
+
+class SequentialRunner:
+    """
+    Execute each :term:`Experimental Run` in an :term:`Experiment`.
+
+    Runs are executed parallel if the selected execution environment supports
+    it, otherwise sequentially. This class is meant for executing experiments
+    within a batch sequentially.
+    """
+
+    def __init__(
+        self,
+        pathset: batchroot.PathSet,
+        cmdopts: types.Cmdopts,
+        exec_times_fpath: pathlib.Path,
+        generator: execenv.ExpShellCmdsGenerator,
+        shell: ExpShell,
+    ) -> None:
+
+        self.exec_times_fpath = exec_times_fpath
+        self.shell = shell
+        self.generator = generator
+        self.cmdopts = cmdopts
+        self.pathset = pathset
+        self.logger = logging.getLogger(__name__)
+
+    def __call__(self, exp_name: str, exp_num: int) -> None:
+        """Execute experimental runs for a single experiment."""
+        exp_input_root = self.pathset.input_root / exp_name
+        exp_scratch_root = self.pathset.scratch_root / exp_name
+        self.logger.info(
+            "Running exp%s in <batchroot>/%s",
+            exp_num,
+            exp_input_root.relative_to(self.pathset.root),
+        )
+        sys.stdout.flush()
+
+        start = time.time()
+
+        utils.dir_create_checked(exp_scratch_root, exist_ok=True)
+
+        # TODO: This restriction should be removed/pushed down to execution
+        # environments that require it.
+        assert (
+            self.cmdopts["exec_jobs_per_node"] is not None
+        ), "# parallel jobs can't be None"
+
+        exec_opts = {
+            "exp_input_root": str(exp_input_root),
+            "work_dir": exp_input_root,
+            "exp_scratch_root": str(exp_scratch_root),
+            "cmdfile_stem_path": str(
+                exp_input_root / config.kGNUParallel["cmdfile_stem"]
+            ),
+            "cmdfile_ext": config.kGNUParallel["cmdfile_ext"],
+            "exec_resume": self.cmdopts["exec_resume"],
+            "n_jobs": self.cmdopts["exec_jobs_per_node"],
+            "nodefile": self.cmdopts["nodefile"],
+        }
+
+        for spec in self.generator.exec_exp_cmds(exec_opts):
+            if not self.shell.run_from_spec(spec):
+                self.logger.error(
+                    "Check outputs in %s for full details",
+                    exec_opts["exp_scratch_root"],
+                )
+
+        elapsed = int(time.time() - start)
+        sec = datetime.timedelta(seconds=elapsed)
+        self.logger.info("Exp%s elapsed time: %s", exp_num, sec)
+
+        with utils.utf8open(self.exec_times_fpath, "a") as f:
+            f.write("exp" + str(exp_num) + ": " + str(sec) + "\n")
+
+
+class ParallelRunner:
+    """
+    Execute all runs in all :term:`Experiments <Experiment>` in parallel.
+
+    This class is meant for executing experiments within a batch concurrently.
+    """
+
+    def __init__(
+        self,
+        pathset: batchroot.PathSet,
+        cmdopts: types.Cmdopts,
+        exec_times_fpath: pathlib.Path,
+        exp_all: tp.List[pathlib.Path],
+        shell: ExpShell,
+    ) -> None:
+
+        self.exec_times_fpath = exec_times_fpath
+        self.shell = shell
+        self.exp_all = exp_all
+        self.cmdopts = cmdopts
+        self.pathset = pathset
+        self.logger = logging.getLogger(__name__)
+
+    def __call__(self, exp_to_run: tp.List[pathlib.Path]) -> None:
+        """Execute all experimental runs for all experiments."""
+
+        self.logger.info("Kicking off all experiments in <batchroot>")
+        sys.stdout.flush()
+
+        start = time.time()
+
+        exp_scratch_root = self.pathset.scratch_root
+        utils.dir_create_checked(exp_scratch_root, exist_ok=True)
+        exec_opts = {
+            "batch_root": str(self.pathset.root),
+            "work_dir": str(self.pathset.root),
+            "batch_scratch_root": str(self.pathset.scratch_root),
+            "cmdfile_stem_path": str(
+                self.pathset.root / config.kGNUParallel["cmdfile_stem"]
+            ),
+            "cmdfile_ext": config.kGNUParallel["cmdfile_ext"],
+            "exec_resume": self.cmdopts["exec_resume"],
+            "n_jobs": self.cmdopts["exec_jobs_per_node"],
+        }
+
+        # Run cmds for engine-specific things to setup the experiment
+        # (e.g., start daemons) if needed.
+        engine_generator = engine.BatchShellCmdsGenerator(self.cmdopts)
+        execenv_generator = execenv.BatchShellCmdsGenerator(self.cmdopts)
+
+        for spec in execenv_generator.pre_batch_cmds():
+            self.shell.run_from_spec(spec)
+
+        for spec in engine_generator.pre_batch_cmds():
+            self.shell.run_from_spec(spec)
+
+        for spec in execenv_generator.exec_batch_cmds(exec_opts):
+            if not self.shell.run_from_spec(spec):
+                self.logger.error(
+                    "Check outputs in %s for full details",
+                    exec_opts["batch_scratch_root"],
+                )
+
+        # Run cmds to cleanup {execenv, engine}-specific things now that
+        # the experiment is done (if needed).
+        for spec in execenv_generator.post_batch_cmds():
+            self.shell.run_from_spec(spec)
+
+        for spec in engine_generator.post_batch_cmds():
+            self.shell.run_from_spec(spec)
+
+        elapsed = int(time.time() - start)
+        sec = datetime.timedelta(seconds=elapsed)
+        self.logger.info("Elapsed time: %s", sec)
+
+        with utils.utf8open(self.exec_times_fpath, "a") as f:
+            f.write(": " + str(sec) + "\n")
+
+
+__all__ = ["BatchExpRunner", "SequentialRunner", "ParallelRunner", "ExpShell"]

sierra/core/pipeline/stage3/__init__.py

@@ -0,0 +1,12 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Container module for stage 3 of the pipeline."""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages