sierra-research 1.3.11__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/proc/imagize/plugin.py

@@ -0,0 +1,270 @@
+# Copyright 2019 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+
+"""Classes for creating image files from ``.mean`` files for experiments.
+
+See :ref:`plugins/proc/imagize` for usage documentation.
+
+"""
+
+# Core packages
+import multiprocessing as mp
+import logging
+import pathlib
+import typing as tp
+
+# 3rd party packages
+import yaml
+
+# Project packages
+import sierra.core.variables.batch_criteria as bc
+from sierra.core import types, utils, batchroot, graphs, config
+from sierra.core.pipeline.stage3 import gather
+from sierra.plugins.proc.statistics import plugin as statistics
+import sierra.core.plugin as pm
+
+_logger = logging.getLogger(__name__)
+
+
+def proc_batch_exp(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """
+    Generate images for each :term:`Experiment` in the :term:`Batch Experiment`.
+
+    Ideally this is done in parallel across experiments, but this can be changed
+    to serial if memory on the SIERRA host machine is limited via
+    ``--processing-parallelism``.
+    """
+    config_path = pathlib.Path(cmdopts["project_config_root"]) / pathlib.Path(
+        config.kYAML.graphs
+    )
+    if utils.path_exists(config_path):
+        _logger.info("Loading imagizing config for project=%s", cmdopts["project"])
+        imagize_config = yaml.load(utils.utf8open(config_path), yaml.FullLoader)[
+            "imagize"
+        ]
+    else:
+        _logger.warning("%s does not exist--cannot imagize", config_path)
+        return
+
+    if not cmdopts["imagize_no_stats"]:
+        statistics.proc_batch_exp(
+            main_config, cmdopts, pathset, criteria, ImagizeInputGatherer
+        )
+
+    exp_to_imagize = utils.exp_range_calc(
+        cmdopts["exp_range"], pathset.output_root, criteria.gen_exp_names()
+    )
+
+    parallelism = cmdopts["processing_parallelism"]
+
+    tasks = []
+    for exp in exp_to_imagize:
+        exp_stat_root = pathset.stat_root / exp.name
+        exp_imagize_root = pathset.imagize_root / exp.name
+
+        tasks.extend(
+            _build_tasklist_for_exp(
+                exp_stat_root, exp_imagize_root, imagize_config, cmdopts["storage"]
+            )
+        )
+
+    # 2025-06-06 [JRH]: This works around what is apparently a nasty memory leak
+    # in hv caused by the hv.save() function for which the usual methods of
+    # clearing memory/figures do not work.  The maxtasksperchild argument +
+    # chunksize kills all the child threads periodically, which fixes the leak
+    # problem, for now.
+    _logger.debug("Starting %s workers, method=%s", parallelism, mp.get_start_method())
+    with mp.Pool(processes=parallelism, maxtasksperchild=1) as pool:
+        processed = pool.starmap_async(_worker, tasks, chunksize=10)
+
+        _logger.debug("Waiting for workers to finish")
+        processed.get()
+
+    _logger.debug("All workers finished")
+
+
+def _build_tasklist_for_exp(
+    exp_stat_root: pathlib.Path,
+    exp_imagize_root: pathlib.Path,
+    imagize_config: types.YAMLDict,
+    storage: str,
+) -> tp.List[tp.Tuple[dict, types.YAMLDict]]:
+    """Add all files from experiment to multiprocessing queue for processing.
+
+    Enqueueing for processing is done at the file-level rather than
+    per-experiment, so that for systems with more CPUs than experiments you
+    still get maximum throughput.
+    """
+    res = []
+    # For each graph in each category
+    for graph in imagize_config:
+        candidate = exp_stat_root / dict(graph)["src_stem"]
+
+        if not candidate.is_dir():
+            _logger.debug(
+                "Configured imagize source <batch stat root>/%s does not exist",
+                (candidate.relative_to(exp_stat_root)),
+            )
+            continue
+
+        imagize_output_root = exp_imagize_root / candidate.relative_to(exp_stat_root)
+        utils.dir_create_checked(imagize_output_root, exist_ok=True)
+
+        for fpath in candidate.iterdir():
+            assert (
+                fpath.is_file()
+            ), f"Imagize directory {candidate} must only contain files!"
+
+            res.append(
+                (
+                    {
+                        "input_path": fpath,
+                        "graph_stem": candidate.relative_to(exp_stat_root),
+                        "imagize_output_root": imagize_output_root,
+                        "batch_root": exp_stat_root.parent.parent,
+                        "storage": storage,
+                    },
+                    imagize_config,
+                )
+            )
+
+    return res
+
+
+def _worker(imagize_opts: dict, imagize_config: types.YAMLDict) -> None:
+    _proc_single_exp(imagize_config, imagize_opts)
+
+
+def _proc_single_exp(imagize_config: types.YAMLDict, imagize_opts: dict) -> None:
+    """Create images from the averaged ``.mean`` files from a single experiment.
+
+    If no ``.mean`` files suitable for averaging are found, nothing is done. See
+    :ref:`plugins/proc/imagize` for per-engine descriptions of what
+    "suitable" means.
+
+    Arguments:
+
+        imagize_config: Parsed YAML configuration for heatmaps.
+
+        imagize_opts: Dictionary of imagizing options.
+    """
+
+    match = None
+    for graph in imagize_config:
+        if dict(graph)["src_stem"] == str(imagize_opts["graph_stem"]):
+            match = graph
+
+    if match is not None:
+        graph_pathset = graphs.PathSet(
+            input_root=imagize_opts["input_path"].parent,
+            output_root=imagize_opts["imagize_output_root"],
+            model_root=None,
+            batchroot=imagize_opts["batch_root"],
+        )
+
+        # 2025-06-05 [JRH]: We always write stage {3,4} output data files as
+        # .csv because that is currently SIERRA's 'native' format; this may
+        # change in the future.
+        #
+        # Input paths are of the form <dir>/<dir>_<NUMBER>.{extension}
+        graphs.heatmap(
+            pathset=graph_pathset,
+            input_stem=imagize_opts["input_path"].stem,
+            output_stem=imagize_opts["input_path"].stem,
+            title=dict(match)["title"],
+            medium="storage.csv",
+            xlabel="X",
+            ylabel="Y",
+            colnames=(
+                match.get("x", "x"),
+                match.get("y", "y"),
+                match.get("z", "z"),
+            ),
+            backend="matplotlib",
+        )
+
+    else:
+        _logger.warning(
+            "No match for graph with src_stem='%s' found in configuration",
+            imagize_opts["graph_stem"],
+        )
+
+
+class ImagizeInputGatherer(gather.BaseGatherer):
+    """Gather :term:`Raw Output Data` files from all runs for imagizing.
+
+    The configured output directory for each run is searched recursively for
+    directories containing files to gather.  To be eligible for gathering and
+    later processing, files must:
+
+        - Be in a directory with the same name as the file, sans extension.
+
+        - Be non-empty
+
+        - Have a suffix which supported by the selected ``--storage`` plugin.
+
+    Recursive nesting of files *within* a directory containing files to imagize
+    is not supported--why would you do this anyway?
+    """
+
+    def __init__(
+        self,
+        main_config: types.YAMLDict,
+        gather_opts: types.SimpleDict,
+        processq: mp.Queue,
+    ) -> None:
+        super().__init__(main_config, gather_opts, processq)
+        self.logger = logging.getLogger(__name__)
+
+        self.config_path = (
+            pathlib.Path(gather_opts["project_config_root"]) / config.kYAML.graphs
+        )
+
+        self.imagize_config = yaml.load(
+            utils.utf8open(self.config_path), yaml.FullLoader
+        )["imagize"]
+
+    def calc_gather_items(
+        self, run_output_root: pathlib.Path, exp_name: str
+    ) -> tp.List[gather.GatherSpec]:
+        to_gather = []
+        proj_output_root = run_output_root / str(self.run_metrics_leaf)
+        plugin = pm.pipeline.get_plugin_module(self.gather_opts["storage"])
+
+        for item in proj_output_root.rglob("*"):
+            if not item.is_dir():
+                continue
+
+            for imagizable in item.iterdir():
+                if (
+                    not any(s in plugin.suffixes() for s in imagizable.suffixes)
+                    and imagizable.stat().st_size > 0
+                ):
+                    continue
+
+                if not any(
+                    g["src_stem"] in str(imagizable) for g in self.imagize_config
+                ):
+                    continue
+
+                if item.name in imagizable.name:
+                    to_gather.append(
+                        gather.GatherSpec(
+                            exp_name=exp_name,
+                            item_stem_path=imagizable.relative_to(proj_output_root),
+                            collate_col=None,
+                        )
+                    )
+
+        return to_gather
+
+
+__all__ = [
+    "proc_batch_exp",
+]

sierra/plugins/proc/modelrunner/__init__.py

@@ -0,0 +1,16 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"Plugin for running models during stage 3."
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/proc/modelrunner/plugin.py

@@ -0,0 +1,250 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Functionality for runner different types of models.
+
+See :ref:`plugins/proc/modelrunner` for usage documentation.
+
+"""
+# Core packages
+import pathlib
+import time
+import datetime
+import typing as tp
+import logging
+
+# 3rd party packages
+import yaml
+
+# Project packages
+from sierra.core import config, utils, types, batchroot, storage, exproot
+from sierra.core.variables import batch_criteria as bc
+from sierra.core import plugin as pm
+from sierra.core.models import interface
+
+_logger = logging.getLogger(__name__)
+
+
+def proc_batch_exp(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """
+    Run all intra- and inter-exp models.
+    """
+    models = _load_models(main_config, cmdopts, "intra")
+
+    _logger.info("Running %d intra-experiment models...", len(models))
+    start = time.time()
+    _run_intra_exp(cmdopts, pathset, models, criteria)
+    elapsed = int(time.time() - start)
+    sec = datetime.timedelta(seconds=elapsed)
+    _logger.info("Intra-experiment models finished in %s", str(sec))
+
+    models = _load_models(main_config, cmdopts, "inter")
+
+    _logger.info("Running %d inter-experiment models...", len(models))
+    start = time.time()
+
+    _run_inter_exp(cmdopts, pathset, models, criteria)
+
+    elapsed = int(time.time() - start)
+    sec = datetime.timedelta(seconds=elapsed)
+    _logger.info("Inter-experiment models finished in %s", str(sec))
+
+
+def _load_models(
+    main_config: types.YAMLDict, cmdopts: types.Cmdopts, model_type: str
+) -> tp.Dict[str, tp.Union[tp.Dict[str, tp.Any]]]:
+    project_models = pathlib.Path(cmdopts["project_config_root"]) / config.kYAML.models
+    loaded = {}
+
+    _logger.info("Loading %s-exp models for project %s", model_type, cmdopts["project"])
+
+    models_config = yaml.load(utils.utf8open(project_models), yaml.FullLoader)
+
+    # This is ALL model plugins found by SIERRA.
+    loaded_model_plugins = [
+        p
+        for p in pm.pipeline.loaded_plugins()
+        if pm.pipeline.loaded_plugins()[p]["type"] == "model"
+    ]
+    _logger.debug(
+        "%d loaded model plugins on SIERRA_PLUGIN_PATH", len(loaded_model_plugins)
+    )
+
+    for plugin_name in loaded_model_plugins:
+        # This is all available models for a specific plugin
+        module = pm.module_load(plugin_name)
+        available_models = getattr(module, "sierra_models")(model_type)
+        _logger.debug(
+            "Loaded model plugin %s has %d %s-exp models: %s",
+            plugin_name,
+            len(available_models) if available_models else 0,
+            model_type,
+            available_models,
+        )
+        if models_config.get(f"{model_type}-exp"):
+            for conf in models_config[f"{model_type}-exp"]:
+
+                # Class name of the model is the last part of the dot-separated
+                # path, and plays no part in module lookup in the python
+                # interpreter.
+                model_name = conf["name"].split(".")[-1]
+
+                # The part of the model name which is relative to the plugin
+                # directory in which it should be found.
+                model_module_relpath = ".".join(conf["name"].split(".")[:-1])
+                model_path = "{0}.{1}".format(plugin_name, model_module_relpath)
+                model_fullpath = f"{model_module_relpath}.{model_name}"
+                if pm.module_exists(model_path):
+                    _logger.debug(
+                        "Loading %s-exp model %s using path %s: YAML configuration match",
+                        model_type,
+                        model_name,
+                        model_module_relpath,
+                    )
+                    model_params = {
+                        k: v for k, v in conf.items() if k not in ["name", "targets"]
+                    }
+                    loaded[model_fullpath] = {
+                        "targets": conf["targets"],
+                        "legend": (
+                            conf["legend"]
+                            if "legend" in conf
+                            else ["Model Prediction" for t in conf["targets"]]
+                        ),
+                        "model": getattr(pm.module_load(model_path), model_name)(
+                            model_params
+                        ),
+                    }
+
+        else:
+            _logger.debug("All %s-exp models disabled: no configuration", model_type)
+
+    if len(loaded) > 0:
+        _logger.info(
+            "Loaded %s-exp %s models for project %s",
+            len(loaded),
+            model_type,
+            cmdopts["project"],
+        )
+
+    return loaded
+
+
+def _run_intra_exp(
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    to_run: tp.Dict[str, tp.Dict[str, tp.Any]],
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """
+    Run all enabled intra-experiment models for all experiments in a batch.
+    """
+    exp_dirnames = criteria.gen_exp_names()
+    exp_to_run = utils.exp_range_calc(
+        cmdopts["exp_range"], pathset.output_root, exp_dirnames
+    )
+
+    for exp in exp_to_run:
+        exp_index = exp_dirnames.index(exp.name)
+        exproots = exproot.PathSet(pathset, exp.name, exp_dirnames[0])
+
+        utils.dir_create_checked(exproots.model_root, exist_ok=True)
+
+        for blob in to_run.values():
+            _run_intra_single_in_exp(criteria, cmdopts, exproots, exp_index, blob)
+
+
+def _run_intra_single_in_exp(
+    criteria: bc.XVarBatchCriteria,
+    cmdopts: types.Cmdopts,
+    pathset: exproot.PathSet,
+    exp_index: int,
+    blob: tp.Dict[str, tp.Union[interface.IIntraExpModel1D, tp.List[str]]],
+) -> None:
+    model = blob["model"]
+    targets = blob["targets"]
+    if not model.should_run(criteria, cmdopts, exp_index):
+        _logger.debug(
+            "Skip running intra-experiment model from '%s' for exp%s",
+            str(model),
+            exp_index,
+        )
+        return
+
+    # Run the model
+    _logger.debug(
+        "Run intra-experiment model %s for %s",
+        str(model),
+        pathset.input_root.name,
+    )
+    dfs = model.run(criteria, exp_index, cmdopts, pathset)
+    legend = blob["legend"]
+
+    for df, target in zip(dfs, targets):
+        path_stem = pathset.model_root / target
+
+        # Write model legend file so the generated graph can find it
+        idx = dfs.index(df)
+        with utils.utf8open(
+            path_stem.with_suffix(config.kModelsExt["legend"]), "w"
+        ) as f:
+            f.write(legend[idx])
+
+        # Write model .csv file
+        storage.df_write(
+            df,
+            path_stem.with_suffix(config.kModelsExt["model"]),
+            "storage.csv",
+            index=False,
+        )
+
+
+def _run_inter_exp(
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    to_run: tp.Dict[str, tp.Any],
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    utils.dir_create_checked(pathset.model_interexp_root, exist_ok=True)
+
+    for blob in to_run.values():
+        model = blob["model"]
+        legend = blob["legend"]
+        targets = blob["targets"]
+
+        if not model.should_run(criteria, cmdopts):
+            _logger.debug("Skip running inter-experiment model '%s'", str(model))
+            continue
+
+        # Run the model
+        _logger.debug("Run inter-experiment model '%s'", str(model))
+
+        dfs = model.run(criteria, cmdopts, pathset)
+
+        for df, csv_stem in zip(dfs, targets):
+            path_stem = pathset.model_interexp_root / csv_stem
+            utils.dir_create_checked(path_stem.parent, exist_ok=True)
+
+            # Write model .csv file
+            storage.df_write(
+                df,
+                path_stem.with_suffix(config.kModelsExt["model"]),
+                "storage.csv",
+                index=True,
+            )
+
+            idx = dfs.index(df)
+            with utils.utf8open(
+                path_stem.with_suffix(config.kModelsExt["legend"]), "w"
+            ) as f:
+                f.write(legend[idx])
+
+
+__all__ = ["proc_batch_exp"]

sierra/plugins/proc/statistics/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Container module for the statistics generation data processing plugin."""
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/proc/statistics/cmdline.py

@@ -0,0 +1,64 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+import typing as tp
+import argparse
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import types
+from sierra.plugins import PluginCmdline
+
+
+def build(
+    parents: tp.List[argparse.ArgumentParser], stages: tp.List[int]
+) -> PluginCmdline:
+    """
+    Get a cmdline supporting the ``proc.statistics`` plugin.
+    """
+    cmdline = PluginCmdline(parents, stages)
+    cmdline.multistage.add_argument(
+        "--dist-stats",
+        choices=["none", "all", "conf95", "bw"],
+        help="""
+             Specify what kinds of statistics, if any, should be calculated on
+             the distribution of experimental data during stage 3 for inclusion
+             on graphs during stage 4:
+
+                 - ``none`` - Only calculate and show raw mean on graphs.
+
+                 - ``conf95`` - Calculate standard deviation of experimental
+                   distribution and show 95%% confidence interval on relevant
+                   graphs.
+
+                 - ``bw`` - Calculate statistics necessary to show box and
+                   whisker plots around each point in the graph (Summary Line
+                   graphs only).
+
+                 - ``all`` - Generate all possible statistics, and plot all
+                   possible statistics on graphs.
+             """
+        + cmdline.graphs_applicable_doc(
+            [
+                ":py:func:`Summary Line <sierra.core.graphs.summary_line.generate>`",
+                ":py:func:`Stacked Line <sierra.core.graphs.stacked_line.generate>`",
+            ]
+        )
+        + cmdline.stage_usage_doc([3, 4, 5]),
+        default="none",
+    )
+
+    return cmdline
+
+
+def to_cmdopts(args: argparse.Namespace) -> types.Cmdopts:
+    return {"dist_stats": args.dist_stats}
+
+
+def sphinx_cmdline_multistage():
+    return build([], [3, 4, 5]).parser